From e97f98e5e27a6df3827564cce594f27f18c89267 Mon Sep 17 00:00:00 2001 From: Evgeny Zinoviev Date: Wed, 14 Jun 2023 14:06:26 +0300 Subject: wip --- bin/ipcam_capture.py | 141 ++++++++++++++++++++++++++++++++++ bin/ipcam_capture.sh | 119 ----------------------------- bin/ipcam_rtsp2hls.sh | 127 ------------------------------- bin/ipcam_server.py | 205 ++++++++++++++++++++------------------------------ bin/web_api.py | 1 - 5 files changed, 223 insertions(+), 370 deletions(-) create mode 100755 bin/ipcam_capture.py delete mode 100755 bin/ipcam_capture.sh delete mode 100755 bin/ipcam_rtsp2hls.sh (limited to 'bin') diff --git a/bin/ipcam_capture.py b/bin/ipcam_capture.py new file mode 100755 index 0000000..5de14af --- /dev/null +++ b/bin/ipcam_capture.py @@ -0,0 +1,141 @@ +#!/usr/bin/env python3 +import __py_include +import sys +import os +import subprocess +import asyncio +import signal + +from typing import TextIO +from argparse import ArgumentParser +from socket import gethostname +from asyncio.streams import StreamReader +from homekit.config import LinuxBoardsConfig, config as homekit_config +from homekit.camera import IpcamConfig, CaptureType +from homekit.camera.util import get_hls_directory, get_hls_channel_name, get_recordings_path + +ipcam_config = IpcamConfig() +lbc_config = LinuxBoardsConfig() +channels = (1, 2) +tasks = [] +restart_delay = 3 +lock = asyncio.Lock() +worker_type: CaptureType + + +async def read_output(stream: StreamReader, + thread_name: str, + output: TextIO): + try: + while True: + line = await stream.readline() + if not line: + break + print(f"[{thread_name}] {line.decode().strip()}", file=output) + + except asyncio.LimitOverrunError: + print(f"[{thread_name}] Output limit exceeded.", file=output) + + except Exception as e: + print(f"[{thread_name}] Error occurred while reading output: {e}", file=sys.stderr) + + +async def run_ffmpeg(cam: int, channel: int): + prefix = get_hls_channel_name(cam, channel) + + if homekit_config.app_config.logging_is_verbose(): + debug_args = ['-v', '-info'] + else: + debug_args = ['-nostats', '-loglevel', 'error'] + + protocol = 'tcp' if ipcam_config.should_use_tcp_for_rtsp(cam) else 'udp' + user, pw = ipcam_config.get_rtsp_creds() + ip = ipcam_config.get_camera_ip(cam) + path = ipcam_config.get_camera_type(cam).get_channel_url(channel) + ext = ipcam_config.get_camera_container(cam) + ffmpeg_command = ['ffmpeg', *debug_args, + '-rtsp_transport', protocol, + '-i', f'rtsp://{user}:{pw}@{ip}:554{path}', + '-c', 'copy',] + + if worker_type == CaptureType.HLS: + ffmpeg_command.extend(['-bufsize', '1835k', + '-pix_fmt', 'yuv420p', + '-flags', '-global_header', + '-hls_time', '2', + '-hls_list_size', '3', + '-hls_flags', 'delete_segments', + os.path.join(get_hls_directory(cam, channel), 'live.m3u8')]) + + elif worker_type == CaptureType.RECORD: + ffmpeg_command.extend(['-f', 'segment', + '-strftime', '1', + '-segment_time', '00:10:00', + '-segment_atclocktime', '1', + os.path.join(get_recordings_path(cam), f'record_%Y-%m-%d-%H.%M.%S.{ext.value}')]) + + else: + raise ValueError(f'invalid worker type: {worker_type}') + + while True: + try: + process = await asyncio.create_subprocess_exec( + *ffmpeg_command, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE + ) + + stdout_task = asyncio.create_task(read_output(process.stdout, prefix, sys.stdout)) + stderr_task = asyncio.create_task(read_output(process.stderr, prefix, sys.stderr)) + + await asyncio.gather(stdout_task, stderr_task) + + # check the return code of the process + if process.returncode != 0: + raise subprocess.CalledProcessError(process.returncode, ffmpeg_command) + + except (FileNotFoundError, PermissionError, subprocess.CalledProcessError) as e: + # an error occurred, print the error message + error_message = f"Error occurred in {prefix}: {e}" + print(error_message, file=sys.stderr) + + # sleep for 5 seconds before restarting the process + await asyncio.sleep(restart_delay) + + +async def run(): + kwargs = {} + if worker_type == CaptureType.RECORD: + kwargs['filter_by_server'] = gethostname() + for cam in ipcam_config.get_all_cam_names(**kwargs): + for channel in channels: + task = asyncio.create_task(run_ffmpeg(cam, channel)) + tasks.append(task) + + try: + await asyncio.gather(*tasks) + except KeyboardInterrupt: + print('KeyboardInterrupt: stopping processes...', file=sys.stderr) + for task in tasks: + task.cancel() + + # wait for subprocesses to terminate + await asyncio.gather(*tasks, return_exceptions=True) + + # send termination signal to all subprocesses + for task in tasks: + process = task.get_stack() + if process: + process.send_signal(signal.SIGTERM) + + +if __name__ == '__main__': + capture_types = [t.value for t in CaptureType] + parser = ArgumentParser() + parser.add_argument('type', type=str, metavar='CAPTURE_TYPE', choices=tuple(capture_types), + help='capture type (variants: '+', '.join(capture_types)+')') + + arg = homekit_config.load_app(no_config=True, parser=parser) + worker_type = CaptureType(arg['type']) + + asyncio.run(run()) diff --git a/bin/ipcam_capture.sh b/bin/ipcam_capture.sh deleted file mode 100755 index b97c856..0000000 --- a/bin/ipcam_capture.sh +++ /dev/null @@ -1,119 +0,0 @@ -#!/bin/bash - -PROGNAME="$0" -PORT=554 -IP= -CREDS= -DEBUG=0 -CHANNEL=1 -FORCE_UDP=0 -FORCE_TCP=0 -EXTENSION="mp4" - -die() { - echo >&2 "error: $@" - exit 1 -} - -usage() { - cat <&2 "error: $@" - exit 1 -} - -usage() { - cat < bool: - return filename.startswith('record_') and filename.endswith('.mp4') - - -def filename_to_datetime(filename: str) -> datetime: - filename = os.path.basename(filename).replace('record_', '').replace('.mp4', '') - return datetime.strptime(filename, datetime_format) - - -def get_all_cams() -> list: - return [cam for cam in config['camera'].keys()] +ipcam_config = IpcamConfig() +lbc_config = LinuxBoardsConfig() # ipcam database # -------------- -class IPCamServerDatabase(SQLiteBase): +class IpcamServerDatabase(SQLiteBase): SCHEMA = 4 def __init__(self, path=None): @@ -67,7 +58,7 @@ class IPCamServerDatabase(SQLiteBase): fix_time INTEGER NOT NULL, motion_time INTEGER NOT NULL )""") - for cam in config['camera'].keys(): + for cam in ipcam_config.get_all_cam_names_for_this_server(): self.add_camera(cam) if version < 2: @@ -135,7 +126,7 @@ class IPCamServerDatabase(SQLiteBase): # ipcam web api # ------------- -class IPCamWebServer(http.HTTPServer): +class IpcamWebServer(http.HTTPServer): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) @@ -146,16 +137,16 @@ class IPCamWebServer(http.HTTPServer): self.get('/api/timestamp/{name}/{type}', self.get_timestamp) self.get('/api/timestamp/all', self.get_all_timestamps) - self.post('/api/debug/migrate-mtimes', self.debug_migrate_mtimes) self.post('/api/debug/fix', self.debug_fix) self.post('/api/debug/cleanup', self.debug_cleanup) + self.post('/api/timestamp/{name}/{type}', self.set_timestamp) self.post('/api/motion/done/{name}', self.submit_motion) self.post('/api/motion/fail/{name}', self.submit_motion_failure) - self.get('/api/motion/params/{name}', self.get_motion_params) - self.get('/api/motion/params/{name}/roi', self.get_motion_roi_params) + # self.get('/api/motion/params/{name}', self.get_motion_params) + # self.get('/api/motion/params/{name}/roi', self.get_motion_roi_params) self.queue_lock = Lock() @@ -173,7 +164,7 @@ class IPCamWebServer(http.HTTPServer): files = get_recordings_files(camera, filter, limit) if files: - time = filename_to_datetime(files[len(files)-1]['name']) + time = datetime_from_filename(files[len(files)-1]['name']) db.set_timestamp(camera, TimeFilterType.MOTION_START, time) return self.ok({'files': files}) @@ -188,7 +179,7 @@ class IPCamWebServer(http.HTTPServer): if files: times_by_cam = {} for file in files: - time = filename_to_datetime(file['name']) + time = datetime_from_filename(file['name']) if file['cam'] not in times_by_cam or times_by_cam[file['cam']] < time: times_by_cam[file['cam']] = time for cam, time in times_by_cam.items(): @@ -200,14 +191,14 @@ class IPCamWebServer(http.HTTPServer): cam = int(req.match_info['name']) file = req.match_info['file'] - fullpath = os.path.join(config['camera'][cam]['recordings_path'], file) + fullpath = os.path.join(get_recordings_path(cam), file) if not os.path.isfile(fullpath): raise ValueError(f'file "{fullpath}" does not exists') return http.FileResponse(fullpath) async def camlist(self, req: http.Request): - return self.ok(config['camera']) + return self.ok(ipcam_config.get_all_cam_names_for_this_server()) async def submit_motion(self, req: http.Request): data = await req.post() @@ -216,7 +207,7 @@ class IPCamWebServer(http.HTTPServer): timecodes = data['timecodes'] filename = data['filename'] - time = filename_to_datetime(filename) + time = datetime_from_filename(filename) try: if timecodes != '': @@ -239,27 +230,10 @@ class IPCamWebServer(http.HTTPServer): message = data['message'] db.add_motion_failure(camera, filename, message) - db.set_timestamp(camera, TimeFilterType.MOTION, filename_to_datetime(filename)) + db.set_timestamp(camera, TimeFilterType.MOTION, datetime_from_filename(filename)) return self.ok() - async def debug_migrate_mtimes(self, req: http.Request): - written = {} - for cam in config['camera'].keys(): - confdir = os.path.join(os.getenv('HOME'), '.config', f'video-util-{cam}') - for time_type in TimeFilterType: - txt_file = os.path.join(confdir, f'{time_type.value}_mtime') - if os.path.isfile(txt_file): - with open(txt_file, 'r') as fd: - data = fd.read() - db.set_timestamp(cam, time_type, int(data.strip())) - - if cam not in written: - written[cam] = [] - written[cam].append(time_type) - - return self.ok({'written': written}) - async def debug_fix(self, req: http.Request): asyncio.ensure_future(fix_job()) return self.ok() @@ -280,26 +254,26 @@ class IPCamWebServer(http.HTTPServer): async def get_all_timestamps(self, req: http.Request): return self.ok(db.get_all_timestamps()) - async def get_motion_params(self, req: http.Request): - data = config['motion_params'][int(req.match_info['name'])] - lines = [ - f'threshold={data["threshold"]}', - f'min_event_length=3s', - f'frame_skip=2', - f'downscale_factor=3', - ] - return self.plain('\n'.join(lines)+'\n') - - async def get_motion_roi_params(self, req: http.Request): - data = config['motion_params'][int(req.match_info['name'])] - return self.plain('\n'.join(data['roi'])+'\n') + # async def get_motion_params(self, req: http.Request): + # data = config['motion_params'][int(req.match_info['name'])] + # lines = [ + # f'threshold={data["threshold"]}', + # f'min_event_length=3s', + # f'frame_skip=2', + # f'downscale_factor=3', + # ] + # return self.plain('\n'.join(lines)+'\n') + # + # async def get_motion_roi_params(self, req: http.Request): + # data = config['motion_params'][int(req.match_info['name'])] + # return self.plain('\n'.join(data['roi'])+'\n') @staticmethod def _getset_timestamp_params(req: http.Request, need_time=False): values = [] cam = int(req.match_info['name']) - assert cam in config['camera'], 'invalid camera' + assert cam in ipcam_config.get_all_cam_names_for_this_server(), 'invalid camera' values.append(cam) values.append(TimeFilterType(req.match_info['type'])) @@ -307,7 +281,7 @@ class IPCamWebServer(http.HTTPServer): if need_time: time = req.query['time'] if time.startswith('record_'): - time = filename_to_datetime(time) + time = datetime_from_filename(time) elif time.isnumeric(): time = int(time) else: @@ -322,30 +296,22 @@ class IPCamWebServer(http.HTTPServer): def open_database(database_path: str): global db - db = IPCamServerDatabase(database_path) + db = IpcamServerDatabase(database_path) # update cams list in database, if needed - cams = db.get_all_timestamps().keys() - for cam in config['camera']: - if cam not in cams: + stored_cams = db.get_all_timestamps().keys() + for cam in ipcam_config.get_all_cam_names_for_this_server(): + if cam not in stored_cams: db.add_camera(cam) -def get_recordings_path(cam: int) -> str: - return config['camera'][cam]['recordings_path'] - - -def get_motion_path(cam: int) -> str: - return config['camera'][cam]['motion_path'] - - def get_recordings_files(cam: Optional[int] = None, time_filter_type: Optional[TimeFilterType] = None, limit=0) -> List[dict]: from_time = 0 to_time = int(time.time()) - cams = [cam] if cam is not None else get_all_cams() + cams = [cam] if cam is not None else ipcam_config.get_all_cam_names_for_this_server() files = [] for cam in cams: if time_filter_type: @@ -362,7 +328,7 @@ def get_recordings_files(cam: Optional[int] = None, 'name': file, 'size': os.path.getsize(os.path.join(recdir, file))} for file in os.listdir(recdir) - if valid_recording_name(file) and from_time < filename_to_datetime(file) <= to_time] + if is_valid_recording_name(file) and from_time < datetime_from_filename(file) <= to_time] cam_files.sort(key=lambda file: file['name']) if cam_files: @@ -382,7 +348,7 @@ def get_recordings_files(cam: Optional[int] = None, async def process_fragments(camera: int, filename: str, fragments: List[Tuple[int, int]]) -> None: - time = filename_to_datetime(filename) + time = datetime_from_filename(filename) rec_dir = get_recordings_path(camera) motion_dir = get_motion_path(camera) @@ -392,8 +358,8 @@ async def process_fragments(camera: int, for fragment in fragments: start, end = fragment - start -= config['motion']['padding'] - end += config['motion']['padding'] + start -= ipcam_config['motion_padding'] + end += ipcam_config['motion_padding'] if start < 0: start = 0 @@ -408,14 +374,14 @@ async def process_fragments(camera: int, start_pos=start, duration=duration) - if fragments and 'telegram' in config['motion'] and config['motion']['telegram']: + if fragments and ipcam_config['motion_telegram']: asyncio.ensure_future(motion_notify_tg(camera, filename, fragments)) async def motion_notify_tg(camera: int, filename: str, fragments: List[Tuple[int, int]]): - dt_file = filename_to_datetime(filename) + dt_file = datetime_from_filename(filename) fmt = '%H:%M:%S' text = f'Camera: {camera}\n' @@ -423,8 +389,8 @@ async def motion_notify_tg(camera: int, text += _tg_links(TelegramLinkType.ORIGINAL_FILE, camera, filename) for start, end in fragments: - start -= config['motion']['padding'] - end += config['motion']['padding'] + start -= ipcam_config['motion_padding'] + end += ipcam_config['motion_padding'] if start < 0: start = 0 @@ -446,7 +412,7 @@ def _tg_links(link_type: TelegramLinkType, camera: int, file: str) -> str: links = [] - for link_name, link_template in config['telegram'][f'{link_type.value}_url_templates']: + for link_name, link_template in ipcam_config[f'{link_type.value}_url_templates']: link = link_template.replace('{camera}', str(camera)).replace('{file}', file) links.append(f'{link_name}') return ' '.join(links) @@ -462,7 +428,7 @@ async def fix_job() -> None: try: fix_job_running = True - for cam in config['camera'].keys(): + for cam in ipcam_config.get_all_cam_names_for_this_server(): files = get_recordings_files(cam, TimeFilterType.FIX) if not files: logger.debug(f'fix_job: no files for camera {cam}') @@ -473,7 +439,7 @@ async def fix_job() -> None: for file in files: fullpath = os.path.join(get_recordings_path(cam), file['name']) await camutil.ffmpeg_recreate(fullpath) - timestamp = filename_to_datetime(file['name']) + timestamp = datetime_from_filename(file['name']) if timestamp: db.set_timestamp(cam, TimeFilterType.FIX, timestamp) @@ -482,21 +448,9 @@ async def fix_job() -> None: async def cleanup_job() -> None: - def fn2dt(name: str) -> datetime: - name = os.path.basename(name) - - if name.startswith('record_'): - return datetime.strptime(re.match(r'record_(.*?)\.mp4', name).group(1), datetime_format) - - m = re.match(rf'({datetime_format_re})__{datetime_format_re}\.mp4', name) - if m: - return datetime.strptime(m.group(1), datetime_format) - - raise ValueError(f'unrecognized filename format: {name}') - def compare(i1: str, i2: str) -> int: - dt1 = fn2dt(i1) - dt2 = fn2dt(i2) + dt1 = datetime_from_filename(i1) + dt2 = datetime_from_filename(i2) if dt1 < dt2: return -1 @@ -516,18 +470,19 @@ async def cleanup_job() -> None: cleanup_job_running = True gb = float(1 << 30) - for storage in config['storages']: + disk_number = 0 + for storage in lbc_config.get_board_disks(gethostname()): + disk_number += 1 if os.path.exists(storage['mountpoint']): total, used, free = shutil.disk_usage(storage['mountpoint']) free_gb = free // gb - if free_gb < config['cleanup_min_gb']: - # print(f"{storage['mountpoint']}: free={free}, free_gb={free_gb}") + if free_gb < ipcam_config['cleanup_min_gb']: cleaned = 0 files = [] - for cam in storage['cams']: - for _dir in (config['camera'][cam]['recordings_path'], config['camera'][cam]['motion_path']): + for cam in ipcam_config.get_all_cam_names_for_this_server(filter_by_disk=disk_number): + for _dir in (get_recordings_path(cam), get_motion_path(cam)): files += list(map(lambda file: os.path.join(_dir, file), os.listdir(_dir))) - files = list(filter(lambda path: os.path.isfile(path) and path.endswith('.mp4'), files)) + files = list(filter(lambda path: os.path.isfile(path) and path.endswith(tuple([f'.{t.value}' for t in VideoContainerType])), files)) files.sort(key=cmp_to_key(compare)) for file in files: @@ -537,7 +492,7 @@ async def cleanup_job() -> None: cleaned += size except OSError as e: logger.exception(e) - if (free + cleaned) // gb >= config['cleanup_min_gb']: + if (free + cleaned) // gb >= ipcam_config['cleanup_min_gb']: break else: logger.error(f"cleanup_job: {storage['mountpoint']} not found") @@ -550,8 +505,8 @@ cleanup_job_running = False datetime_format = '%Y-%m-%d-%H.%M.%S' datetime_format_re = r'\d{4}-\d{2}-\d{2}-\d{2}\.\d{2}.\d{2}' -db: Optional[IPCamServerDatabase] = None -server: Optional[IPCamWebServer] = None +db: Optional[IpcamServerDatabase] = None +server: Optional[IpcamWebServer] = None logger = logging.getLogger(__name__) @@ -562,7 +517,7 @@ if __name__ == '__main__': parser = ArgumentParser() parser.add_argument('--listen', type=str, required=True) parser.add_argument('--database-path', type=str, required=True) - arg = config.load_app(no_config=True, parser=parser) + arg = homekit_config.load_app(no_config=True, parser=parser) open_database(arg.database_path) @@ -570,10 +525,14 @@ if __name__ == '__main__': try: scheduler = AsyncIOScheduler(event_loop=loop) - if config['fix_enabled']: - scheduler.add_job(fix_job, 'interval', seconds=config['fix_interval'], misfire_grace_time=None) - - scheduler.add_job(cleanup_job, 'interval', seconds=config['cleanup_interval'], misfire_grace_time=None) + if ipcam_config['fix_enabled']: + scheduler.add_job(fix_job, 'interval', + seconds=ipcam_config['fix_interval'], + misfire_grace_time=None) + + scheduler.add_job(cleanup_job, 'interval', + seconds=ipcam_config['cleanup_interval'], + misfire_grace_time=None) scheduler.start() except KeyError: pass @@ -581,5 +540,5 @@ if __name__ == '__main__': asyncio.ensure_future(fix_job()) asyncio.ensure_future(cleanup_job()) - server = IPCamWebServer(config.get_addr('server.listen')) + server = IpcamWebServer(Addr.fromstring(arg.listen)) server.run() diff --git a/bin/web_api.py b/bin/web_api.py index e543d22..d221838 100755 --- a/bin/web_api.py +++ b/bin/web_api.py @@ -42,7 +42,6 @@ class WebAPIServer(http.HTTPServer): self.get('/sound_sensors/hits/', self.GET_sound_sensors_hits) self.post('/sound_sensors/hits/', self.POST_sound_sensors_hits) - self.post('/log/bot_request/', self.POST_bot_request_log) self.post('/log/openwrt/', self.POST_openwrt_log) self.get('/inverter/consumed_energy/', self.GET_consumed_energy) -- cgit v1.2.3