ipcam_server: centralized config, universal workers

3 files changed, 254 insertions, 137 deletions

diff --git a/src/home/http/http.py b/src/home/http/http.py
index 76197d2..3e70751 100644
--- a/src/home/http/http.py
+++ b/src/home/http/http.py
@@ -2,6 +2,7 @@ import logging
 import asyncio
 
 from aiohttp import web
+from aiohttp.web import Response
 from aiohttp.web_exceptions import HTTPNotFound
 
 from ..util import stringify, format_tb, Addr
@@ -99,4 +100,7 @@ class HTTPServer:
         event_loop.run_forever()
 
     def ok(self, data=None):
-        return ok(data)
\ No newline at end of file
+        return ok(data)
+
+    def plain(self, text: str):
+        return Response(text=text, content_type='text/plain')
diff --git a/src/ipcam_server.py b/src/ipcam_server.py
index 3fdf888..0159b77 100755
--- a/src/ipcam_server.py
+++ b/src/ipcam_server.py
@@ -39,6 +39,10 @@ def filename_to_datetime(filename: str) -> datetime:
     return datetime.strptime(filename, datetime_format)
 
 
+def get_all_cams() -> list:
+    return [cam for cam in config['camera'].keys()]
+
+
 # ipcam database
 # --------------
 
@@ -125,6 +129,7 @@ class IPCamWebServer(http.HTTPServer):
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
 
+        self.get('/api/recordings', self.get_motion_queue)
         self.get('/api/recordings/{name}', self.get_camera_recordings)
         self.get('/api/recordings/{name}/download/{file}', self.download_recording)
         self.get('/api/camera/list', self.camlist)
@@ -139,6 +144,9 @@ class IPCamWebServer(http.HTTPServer):
         self.post('/api/motion/done/{name}', self.submit_motion)
         self.post('/api/motion/fail/{name}', self.submit_motion_failure)
 
+        self.get('/api/motion/params/{name}', self.get_motion_params)
+        self.get('/api/motion/params/{name}/roi', self.get_motion_roi_params)
+
     async def get_camera_recordings(self, req):
         cam = int(req.match_info['name'])
         try:
@@ -146,8 +154,21 @@ class IPCamWebServer(http.HTTPServer):
         except KeyError:
             filter = None
 
-        files = get_recordings_files(cam, filter)
+        try:
+            limit = int(req.query['limit'])
+        except KeyError:
+            limit = 0
+
+        files = get_recordings_files(cam, filter, limit)
+        return self.ok({'files': files})
+
+    async def get_motion_queue(self, req):
+        try:
+            limit = int(req.query['limit'])
+        except KeyError:
+            limit = 0
 
+        files = get_recordings_files(None, TimeFilterType.MOTION, limit)
         return self.ok({'files': files})
 
     async def download_recording(self, req: http.Request):
@@ -234,6 +255,20 @@ class IPCamWebServer(http.HTTPServer):
     async def get_all_timestamps(self, req: http.Request):
         return self.ok(db.get_all_timestamps())
 
+    async def get_motion_params(self, req: http.Request):
+        data = config['motion_params'][int(req.match_info['name'])]
+        lines = [
+            f'threshold={data["threshold"]}',
+            f'min_event_length=3s',
+            f'frame_skip=2',
+            f'downscale_factor=3',
+        ]
+        return self.plain('\n'.join(lines)+'\n')
+
+    async def get_motion_roi_params(self, req: http.Request):
+        data = config['motion_params'][int(req.match_info['name'])]
+        return self.plain('\n'.join(data['roi'])+'\n')
+
     @staticmethod
     def _getset_timestamp_params(req: http.Request, need_time=False):
         values = []
@@ -279,33 +314,42 @@ def get_motion_path(cam: int) -> str:
     return config['camera'][cam]['motion_path']
 
 
-def get_recordings_files(cam: int,
-                         time_filter_type: Optional[TimeFilterType] = None) -> List[dict]:
+def get_recordings_files(cam: Optional[int] = None,
+                         time_filter_type: Optional[TimeFilterType] = None,
+                         limit=0) -> List[dict]:
     from_time = 0
     to_time = int(time.time())
 
-    if time_filter_type:
-        from_time = db.get_timestamp(cam, time_filter_type)
-        if time_filter_type == TimeFilterType.MOTION:
-            to_time = db.get_timestamp(cam, TimeFilterType.FIX)
-
-    from_time = datetime.fromtimestamp(from_time)
-    to_time = datetime.fromtimestamp(to_time)
-
-    recdir = get_recordings_path(cam)
-    files = [{
-        'name': file,
-        'size': os.path.getsize(os.path.join(recdir, file))}
-             for file in os.listdir(recdir)
-             if valid_recording_name(file) and from_time < filename_to_datetime(file) <= to_time]
-    files.sort(key=lambda file: file['name'])
-
-    if files:
-        last = files[len(files)-1]
-        fullpath = os.path.join(recdir, last['name'])
-        if camutil.has_handle(fullpath):
-            logger.debug(f'get_recordings_files: file {fullpath} has opened handle, ignoring it')
-            files.pop()
+    cams = [cam] if cam is not None else get_all_cams()
+    files = []
+    for cam in cams:
+        if time_filter_type:
+            from_time = db.get_timestamp(cam, time_filter_type)
+            if time_filter_type == TimeFilterType.MOTION:
+                to_time = db.get_timestamp(cam, TimeFilterType.FIX)
+
+        from_time = datetime.fromtimestamp(from_time)
+        to_time = datetime.fromtimestamp(to_time)
+
+        recdir = get_recordings_path(cam)
+        cam_files = [{
+            'cam': cam,
+            'name': file,
+            'size': os.path.getsize(os.path.join(recdir, file))}
+                 for file in os.listdir(recdir)
+                 if valid_recording_name(file) and from_time < filename_to_datetime(file) <= to_time]
+        cam_files.sort(key=lambda file: file['name'])
+
+        if cam_files:
+            last = cam_files[len(cam_files)-1]
+            fullpath = os.path.join(recdir, last['name'])
+            if camutil.has_handle(fullpath):
+                logger.debug(f'get_recordings_files: file {fullpath} has opened handle, ignoring it')
+                cam_files.pop()
+            files.extend(cam_files)
+
+    if limit > 0:
+        files = files[:limit]
 
     return files
 
diff --git a/tools/ipcam_motion_worker.sh b/tools/ipcam_motion_worker.sh
index e407f51..c5f711d 100755
--- a/tools/ipcam_motion_worker.sh
+++ b/tools/ipcam_motion_worker.sh
@@ -7,9 +7,19 @@ PROGNAME="$0"
 
 . "$DIR/lib.bash"
 
-curl_opts="-s --connect-timeout 5 --retry 5 --max-time 10 --retry-delay 0 --retry-max-time 40"
+curl_opts="-s --connect-timeout 10 --retry 5 --max-time 180 --retry-delay 0 --retry-max-time 180"
 allow_multiple=
-config_file="$HOME/.config/ipcam_motion_worker/config.txt"
+fetch_limit=10
+
+config=
+config_camera=
+is_remote=
+api_url=
+
+dvr_scan_path="$HOME/.local/bin/dvr-scan"
+fs_root="/var/ipcam_motion_fs"
+fs_max_filesize=146800640
+
 declare -A config=()
 
 usage() {
@@ -17,42 +27,101 @@ usage() {
 usage: $PROGNAME OPTIONS
 
 Options:
-    -c|--config FILE  configuration file, default is $config_file
-    -v, -vx           be verbose.
-                      -v enables debug logs.
-                      -vx does \`set -x\`, may be used to debug the script.
-    --allow-multiple  don't check for another instance
+    -v, -vx             be verbose.
+                        -v enables debug logs.
+                        -vx does \`set -x\`, may be used to debug the script.
+    --allow-multiple    don't check for another instance
+    --L, --fetch-limit  default: $fetch_limit
+    --remote
+    --local
+    --dvr-scan-path     default: $dvr_scan_path
+    --fs-root           default: $fs_root
+    --fs-max-filesize   default: $fs_max_filesize
 EOF
 	exit 1
 }
 
 get_recordings_dir() {
-	curl $curl_opts "${config[api_url]}/api/camera/list" \
-		| jq ".response.\"${config[camera]}\".recordings_path" | tr -d '"'
+	local camera="$1"
+	curl $curl_opts "${api_url}/api/camera/list" \
+		| jq ".response.\"${camera}\".recordings_path" | tr -d '"'
 }
 
-# returns two words per line:
-# filename filesize
+# returns three words per line:
+# filename filesize camera
 get_recordings_list() {
-	curl $curl_opts "${config[api_url]}/api/recordings/${config[camera]}?filter=motion" \
-		| jq '.response.files[] | [.name, .size] | join(" ")' | tr -d '"'
+	curl $curl_opts "${api_url}/api/recordings?limit=${fetch_limit}" \
+		| jq '.response.files[] | [.name, .size, .cam] | join(" ")' | tr -d '"'
+}
+
+read_camera_motion_config() {
+	local camera="$1"
+	local dst=config
+
+	if [ "$config_camera" != "$camera" ]; then
+		local n=0
+		local failed=
+		local key
+		local value
+
+		while read line; do
+			n=$(( n+1 ))
+
+			# skip empty lines or comments
+			if [ -z "$line" ] || [[ "$line" =~ ^#.*  ]]; then
+				continue
+			fi
+
+			if [[ $line = *"="* ]]; then
+				key="${line%%=*}"
+				value="${line#*=}"
+				eval "$dst[$key]=\"$value\""
+			else
+				echoerr "config: invalid line $n"
+				failed=1
+			fi
+		done < <(curl $curl_opts "${api_url}/api/motion/params/${camera}")
+
+		config_camera="$camera"
+
+		[ -z "$failed" ]
+	else
+		debug "read_camera_motion_config: config for $camera already loaded"
+	fi
+}
+
+dump_config() {
+	for key in min_event_length downscale_factor frame_skip threshold; do
+		debug "config[$key]=${config[$key]}"
+	done
+}
+
+get_camera_roi_config() {
+	local camera="$1"
+	curl $curl_opts "${api_url}/api/motion/params/${camera}/roi"
 }
 
 report_failure() {
-	local file="$1"
-	local message="$2"
-	local response=$(curl $curl_opts -X POST "${config[api_url]}/api/motion/fail/${config[camera]}" \
+	local camera="$1"
+	local file="$2"
+	local message="$3"
+
+	local response=$(curl $curl_opts -X POST "${api_url}/api/motion/fail/${camera}" \
 		-F "filename=$file" \
 		-F "message=$message")
+
 	print_response_error "$response" "report_failure"
 }
 
 report_timecodes() {
-	local file="$1"
-	local timecodes="$2"
-	local response=$(curl $curl_opts -X POST "${config[api_url]}/api/motion/done/${config[camera]}" \
+	local camera="$1"
+	local file="$2"
+	local timecodes="$3"
+
+	local response=$(curl $curl_opts -X POST "${api_url}/api/motion/done/${camera}" \
 		-F "filename=$file" \
 		-F "timecodes=$timecodes")
+
 	print_response_error "$response" "report_timecodes"
 }
 
@@ -71,109 +140,92 @@ print_response_error() {
 	fi
 }
 
-get_roi_file() {
-	if [ -n "${config[roi_file]}" ]; then
-		file="${config[roi_file]}"
-		if ! [[ "$file" =~ ^/.* ]]; then
-			file="$(dirname "$config_file")/$file"
-		fi
-
-		debug "get_roi_file: detected file $file"
-		[ -f "$file" ] || die "invalid roi_file: $file: no such file"
-
-		echo "$file"
-	fi
-}
-
-process_local() {
-	local recdir="$(get_recordings_dir)"
+process_queue() {
 	local tc
+	local url
 	local words
 	local file
+	local size
+	local camera
+	local local_recs_dir
+
+	if [ "$is_remote" = "1" ]; then
+		pushd "${fs_root}" >/dev/null || die "failed to change to ${fs_root}"
+		touch tmp || die "directory '${fs_root}' is not writable"
+		rm tmp
+
+		[ -f "video.mp4" ] && {
+			echowarn "video.mp4 already exists in ${fs_root}, removing.."
+			rm "video.mp4"
+		}
+	fi
 
 	while read line; do
 		words=($line)
 		file=${words[0]}
+		size=${words[1]}
+		camera=${words[2]}
 
-		debug "processing $file..."
+		debug "next video: cam=$camera file=$file"
 
-		tc=$(do_motion "${recdir}/$file")
-		debug "$file: timecodes=$tc"
+		read_camera_motion_config "$camera"
+#		dump_config
 
-		report_timecodes "$file" "$tc"
-	done < <(get_recordings_list)
-}
+		if [ "$is_remote" = "0" ]; then
+			local_recs_dir="$(get_recordings_dir "$camera")"
 
-process_remote() {
-	local tc
-	local url
-	local words
-	local file
-	local size
-
-	pushd "${config[fs_root]}" >/dev/null || die "failed to change to ${config[fs_root]}"
-	touch tmp || die "directory '${config[fs_root]}' is not writable"
-	rm tmp
-
-	[ -f "video.mp4" ] && {
-		echowarn "video.mp4 already exists in ${config[fs_root]}, removing.."
-		rm "video.mp4"
-	}
+			debug "[$camera] processing $file..."
 
-	while read line; do
-		words=($line)
-		file=${words[0]}
-		size=${words[1]}
+			tc=$(do_motion "$camera" "${local_recs_dir}/$file")
+			debug "[$camera] $file: timecodes=$tc"
 
-		if (( size > config[fs_max_filesize] )); then
-			echoerr "won't download $file, size exceedes fs_max_filesize ($size > ${config[fs_max_filesize]})"
-			report_failure "$file" "too large file"
-			continue
-		fi
+			report_timecodes "$camera" "$file" "$tc"
+		else
+			if (( size > fs_max_filesize )); then
+				echoerr "[$camera] won't download $file, size exceeds fs_max_filesize ($size > ${fs_max_filesize})"
+				report_failure "$camera" "$file" "too large file"
+				continue
+			fi
 
-		url="${config[api_url]}/api/recordings/${config[camera]}/download/${file}"
-		debug "downloading $url..."
+			url="${api_url}/api/recordings/${camera}/download/${file}"
+			debug "[$camera] downloading $url..."
 
-		if ! download "$url" "video.mp4"; then
-			echoerr "failed to download $file"
-			report_failure "$file" "download error"
-			continue
-		fi
+			if ! download "$url" "video.mp4"; then
+				echoerr "[$camera] failed to download $file"
+				report_failure "$camera" "$file" "download error"
+				continue
+			fi
 
-		tc=$(do_motion "video.mp4")
-		debug "$file: timecodes=$tc"
+			tc=$(do_motion "$camera" "video.mp4")
+			debug "[$camera] $file: timecodes=$tc"
 
-		report_timecodes "$file" "$tc"
+			report_timecodes "$camera" "$file" "$tc"
 
-		rm "video.mp4"
+			rm "video.mp4"
+		fi
 	done < <(get_recordings_list)
 
-	popd >/dev/null
+	if [ "$is_remote" = "1" ]; then popd >/dev/null; fi
 }
 
 do_motion() {
-	local input="$1"
-	local roi_file="$(get_roi_file)"
+	local camera="$1"
+	local input="$2"
 	local tc
 
 	local timecodes=()
 
 	time_start
-	if [ -z "$roi_file" ]; then
-		timecodes+=($(do_dvr_scan "$input"))
-	else
-		echoinfo "using roi sets from file: ${BOLD}$roi_file"
-		while read line; do
-			if ! [[ "$line" =~ ^#.*  ]]; then
-				tc="$(do_dvr_scan "$input" "$line")"
-				if [ -n "$tc" ]; then
-					timecodes+=("$tc")
-				fi
+	while read line; do
+		if ! [[ "$line" =~ ^#.*  ]]; then
+			tc="$(do_dvr_scan "$input" "$line")"
+			if [ -n "$tc" ]; then
+				timecodes+=("$tc")
 			fi
-		done < <(cat "$roi_file")
-	fi
+		fi
+	done < <(get_camera_roi_config "$camera")
 
-	debug "do_motion: finished in $(time_elapsed)s"
+	debug "[$camera] do_motion: finished in $(time_elapsed)s"
 
 	timecodes="$(echo "${timecodes[@]}" | sed 's/  */ /g' | xargs)"
 	timecodes="${timecodes// /,}"
@@ -182,7 +234,7 @@ do_motion() {
 }
 
 dvr_scan() {
-	"${config[dvr_scan_path]}" "$@"
+	"${dvr_scan_path}" "$@"
 }
 
 do_dvr_scan() {
@@ -207,8 +259,8 @@ do_dvr_scan() {
 
 while [[ $# -gt 0 ]]; do
 	case $1 in
-		-c|--config)
-			config_file="$2"
+		-L|--fetch-limit)
+			fetch_limit="$2"
 			shift; shift
 			;;
 
@@ -217,6 +269,36 @@ while [[ $# -gt 0 ]]; do
 			shift
 			;;
 
+		--remote)
+			is_remote=1
+			shift
+			;;
+
+		--local)
+			is_remote=0
+			shift
+			;;
+
+		--dvr-scan-path)
+			dvr_scan_path="$2"
+			shift; shift
+			;;
+
+		--fs-root)
+			fs_root="$2"
+			shift; shift
+			;;
+
+		--fs-max-filesize)
+			fs_max_filesize="$2"
+			shift; shift
+			;;
+
+		--api-url)
+			api_url="$2"
+			shift; shift
+			;;
+
 		-v)
 			VERBOSE=1
 			shift
@@ -239,20 +321,7 @@ if [ -z "$allow_multiple" ] && pidof -o %PPID -x "$(basename "${BASH_SOURCE[0]}"
 	die "process already running"
 fi
 
-read_config "$config_file" config
-check_config config "api_url camera"
-if [ -n "${config[remote]}" ]; then
-	check_config config "fs_root fs_max_filesize"
-fi
-
-[ -z "${config[threshold]}" ] && config[threshold]=1
-[ -z "${conifg[min_event_length]}" ] && config[min_event_length]="3s"
-[ -z "${conifg[frame_skip]}" ] && config[frame_skip]=2
-[ -z "${conifg[downscale_factor]}" ] && config[downscale_factor]=3
-[ -z "${conifg[dvr_scan_path]}" ] && config[dvr_scan_path]="dvr-scan"
+[ -z "$is_remote" ] && die "either --remote or --local is required"
+[ -z "$api_url" ] && die "--api-url is required"
 
-if [ -z "${config[remote]}" ]; then
-	process_local
-else
-	process_remote
-fi
+process_queue
\ No newline at end of file