initial public

5 files changed, 337 insertions, 0 deletions

diff --git a/tools/clickhouse-backup.sh b/tools/clickhouse-backup.sh
new file mode 100644
index 0000000..6e938e4
--- /dev/null
+++ b/tools/clickhouse-backup.sh
@@ -0,0 +1,31 @@
+#!/bin/bash
+
+DIR=/var/lib/clickhouse/backup
+MAX_COUNT=3
+NAME=backup_$(date -u +%Y-%m-%d)
+
+create() {
+	local name="$1"
+	clickhouse-backup create "$name"
+}
+
+del() {
+	local name="$1"
+	clickhouse-backup delete local "$name"
+}
+
+# create a backup
+create "$NAME"
+
+# compress backup
+cd "$DIR"
+tar czvf $NAME.tar.gz $NAME
+
+# delete uncompressed files
+del "$NAME"
+
+# delete old backups
+for file in $(ls -t "${DIR}" | tail -n +$(( MAX_COUNT+1 ))); do
+	echo "removing $file..."
+	rm "$file"
+done
\ No newline at end of file
diff --git a/tools/merge-recordings.py b/tools/merge-recordings.py
new file mode 100755
index 0000000..637858e
--- /dev/null
+++ b/tools/merge-recordings.py
@@ -0,0 +1,130 @@
+#!/usr/bin/env python3
+import os
+import re
+import subprocess
+import tempfile
+import sys
+
+from datetime import datetime, timedelta
+from argparse import ArgumentParser
+
+
+fmt = '%d%m%y-%H%M%S'
+
+File = dict
+FileList = list[File]
+
+
+def get_files(source_directory: str) -> FileList:
+    files = []
+    for f in os.listdir(source_directory):
+        m = re.match(r'^(\d{6}-\d{6})_(\d{6}-\d{6})_id(\d+)(_\w+)?\.mp3$', f)
+        if not m:
+            continue
+
+        files.append({
+            'filename': os.path.join(source_directory, f),
+            'start': datetime.strptime(m.group(1), fmt),
+            'stop':  datetime.strptime(m.group(2), fmt)
+        })
+    files.sort(key=lambda f: f['start'].timestamp())
+    return files
+
+
+def group_files(files: FileList) -> list[FileList]:
+    groups = []
+    group_idx = None
+    
+    for info in files:
+        # if group_idx is not None:
+        #     print(info['start'], groups[group_idx][-1]['stop'])
+        #     print('    ', info['start'] - groups[group_idx][-1]['stop'])
+        #     print()
+
+        if group_idx is None or \
+                not groups[group_idx] or \
+                info['start'] - groups[group_idx][-1]['stop'] <= timedelta(seconds=1):
+            if group_idx is None:
+                groups.append([])
+                group_idx = 0
+        else:
+            group_idx += 1
+            groups.append([])
+        groups[group_idx].append(info)
+
+    return groups
+
+
+def merge(groups: list[FileList],
+          output_directory: str,
+          delete_source_files=False,
+          vbr=False) -> None:
+    for g in groups:
+        success = False
+
+        fd = tempfile.NamedTemporaryFile(delete=False)
+        try:
+            for file in g:
+                line = f'file \'{file["filename"]}\'\n'
+                # print(line.strip())
+                fd.write(line.encode())
+            fd.close()
+
+            start = g[0]['start'].strftime(fmt)
+            stop = g[-1]['stop'].strftime(fmt)
+            fn = f'{start}_{stop}_merged.mp3'
+            output = os.path.join(output_directory, fn)
+
+            cmd = ['ffmpeg', '-y',
+                   '-f', 'concat',
+                   '-safe', '0',
+                   '-i', fd.name,
+                   '-map_metadata', '-1',
+                   '-codec:a', 'libmp3lame']
+            if vbr:
+                cmd.extend(['-codec:a', 'libmp3lame', '-q:a', '4'])
+            else:
+                cmd.extend(['-codec:a', 'copy'])
+            cmd.append(output)
+
+            p = subprocess.run(cmd, capture_output=False)
+            if p.returncode != 0:
+                print(f'error: ffmpeg returned {p.returncode}')
+            else:
+                success = True
+        finally:
+            os.unlink(fd.name)
+
+        if success and delete_source_files:
+            for file in g:
+                os.unlink(file['filename'])
+
+
+def main():
+    default_dir = os.getcwd()
+
+    parser = ArgumentParser()
+    parser.add_argument('--input-directory', '-i', type=str, default=default_dir,
+                        help='Directory with files')
+    parser.add_argument('--output-directory', '-o', type=str, default=default_dir,
+                        help='Output directory')
+    parser.add_argument('-D', '--delete-source-files', action='store_true')
+    parser.add_argument('--vbr', action='store_true',
+                        help='Re-encode using VBR (-q:a 4)')
+    args = parser.parse_args()
+
+    files = get_files(os.path.realpath(args.input_directory))
+    if not len(files):
+        print(f"No mp3 files found in {args.input_directory}.")
+        sys.exit()
+
+    groups = group_files(files)
+
+    merge(groups,
+          os.path.realpath(args.output_directory),
+          delete_source_files=args.delete_source_files,
+          vbr=args.vbr)
+
+
+if __name__ == '__main__':
+    main()
diff --git a/tools/remove-old-recordings.sh b/tools/remove-old-recordings.sh
new file mode 100644
index 0000000..d376572
--- /dev/null
+++ b/tools/remove-old-recordings.sh
@@ -0,0 +1,5 @@
+#!/bin/bash
+
+# to be launched by cron on remote server
+
+find /var/recordings -type f -mtime +14 -delete
diff --git a/tools/sync-recordings-to-remote.sh b/tools/sync-recordings-to-remote.sh
new file mode 100755
index 0000000..cf979d1
--- /dev/null
+++ b/tools/sync-recordings-to-remote.sh
@@ -0,0 +1,72 @@
+#!/bin/bash
+
+PROGNAME="$0"
+NODE_CONFIG="/etc/sound_node.toml"
+REMOTE_USER=user
+REMOTE_SERVER=solarmon.ru
+REMOTE_DIRECTORY=/var/recordings
+
+set -e
+
+echoerr() {
+	>&2 echo "error: $@"
+}
+
+echowarn() {
+	>&2 echo "warning: $@"
+}
+
+telegram_alert() {
+	if [ -z "$TG_TOKEN" ] || [ -z "$TG_CHAT_ID" ]; then return; fi
+	curl -X POST \
+		-F "chat_id=${TG_CHAT_ID}" \
+		-F "text=$1" \
+		"https://api.telegram.org/bot${TG_TOKEN}/sendMessage"
+}
+
+fatal() {
+	echoerr "$@"
+	telegram_alert "$PROGNAME: $@"
+	exit 1
+}
+
+get_config_var() {
+	local varname="$1"
+	cat "$NODE_CONFIG"  | grep "^$varname = \"" | awk '{print $3}' | tr -d '"'
+}
+
+get_mp3_count() {
+	find "$LOCAL_DIR" -mindepth 1 -type f -name "*.mp3" -printf x | wc -c
+}
+
+[ -z "$TG_TOKEN" ] && echowarn "TG_TOKEN is not set"
+[ -z "$TG_CHAT_ID" ] && echowarn "TG_CHAT_ID is not set"
+
+NODE_NAME=$(get_config_var name)
+LOCAL_DIR=$(get_config_var storage)
+
+[ -z "$NODE_NAME" ] && fatal "failed to parse NODE_NAME"
+[ -z "$LOCAL_DIR" ] && fatal "failed to parse LOCAL_DIR"
+
+[ -d "$LOCAL_DIR" ] || fatal "$LOCAL_DIR is not a directory"
+
+COUNT=$(get_mp3_count)
+(( $COUNT < 1 )) && {
+	echo "seems there's nothing to sync"
+	exit
+}
+
+cd "$LOCAL_DIR" || fatal "failed to change to $LOCAL_DIR"
+
+rsync -azPv -e "ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o LogLevel=ERROR" \
+	*.mp3 \
+	${REMOTE_USER}@${REMOTE_SERVER}:"${REMOTE_DIRECTORY}/${NODE_NAME}/" \
+	--exclude temp.mp3
+
+RC=$?
+
+if [ $RC -eq 0 ]; then
+	find "$LOCAL_DIR" -name "*.mp3" -type f -mmin +1440 -delete || fatal "find failed to delete old files"
+else
+	fatal "failed to rsync: code $RC"
+fi
diff --git a/tools/vkos.sh b/tools/vkos.sh
new file mode 100755
index 0000000..ebe0d66
--- /dev/null
+++ b/tools/vkos.sh
@@ -0,0 +1,99 @@
+#!/bin/bash
+
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"
+PROGNAME="$0"
+
+die() {
+	>&2 echo "error: $@"
+	exit 1
+}
+
+usage() {
+	cat <<EOF
+usage: $PROGNAME [OPTIONS] COMMAND
+
+Options:
+	-b  use backup server
+	-d  don't delete files after merge
+
+Supported commands:
+	list NODE
+	fetch NODE PREFIX
+	merge
+EOF
+	exit
+}
+
+[ -z "$1" ] && usage
+
+COMMAND=
+NODE=
+PREFIX=
+FROM_BACKUP=0
+DONT_DELETE=0
+while [[ $# -gt 0 ]]; do
+	case "$1" in
+		list)
+			COMMAND="$1"
+			NODE="$2"
+			shift
+			;;
+
+		fetch)
+			COMMAND="$1"
+			NODE="$2"
+			PREFIX="$3"
+			shift; shift
+			;;
+
+		merge)
+			COMMAND="$1"
+			;;
+
+		-b)
+			FROM_BACKUP=1
+			;;
+
+		-d)
+			DONT_DELETE=1
+			;;
+
+		*)
+			die "unrecognized argument: $1"
+			;;
+	esac
+	shift
+done
+
+[ -z "$COMMAND" ] && usage
+
+if [ "$FROM_BACKUP" = "0" ]; then
+	SRV_HOST=solarmon.ru
+	SRV_PORT=60681
+	SRV_USER=user
+	SRV_DIR=/var/recordings
+else
+	SRV_HOST=srv_nas4
+	SRV_PORT=22
+	SRV_USER=root
+	SRV_DIR=/var/storage1/solarmon/recordings
+fi
+
+case "$COMMAND" in
+	list)
+		[ -z "$NODE" ] && usage
+		ssh -p${SRV_PORT} ${SRV_USER}@${SRV_HOST} "ls -rt --time creation \"${SRV_DIR}/${NODE}\""
+		;;
+
+	fetch)
+		[ -z "$NODE" ] && usage
+		[ -z "$PREFIX" ] && usage
+		rsync -azPv -e "ssh -p${SRV_PORT}" ${SRV_USER}@${SRV_HOST}:"${SRV_DIR}/${NODE}/${PREFIX}*" .
+		;;
+
+	merge)
+		args=
+		if [ "$DONT_DELETE" = "0" ]; then args="-D"; fi
+		$DIR/merge-recordings.py $args
+		;;
+esac