summaryrefslogtreecommitdiff
path: root/tools
diff options
context:
space:
mode:
Diffstat (limited to 'tools')
-rw-r--r--tools/clickhouse-backup.sh31
-rwxr-xr-xtools/merge-recordings.py130
-rw-r--r--tools/remove-old-recordings.sh5
-rwxr-xr-xtools/sync-recordings-to-remote.sh72
-rwxr-xr-xtools/vkos.sh99
5 files changed, 337 insertions, 0 deletions
diff --git a/tools/clickhouse-backup.sh b/tools/clickhouse-backup.sh
new file mode 100644
index 0000000..6e938e4
--- /dev/null
+++ b/tools/clickhouse-backup.sh
@@ -0,0 +1,31 @@
+#!/bin/bash
+
+DIR=/var/lib/clickhouse/backup
+MAX_COUNT=3
+NAME=backup_$(date -u +%Y-%m-%d)
+
+create() {
+ local name="$1"
+ clickhouse-backup create "$name"
+}
+
+del() {
+ local name="$1"
+ clickhouse-backup delete local "$name"
+}
+
+# create a backup
+create "$NAME"
+
+# compress backup
+cd "$DIR"
+tar czvf $NAME.tar.gz $NAME
+
+# delete uncompressed files
+del "$NAME"
+
+# delete old backups
+for file in $(ls -t "${DIR}" | tail -n +$(( MAX_COUNT+1 ))); do
+ echo "removing $file..."
+ rm "$file"
+done \ No newline at end of file
diff --git a/tools/merge-recordings.py b/tools/merge-recordings.py
new file mode 100755
index 0000000..637858e
--- /dev/null
+++ b/tools/merge-recordings.py
@@ -0,0 +1,130 @@
+#!/usr/bin/env python3
+import os
+import re
+import subprocess
+import tempfile
+import sys
+
+from datetime import datetime, timedelta
+from argparse import ArgumentParser
+
+
+fmt = '%d%m%y-%H%M%S'
+
+File = dict
+FileList = list[File]
+
+
+def get_files(source_directory: str) -> FileList:
+ files = []
+ for f in os.listdir(source_directory):
+ m = re.match(r'^(\d{6}-\d{6})_(\d{6}-\d{6})_id(\d+)(_\w+)?\.mp3$', f)
+ if not m:
+ continue
+
+ files.append({
+ 'filename': os.path.join(source_directory, f),
+ 'start': datetime.strptime(m.group(1), fmt),
+ 'stop': datetime.strptime(m.group(2), fmt)
+ })
+ files.sort(key=lambda f: f['start'].timestamp())
+ return files
+
+
+def group_files(files: FileList) -> list[FileList]:
+ groups = []
+ group_idx = None
+
+ for info in files:
+ # if group_idx is not None:
+ # print(info['start'], groups[group_idx][-1]['stop'])
+ # print(' ', info['start'] - groups[group_idx][-1]['stop'])
+ # print()
+
+ if group_idx is None or \
+ not groups[group_idx] or \
+ info['start'] - groups[group_idx][-1]['stop'] <= timedelta(seconds=1):
+ if group_idx is None:
+ groups.append([])
+ group_idx = 0
+ else:
+ group_idx += 1
+ groups.append([])
+ groups[group_idx].append(info)
+
+ return groups
+
+
+def merge(groups: list[FileList],
+ output_directory: str,
+ delete_source_files=False,
+ vbr=False) -> None:
+ for g in groups:
+ success = False
+
+ fd = tempfile.NamedTemporaryFile(delete=False)
+ try:
+ for file in g:
+ line = f'file \'{file["filename"]}\'\n'
+ # print(line.strip())
+ fd.write(line.encode())
+ fd.close()
+
+ start = g[0]['start'].strftime(fmt)
+ stop = g[-1]['stop'].strftime(fmt)
+ fn = f'{start}_{stop}_merged.mp3'
+ output = os.path.join(output_directory, fn)
+
+ cmd = ['ffmpeg', '-y',
+ '-f', 'concat',
+ '-safe', '0',
+ '-i', fd.name,
+ '-map_metadata', '-1',
+ '-codec:a', 'libmp3lame']
+ if vbr:
+ cmd.extend(['-codec:a', 'libmp3lame', '-q:a', '4'])
+ else:
+ cmd.extend(['-codec:a', 'copy'])
+ cmd.append(output)
+
+ p = subprocess.run(cmd, capture_output=False)
+ if p.returncode != 0:
+ print(f'error: ffmpeg returned {p.returncode}')
+ else:
+ success = True
+ finally:
+ os.unlink(fd.name)
+
+ if success and delete_source_files:
+ for file in g:
+ os.unlink(file['filename'])
+
+
+def main():
+ default_dir = os.getcwd()
+
+ parser = ArgumentParser()
+ parser.add_argument('--input-directory', '-i', type=str, default=default_dir,
+ help='Directory with files')
+ parser.add_argument('--output-directory', '-o', type=str, default=default_dir,
+ help='Output directory')
+ parser.add_argument('-D', '--delete-source-files', action='store_true')
+ parser.add_argument('--vbr', action='store_true',
+ help='Re-encode using VBR (-q:a 4)')
+ args = parser.parse_args()
+
+ files = get_files(os.path.realpath(args.input_directory))
+ if not len(files):
+ print(f"No mp3 files found in {args.input_directory}.")
+ sys.exit()
+
+ groups = group_files(files)
+
+ merge(groups,
+ os.path.realpath(args.output_directory),
+ delete_source_files=args.delete_source_files,
+ vbr=args.vbr)
+
+
+if __name__ == '__main__':
+ main()
diff --git a/tools/remove-old-recordings.sh b/tools/remove-old-recordings.sh
new file mode 100644
index 0000000..d376572
--- /dev/null
+++ b/tools/remove-old-recordings.sh
@@ -0,0 +1,5 @@
+#!/bin/bash
+
+# to be launched by cron on remote server
+
+find /var/recordings -type f -mtime +14 -delete
diff --git a/tools/sync-recordings-to-remote.sh b/tools/sync-recordings-to-remote.sh
new file mode 100755
index 0000000..cf979d1
--- /dev/null
+++ b/tools/sync-recordings-to-remote.sh
@@ -0,0 +1,72 @@
+#!/bin/bash
+
+PROGNAME="$0"
+NODE_CONFIG="/etc/sound_node.toml"
+REMOTE_USER=user
+REMOTE_SERVER=solarmon.ru
+REMOTE_DIRECTORY=/var/recordings
+
+set -e
+
+echoerr() {
+ >&2 echo "error: $@"
+}
+
+echowarn() {
+ >&2 echo "warning: $@"
+}
+
+telegram_alert() {
+ if [ -z "$TG_TOKEN" ] || [ -z "$TG_CHAT_ID" ]; then return; fi
+ curl -X POST \
+ -F "chat_id=${TG_CHAT_ID}" \
+ -F "text=$1" \
+ "https://api.telegram.org/bot${TG_TOKEN}/sendMessage"
+}
+
+fatal() {
+ echoerr "$@"
+ telegram_alert "$PROGNAME: $@"
+ exit 1
+}
+
+get_config_var() {
+ local varname="$1"
+ cat "$NODE_CONFIG" | grep "^$varname = \"" | awk '{print $3}' | tr -d '"'
+}
+
+get_mp3_count() {
+ find "$LOCAL_DIR" -mindepth 1 -type f -name "*.mp3" -printf x | wc -c
+}
+
+[ -z "$TG_TOKEN" ] && echowarn "TG_TOKEN is not set"
+[ -z "$TG_CHAT_ID" ] && echowarn "TG_CHAT_ID is not set"
+
+NODE_NAME=$(get_config_var name)
+LOCAL_DIR=$(get_config_var storage)
+
+[ -z "$NODE_NAME" ] && fatal "failed to parse NODE_NAME"
+[ -z "$LOCAL_DIR" ] && fatal "failed to parse LOCAL_DIR"
+
+[ -d "$LOCAL_DIR" ] || fatal "$LOCAL_DIR is not a directory"
+
+COUNT=$(get_mp3_count)
+(( $COUNT < 1 )) && {
+ echo "seems there's nothing to sync"
+ exit
+}
+
+cd "$LOCAL_DIR" || fatal "failed to change to $LOCAL_DIR"
+
+rsync -azPv -e "ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o LogLevel=ERROR" \
+ *.mp3 \
+ ${REMOTE_USER}@${REMOTE_SERVER}:"${REMOTE_DIRECTORY}/${NODE_NAME}/" \
+ --exclude temp.mp3
+
+RC=$?
+
+if [ $RC -eq 0 ]; then
+ find "$LOCAL_DIR" -name "*.mp3" -type f -mmin +1440 -delete || fatal "find failed to delete old files"
+else
+ fatal "failed to rsync: code $RC"
+fi
diff --git a/tools/vkos.sh b/tools/vkos.sh
new file mode 100755
index 0000000..ebe0d66
--- /dev/null
+++ b/tools/vkos.sh
@@ -0,0 +1,99 @@
+#!/bin/bash
+
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"
+PROGNAME="$0"
+
+die() {
+ >&2 echo "error: $@"
+ exit 1
+}
+
+usage() {
+ cat <<EOF
+usage: $PROGNAME [OPTIONS] COMMAND
+
+Options:
+ -b use backup server
+ -d don't delete files after merge
+
+Supported commands:
+ list NODE
+ fetch NODE PREFIX
+ merge
+EOF
+ exit
+}
+
+[ -z "$1" ] && usage
+
+COMMAND=
+NODE=
+PREFIX=
+FROM_BACKUP=0
+DONT_DELETE=0
+while [[ $# -gt 0 ]]; do
+ case "$1" in
+ list)
+ COMMAND="$1"
+ NODE="$2"
+ shift
+ ;;
+
+ fetch)
+ COMMAND="$1"
+ NODE="$2"
+ PREFIX="$3"
+ shift; shift
+ ;;
+
+ merge)
+ COMMAND="$1"
+ ;;
+
+ -b)
+ FROM_BACKUP=1
+ ;;
+
+ -d)
+ DONT_DELETE=1
+ ;;
+
+ *)
+ die "unrecognized argument: $1"
+ ;;
+ esac
+ shift
+done
+
+[ -z "$COMMAND" ] && usage
+
+if [ "$FROM_BACKUP" = "0" ]; then
+ SRV_HOST=solarmon.ru
+ SRV_PORT=60681
+ SRV_USER=user
+ SRV_DIR=/var/recordings
+else
+ SRV_HOST=srv_nas4
+ SRV_PORT=22
+ SRV_USER=root
+ SRV_DIR=/var/storage1/solarmon/recordings
+fi
+
+case "$COMMAND" in
+ list)
+ [ -z "$NODE" ] && usage
+ ssh -p${SRV_PORT} ${SRV_USER}@${SRV_HOST} "ls -rt --time creation \"${SRV_DIR}/${NODE}\""
+ ;;
+
+ fetch)
+ [ -z "$NODE" ] && usage
+ [ -z "$PREFIX" ] && usage
+ rsync -azPv -e "ssh -p${SRV_PORT}" ${SRV_USER}@${SRV_HOST}:"${SRV_DIR}/${NODE}/${PREFIX}*" .
+ ;;
+
+ merge)
+ args=
+ if [ "$DONT_DELETE" = "0" ]; then args="-D"; fi
+ $DIR/merge-recordings.py $args
+ ;;
+esac
generated by cgit v1.2.3 (git 2.25.1) at 2024-11-22 01:55:12 +0000