From 6b045fd2259b0dc3dfffb8f3a3142f0ff9a59028 Mon Sep 17 00:00:00 2001 From: auricom <27022259+auricom@users.noreply.github.com> Date: Wed, 5 Mar 2025 09:21:59 +0100 Subject: [PATCH] feat: homelab-music-transcode optimizations --- .../homelab/music-transcode/helmrelease.yaml | 2 +- .../music-transcode/scripts/transcode.sh | 229 ++++++++++-------- 2 files changed, 131 insertions(+), 100 deletions(-) diff --git a/kubernetes/apps/default/homelab/music-transcode/helmrelease.yaml b/kubernetes/apps/default/homelab/music-transcode/helmrelease.yaml index 23c3b2b9e..409e76d8c 100644 --- a/kubernetes/apps/default/homelab/music-transcode/helmrelease.yaml +++ b/kubernetes/apps/default/homelab/music-transcode/helmrelease.yaml @@ -37,7 +37,7 @@ spec: app: image: repository: ghcr.io/auricom/freac - tag: 1.1.7@sha256:d70f65290ffd63e10f7a864f397203510ccdf166208bbbab405a889adceaeecd + tag: 1.1.7@sha256:faf620257ba1fdcb16b0b18e4d58a518f7a3ce6e886abb4d610b5ba374a503bf command: - /bin/bash - -c diff --git a/kubernetes/apps/default/homelab/music-transcode/scripts/transcode.sh b/kubernetes/apps/default/homelab/music-transcode/scripts/transcode.sh index feb49e2db..bfebfd03f 100755 --- a/kubernetes/apps/default/homelab/music-transcode/scripts/transcode.sh +++ b/kubernetes/apps/default/homelab/music-transcode/scripts/transcode.sh @@ -11,12 +11,6 @@ set -o pipefail export LANG=C.UTF-8 export LC_ALL=C.UTF-8 -# # Send start ping to healthchecks -# if [[ -n "${HEALTHCHECKS_ID:-}" ]]; then -# curl --silent --max-time 10 --retry 5 "https://hc-ping.com/${HEALTHCHECKS_ID}/start" -# fi - - # Create a logging function log() { echo "[$(date '+%Y-%m-%d %H:%M:%S')] $*" @@ -169,7 +163,7 @@ transcode() { fi mkdir -p "$(dirname "$md5_file")" - md5sum "$input_file" | awk '{ print $1 }' > "$md5_file" + md5sum "$input_file" | cut -d' ' -f1 > "$md5_file" log "Successfully transcoded: $input_file -> $output_file" fi } @@ -188,7 +182,7 @@ write_cue() { fi mkdir -p "$(dirname "$md5_file")" - md5sum "$input_file" | awk '{ print $1 }' > "$md5_file" + md5sum "$input_file" | cut -d' ' -f1 > "$md5_file" log "Successfully wrote cue: $output_file" fi } @@ -206,32 +200,18 @@ write_jpg() { fi mkdir -p "$(dirname "$md5_file")" - md5sum "$input_file" | awk '{ print $1 }' > "$md5_file" + md5sum "$input_file" | cut -d' ' -f1 > "$md5_file" log "Successfully converted cover: $input_file -> $output_file" fi } -directory_structure() { - local dryrun_flag="" - [[ $MODE_DRY_RUN == true ]] && dryrun_flag="--dry-run" - - log "INFO: Creating directory structure with rsync..." - if ! rsync -rvz $dryrun_flag --exclude-from="./transcode_exclude.cfg" \ - --include="*/" --exclude="*" "$TRANSCODE_INPUT_DIR" "$TRANSCODE_OUTPUT_DIR"; then - log "ERROR: rsync failed" - return 1 - fi -} - -convert_covers() { - log "INFO: Looking for covers to convert..." - cd "$TRANSCODE_INPUT_DIR" || exit 1 - - for ext in $TRANSCODE_COVER_EXTENSIONS; do - log "INFO: Searching for .$ext files..." - while IFS= read -r val; do - [[ -z "$val" ]] && continue +process_file() { + local val="$1" + local ext="$2" + local type="$3" # cover, music, or cue + case "$type" in + cover) local filename="$TRANSCODE_OUTPUT_DIR/${val%.*}.jpg" local md5_filename="$TRANSCODE_DB/${val}.md5" local process_file=false @@ -245,7 +225,7 @@ convert_covers() { process_file=true log "Processing new file: $val" elif [[ $MODE_CHECKSUM == true ]]; then - if [[ ! -f "$md5_filename" ]] || [[ "$(cat "$md5_filename" 2>/dev/null)" != "$(md5sum "$val" | awk '{ print $1 }')" ]]; then + if [[ ! -f "$md5_filename" ]] || [[ "$(cat "$md5_filename" 2>/dev/null)" != "$(md5sum "$val" | cut -d' ' -f1)" ]]; then process_file=true log "File changed, reprocessing: $val" fi @@ -254,19 +234,9 @@ convert_covers() { if [[ $process_file == true ]]; then write_jpg "$val" "$filename" "$md5_filename" fi - done < <("$TRANSCODE_FD_BIN" --extension "$ext" $TRANSCODE_FD_FILTERS --type f) - done -} - -convert_music() { - log "INFO: Looking for music to transcode..." - cd "$TRANSCODE_INPUT_DIR" || exit 1 - - for ext in $TRANSCODE_MUSIC_EXTENSIONS; do - log "INFO: Searching for .$ext files..." - while IFS= read -r val; do - [[ -z "$val" ]] && continue + ;; + music) local filebasename="$TRANSCODE_OUTPUT_DIR/${val%.*}" local filename="${filebasename}.opus" local md5_filename="$TRANSCODE_DB/${val}.md5" @@ -281,7 +251,7 @@ convert_music() { process_file=true log "Processing new file: $val" elif [[ $MODE_CHECKSUM == true ]]; then - if [[ ! -f "$md5_filename" ]] || [[ "$(cat "$md5_filename" 2>/dev/null)" != "$(md5sum "$val" | awk '{ print $1 }')" ]]; then + if [[ ! -f "$md5_filename" ]] || [[ "$(cat "$md5_filename" 2>/dev/null)" != "$(md5sum "$val" | cut -d' ' -f1)" ]]; then process_file=true log "File changed, reprocessing: $val" fi @@ -290,7 +260,91 @@ convert_music() { if [[ $process_file == true ]]; then transcode "$val" "$filename" "$md5_filename" fi - done < <(fd_safe --extension "$ext" $TRANSCODE_FD_FILTERS --type f) + ;; + + cue) + local output_file="$TRANSCODE_OUTPUT_DIR/$val" + local md5_filename="$TRANSCODE_DB/${val}.md5" + local replacement_text_string="FILE \"$(basename "${val%.*}").opus\" MP3" + local process_file=false + + # Create output directory if it doesn't exist + mkdir -p "$(dirname "$output_file")" + mkdir -p "$(dirname "$md5_filename")" + + # Check if we need to process this file + if [[ ! -f "$md5_filename" ]]; then + process_file=true + log "Processing new cuefile: $val" + elif [[ $MODE_CHECKSUM == true ]]; then + if [[ ! -f "$md5_filename" ]] || [[ "$(cat "$md5_filename" 2>/dev/null)" != "$(md5sum "$val" | cut -d' ' -f1)" ]]; then + process_file=true + log "Cuefile changed, reprocessing: $val" + fi + fi + + if [[ $process_file == true ]]; then + cp -p "$val" "$output_file" + write_cue "$val" "$output_file" "$replacement_text_string" "$md5_filename" + fi + ;; + esac +} + +directory_structure() { + local dryrun_flag="" + [[ $MODE_DRY_RUN == true ]] && dryrun_flag="--dry-run" + + log "INFO: Creating directory structure with rsync..." + if ! rsync -rvz $dryrun_flag --exclude-from="./transcode_exclude.cfg" \ + --include="*/" --exclude="*" "$TRANSCODE_INPUT_DIR/" "$TRANSCODE_OUTPUT_DIR/"; then + log "ERROR: rsync failed" + return 1 + fi +} + +# Export functions so they're available to subshells +export -f log +export -f transcode +export -f write_cue +export -f write_jpg +export -f process_file + +convert_covers() { + log "INFO: Looking for covers to convert..." + cd "$TRANSCODE_INPUT_DIR" || exit 1 + + for ext in $TRANSCODE_COVER_EXTENSIONS; do + log "INFO: Searching for .$ext files..." + # Create a temporary script for processing + local temp_script=$(mktemp) + cat > "$temp_script" << 'EOF' +#!/bin/bash +process_file "$1" "$2" "$3" +EOF + chmod +x "$temp_script" + + fd_safe --extension "$ext" $TRANSCODE_FD_FILTERS --type f -x "$temp_script" {} "$ext" cover \; + rm -f "$temp_script" + done +} + +convert_music() { + log "INFO: Looking for music to transcode..." + cd "$TRANSCODE_INPUT_DIR" || exit 1 + + for ext in $TRANSCODE_MUSIC_EXTENSIONS; do + log "INFO: Searching for .$ext files..." + # Create a temporary script for processing + local temp_script=$(mktemp) + cat > "$temp_script" << 'EOF' +#!/bin/bash +process_file "$1" "$2" "$3" +EOF + chmod +x "$temp_script" + + fd_safe --extension "$ext" $TRANSCODE_FD_FILTERS --type f -x "$temp_script" {} "$ext" music \; + rm -f "$temp_script" done } @@ -298,68 +352,50 @@ fix_cuefiles() { log "INFO: Looking for cuefiles..." cd "$TRANSCODE_INPUT_DIR" || exit 1 - while IFS= read -r val; do - [[ -z "$val" ]] && continue + # Create a temporary script for processing + local temp_script=$(mktemp) + cat > "$temp_script" << 'EOF' +#!/bin/bash +process_file "$1" "$2" "$3" +EOF + chmod +x "$temp_script" - local output_file="$TRANSCODE_OUTPUT_DIR/$val" - local md5_filename="$TRANSCODE_DB/${val}.md5" - local replacement_text_string="FILE \"$(basename "${val%.*}").opus\" MP3" - local process_file=false - - # Create output directory if it doesn't exist - mkdir -p "$(dirname "$output_file")" - mkdir -p "$(dirname "$md5_filename")" - - # Check if we need to process this file - if [[ ! -f "$md5_filename" ]]; then - process_file=true - log "Processing new cuefile: $val" - elif [[ $MODE_CHECKSUM == true ]]; then - if [[ ! -f "$md5_filename" ]] || [[ "$(cat "$md5_filename" 2>/dev/null)" != "$(md5sum "$val" | awk '{ print $1 }')" ]]; then - process_file=true - log "Cuefile changed, reprocessing: $val" - fi - fi - - if [[ $process_file == true ]]; then - cp -p "$val" "$output_file" - write_cue "$val" "$output_file" "$replacement_text_string" "$md5_filename" - fi - done < <("$TRANSCODE_FD_BIN" --extension cue $TRANSCODE_FD_FILTERS --type f) + fd_safe --extension cue $TRANSCODE_FD_FILTERS --type f -x "$temp_script" {} cue cue \; + rm -f "$temp_script" } remove_absent_from_source() { cd "$TRANSCODE_DB" || exit 1 - local files - files=$("$TRANSCODE_FD_BIN" --extension md5) || return 0 + # Create a temporary script file for the removal operation + local temp_script=$(mktemp) + cat > "$temp_script" << 'EOF' +#!/bin/bash +val="$1" +[[ -z "$val" ]] && exit 0 - while IFS= read -r val; do - [[ -z "$val" ]] && continue +filename="$(dirname "$val")/$(basename "$val" .md5)" +source_path="$TRANSCODE_INPUT_DIR/$filename" - local filename="$(dirname "$val")/$(basename "$val" .md5)" - # Use printf to properly handle special characters - local source_path - source_path=$(printf "%s/%s" "$TRANSCODE_INPUT_DIR" "$filename") - - if [[ ! -e "$source_path" ]]; then - # Double check with find to handle special characters - if ! find "$TRANSCODE_INPUT_DIR/$(dirname "$filename")" -maxdepth 1 -name "$(basename "$filename")*" 2>/dev/null | grep -q .; then - log "INFO: Confirmed - Transcoded file $filename doesn't have a source file: delete" - if [[ $MODE_DRY_RUN == false ]]; then - rm -f "$TRANSCODE_OUTPUT_DIR/$filename"* - rm -f "$TRANSCODE_DB/$filename"* - fi - fi +if [[ ! -e "$source_path" ]]; then + if ! find "$TRANSCODE_INPUT_DIR/$(dirname "$filename")" -maxdepth 1 -name "$(basename "$filename")*" 2>/dev/null | grep -q .; then + echo "[$(date "+%Y-%m-%d %H:%M:%S")] INFO: Confirmed - Transcoded file $filename doesnt have a source file: delete" + if [[ $MODE_DELETE == true && $MODE_DRY_RUN == false ]]; then + rm -f "$TRANSCODE_OUTPUT_DIR/$filename"* + rm -f "$TRANSCODE_DB/$filename"* fi - done <<< "$files" + fi +fi +EOF + chmod +x "$temp_script" + + "$TRANSCODE_FD_BIN" --extension md5 -x "$temp_script" {} \; + rm -f "$temp_script" log "INFO: removing empty directories..." if [[ $MODE_DRY_RUN == false ]]; then - cd "$TRANSCODE_OUTPUT_DIR" || exit 1 - find . -type d -empty -delete 2>/dev/null || true - cd "$TRANSCODE_DB" || exit 1 - find . -type d -empty -delete 2>/dev/null || true + find "$TRANSCODE_OUTPUT_DIR" -type d -empty -delete 2>/dev/null || true + find "$TRANSCODE_DB" -type d -empty -delete 2>/dev/null || true fi } @@ -379,8 +415,3 @@ fi if [[ $MODE_DRY_RUN == false ]]; then manage_execution_time write fi - -# # Send completion ping to healthchecks -# if [[ -n "${HEALTHCHECKS_ID:-}" ]]; then -# curl --silent --max-time 10 --retry 5 "https://hc-ping.com/${HEALTHCHECKS_ID}" -# fi