shpaq/auricom-home-cluster
1
0
Fork 0
mirror of https://github.com/auricom/home-cluster.git synced 2025-09-17 18:24:14 +02:00
Code Issues Packages Projects Releases Wiki Activity

feat: homelab-music-transcode optimizations

Browse Source
This commit is contained in:
auricom
2025-03-05 09:21:59 +01:00
parent 805163b280
commit 6b045fd225
2 changed files with 131 additions and 100 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
kubernetes/apps/default/homelab/music-transcode/helmrelease.yaml
View File

@@ -37,7 +37,7 @@ spec:
app:
image:
repository: ghcr.io/auricom/freac
tag: 1.1.7@sha256:d70f65290ffd63e10f7a864f397203510ccdf166208bbbab405a889adceaeecd
tag: 1.1.7@sha256:faf620257ba1fdcb16b0b18e4d58a518f7a3ce6e886abb4d610b5ba374a503bf
command:
- /bin/bash
- -c

229
kubernetes/apps/default/homelab/music-transcode/scripts/transcode.sh
View File

@@ -11,12 +11,6 @@ set -o pipefail
export LANG=C.UTF-8
export LC_ALL=C.UTF-8
# # Send start ping to healthchecks
# if [[ -n "${HEALTHCHECKS_ID:-}" ]]; then
# curl --silent --max-time 10 --retry 5 "https://hc-ping.com/${HEALTHCHECKS_ID}/start"
# fi
# Create a logging function
log() {
echo "[$(date '+%Y-%m-%d %H:%M:%S')] $*"
@@ -169,7 +163,7 @@ transcode() {
fi
mkdir -p "$(dirname "$md5_file")"
md5sum "$input_file" | awk '{ print $1 }' > "$md5_file"
md5sum "$input_file" | cut -d' ' -f1 > "$md5_file"
log "Successfully transcoded: $input_file -> $output_file"
fi
}
@@ -188,7 +182,7 @@ write_cue() {
fi
mkdir -p "$(dirname "$md5_file")"
md5sum "$input_file" | awk '{ print $1 }' > "$md5_file"
md5sum "$input_file" | cut -d' ' -f1 > "$md5_file"
log "Successfully wrote cue: $output_file"
fi
}
@@ -206,32 +200,18 @@ write_jpg() {
fi
mkdir -p "$(dirname "$md5_file")"
md5sum "$input_file" | awk '{ print $1 }' > "$md5_file"
md5sum "$input_file" | cut -d' ' -f1 > "$md5_file"
log "Successfully converted cover: $input_file -> $output_file"
fi
}
directory_structure() {
local dryrun_flag=""
[[ $MODE_DRY_RUN == true ]] && dryrun_flag="--dry-run"
log "INFO: Creating directory structure with rsync..."
if ! rsync -rvz $dryrun_flag --exclude-from="./transcode_exclude.cfg" \
--include="*/" --exclude="*" "$TRANSCODE_INPUT_DIR" "$TRANSCODE_OUTPUT_DIR"; then
log "ERROR: rsync failed"
return 1
fi
}
convert_covers() {
log "INFO: Looking for covers to convert..."
cd "$TRANSCODE_INPUT_DIR" || exit 1
for ext in $TRANSCODE_COVER_EXTENSIONS; do
log "INFO: Searching for .$ext files..."
while IFS= read -r val; do
[[ -z "$val" ]] && continue
process_file() {
local val="$1"
local ext="$2"
local type="$3" # cover, music, or cue
case "$type" in
cover)
local filename="$TRANSCODE_OUTPUT_DIR/${val%.*}.jpg"
local md5_filename="$TRANSCODE_DB/${val}.md5"
local process_file=false
@@ -245,7 +225,7 @@ convert_covers() {
process_file=true
log "Processing new file: $val"
elif [[ $MODE_CHECKSUM == true ]]; then
if [[ ! -f "$md5_filename" ]] || [[ "$(cat "$md5_filename" 2>/dev/null)" != "$(md5sum "$val" | awk '{ print $1 }')" ]]; then
if [[ ! -f "$md5_filename" ]] || [[ "$(cat "$md5_filename" 2>/dev/null)" != "$(md5sum "$val" | cut -d' ' -f1)" ]]; then
process_file=true
log "File changed, reprocessing: $val"
fi
@@ -254,19 +234,9 @@ convert_covers() {
if [[ $process_file == true ]]; then
write_jpg "$val" "$filename" "$md5_filename"
fi
done < <("$TRANSCODE_FD_BIN" --extension "$ext" $TRANSCODE_FD_FILTERS --type f)
done
}
convert_music() {
log "INFO: Looking for music to transcode..."
cd "$TRANSCODE_INPUT_DIR" || exit 1
for ext in $TRANSCODE_MUSIC_EXTENSIONS; do
log "INFO: Searching for .$ext files..."
while IFS= read -r val; do
[[ -z "$val" ]] && continue
;;
music)
local filebasename="$TRANSCODE_OUTPUT_DIR/${val%.*}"
local filename="${filebasename}.opus"
local md5_filename="$TRANSCODE_DB/${val}.md5"
@@ -281,7 +251,7 @@ convert_music() {
process_file=true
log "Processing new file: $val"
elif [[ $MODE_CHECKSUM == true ]]; then
if [[ ! -f "$md5_filename" ]] || [[ "$(cat "$md5_filename" 2>/dev/null)" != "$(md5sum "$val" | awk '{ print $1 }')" ]]; then
if [[ ! -f "$md5_filename" ]] || [[ "$(cat "$md5_filename" 2>/dev/null)" != "$(md5sum "$val" | cut -d' ' -f1)" ]]; then
process_file=true
log "File changed, reprocessing: $val"
fi
@@ -290,7 +260,91 @@ convert_music() {
if [[ $process_file == true ]]; then
transcode "$val" "$filename" "$md5_filename"
fi
done < <(fd_safe --extension "$ext" $TRANSCODE_FD_FILTERS --type f)
;;
cue)
local output_file="$TRANSCODE_OUTPUT_DIR/$val"
local md5_filename="$TRANSCODE_DB/${val}.md5"
local replacement_text_string="FILE \"$(basename "${val%.*}").opus\" MP3"
local process_file=false
# Create output directory if it doesn't exist
mkdir -p "$(dirname "$output_file")"
mkdir -p "$(dirname "$md5_filename")"
# Check if we need to process this file
if [[ ! -f "$md5_filename" ]]; then
process_file=true
log "Processing new cuefile: $val"
elif [[ $MODE_CHECKSUM == true ]]; then
if [[ ! -f "$md5_filename" ]] || [[ "$(cat "$md5_filename" 2>/dev/null)" != "$(md5sum "$val" | cut -d' ' -f1)" ]]; then
process_file=true
log "Cuefile changed, reprocessing: $val"
fi
fi
if [[ $process_file == true ]]; then
cp -p "$val" "$output_file"
write_cue "$val" "$output_file" "$replacement_text_string" "$md5_filename"
fi
;;
esac
}
directory_structure() {
local dryrun_flag=""
[[ $MODE_DRY_RUN == true ]] && dryrun_flag="--dry-run"
log "INFO: Creating directory structure with rsync..."
if ! rsync -rvz $dryrun_flag --exclude-from="./transcode_exclude.cfg" \
--include="*/" --exclude="*" "$TRANSCODE_INPUT_DIR/" "$TRANSCODE_OUTPUT_DIR/"; then
log "ERROR: rsync failed"
return 1
fi
}
# Export functions so they're available to subshells
export -f log
export -f transcode
export -f write_cue
export -f write_jpg
export -f process_file
convert_covers() {
log "INFO: Looking for covers to convert..."
cd "$TRANSCODE_INPUT_DIR" || exit 1
for ext in $TRANSCODE_COVER_EXTENSIONS; do
log "INFO: Searching for .$ext files..."
# Create a temporary script for processing
local temp_script=$(mktemp)
cat > "$temp_script" << 'EOF'
#!/bin/bash
process_file "$1" "$2" "$3"
EOF
chmod +x "$temp_script"
fd_safe --extension "$ext" $TRANSCODE_FD_FILTERS --type f -x "$temp_script" {} "$ext" cover \;
rm -f "$temp_script"
done
}
convert_music() {
log "INFO: Looking for music to transcode..."
cd "$TRANSCODE_INPUT_DIR" || exit 1
for ext in $TRANSCODE_MUSIC_EXTENSIONS; do
log "INFO: Searching for .$ext files..."
# Create a temporary script for processing
local temp_script=$(mktemp)
cat > "$temp_script" << 'EOF'
#!/bin/bash
process_file "$1" "$2" "$3"
EOF
chmod +x "$temp_script"
fd_safe --extension "$ext" $TRANSCODE_FD_FILTERS --type f -x "$temp_script" {} "$ext" music \;
rm -f "$temp_script"
done
}
@@ -298,68 +352,50 @@ fix_cuefiles() {
log "INFO: Looking for cuefiles..."
cd "$TRANSCODE_INPUT_DIR" || exit 1
while IFS= read -r val; do
[[ -z "$val" ]] && continue
# Create a temporary script for processing
local temp_script=$(mktemp)
cat > "$temp_script" << 'EOF'
#!/bin/bash
process_file "$1" "$2" "$3"
EOF
chmod +x "$temp_script"
local output_file="$TRANSCODE_OUTPUT_DIR/$val"
local md5_filename="$TRANSCODE_DB/${val}.md5"
local replacement_text_string="FILE \"$(basename "${val%.*}").opus\" MP3"
local process_file=false
# Create output directory if it doesn't exist
mkdir -p "$(dirname "$output_file")"
mkdir -p "$(dirname "$md5_filename")"
# Check if we need to process this file
if [[ ! -f "$md5_filename" ]]; then
process_file=true
log "Processing new cuefile: $val"
elif [[ $MODE_CHECKSUM == true ]]; then
if [[ ! -f "$md5_filename" ]] || [[ "$(cat "$md5_filename" 2>/dev/null)" != "$(md5sum "$val" | awk '{ print $1 }')" ]]; then
process_file=true
log "Cuefile changed, reprocessing: $val"
fi
fi
if [[ $process_file == true ]]; then
cp -p "$val" "$output_file"
write_cue "$val" "$output_file" "$replacement_text_string" "$md5_filename"
fi
done < <("$TRANSCODE_FD_BIN" --extension cue $TRANSCODE_FD_FILTERS --type f)
fd_safe --extension cue $TRANSCODE_FD_FILTERS --type f -x "$temp_script" {} cue cue \;
rm -f "$temp_script"
}
remove_absent_from_source() {
cd "$TRANSCODE_DB" || exit 1
local files
files=$("$TRANSCODE_FD_BIN" --extension md5) || return 0
# Create a temporary script file for the removal operation
local temp_script=$(mktemp)
cat > "$temp_script" << 'EOF'
#!/bin/bash
val="$1"
[[ -z "$val" ]] && exit 0
while IFS= read -r val; do
[[ -z "$val" ]] && continue
filename="$(dirname "$val")/$(basename "$val" .md5)"
source_path="$TRANSCODE_INPUT_DIR/$filename"
local filename="$(dirname "$val")/$(basename "$val" .md5)"
# Use printf to properly handle special characters
local source_path
source_path=$(printf "%s/%s" "$TRANSCODE_INPUT_DIR" "$filename")
if [[ ! -e "$source_path" ]]; then
# Double check with find to handle special characters
if ! find "$TRANSCODE_INPUT_DIR/$(dirname "$filename")" -maxdepth 1 -name "$(basename "$filename")*" 2>/dev/null | grep -q .; then
log "INFO: Confirmed - Transcoded file $filename doesn't have a source file: delete"
if [[ $MODE_DRY_RUN == false ]]; then
rm -f "$TRANSCODE_OUTPUT_DIR/$filename"*
rm -f "$TRANSCODE_DB/$filename"*
fi
fi
if [[ ! -e "$source_path" ]]; then
if ! find "$TRANSCODE_INPUT_DIR/$(dirname "$filename")" -maxdepth 1 -name "$(basename "$filename")*" 2>/dev/null | grep -q .; then
echo "[$(date "+%Y-%m-%d %H:%M:%S")] INFO: Confirmed - Transcoded file $filename doesnt have a source file: delete"
if [[ $MODE_DELETE == true && $MODE_DRY_RUN == false ]]; then
rm -f "$TRANSCODE_OUTPUT_DIR/$filename"*
rm -f "$TRANSCODE_DB/$filename"*
fi
done <<< "$files"
fi
fi
EOF
chmod +x "$temp_script"
"$TRANSCODE_FD_BIN" --extension md5 -x "$temp_script" {} \;
rm -f "$temp_script"
log "INFO: removing empty directories..."
if [[ $MODE_DRY_RUN == false ]]; then
cd "$TRANSCODE_OUTPUT_DIR" || exit 1
find . -type d -empty -delete 2>/dev/null || true
cd "$TRANSCODE_DB" || exit 1
find . -type d -empty -delete 2>/dev/null || true
find "$TRANSCODE_OUTPUT_DIR" -type d -empty -delete 2>/dev/null || true
find "$TRANSCODE_DB" -type d -empty -delete 2>/dev/null || true
fi
}
@@ -379,8 +415,3 @@ fi
if [[ $MODE_DRY_RUN == false ]]; then
manage_execution_time write
fi
# # Send completion ping to healthchecks
# if [[ -n "${HEALTHCHECKS_ID:-}" ]]; then
# curl --silent --max-time 10 --retry 5 "https://hc-ping.com/${HEALTHCHECKS_ID}"
# fi