From 72f0c1aed82d1956f9a409ac0342fbfe50711f6a Mon Sep 17 00:00:00 2001 From: ScriptFanix <62394-ScriptFanix@users.noreply.framagit.org> Date: Sun, 15 Feb 2026 05:10:36 +0100 Subject: [PATCH] Resolve "Filenames can have linefeeds in them" --- CHANGELOG.md | 1 + atom | 24 ++++++++++++------------ lib/copy/action | 7 ++++--- lib/copy/guessPath | 4 ++-- lib/database/Insert | 3 ++- lib/database/Select | 3 ++- lib/database/open | 12 ++++++++---- lib/database/upgradedatabase_1_2 | 4 ++-- lib/decode/mpcdec | 2 +- lib/decode/opusdec | 2 +- lib/decode/sox | 2 +- lib/files/getDestDir | 3 ++- lib/files/getDestFile | 3 ++- lib/files/getFiles | 10 +++++----- lib/tags/update | 4 ++-- lib/tasks/gettaskinfos | 2 +- lib/workers/create | 2 +- lib/workers/master | 8 ++++---- toys/checkextensions | 4 ++-- toys/checkmissing | 6 +++--- toys/cleandestinations | 13 +++++-------- toys/createindex | 14 +++++++------- toys/lowquality | 4 ++-- toys/missingtags | 4 ++-- 24 files changed, 74 insertions(+), 67 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 68312d8..3be9d2f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,5 @@ # DEV +* Support for newline character in filenames # 1.0.5 ### BUGS (Minor) diff --git a/atom b/atom index 6c5b510..bb5e63a 100755 --- a/atom +++ b/atom @@ -202,7 +202,7 @@ echo ' FROM destination_files WHERE source_file_id is NULL;' >&3 -read -u4 removecount +read -u4 -r -d $'\0' removecount until (( ${#removefile[@]} == removecount )) do echo ' @@ -219,7 +219,7 @@ do SELECT "AtOM:NoMoreFiles"; ' >&3 - read -u4 line + read -u4 -r -d $'\0' line until [[ $line == AtOM:NoMoreFiles ]] do removeFileId=${line%%::AtOM:SQL:Sep::*} @@ -227,7 +227,7 @@ do removeFileDestName=${line%%::AtOM:SQL:Sep::*} rest=${line#*::AtOM:SQL:Sep::} removefile[$removeFileId]="${destinationpath["$removeFileDestName"]}/${rest%%::AtOM:SQL:Sep::*}" - read -u4 line + read -u4 -r -d $'\0' line done done @@ -387,7 +387,7 @@ echo ' <> CAST(source_files.last_change AS TEXT) AND mime_type_actions.destination_id = destinations.id AND mime_type_actions.action = 1;' >&3 -read -u4 filecount +read -u4 -r -d $'\0' filecount if [ -n "$maxbatch" ] && (( maxbatch < filecount )) then (( togo = filecount - maxbatch )) @@ -436,11 +436,11 @@ echo ' (( maxbatch )) && echo "LIMIT $maxbatch" >&3 echo '; SELECT "AtOM:NoMoreFiles";' >&3 -read -u4 line +read -u4 -r -d $'\0' line while ! [[ $line = AtOM:NoMoreFiles ]] do decodefiles+=("$line::AtOM:SQL:Sep::") - read -u4 line + read -u4 -r -d $'\0' line done (( cron )) || echo -n $'Creating tasks...\033[K' @@ -773,11 +773,11 @@ then WHERE tasks.status = 2; SELECT "AtOM:NoMoreFiles";' >&3 - read -u4 line + read -u4 -r -d $'\0' line while ! [[ $line = AtOM:NoMoreFiles ]] do failedtasks+=("$line") - read -u4 line + read -u4 -r -d $'\0' line done for line in "${failedtasks[@]}" do @@ -833,11 +833,11 @@ do SELECT "AtOM:NoMoreFiles"; ' >&3 - read -u4 line + read -u4 -r -d $'\0' line while [[ $line != AtOM:NoMoreFiles ]] do renamefiles+=("$line") - read -u4 line + read -u4 -r -d $'\0' line done if (( ${#renamefiles[@]} )) then @@ -940,11 +940,11 @@ echo ' (( cron )) || echo -n 'Removing obsolete files...'$'\033[K' lines=() -read -u4 line +read -u4 -r -d $'\0' line while [[ $line != AtOM:NoMoreFiles ]] do lines+=("$line") - read -u4 line + read -u4 -r -d $'\0' line done echo 'BEGIN TRANSACTION;' >&3 for line in "${lines[@]}" diff --git a/lib/copy/action b/lib/copy/action index 623d148..b49aea7 100644 --- a/lib/copy/action +++ b/lib/copy/action @@ -22,11 +22,11 @@ copyFiles_action() { AND mime_type_actions.action = 2; SELECT "AtOM:NoMoreFiles";' >&3 - read -u4 line + read -u4 -r -d $'\0' line while ! [[ $line = AtOM:NoMoreFiles ]] do copyfiles+=("$line") - read -u4 line + read -u4 -r -d $'\0' line done echo 'BEGIN TRANSACTION;' >&3 @@ -91,9 +91,10 @@ copyFiles_action() { "$sourcepath/$sourcefilename" \ "$destdir" then + destfilename=${sourcefilename//$'\n'/::AtOM:NewLine:SQL:Inline::} Update destination_files \ filename \ - "$destdir/${sourcefilename##*/}"\ + "$destdir/${destfilename##*/}"\ rename_pattern \ "${destinationrenamepath[$destination]}/${destinationrename[$destination]}"\ fat32compat \ diff --git a/lib/copy/guessPath b/lib/copy/guessPath index 75e1914..f7f0926 100644 --- a/lib/copy/guessPath +++ b/lib/copy/guessPath @@ -21,7 +21,7 @@ guessPath() { LIMIT 1 ),"0.0"); '>&3 - read -u4 timestamp + read -u4 -r -d $'\0' timestamp if (( ${timestamp/./} == 0 )) then return 2 @@ -46,7 +46,7 @@ guessPath() { LIMIT 1 ),"AtOM:NotFound"); '>&3 - read -u4 filename + read -u4 -r -d $'\0' filename if [[ $filename != AtOM:NotFound ]] then echo "${filename%/*}" diff --git a/lib/database/Insert b/lib/database/Insert index 32d09be..3a7bafd 100644 --- a/lib/database/Insert +++ b/lib/database/Insert @@ -26,6 +26,7 @@ Insert() { insert_values+=$value ;; *) + value=${value//::AtOM:NewLine:SQL:Inline::/$'\n'} insert_values+='"'"${value//\"/\"\"}"'"' ;; esac @@ -36,7 +37,7 @@ Insert() { "( $insert_values );" >&3 (( no_id )) || { echo 'SELECT LAST_INSERT_ROWID();' >&3 - read -u 4 results + read -u4 -r -d $'\0' results echo "$results" } } diff --git a/lib/database/Select b/lib/database/Select index b096953..cce3646 100644 --- a/lib/database/Select +++ b/lib/database/Select @@ -19,6 +19,7 @@ Select() { while read key operator value do (( ${#where_statement} )) && where_statement+=( "AND" ) + value=${value//::AtOM:NewLine:SQL:Inline::/$'\n'} where_statement+=( "$key $operator "'"'"${value//\"/\"\"}"'"' ) done echo "SELECT IFNULL(" \ @@ -26,7 +27,7 @@ Select() { "WHERE ${where_statement[@]})" \ ",'SQL::Select:not found'" \ ");" >&3 - read -u 4 results + read -u 4 -r -d $'\0' results if ! [[ $results == "SQL::Select:not found" ]] then echo "$results" diff --git a/lib/database/open b/lib/database/open index e9d2b6a..c63d148 100644 --- a/lib/database/open +++ b/lib/database/open @@ -6,13 +6,17 @@ openDatabase() { [[ -f "$database" ]] || populate_db=1 rm -f "$tempdir"/sqlite.{in,out} mkfifo "$tempdir"/sqlite.{in,out} - sqlite3 -bail "$database" \ - < "$tempdir/sqlite.in" \ + stdbuf -o0 sqlite3 -bail \ + -newline $'::AtOM:SQL:EOL::\n' \ + "$database" \ + < "$tempdir/sqlite.in" \ + | stdbuf -o0 \ + sed 's/::AtOM:SQL:EOL::/\x0/g;s/\(\x0\)\xA/\1/g' \ > "$tempdir/sqlite.out" & db_pid=$! exec 3> "$tempdir"/sqlite.in exec 4< "$tempdir"/sqlite.out - rm "$tempdir"/sqlite.in "$tempdir"/sqlite.out + rm "$tempdir"/sqlite.{in,out} if (( debug > 2 )) then exec 5>&3 @@ -24,7 +28,7 @@ openDatabase() { echo 'PRAGMA recursive_triggers = ON;' >&3 echo 'PRAGMA temp_store = 2;' >&3 echo 'PRAGMA locking_mode = EXCLUSIVE;' >&3 - read -u4 + read -u4 -r -d $'\0' unset REPLY checkDatabaseVersion } diff --git a/lib/database/upgradedatabase_1_2 b/lib/database/upgradedatabase_1_2 index 0985774..b4cd510 100644 --- a/lib/database/upgradedatabase_1_2 +++ b/lib/database/upgradedatabase_1_2 @@ -18,11 +18,11 @@ FROM destination_files; SELECT "AtOM::NoMoreData";' >&3 - read -u4 data + read -u4 -r -d $'\0' data while [[ $data != AtOM::NoMoreData ]] do datas+=( "$data" ) - read -u4 data + read -u4 -r -d $'\0' data done echo 'BEGIN TRANSACTION;' >&3 for data in "${datas[@]}" diff --git a/lib/decode/mpcdec b/lib/decode/mpcdec index 3bbd71c..c0621ff 100644 --- a/lib/decode/mpcdec +++ b/lib/decode/mpcdec @@ -2,5 +2,5 @@ decodeMpcdec() { tmpfile="${fileid}mpcdec" commandline=(${ionice}mpcdec) - commandline+=("$sourcepath/$filename" "$tempdir/$tmpfile.wav") + commandline+=("$sourcepath/${filename//$'\n'/::AtOM:NewLine:SQL:Inline::}" "$tempdir/$tmpfile.wav") } diff --git a/lib/decode/opusdec b/lib/decode/opusdec index 46f08bd..b2cac97 100644 --- a/lib/decode/opusdec +++ b/lib/decode/opusdec @@ -2,5 +2,5 @@ decodeOpusdec() { tmpfile="${fileid}opusdec" commandline=(${ionice}opusdec) - commandline+=("$sourcepath/$filename" "$tempdir/$tmpfile.wav") + commandline+=("$sourcepath/${filename//$'\n'/::AtOM:NewLine:SQL:Inline::}" "$tempdir/$tmpfile.wav") } diff --git a/lib/decode/sox b/lib/decode/sox index b9d4c0e..e12bf4c 100644 --- a/lib/decode/sox +++ b/lib/decode/sox @@ -12,7 +12,7 @@ decodeSox() { then commandline+=("$1") else - commandline+=("$sourcepath/$filename") + commandline+=("$sourcepath/${filename//$'\n'/::AtOM:NewLine:SQL:Inline::}") fi if [ -n "${destinationfrequency["$destination"]}" ] \ && (( ${rate:-0} != ${destinationfrequency["$destination"]} )) diff --git a/lib/files/getDestDir b/lib/files/getDestDir index cb9cd89..79d2c32 100644 --- a/lib/files/getDestDir +++ b/lib/files/getDestDir @@ -88,6 +88,7 @@ getDestDir() { while [[ $part =~ / ]] do thispart="${part%%/*}" + thispart=${thispart//$'\n'/::AtOM:NewLine:SQL:Inline::} if (( ${destinationascii["$destination"]} )) then echo "$thispart" >&${toascii[1]} @@ -100,7 +101,7 @@ getDestDir() { fi if ! [ -d "${destinationpath[$destination]}/$destdir" ] then - mkdir -p "${destinationpath[$destination]}/$destdir" + mkdir -p "${destinationpath[$destination]}/${destdir//::AtOM:NewLine:SQL:Inline::/$'\n'}" fi destdir="${destdir//+(\/)//}" } diff --git a/lib/files/getDestFile b/lib/files/getDestFile index 03f4e53..5180c94 100644 --- a/lib/files/getDestFile +++ b/lib/files/getDestFile @@ -55,10 +55,11 @@ getDestFile() { destfile="${filename##*/}" destfile="${destfile%.*}" fi + destfile=$(sanitizeFile "$destfile") + destfile=${destfile//$'\n'/::AtOM:NewLine:SQL:Inline::} if (( ${destinationascii["$destination"]} )) then echo "$destfile" >&${toascii[1]} read -r -u${toascii[0]} destfile fi - destfile=$(sanitizeFile "$destfile") } diff --git a/lib/files/getFiles b/lib/files/getFiles index 8dae04f..7f04616 100644 --- a/lib/files/getFiles +++ b/lib/files/getFiles @@ -8,7 +8,7 @@ getFiles() { (( cron )) || echo -n "Scanning $sourcepath... " # We probably have thousands of files, don't waste time on disk writes echo 'BEGIN TRANSACTION;' >&3 - while read time size filename + while read -d $'\0' time size filename do if (( skip_us_timestamp )) then @@ -17,7 +17,7 @@ getFiles() { compare_time=$time fi if ! Select source_files id >/dev/null <<-EOWhere - filename = $filename + filename = ${filename//$'\n'/::AtOM:NewLine:SQL:Inline::} mime_type > 0 last_change LIKE $compare_time size = $size @@ -47,7 +47,7 @@ getFiles() { mime_type $mimetypeid \ >/dev/null \ <<-EOWhere - filename $filename + filename ${filename//$'\n'/::AtOM:NewLine:SQL:Inline::} EOWhere (( ++new )) if (( new % 1000 == 0 )) @@ -58,12 +58,12 @@ getFiles() { fi else Update source_files last_seen $scantime <<-EOWhere - filename = $filename + filename = ${filename//$'\n'/::AtOM:NewLine:SQL:Inline::} EOWhere fi progressSpin done < <( - find "$sourcepath" "${prunes[@]}" -type f -not -name '.*' -printf "%T@ %s %P\n" + find "$sourcepath" "${prunes[@]}" -type f -not -name '.*' -printf "%T@ %s %P\0" ) echo 'COMMIT;' >&3 (( cron )) || echo -n $'\r' diff --git a/lib/tags/update b/lib/tags/update index 3286e26..ad82207 100644 --- a/lib/tags/update +++ b/lib/tags/update @@ -52,12 +52,12 @@ echo ' ; SELECT "AtOM:NoMoreFiles";' >&3 - read -u4 line + read -u4 -r -d $'\0' line while ! [[ $line = AtOM:NoMoreFiles ]] do tagfiles+=("$line") (( filecount++ )) - read -u4 line + read -u4 -r -d $'\0' line done echo 'BEGIN TRANSACTION;' >&3 for line in "${tagfiles[@]}" diff --git a/lib/tasks/gettaskinfos b/lib/tasks/gettaskinfos index 648e4ab..37cb859 100644 --- a/lib/tasks/gettaskinfos +++ b/lib/tasks/gettaskinfos @@ -10,7 +10,7 @@ gettaskinfos() { FROM tasks WHERE id='$1'; ' >&3 - read -u4 line + read -u4 -r -d $'\0' line taskid=${line%%::AtOM:SQL:Sep::*} rest="${line#*::AtOM:SQL:Sep::}::AtOM:SQL:Sep::" sourcefileid=${rest%%::AtOM:SQL:Sep::*} diff --git a/lib/workers/create b/lib/workers/create index f4c9cff..e4be183 100644 --- a/lib/workers/create +++ b/lib/workers/create @@ -2,7 +2,7 @@ createworker() { (( ++active )) - read -u4 line + read -u4 -r -d $'\0' line taskid=${line%%::AtOM:SQL:Sep::*} rest="${line#*::AtOM:SQL:Sep::}::AtOM:SQL:Sep::" sourcefileid=${rest%%::AtOM:SQL:Sep::*} diff --git a/lib/workers/master b/lib/workers/master index 0ef64e8..fb633f5 100644 --- a/lib/workers/master +++ b/lib/workers/master @@ -10,7 +10,7 @@ master() { WHERE status = 0; '>&3 - read -u4 remaining + read -u4 -r -d $'\0' remaining if (( remaining == 0 )) then sleep 0.1 @@ -108,7 +108,7 @@ master() { LIMIT 1; '>&3 - read -u4 ready + read -u4 -r -d $'\0' ready if (( ready > 0 )) then createworker @@ -194,7 +194,7 @@ master() { LIMIT 1; ' >&3 - read -u4 ready + read -u4 -r -d $'\0' ready if (( active == 0 && ready == 0 )) then @@ -234,7 +234,7 @@ master() { WHERE status = 0; '>&3 - read -u4 remaining + read -u4 -r -d $'\0' remaining done fi } diff --git a/toys/checkextensions b/toys/checkextensions index 6c8d874..be61773 100755 --- a/toys/checkextensions +++ b/toys/checkextensions @@ -121,7 +121,7 @@ getdstfiles() { ; SELECT "AtOM:NoMoreFiles"; '>&3 - while read -u4 line + while read -u4 -r -d $'\0' line do if [[ $line == AtOM:NoMoreFiles ]] then @@ -159,7 +159,7 @@ renameFile() { fi } -while read -u4 line +while read -u4 -r -d $'\0' line do if [[ $line == AtOM:NoMoreFiles ]] then diff --git a/toys/checkmissing b/toys/checkmissing index 13215f0..d5db9fd 100755 --- a/toys/checkmissing +++ b/toys/checkmissing @@ -65,6 +65,7 @@ getConfig sanityCheck openDatabase +echo -n "Checking for missing files... " echo ' SELECT destination_files.id, @@ -79,7 +80,7 @@ echo 'SELECT "AtOM:NoMoreFiles";' >&3 declare -a \ destination_names \ files -read -u4 line +read -u4 -r -d $'\0' line until [[ $line == AtOM:NoMoreFiles ]] do id=${line%%::AtOM:SQL:Sep::*} @@ -87,12 +88,11 @@ do destination_names[id]=${rest%%::AtOM:SQL:Sep::*} rest=${rest#*::AtOM:SQL:Sep::} files[id]=${rest} - read -u4 line + read -u4 -r -d $'\0' line done echo 'BEGIN TRANSACTION;' >&3 -echo -n "Checking for missing files... " for index in "${!files[@]}" do destination=${destination_names[index]} diff --git a/toys/cleandestinations b/toys/cleandestinations index e45ef59..1f01b62 100755 --- a/toys/cleandestinations +++ b/toys/cleandestinations @@ -65,25 +65,22 @@ getConfig sanityCheck openDatabase -checkwanted() { - Select id <<<"filename = $1" -} - for destination in "${!destinationpath[@]}" do echo -ne "\rScanning destination $destination... \033[K" - while read -r filename + while read -r -d $'\0' filename do + sqlfile=${filename//$'\n'/::AtOM:NewLine:SQL:Inline::} if ! Select destination_files id \ >/dev/null \ - <<<"filename = ${filename#${destinationpath["$destination"]}/}" + <<<"filename = ${sqlfile#${destinationpath["$destination"]}/}" then echo -e $'\r'"$filename\033[K" - (( remove )) && rm -f "$filename" + (( remove )) && rm "$filename" echo -n "Scanning destination $destination... " fi progressSpin - done < <(find "${destinationpath["$destination"]}" -type f) + done < <(find "${destinationpath["$destination"]}" -type f -print0) done echo -en "\r\033[K" diff --git a/toys/createindex b/toys/createindex index d1a01dd..b9cabf5 100755 --- a/toys/createindex +++ b/toys/createindex @@ -274,7 +274,7 @@ fi echo 'SELECT IFNULL( (SELECT last_seen FROM source_files ORDER BY last_seen DESC LIMIT 1), 0);' >&3 -read -u4 lastupdate +read -u4 -r -d $'\0' lastupdate if ! [[ "$output" == - ]] then @@ -369,11 +369,11 @@ COLLATE NOCASE; SELECT "AtOM:NoMoreFiles";' >&3 -read -u4 line +read -u4 -r -d $'\0' line until [[ $line == AtOM:NoMoreFiles ]] do files+=("$line") - read -u4 line + read -u4 -r -d $'\0' line done for line in "${files[@]}" @@ -675,7 +675,7 @@ echo ' SELECT "AtOM:NoMoreFiles";' >&3 -read -u4 line +read -u4 -r -d $'\0' line until [[ $line == AtOM:NoMoreFiles ]] do artist="${line%%::AtOM:SQL:Sep::*}" @@ -685,7 +685,7 @@ do artists+=( "$artist" ) maxcountlen=$(( ${#count} > maxcountlen ? ${#count} : maxcountlen )) maxartistlen=$(( ${#artist} > maxartistlen ? ${#artist} : maxartistlen )) - read -u4 line + read -u4 -r -d $'\0' line done head=$( printf "| # | %'${maxcoutlen}s | %-${maxartistlen}s |" \ @@ -714,7 +714,7 @@ echo ' FROM source_files INNER JOIN mime_types ON source_files.mime_type=mime_types.id;' >&3 -read -u4 line +read -u4 -r -d $'\0' line totalcount="${line%%::AtOM:SQL:Sep::*}" maxcountlen=$(printf "%'i" $totalcount) maxcountlen=${#maxcountlen} @@ -743,7 +743,7 @@ do INNER JOIN mime_types ON source_files.mime_type=mime_types.id WHERE mime_text LIKE "'"$format"'";' >&3 - read -u4 line + read -u4 -r -d $'\0' line count="${line%%::AtOM:SQL:Sep::*}" rest="${line#*::AtOM:SQL:Sep::}::AtOM:SQL:Sep::" size="${rest%%::AtOM:SQL:Sep::*}" diff --git a/toys/lowquality b/toys/lowquality index 665b30d..46839df 100755 --- a/toys/lowquality +++ b/toys/lowquality @@ -131,11 +131,11 @@ echo ') ORDER BY bitrate;' >&3 echo 'SELECT "AtOM:NoMoreFiles";' >&3 -read -u4 line +read -u4 -r -d $'\0' line until [[ $line == AtOM:NoMoreFiles ]] do echo "${line//::AtOM:SQL:Sep::/$'\t'}" - read -u4 line + read -u4 -r -d $'\0' line done closeDatabase diff --git a/toys/missingtags b/toys/missingtags index f523ff2..f848105 100755 --- a/toys/missingtags +++ b/toys/missingtags @@ -149,11 +149,11 @@ cat >&3 <<-EOSelect SELECT "AtOM:NoMoreFiles"; EOSelect -read -u4 line +read -u4 -r -d $'\0' line until [[ $line == AtOM:NoMoreFiles ]] do lines+=( "$line" ) - read -u4 line + read -u4 -r -d $'\0' line done for line in "${lines[@]}"