ScriptFanix/AtOM
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Resolve "Filenames can have linefeeds in them"

Browse Source
This commit is contained in:
ScriptFanix 2026-02-15 05:10:36 +01:00
parent c4f92ef3ad
commit 72f0c1aed8
24 changed files with 74 additions and 67 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
CHANGELOG.md
View File

@ -1,4 +1,5 @@
# DEV
* Support for newline character in filenames
# 1.0.5
### BUGS (Minor)

24
atom
View File

@ -202,7 +202,7 @@ echo '
FROM destination_files
WHERE source_file_id is NULL;' >&3
read -u4 removecount
read -u4 -r -d $'\0' removecount
until (( ${#removefile[@]} == removecount ))
do
echo '
@ -219,7 +219,7 @@ do
SELECT "AtOM:NoMoreFiles";
' >&3
read -u4 line
read -u4 -r -d $'\0' line
until [[ $line == AtOM:NoMoreFiles ]]
do
removeFileId=${line%%::AtOM:SQL:Sep::*}
@ -227,7 +227,7 @@ do
removeFileDestName=${line%%::AtOM:SQL:Sep::*}
rest=${line#*::AtOM:SQL:Sep::}
removefile[$removeFileId]="${destinationpath["$removeFileDestName"]}/${rest%%::AtOM:SQL:Sep::*}"
read -u4 line
read -u4 -r -d $'\0' line
done
done
@ -387,7 +387,7 @@ echo '
<> CAST(source_files.last_change AS TEXT)
AND mime_type_actions.destination_id = destinations.id
AND mime_type_actions.action = 1;' >&3
read -u4 filecount
read -u4 -r -d $'\0' filecount
if [ -n "$maxbatch" ] && (( maxbatch < filecount ))
then
(( togo = filecount - maxbatch ))
@ -436,11 +436,11 @@ echo '
(( maxbatch )) && echo "LIMIT $maxbatch" >&3
echo ';
SELECT "AtOM:NoMoreFiles";' >&3
read -u4 line
read -u4 -r -d $'\0' line
while ! [[ $line = AtOM:NoMoreFiles ]]
do
decodefiles+=("$line::AtOM:SQL:Sep::")
read -u4 line
read -u4 -r -d $'\0' line
done
(( cron )) || echo -n $'Creating tasks...\033[K'
@ -773,11 +773,11 @@ then
WHERE tasks.status = 2;
SELECT "AtOM:NoMoreFiles";' >&3
read -u4 line
read -u4 -r -d $'\0' line
while ! [[ $line = AtOM:NoMoreFiles ]]
do
failedtasks+=("$line")
read -u4 line
read -u4 -r -d $'\0' line
done
for line in "${failedtasks[@]}"
do
@ -833,11 +833,11 @@ do
SELECT "AtOM:NoMoreFiles";
' >&3
read -u4 line
read -u4 -r -d $'\0' line
while [[ $line != AtOM:NoMoreFiles ]]
do
renamefiles+=("$line")
read -u4 line
read -u4 -r -d $'\0' line
done
if (( ${#renamefiles[@]} ))
then
@ -940,11 +940,11 @@ echo '
(( cron )) || echo -n 'Removing obsolete files...'$'\033[K'
lines=()
read -u4 line
read -u4 -r -d $'\0' line
while [[ $line != AtOM:NoMoreFiles ]]
do
lines+=("$line")
read -u4 line
read -u4 -r -d $'\0' line
done
echo 'BEGIN TRANSACTION;' >&3
for line in "${lines[@]}"

7
lib/copy/action
View File

@ -22,11 +22,11 @@ copyFiles_action() {
AND mime_type_actions.action = 2;
SELECT "AtOM:NoMoreFiles";' >&3
read -u4 line
read -u4 -r -d $'\0' line
while ! [[ $line = AtOM:NoMoreFiles ]]
do
copyfiles+=("$line")
read -u4 line
read -u4 -r -d $'\0' line
done
echo 'BEGIN TRANSACTION;' >&3
@ -91,9 +91,10 @@ copyFiles_action() {
"$sourcepath/$sourcefilename" \
"$destdir"
then
destfilename=${sourcefilename//$'\n'/::AtOM:NewLine:SQL:Inline::}
Update destination_files \
filename \
"$destdir/${sourcefilename##*/}"\
"$destdir/${destfilename##*/}"\
rename_pattern \
"${destinationrenamepath[$destination]}/${destinationrename[$destination]}"\
fat32compat \

4
lib/copy/guessPath
View File

@ -21,7 +21,7 @@ guessPath() {
LIMIT 1
),"0.0");
'>&3
read -u4 timestamp
read -u4 -r -d $'\0' timestamp
if (( ${timestamp/./} == 0 ))
then
return 2
@ -46,7 +46,7 @@ guessPath() {
LIMIT 1
),"AtOM:NotFound");
'>&3
read -u4 filename
read -u4 -r -d $'\0' filename
if [[ $filename != AtOM:NotFound ]]
then
echo "${filename%/*}"

3
lib/database/Insert
View File

@ -26,6 +26,7 @@ Insert() {
insert_values+=$value
;;
*)
value=${value//::AtOM:NewLine:SQL:Inline::/$'\n'}
insert_values+='"'"${value//\"/\"\"}"'"'
;;
esac
@ -36,7 +37,7 @@ Insert() {
"( $insert_values );" >&3
(( no_id )) || {
echo 'SELECT LAST_INSERT_ROWID();' >&3
read -u 4 results
read -u4 -r -d $'\0' results
echo "$results"
}
}

3
lib/database/Select
View File

@ -19,6 +19,7 @@ Select() {
while read key operator value
do
(( ${#where_statement} )) && where_statement+=( "AND" )
value=${value//::AtOM:NewLine:SQL:Inline::/$'\n'}
where_statement+=( "$key $operator "'"'"${value//\"/\"\"}"'"' )
done
echo "SELECT IFNULL(" \
@ -26,7 +27,7 @@ Select() {
"WHERE ${where_statement[@]})" \
",'SQL::Select:not found'" \
");" >&3
read -u 4 results
read -u 4 -r -d $'\0' results
if ! [[ $results == "SQL::Select:not found" ]]
then
echo "$results"

12
lib/database/open
View File

@ -6,13 +6,17 @@ openDatabase() {
[[ -f "$database" ]] || populate_db=1
rm -f "$tempdir"/sqlite.{in,out}
mkfifo "$tempdir"/sqlite.{in,out}
sqlite3 -bail "$database" \
< "$tempdir/sqlite.in" \
stdbuf -o0 sqlite3 -bail \
-newline $'::AtOM:SQL:EOL::\n' \
"$database" \
< "$tempdir/sqlite.in" \
| stdbuf -o0 \
sed 's/::AtOM:SQL:EOL::/\x0/g;s/\(\x0\)\xA/\1/g' \
> "$tempdir/sqlite.out" &
db_pid=$!
exec 3> "$tempdir"/sqlite.in
exec 4< "$tempdir"/sqlite.out
rm "$tempdir"/sqlite.in "$tempdir"/sqlite.out
rm "$tempdir"/sqlite.{in,out}
if (( debug > 2 ))
then
exec 5>&3
@ -24,7 +28,7 @@ openDatabase() {
echo 'PRAGMA recursive_triggers = ON;' >&3
echo 'PRAGMA temp_store = 2;' >&3
echo 'PRAGMA locking_mode = EXCLUSIVE;' >&3
read -u4
read -u4 -r -d $'\0'
unset REPLY
checkDatabaseVersion
}

4
lib/database/upgradedatabase_1_2
View File

@ -18,11 +18,11 @@ FROM destination_files;
SELECT "AtOM::NoMoreData";' >&3
read -u4 data
read -u4 -r -d $'\0' data
while [[ $data != AtOM::NoMoreData ]]
do
datas+=( "$data" )
read -u4 data
read -u4 -r -d $'\0' data
done
echo 'BEGIN TRANSACTION;' >&3
for data in "${datas[@]}"

2
lib/decode/mpcdec
View File

@ -2,5 +2,5 @@
decodeMpcdec() {
tmpfile="${fileid}mpcdec"
commandline=(${ionice}mpcdec)
commandline+=("$sourcepath/$filename" "$tempdir/$tmpfile.wav")
commandline+=("$sourcepath/${filename//$'\n'/::AtOM:NewLine:SQL:Inline::}" "$tempdir/$tmpfile.wav")
}

2
lib/decode/opusdec
View File

@ -2,5 +2,5 @@
decodeOpusdec() {
tmpfile="${fileid}opusdec"
commandline=(${ionice}opusdec)
commandline+=("$sourcepath/$filename" "$tempdir/$tmpfile.wav")
commandline+=("$sourcepath/${filename//$'\n'/::AtOM:NewLine:SQL:Inline::}" "$tempdir/$tmpfile.wav")
}

2
lib/decode/sox
View File

@ -12,7 +12,7 @@ decodeSox() {
then
commandline+=("$1")
else
commandline+=("$sourcepath/$filename")
commandline+=("$sourcepath/${filename//$'\n'/::AtOM:NewLine:SQL:Inline::}")
fi
if [ -n "${destinationfrequency["$destination"]}" ] \
&& (( ${rate:-0} != ${destinationfrequency["$destination"]} ))

3
lib/files/getDestDir
View File

@ -88,6 +88,7 @@ getDestDir() {
while [[ $part =~ / ]]
do
thispart="${part%%/*}"
thispart=${thispart//$'\n'/::AtOM:NewLine:SQL:Inline::}
if (( ${destinationascii["$destination"]} ))
then
echo "$thispart" >&${toascii[1]}
@ -100,7 +101,7 @@ getDestDir() {
fi
if ! [ -d "${destinationpath[$destination]}/$destdir" ]
then
mkdir -p "${destinationpath[$destination]}/$destdir"
mkdir -p "${destinationpath[$destination]}/${destdir//::AtOM:NewLine:SQL:Inline::/$'\n'}"
fi
destdir="${destdir//+(\/)//}"
}

3
lib/files/getDestFile
View File

@ -55,10 +55,11 @@ getDestFile() {
destfile="${filename##*/}"
destfile="${destfile%.*}"
fi
destfile=$(sanitizeFile "$destfile")
destfile=${destfile//$'\n'/::AtOM:NewLine:SQL:Inline::}
if (( ${destinationascii["$destination"]} ))
then
echo "$destfile" >&${toascii[1]}
read -r -u${toascii[0]} destfile
fi
destfile=$(sanitizeFile "$destfile")
}

10
lib/files/getFiles
View File

@ -8,7 +8,7 @@ getFiles() {
(( cron )) || echo -n "Scanning $sourcepath... "
# We probably have thousands of files, don't waste time on disk writes
echo 'BEGIN TRANSACTION;' >&3
while read time size filename
while read -d $'\0' time size filename
do
if (( skip_us_timestamp ))
then
@ -17,7 +17,7 @@ getFiles() {
compare_time=$time
fi
if ! Select source_files id >/dev/null <<-EOWhere
filename = $filename
filename = ${filename//$'\n'/::AtOM:NewLine:SQL:Inline::}
mime_type > 0
last_change LIKE $compare_time
size = $size
@ -47,7 +47,7 @@ getFiles() {
mime_type $mimetypeid \
>/dev/null \
<<-EOWhere
filename $filename
filename ${filename//$'\n'/::AtOM:NewLine:SQL:Inline::}
EOWhere
(( ++new ))
if (( new % 1000 == 0 ))
@ -58,12 +58,12 @@ getFiles() {
fi
else
Update source_files last_seen $scantime <<-EOWhere
filename = $filename
filename = ${filename//$'\n'/::AtOM:NewLine:SQL:Inline::}
EOWhere
fi
progressSpin
done < <(
find "$sourcepath" "${prunes[@]}" -type f -not -name '.*' -printf "%T@ %s %P\n"
find "$sourcepath" "${prunes[@]}" -type f -not -name '.*' -printf "%T@ %s %P\0"
)
echo 'COMMIT;' >&3
(( cron )) || echo -n $'\r'

4
lib/tags/update
View File

@ -52,12 +52,12 @@ echo '
;
SELECT "AtOM:NoMoreFiles";' >&3
read -u4 line
read -u4 -r -d $'\0' line
while ! [[ $line = AtOM:NoMoreFiles ]]
do
tagfiles+=("$line")
(( filecount++ ))
read -u4 line
read -u4 -r -d $'\0' line
done
echo 'BEGIN TRANSACTION;' >&3
for line in "${tagfiles[@]}"

2
lib/tasks/gettaskinfos
View File

@ -10,7 +10,7 @@ gettaskinfos() {
FROM tasks
WHERE id='$1';
' >&3
read -u4 line
read -u4 -r -d $'\0' line
taskid=${line%%::AtOM:SQL:Sep::*}
rest="${line#*::AtOM:SQL:Sep::}::AtOM:SQL:Sep::"
sourcefileid=${rest%%::AtOM:SQL:Sep::*}

2
lib/workers/create
View File

@ -2,7 +2,7 @@
createworker() {
(( ++active ))
read -u4 line
read -u4 -r -d $'\0' line
taskid=${line%%::AtOM:SQL:Sep::*}
rest="${line#*::AtOM:SQL:Sep::}::AtOM:SQL:Sep::"
sourcefileid=${rest%%::AtOM:SQL:Sep::*}

8
lib/workers/master
View File

@ -10,7 +10,7 @@ master() {
WHERE status = 0;
'>&3
read -u4 remaining
read -u4 -r -d $'\0' remaining
if (( remaining == 0 ))
then
sleep 0.1
@ -108,7 +108,7 @@ master() {
LIMIT 1;
'>&3
read -u4 ready
read -u4 -r -d $'\0' ready
if (( ready > 0 ))
then
createworker
@ -194,7 +194,7 @@ master() {
LIMIT 1;
' >&3
read -u4 ready
read -u4 -r -d $'\0' ready
if (( active == 0 && ready == 0 ))
then
@ -234,7 +234,7 @@ master() {
WHERE status = 0;
'>&3
read -u4 remaining
read -u4 -r -d $'\0' remaining
done
fi
}

4
toys/checkextensions
View File

@ -121,7 +121,7 @@ getdstfiles() {
;
SELECT "AtOM:NoMoreFiles";
'>&3
while read -u4 line
while read -u4 -r -d $'\0' line
do
if [[ $line == AtOM:NoMoreFiles ]]
then
@ -159,7 +159,7 @@ renameFile() {
fi
}
while read -u4 line
while read -u4 -r -d $'\0' line
do
if [[ $line == AtOM:NoMoreFiles ]]
then

6
toys/checkmissing
View File

@ -65,6 +65,7 @@ getConfig
sanityCheck
openDatabase
echo -n "Checking for missing files... "
echo '
SELECT
destination_files.id,
@ -79,7 +80,7 @@ echo 'SELECT "AtOM:NoMoreFiles";' >&3
declare -a \
destination_names \
files
read -u4 line
read -u4 -r -d $'\0' line
until [[ $line == AtOM:NoMoreFiles ]]
do
id=${line%%::AtOM:SQL:Sep::*}
@ -87,12 +88,11 @@ do
destination_names[id]=${rest%%::AtOM:SQL:Sep::*}
rest=${rest#*::AtOM:SQL:Sep::}
files[id]=${rest}
read -u4 line
read -u4 -r -d $'\0' line
done
echo 'BEGIN TRANSACTION;' >&3
echo -n "Checking for missing files... "
for index in "${!files[@]}"
do
destination=${destination_names[index]}

13
toys/cleandestinations
View File

@ -65,25 +65,22 @@ getConfig
sanityCheck
openDatabase
checkwanted() {
Select id <<<"filename = $1"
}
for destination in "${!destinationpath[@]}"
do
echo -ne "\rScanning destination $destination... \033[K"
while read -r filename
while read -r -d $'\0' filename
do
sqlfile=${filename//$'\n'/::AtOM:NewLine:SQL:Inline::}
if ! Select destination_files id \
>/dev/null \
<<<"filename = ${filename#${destinationpath["$destination"]}/}"
<<<"filename = ${sqlfile#${destinationpath["$destination"]}/}"
then
echo -e $'\r'"$filename\033[K"
(( remove )) && rm -f "$filename"
(( remove )) && rm "$filename"
echo -n "Scanning destination $destination... "
fi
progressSpin
done < <(find "${destinationpath["$destination"]}" -type f)
done < <(find "${destinationpath["$destination"]}" -type f -print0)
done
echo -en "\r\033[K"

14
toys/createindex
View File

@ -274,7 +274,7 @@ fi
echo 'SELECT IFNULL(
(SELECT last_seen FROM source_files ORDER BY last_seen DESC LIMIT 1),
0);' >&3
read -u4 lastupdate
read -u4 -r -d $'\0' lastupdate
if ! [[ "$output" == - ]]
then
@ -369,11 +369,11 @@ COLLATE NOCASE;
SELECT "AtOM:NoMoreFiles";' >&3
read -u4 line
read -u4 -r -d $'\0' line
until [[ $line == AtOM:NoMoreFiles ]]
do
files+=("$line")
read -u4 line
read -u4 -r -d $'\0' line
done
for line in "${files[@]}"
@ -675,7 +675,7 @@ echo '
SELECT "AtOM:NoMoreFiles";' >&3
read -u4 line
read -u4 -r -d $'\0' line
until [[ $line == AtOM:NoMoreFiles ]]
do
artist="${line%%::AtOM:SQL:Sep::*}"
@ -685,7 +685,7 @@ do
artists+=( "$artist" )
maxcountlen=$(( ${#count} > maxcountlen ? ${#count} : maxcountlen ))
maxartistlen=$(( ${#artist} > maxartistlen ? ${#artist} : maxartistlen ))
read -u4 line
read -u4 -r -d $'\0' line
done
head=$(
printf "| # | %'${maxcoutlen}s | %-${maxartistlen}s |" \
@ -714,7 +714,7 @@ echo '
FROM source_files
INNER JOIN mime_types
ON source_files.mime_type=mime_types.id;' >&3
read -u4 line
read -u4 -r -d $'\0' line
totalcount="${line%%::AtOM:SQL:Sep::*}"
maxcountlen=$(printf "%'i" $totalcount)
maxcountlen=${#maxcountlen}
@ -743,7 +743,7 @@ do
INNER JOIN mime_types
ON source_files.mime_type=mime_types.id
WHERE mime_text LIKE "'"$format"'";' >&3
read -u4 line
read -u4 -r -d $'\0' line
count="${line%%::AtOM:SQL:Sep::*}"
rest="${line#*::AtOM:SQL:Sep::}::AtOM:SQL:Sep::"
size="${rest%%::AtOM:SQL:Sep::*}"

4
toys/lowquality
View File

@ -131,11 +131,11 @@ echo ') ORDER BY bitrate;' >&3
echo 'SELECT "AtOM:NoMoreFiles";' >&3
read -u4 line
read -u4 -r -d $'\0' line
until [[ $line == AtOM:NoMoreFiles ]]
do
echo "${line//::AtOM:SQL:Sep::/$'\t'}"
read -u4 line
read -u4 -r -d $'\0' line
done
closeDatabase

4
toys/missingtags
View File

@ -149,11 +149,11 @@ cat >&3 <<-EOSelect
SELECT "AtOM:NoMoreFiles";
EOSelect
read -u4 line
read -u4 -r -d $'\0' line
until [[ $line == AtOM:NoMoreFiles ]]
do
lines+=( "$line" )
read -u4 line
read -u4 -r -d $'\0' line
done
for line in "${lines[@]}"