From 2d62dfa50b07647fbd43565a019e20a332a7d77f Mon Sep 17 00:00:00 2001 From: Vincent Riquer Date: Sun, 31 Mar 2013 00:00:02 +0100 Subject: [PATCH 01/13] fat32compat --- atom | 18 +++++++++++++++++- doc/config | 2 ++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/atom b/atom index c2a43d0..d58b74a 100755 --- a/atom +++ b/atom @@ -14,6 +14,7 @@ EFMTINVPARM=49 # config structures declare -A \ destinationchannels \ + destinationfat32compat \ destinationcopymime \ destinationformat \ destinationfrequency \ @@ -233,7 +234,7 @@ getConfigDestination() { "'off'" ;; esac - ;; + ;; 'rename') case "$value" in */*) @@ -242,6 +243,21 @@ getConfigDestination() { esac destinationrename["$destination"]="${value##*/}" ;; + 'fat32compat') + case $value in + 'true'|'on'|'yes') + destinationfat32compat["$destination"]=1 + ;; + 'false'|'off'|'no') + destinationfat32compat["$destination"]=0 + ;; + *) + echo "fat32compat takes values:" \ + "'yes' ,'true' ,'on', 'no', 'false',"\ + "'off'" + ;; + esac + ;; 'skip_mime-type') destinationskipmime[$destination]="${destinationskipmime[$destination]:+${destinationskipmime[$destination]}|}$value" ;; diff --git a/doc/config b/doc/config index 9b905df..5494d61 100644 --- a/doc/config +++ b/doc/config @@ -66,6 +66,8 @@ Sections: %{track}, %{year}. Untagged files or files in unrecognized formats will not be changed. + * fat32compat /: Rename files for compatibility with FAT32 + filesystems. * skip_mime-type : Files with mime-type will not be included in that destination. For more than one mime-type, use multiple times, as needed. The '*' character is a wildcard. From d8adee3177b8f957648555df256674eb7f1e437d Mon Sep 17 00:00:00 2001 From: Vincent Riquer Date: Sun, 31 Mar 2013 01:19:35 +0100 Subject: [PATCH 02/13] rename on fat32compat change --- atom | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/atom b/atom index d58b74a..2147030 100755 --- a/atom +++ b/atom @@ -1082,7 +1082,7 @@ copyFile() { " WHERE id=$destfileid" \ " )," \ " rename_pattern=" \ -"\"${destinationrenamepath[$destination]}/${destinationrename[$destination]}\""\ +"\"${destinationrenamepath[$destination]}/${destinationrename[$destination]}:${destinationfat32compat["$destination"]}\""\ "WHERE id=$destfileid;" \ >&3 (( ++copies )) @@ -1217,7 +1217,7 @@ encodeFile::mp3() { cleanup $tempdir/$tmpfile.wav source_file $fileid status 0 - rename_pattern "${destinationrenamepath[$destination]}/${destinationrename[$destination]}" + rename_pattern "${destinationrenamepath[$destination]}/${destinationrename[$destination]}:${destinationfat32compat["$destination"]}" EOInsert ) progressSpin @@ -1252,7 +1252,7 @@ encodeFile::vorbis() { cleanup $tempdir/$tmpfile.wav source_file $fileid status 0 - rename_pattern "${destinationrenamepath[$destination]}/${destinationrename[$destination]}" + rename_pattern "${destinationrenamepath[$destination]}/${destinationrename[$destination]}:${destinationfat32compat["$destination"]}" EOInsert ) progressSpin @@ -2182,7 +2182,7 @@ do WHERE destinations.name="'"$destination"'" AND (destination_files.rename_pattern != -"'"${destinationrenamepath[$destination]}/${destinationrename[$destination]}"'" +"'"${destinationrenamepath[$destination]}/${destinationrename[$destination]}:${destinationfat32compat["$destination"]}"'" OR destination_files.rename_pattern is NULL) ; @@ -2230,7 +2230,7 @@ do echo "UPDATE destination_files" \ "SET filename=\"${destfilename//\"/\"\"}\"," \ " rename_pattern=" \ -"\"${destinationrenamepath[$destination]}/${destinationrename[$destination]}\"" \ +"\"${destinationrenamepath[$destination]}/${destinationrename[$destination]}:${destinationfat32compat["$destination"]}\"" \ "WHERE id=$destfileid;" \ >&3 progressSpin From 63e01692fcec037bca0b48d16958044f2b9f46fd Mon Sep 17 00:00:00 2001 From: Vincent Riquer Date: Mon, 1 Apr 2013 23:10:37 +0200 Subject: [PATCH 03/13] createworker() destroyworker() --- atom | 83 ++++++++++++++++++++++++++++++++---------------------------- 1 file changed, 45 insertions(+), 38 deletions(-) diff --git a/atom b/atom index 2147030..7620990 100755 --- a/atom +++ b/atom @@ -1260,6 +1260,8 @@ encodeFile::vorbis() { worker() { trap "kill -USR1 $masterpid" EXIT + trap - USR1 ALRM PIPE + exec 2>>"$tempdir/errors.log" while : do echo work @@ -1271,6 +1273,7 @@ worker() { done if [[ $line == AtOM:Die ]] then + trap EXIT break elif [[ $line == AtOM:Sleep ]] then @@ -1353,8 +1356,8 @@ worker() { do [ -z "${cmd_arg[key]}" ] && unset cmd_arg[key] done - (( debug >= 2 )) && echo "${cmd_arg[@]}" >>"$tempdir/errors.log" - if "${cmd_arg[@]}" >/dev/null 2>>"$tempdir/errors.log" + (( debug >= 2 )) && echo "${cmd_arg[@]}" >&2 + if "${cmd_arg[@]}" >/dev/null then echo "finished $taskid|$sourcefileid|$destfileid|$destfilename" read line @@ -1408,13 +1411,7 @@ master() { 'work') if [ -n "$quit" ] then - dyingworker=${workers[workerid]} - unset workers[workerid] - eval echo AtOM:Die '>&'$((100+workerid)) - wait $dyingworker - eval $((100+workerid))'>&-' - eval $((200+workerid))'<&-' - rm "$tempdir"/worker${workerid}{in,out} + destroyworker $workerid elif (( active < concurrency )) then echo ' @@ -1474,13 +1471,7 @@ master() { read -u4 ready if (( remaining == 0 )) then - dyingworker=${workers[workerid]} - unset workers[workerid] - eval echo AtOM:Die '>&'$((100+workerid)) - wait $dyingworker - eval $((100+workerid))'>&-' - eval $((200+workerid))'<&-' - rm "$tempdir"/worker${workerid}{in,out} + destroyworker $workerid continue elif (( ready == 0 )) then @@ -1572,17 +1563,45 @@ master() { fi } +getworkerid() { + local i + for (( i=0 ; i < 100 ; i++ )) + do + if [ -z "${workers[i]}" ] + then + echo $i + break + fi + done + # If we reach this, we have reached the hardcoded 100 workers limit + (( concurrency-- )) +} + +createworker() { + mkfifo "$tempdir"/worker$1{in,out} + worker $1 <"$tempdir"/worker$1in >"$tempdir"/worker$1out & + workers[$1]=$! + eval exec $((100+$1))'>"$tempdir"/worker$1in' + eval exec $((200+$1))'<"$tempdir"/worker$1out' +} + +destroyworker() { + dyingworker=${workers[$1]} + unset workers[$1] + [ -z "$2" ] && eval echo AtOM:Die '>&'$((100+$1)) + wait $dyingworker + eval $((100+$1))'>&-' + eval $((200+$1))'<&-' + rm "$tempdir"/worker$1{in,out} +} + checkworkers() { for key in ${!workers[@]} do - if ! kill -0 ${workers[key]} + if ! kill -0 ${workers[key]} 2>/dev/null then - worker $key \ - <"$tempdir"/worker${key}in \ - >"$tempdir"/worker${key}out & - workers[key]=$! - eval exec $((100+key))'>"$tempdir"/worker${key}in' - eval exec $((200+key))'<"$tempdir"/worker${key}out' + destroyworker $key nokill + createworker $key (( ++failed )) fi done @@ -2031,19 +2050,14 @@ echo 'COMMIT;' >&3 echo -e "\rCreated ${count:-0} tasks for $filecount files (${copies:-0} immediate copies)" masterpid=$$ -trap checkworkers USR1 ALRM +trap checkworkers USR1 ALRM PIPE rm -f "$tempdir"/worker* concurrency=$(( maxload / 2 )) (( concurrency )) || concurrency=1 active=0 for (( i=0 ; i < concurrency ; i++ )) do - (( ++wnum )) - mkfifo "$tempdir"/worker${wnum}{in,out} - worker $wnum <"$tempdir"/worker${wnum}in >"$tempdir"/worker${wnum}out & - workers[wnum]=$! - eval exec $((100+wnum))'>"$tempdir"/worker${wnum}in' - eval exec $((200+wnum))'<"$tempdir"/worker${wnum}out' + createworker $(getworkerid) done concurrencychange=$(date +%s) starttime=$concurrencychange @@ -2078,14 +2092,7 @@ do then concurrencychange=$(date +%s) (( ++concurrency )) - (( ++wnum )) - mkfifo "$tempdir"/worker${wnum}{in,out} - worker $wnum \ - <"$tempdir"/worker${wnum}in \ - >"$tempdir"/worker${wnum}out & - workers[wnum]=$! - eval exec $((100+wnum))'>"$tempdir"/worker${wnum}in' - eval exec $((200+wnum))'<"$tempdir"/worker${wnum}out' + createworker $(getworkerid) fi fi master From 628e5db3f7c0e706e2c6eb1ecc5e75bb2db0b266 Mon Sep 17 00:00:00 2001 From: Vincent Riquer Date: Tue, 2 Apr 2013 00:02:42 +0200 Subject: [PATCH 04/13] sanitize path --- atom | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/atom b/atom index 7620990..caed8a0 100755 --- a/atom +++ b/atom @@ -1089,6 +1089,7 @@ copyFile() { } sanitizeFile() { + shopt -s extglob string="$1" # Filenames can't contain / string="${string//\// }" @@ -1106,6 +1107,7 @@ sanitizeFile() { # Filenames can't begin or end with ' ' string=${string/#+( )/} + string=${string/%+( )/} string=${string//+( )./.} string=${string//.+( )/.} fi @@ -1133,7 +1135,14 @@ getDestDir() { replace=$(sanitizeFile "$disc") destdir="${destdir//%\{disc\}/$replace}" else - destdir+="${filename%/*}" + destdir+=(sanitizeFile "${filename%%/*}") + part="${filename#*/}/" + while : + do + destdir+="/$(sanitizeFile "${part%%/*}")" + part=${part#*/} + [ -z "$part" ] && break + done fi if ! [ -d "$destdir" ] then From 799409aa0c932802a0aa3c408a05d9b5c21bc80e Mon Sep 17 00:00:00 2001 From: Vincent Riquer Date: Tue, 2 Apr 2013 02:19:26 +0200 Subject: [PATCH 05/13] typo: sanitize path --- atom | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/atom b/atom index caed8a0..a879217 100755 --- a/atom +++ b/atom @@ -1135,13 +1135,12 @@ getDestDir() { replace=$(sanitizeFile "$disc") destdir="${destdir//%\{disc\}/$replace}" else - destdir+=(sanitizeFile "${filename%%/*}") - part="${filename#*/}/" - while : + destdir+=$(sanitizeFile "${filename%%/*}") + part=${filename#*/} + while [[ $part =~ / ]] do destdir+="/$(sanitizeFile "${part%%/*}")" part=${part#*/} - [ -z "$part" ] && break done fi if ! [ -d "$destdir" ] From ec9201caa901c7b76b9d23545956d53dc703c0b9 Mon Sep 17 00:00:00 2001 From: Vincent Riquer Date: Wed, 3 Apr 2013 15:11:14 +0200 Subject: [PATCH 06/13] minor fixes --- atom | 112 +++++++++++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 86 insertions(+), 26 deletions(-) diff --git a/atom b/atom index a879217..eec547f 100755 --- a/atom +++ b/atom @@ -753,7 +753,7 @@ gettag() { | sed -n "/^${1}=/I{s/^${1}=//I;p;q}" } -getInfosMP3_version='ID3-1' +getInfosMP3_version='ID3-2' tagreaders+=( "$getInfosMP3_version" ) getInfos::MP3() { tagreader="$getInfosMP3_version" @@ -767,9 +767,11 @@ getInfos::MP3() { title=$(gettag title) tracknum=$(gettag tracknumber) year=$(gettag year) - expr='^[0-9]*$' + expr='^\([0-9]*\)$' if [[ $genre =~ $expr ]] then + genre=${genre%)} + genre=${genre#(} genre="${id3genres[$genre]}" fi infos="${infos//: /=}" @@ -1225,7 +1227,7 @@ encodeFile::mp3() { cleanup $tempdir/$tmpfile.wav source_file $fileid status 0 - rename_pattern "${destinationrenamepath[$destination]}/${destinationrename[$destination]}:${destinationfat32compat["$destination"]}" + rename_pattern ${destinationrenamepath[$destination]}/${destinationrename[$destination]}:${destinationfat32compat["$destination"]} EOInsert ) progressSpin @@ -1260,7 +1262,7 @@ encodeFile::vorbis() { cleanup $tempdir/$tmpfile.wav source_file $fileid status 0 - rename_pattern "${destinationrenamepath[$destination]}/${destinationrename[$destination]}:${destinationfat32compat["$destination"]}" + rename_pattern ${destinationrenamepath[$destination]}/${destinationrename[$destination]}:${destinationfat32compat["$destination"]} EOInsert ) progressSpin @@ -1273,11 +1275,11 @@ worker() { while : do echo work - read line - until [[ $line != AtOM:ComFail ]] + read -t 10 line + until [ -n "$line" ] && [[ $line != AtOM:ComFail ]] do echo work - read line + read -t 10 line done if [[ $line == AtOM:Die ]] then @@ -1368,19 +1370,19 @@ worker() { if "${cmd_arg[@]}" >/dev/null then echo "finished $taskid|$sourcefileid|$destfileid|$destfilename" - read line + read -t 10 line until [[ $line == AtOM:OK ]] do echo "finished $taskid|$sourcefileid|$destfileid|$destfilename" - read line + read -t 10 line done else echo "failed $taskid" - read line + read -t 10 line until [[ $line == AtOM:OK ]] do echo "failed $taskid" - read line + read -t 10 line done [ -n "$filename" ] \ && eval rm -f $filename @@ -1389,11 +1391,11 @@ worker() { if [ -n "$cleanup" -a -n "$required" ] then echo "cleanup $required" - read answer + read -t 10 answer until [[ $answer != AtOM:ComFail ]] do echo "cleanup $required" - read answer + read -t 10 answer done if (( answer == 1 )) then @@ -1420,6 +1422,52 @@ master() { if [ -n "$quit" ] then destroyworker $workerid + elif [ -n "${workertasks[workerid]}" ] + then + echo ' + SELECT + id, + source_file, + required, + cmd_arg0, + cmd_arg1, + cmd_arg2, + cmd_arg3, + cmd_arg4, + cmd_arg5, + cmd_arg6, + cmd_arg7, + cmd_arg8, + cmd_arg9, + cmd_arg10, + cmd_arg11, + cmd_arg12, + cmd_arg13, + cmd_arg14, + cmd_arg15, + cmd_arg16, + cmd_arg17, + cmd_arg18, + cmd_arg19, + cmd_arg20, + cmd_arg21, + cmd_arg22, + cmd_arg23, + cmd_arg24, + cmd_arg25, + cmd_arg26, + cmd_arg27, + cmd_arg28, + cmd_arg29, + cleanup, + fileid, + filename + FROM tasks + WHERE + id='${workertasks[workerid]}'; + ' >&3 + read -u4 line + eval echo '"$line" >&'$((100+workerid)) elif (( active < concurrency )) then echo ' @@ -1488,21 +1536,19 @@ master() { (( ++active )) read -u4 line taskid=${line%%|*} + workertasks[workerid]=$taskid Update tasks status 1 <<<"id = $taskid" fi eval echo '"$line" >&'$((100+workerid)) else - dyingworker=${workers[workerid]} - unset workers[workerid] - eval echo AtOM:Die '>&'$((100+workerid)) - wait $dyingworker - eval $((100+workerid))'>&-' - eval $((200+workerid))'<&-' - rm "$tempdir"/worker${workerid}{in,out} + destroyworker $workerid fi ;; 'finished') eval 'echo AtOM:OK >&'$((workerid+100)) + [ -z "${workertasks[workerid]}" ] && continue + (( ++ran )) + unset workertasks[workerid] (( active-- )) || true taskid=${workerquery%%|*} rest="${workerquery#*|}|" @@ -1538,12 +1584,15 @@ master() { ;; 'failed') eval 'echo AtOM:OK >&'$((workerid+100)) + [ -z "${workertasks[workerid]}" ] && continue + unset workertasks[workerid] (( --active )) || true (( ++failed )) + (( ++ran )) taskid=$workerquery faildepends=$( Select tasks 'COUNT(*)' <<-EOWhere - requires = taskid + requires = $taskid EOWhere ) (( failed+=faildepends )) @@ -1609,8 +1658,19 @@ checkworkers() { if ! kill -0 ${workers[key]} 2>/dev/null then destroyworker $key nokill + if [ -n "${workertasks[key]}" ] + then + faildepends=$( + Select tasks 'COUNT(*)' <<-EOWhere + requires = ${workertasks[key]} + EOWhere + ) + (( ++failed )) + (( failed+=faildepends )) + Update tasks status 2 <<<"id = ${workertasks[key]}" + Update tasks status 2 <<<"requires = ${workertasks[key]}" + fi createworker $key - (( ++failed )) fi done } @@ -2132,9 +2192,9 @@ do fi echo -en "\rload: $humanload / $maxload" \ "workers: $active / $concurrency" \ - "done: $(( (taskcount - remaining ) * 100 / taskcount ))%" \ - "- $((taskcount - remaining)) of $taskcount ($failed failed)" \ - "${days}d ${hours}h${minutes}m " + "done: $(( ran * 100 / taskcount ))%" \ + "- ${ran:-0} of $taskcount ($failed failed)" \ + "${days}d ${hours}h${minutes}m\c" done unset count @@ -2162,7 +2222,7 @@ endtime=$(date +%s) ( ( ( ( days*24 + hours ) *60 ) + minutes ) *60 ) )) || true -echo -e "\rRan $taskcount tasks, $failed of which failed, in $days" \ +echo -e "\rRan ${ran:=0} tasks, $failed of which failed, in $days" \ "days, $hours hours, $minutes minutes and $seconds seconds." if [ -n "$quit" ] From be718ad18c0bc2712f48da2852f7fbb2839a3985 Mon Sep 17 00:00:00 2001 From: Vincent Riquer Date: Wed, 3 Apr 2013 16:18:06 +0200 Subject: [PATCH 07/13] extension is stripped before passing through sanitizeFile() --- atom | 2 -- 1 file changed, 2 deletions(-) diff --git a/atom b/atom index eec547f..97d2d04 100755 --- a/atom +++ b/atom @@ -1110,8 +1110,6 @@ sanitizeFile() { # Filenames can't begin or end with ' ' string=${string/#+( )/} string=${string/%+( )/} - string=${string//+( )./.} - string=${string//.+( )/.} fi echo "$string" } From 71417877284e30757391eb7f3b64b5bee7c8bcb6 Mon Sep 17 00:00:00 2001 From: Vincent Riquer Date: Wed, 3 Apr 2013 16:15:56 +0200 Subject: [PATCH 08/13] get rid of pipes --- atom | 618 +++++++++++++++++++++++++---------------------------------- 1 file changed, 256 insertions(+), 362 deletions(-) diff --git a/atom b/atom index eec547f..3c34fe0 100755 --- a/atom +++ b/atom @@ -1269,294 +1269,243 @@ encodeFile::vorbis() { } worker() { + debug=2 trap "kill -USR1 $masterpid" EXIT trap - USR1 ALRM PIPE - exec 2>>"$tempdir/errors.log" - while : - do - echo work - read -t 10 line - until [ -n "$line" ] && [[ $line != AtOM:ComFail ]] - do - echo work - read -t 10 line - done - if [[ $line == AtOM:Die ]] - then - trap EXIT - break - elif [[ $line == AtOM:Sleep ]] - then - sleep 1 - continue - fi - taskid=${line%%|*} - rest="${line#*|}|" - sourcefileid=${rest%%|*} - rest=${rest#*|} - required=${rest%%|*} - rest=${rest#*|} - cmd_arg=("${rest%%|*}") - rest=${rest#*|} - cmd_arg+=("${rest%%|*}") - rest=${rest#*|} - cmd_arg+=("${rest%%|*}") - rest=${rest#*|} - cmd_arg+=("${rest%%|*}") - rest=${rest#*|} - cmd_arg+=("${rest%%|*}") - rest=${rest#*|} - cmd_arg+=("${rest%%|*}") - rest=${rest#*|} - cmd_arg+=("${rest%%|*}") - rest=${rest#*|} - cmd_arg+=("${rest%%|*}") - rest=${rest#*|} - cmd_arg+=("${rest%%|*}") - rest=${rest#*|} - cmd_arg+=("${rest%%|*}") - rest=${rest#*|} - cmd_arg+=("${rest%%|*}") - rest=${rest#*|} - cmd_arg+=("${rest%%|*}") - rest=${rest#*|} - cmd_arg+=("${rest%%|*}") - rest=${rest#*|} - cmd_arg+=("${rest%%|*}") - rest=${rest#*|} - cmd_arg+=("${rest%%|*}") - rest=${rest#*|} - cmd_arg+=("${rest%%|*}") - rest=${rest#*|} - cmd_arg+=("${rest%%|*}") - rest=${rest#*|} - cmd_arg+=("${rest%%|*}") - rest=${rest#*|} - cmd_arg+=("${rest%%|*}") - rest=${rest#*|} - cmd_arg+=("${rest%%|*}") - rest=${rest#*|} - cmd_arg+=("${rest%%|*}") - rest=${rest#*|} - cmd_arg+=("${rest%%|*}") - rest=${rest#*|} - cmd_arg+=("${rest%%|*}") - rest=${rest#*|} - cmd_arg+=("${rest%%|*}") - rest=${rest#*|} - cmd_arg+=("${rest%%|*}") - rest=${rest#*|} - cmd_arg+=("${rest%%|*}") - rest=${rest#*|} - cmd_arg+=("${rest%%|*}") - rest=${rest#*|} - cmd_arg+=("${rest%%|*}") - rest=${rest#*|} - cmd_arg+=("${rest%%|*}") - rest=${rest#*|} - cmd_arg+=("${rest%%|*}") - rest=${rest#*|} - cleanup=${rest%%|*} - rest=${rest#*|} - destfileid=${rest%%|*} - rest=${rest#*|} - destfilename=${rest%%|*} - rest=${rest#*|} - for key in ${!cmd_arg[@]} - do - [ -z "${cmd_arg[key]}" ] && unset cmd_arg[key] - done - (( debug >= 2 )) && echo "${cmd_arg[@]}" >&2 - if "${cmd_arg[@]}" >/dev/null - then - echo "finished $taskid|$sourcefileid|$destfileid|$destfilename" - read -t 10 line - until [[ $line == AtOM:OK ]] - do - echo "finished $taskid|$sourcefileid|$destfileid|$destfilename" - read -t 10 line - done - else - echo "failed $taskid" - read -t 10 line - until [[ $line == AtOM:OK ]] - do - echo "failed $taskid" - read -t 10 line - done - [ -n "$filename" ] \ - && eval rm -f $filename - fi - unset cmd_arg - if [ -n "$cleanup" -a -n "$required" ] - then - echo "cleanup $required" - read -t 10 answer - until [[ $answer != AtOM:ComFail ]] - do - echo "cleanup $required" - read -t 10 answer - done - if (( answer == 1 )) - then - eval rm -f $cleanup - fi - fi - done - return + exec 2>>"$tempdir/worker$1.log" + (( debug >= 2 )) && echo "${cmd_arg[@]}" >&2 + "${cmd_arg[@]}" >/dev/null + status=$? + kill -USR1 $masterpid + exit $status } master() { - for workerid in ${!workers[@]} - do - if read -t0.001 -u$((200+workerid)) workercommand workerquery + if (( active >= concurrency)) || [ -n "$quit" ] + then + sleep 0.1 + else + echo ' + SELECT COUNT(*) + FROM tasks + WHERE status = 0; + + SELECT COUNT(*) + FROM tasks + WHERE status = 0 + AND requires is NULL; + + SELECT + id, + source_file, + required, + cmd_arg0, + cmd_arg1, + cmd_arg2, + cmd_arg3, + cmd_arg4, + cmd_arg5, + cmd_arg6, + cmd_arg7, + cmd_arg8, + cmd_arg9, + cmd_arg10, + cmd_arg11, + cmd_arg12, + cmd_arg13, + cmd_arg14, + cmd_arg15, + cmd_arg16, + cmd_arg17, + cmd_arg18, + cmd_arg19, + cmd_arg20, + cmd_arg21, + cmd_arg22, + cmd_arg23, + cmd_arg24, + cmd_arg25, + cmd_arg26, + cmd_arg27, + cmd_arg28, + cmd_arg29, + cleanup, + fileid, + filename + FROM tasks + WHERE status = 0 + AND requires is NULL + ORDER BY source_file + LIMIT 1; + ' >&3 + read -u4 remaining + read -u4 ready + if (( remaining == 0 )) then - break + sleep 0.1 + continue + elif (( ready == 0 )) + then + sleep 0.1 + else + (( ++active )) + read -u4 line + taskid=${line%%|*} + rest="${line#*|}|" + sourcefileid=${rest%%|*} + rest=${rest#*|} + required=${rest%%|*} + rest=${rest#*|} + cmd_arg=("${rest%%|*}") + rest=${rest#*|} + cmd_arg+=("${rest%%|*}") + rest=${rest#*|} + cmd_arg+=("${rest%%|*}") + rest=${rest#*|} + cmd_arg+=("${rest%%|*}") + rest=${rest#*|} + cmd_arg+=("${rest%%|*}") + rest=${rest#*|} + cmd_arg+=("${rest%%|*}") + rest=${rest#*|} + cmd_arg+=("${rest%%|*}") + rest=${rest#*|} + cmd_arg+=("${rest%%|*}") + rest=${rest#*|} + cmd_arg+=("${rest%%|*}") + rest=${rest#*|} + cmd_arg+=("${rest%%|*}") + rest=${rest#*|} + cmd_arg+=("${rest%%|*}") + rest=${rest#*|} + cmd_arg+=("${rest%%|*}") + rest=${rest#*|} + cmd_arg+=("${rest%%|*}") + rest=${rest#*|} + cmd_arg+=("${rest%%|*}") + rest=${rest#*|} + cmd_arg+=("${rest%%|*}") + rest=${rest#*|} + cmd_arg+=("${rest%%|*}") + rest=${rest#*|} + cmd_arg+=("${rest%%|*}") + rest=${rest#*|} + cmd_arg+=("${rest%%|*}") + rest=${rest#*|} + cmd_arg+=("${rest%%|*}") + rest=${rest#*|} + cmd_arg+=("${rest%%|*}") + rest=${rest#*|} + cmd_arg+=("${rest%%|*}") + rest=${rest#*|} + cmd_arg+=("${rest%%|*}") + rest=${rest#*|} + cmd_arg+=("${rest%%|*}") + rest=${rest#*|} + cmd_arg+=("${rest%%|*}") + rest=${rest#*|} + cmd_arg+=("${rest%%|*}") + rest=${rest#*|} + cmd_arg+=("${rest%%|*}") + rest=${rest#*|} + cmd_arg+=("${rest%%|*}") + rest=${rest#*|} + cmd_arg+=("${rest%%|*}") + rest=${rest#*|} + cmd_arg+=("${rest%%|*}") + rest=${rest#*|} + cmd_arg+=("${rest%%|*}") + rest=${rest#*|} + cleanup=${rest%%|*} + rest=${rest#*|} + destfileid=${rest%%|*} + rest=${rest#*|} + destfilename=${rest%%|*} + rest=${rest#*|} + for key in ${!cmd_arg[@]} + do + [ -z "${cmd_arg[key]}" ] && unset cmd_arg[key] + done + workerid=$(getworkerid) + workertasks[workerid]=$taskid + Update tasks status 1 <<<"id = $taskid" + export cmd_arg + createworker $workerid + unset cmd_arg + fi + fi +} + +getworkerid() { + local i + for (( i=0 ; i >= 0 ; i++ )) + do + if [ -z "${workers[i]}" ] + then + echo $i + return fi done - (( ${#workers[@]} == 0 )) && break - if [ -n "$workercommand" ] - then - case $workercommand in - 'work') - if [ -n "$quit" ] - then - destroyworker $workerid - elif [ -n "${workertasks[workerid]}" ] - then - echo ' - SELECT - id, - source_file, - required, - cmd_arg0, - cmd_arg1, - cmd_arg2, - cmd_arg3, - cmd_arg4, - cmd_arg5, - cmd_arg6, - cmd_arg7, - cmd_arg8, - cmd_arg9, - cmd_arg10, - cmd_arg11, - cmd_arg12, - cmd_arg13, - cmd_arg14, - cmd_arg15, - cmd_arg16, - cmd_arg17, - cmd_arg18, - cmd_arg19, - cmd_arg20, - cmd_arg21, - cmd_arg22, - cmd_arg23, - cmd_arg24, - cmd_arg25, - cmd_arg26, - cmd_arg27, - cmd_arg28, - cmd_arg29, - cleanup, - fileid, - filename - FROM tasks - WHERE - id='${workertasks[workerid]}'; - ' >&3 - read -u4 line - eval echo '"$line" >&'$((100+workerid)) - elif (( active < concurrency )) - then - echo ' - SELECT COUNT(*) - FROM tasks - WHERE status = 0; + # If we reach this, we have reached the signed long limit + # (2^63 - 1 = 9223372036854775807 - Got a supercomputer?) + (( concurrency-- )) +} - SELECT COUNT(*) - FROM tasks - WHERE status = 0 - AND requires is NULL; +createworker() { + worker & + workers[$1]=$! +} - SELECT - id, - source_file, - required, - cmd_arg0, - cmd_arg1, - cmd_arg2, - cmd_arg3, - cmd_arg4, - cmd_arg5, - cmd_arg6, - cmd_arg7, - cmd_arg8, - cmd_arg9, - cmd_arg10, - cmd_arg11, - cmd_arg12, - cmd_arg13, - cmd_arg14, - cmd_arg15, - cmd_arg16, - cmd_arg17, - cmd_arg18, - cmd_arg19, - cmd_arg20, - cmd_arg21, - cmd_arg22, - cmd_arg23, - cmd_arg24, - cmd_arg25, - cmd_arg26, - cmd_arg27, - cmd_arg28, - cmd_arg29, - cleanup, - fileid, - filename - FROM tasks - WHERE status = 0 - AND requires is NULL - ORDER BY source_file - LIMIT 1; - ' >&3 - read -u4 remaining - read -u4 ready - if (( remaining == 0 )) - then - destroyworker $workerid - continue - elif (( ready == 0 )) - then - line=AtOM:Sleep - else - (( ++active )) - read -u4 line - taskid=${line%%|*} - workertasks[workerid]=$taskid - Update tasks status 1 <<<"id = $taskid" - fi - eval echo '"$line" >&'$((100+workerid)) - else - destroyworker $workerid - fi - ;; - 'finished') - eval 'echo AtOM:OK >&'$((workerid+100)) - [ -z "${workertasks[workerid]}" ] && continue +destroyworker() { + dyingworker=${workers[$1]} + unset workers[$1] + wait $dyingworker +} + +checkworkers() { + for key in ${!workers[@]} + do + if ! kill -0 ${workers[key]} 2>/dev/null + then + if read -u4 -t 0.01 alienquery + then + alienresults=(${alienquery%%|*}) + rest="${alienquery#*|}|" + while [ -n "$rest" ] + do + alienresults+=("${rest%%|*}") + rest=${rest#*|} + done + fi + echo ' + SELECT + id, + source_file, + required, + cleanup, + fileid, + filename + FROM tasks + WHERE + id='${workertasks[key]}'; + ' >&3 + read -u4 line + taskid=${line%%|*} + rest="${line#*|}|" + sourcefileid=${rest%%|*} + rest=${rest#*|} + required=${rest%%|*} + rest=${rest#*|} + cleanup=${rest%%|*} + rest=${rest#*|} + destfileid=${rest%%|*} + rest=${rest#*|} + destfilename=${rest%%|*} + rest=${rest#*|} + if destroyworker $key + then (( ++ran )) - unset workertasks[workerid] (( active-- )) || true - taskid=${workerquery%%|*} - rest="${workerquery#*|}|" - sourcefileid=${rest%%|*} - rest=${rest#*|} - destfileid=${rest%%|*} - rest=${rest#*|} - destfilename=${rest%%|*} Delete tasks <<<"id = $taskid" if [ -n "$destfilename" ] then @@ -1581,15 +1530,7 @@ master() { "WHERE id=$destfileid;" \ >&3 fi - ;; - 'failed') - eval 'echo AtOM:OK >&'$((workerid+100)) - [ -z "${workertasks[workerid]}" ] && continue - unset workertasks[workerid] - (( --active )) || true - (( ++failed )) - (( ++ran )) - taskid=$workerquery + else faildepends=$( Select tasks 'COUNT(*)' <<-EOWhere requires = $taskid @@ -1598,79 +1539,33 @@ master() { (( failed+=faildepends )) Update tasks status 2 <<<"id = $taskid" Update tasks status 2 <<<"requires = $taskid" - ;; - 'cleanup') - required=$workerquery - echo "SELECT COUNT(*) - FROM tasks - WHERE ( status = 0 OR status = 1 ) - AND required = $required;">&3 - read -u4 count - if (( count == 0 )) - then - eval echo 1 '>&'$((100+workerid)) - else - eval echo 0 '>&'$((100+workerid)) - fi - ;; - *) - eval 'echo "AtOM:ComFail" >&'$((100+workerid)) - ;; - esac - fi -} - -getworkerid() { - local i - for (( i=0 ; i < 100 ; i++ )) - do - if [ -z "${workers[i]}" ] - then - echo $i - break - fi - done - # If we reach this, we have reached the hardcoded 100 workers limit - (( concurrency-- )) -} - -createworker() { - mkfifo "$tempdir"/worker$1{in,out} - worker $1 <"$tempdir"/worker$1in >"$tempdir"/worker$1out & - workers[$1]=$! - eval exec $((100+$1))'>"$tempdir"/worker$1in' - eval exec $((200+$1))'<"$tempdir"/worker$1out' -} - -destroyworker() { - dyingworker=${workers[$1]} - unset workers[$1] - [ -z "$2" ] && eval echo AtOM:Die '>&'$((100+$1)) - wait $dyingworker - eval $((100+$1))'>&-' - eval $((200+$1))'<&-' - rm "$tempdir"/worker$1{in,out} -} - -checkworkers() { - for key in ${!workers[@]} - do - if ! kill -0 ${workers[key]} 2>/dev/null - then - destroyworker $key nokill - if [ -n "${workertasks[key]}" ] - then - faildepends=$( - Select tasks 'COUNT(*)' <<-EOWhere - requires = ${workertasks[key]} - EOWhere - ) - (( ++failed )) - (( failed+=faildepends )) - Update tasks status 2 <<<"id = ${workertasks[key]}" - Update tasks status 2 <<<"requires = ${workertasks[key]}" fi - createworker $key + echo "SELECT COUNT(*) + FROM tasks + WHERE ( status = 0 OR status = 1 ) + AND required = $taskid;">&3 + read -u4 count + if (( count == 0 )) + then + rm -f "$cleanup" + fi + if (( ${#alienresults[@]} )) + then + alienquery='SELECT ' + for key in ${!alienresults[@]} + do + (( key > 0 )) && alienquery+=, + expr='^[0-9]*$' + if [[ ${alienresults[key]} =~ $expr ]] + then + alienquery+=${alienresults[key]} + else + alienquery+="\"${alienresults[@]//\"/\"\"}\"" + fi + done + echo "$alienquery;" >&3 + unset alienquery alienresults + fi fi done } @@ -2119,21 +2014,21 @@ echo -e "\rCreated ${count:-0} tasks for $filecount files (${copies:-0} immediat masterpid=$$ trap checkworkers USR1 ALRM PIPE -rm -f "$tempdir"/worker* concurrency=$(( maxload / 2 )) (( concurrency )) || concurrency=1 active=0 +#set -x for (( i=0 ; i < concurrency ; i++ )) do - createworker $(getworkerid) + master done concurrencychange=$(date +%s) starttime=$concurrencychange taskcount=$count failed=0 -while : +while (( ${#workers[@]} )) do - if read -n 1 -t 0.01 userinput + if read -n 1 -t 1 userinput then case $userinput in '+') @@ -2160,7 +2055,6 @@ do then concurrencychange=$(date +%s) (( ++concurrency )) - createworker $(getworkerid) fi fi master From 6d575ba53e703cbac5211a92d59f21d1f090ee6a Mon Sep 17 00:00:00 2001 From: Vincent Riquer Date: Thu, 4 Apr 2013 11:30:25 +0200 Subject: [PATCH 09/13] overzealous search and replace --- atom | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/atom b/atom index 97d2d04..81ded60 100755 --- a/atom +++ b/atom @@ -774,7 +774,7 @@ getInfos::MP3() { genre=${genre#(} genre="${id3genres[$genre]}" fi - infos="${infos//: /=}" + infos="${infos/: /=}" channels=$(gettag channels) rate=$(gettag 'sample rate') bitrate=$(gettag 'bit rate') @@ -805,7 +805,7 @@ getInfos::Ogg() { tracknum="$tracknum/$tracktotal" fi year=$(gettag date) - infos="${infos//: /=}" + infos="${infos/: /=}" rate=$(gettag rate|head -n1) channels=$(gettag channels|head -n1) bitrate=$(gettag 'nominal bitrate') From 460e2445360744eaeed67bd10030ddf1ee7b5b6d Mon Sep 17 00:00:00 2001 From: Vincent Riquer Date: Thu, 4 Apr 2013 13:21:51 +0200 Subject: [PATCH 10/13] no SQL inside the signal-handler (race-condition) --- atom | 194 ++++++++++++++++++++++++++++++----------------------------- 1 file changed, 99 insertions(+), 95 deletions(-) diff --git a/atom b/atom index 3c34fe0..444137b 100755 --- a/atom +++ b/atom @@ -1462,109 +1462,112 @@ destroyworker() { wait $dyingworker } +gettaskinfos() { + echo ' + SELECT + id, + source_file, + required, + cleanup, + fileid, + filename + FROM tasks + WHERE + id='$1'; + ' >&3 + read -u4 line + taskid=${line%%|*} + rest="${line#*|}|" + sourcefileid=${rest%%|*} + rest=${rest#*|} + required=${rest%%|*} + rest=${rest#*|} + cleanup=${rest%%|*} + rest=${rest#*|} + destfileid=${rest%%|*} + rest=${rest#*|} + destfilename=${rest%%|*} + rest=${rest#*|} +} + +cleaner() { + for taskid in ${failedtasks[@]} + do + gettaskinfos $taskid + faildepends=$( + Select tasks 'COUNT(*)' <<-EOWhere + requires = $taskid + EOWhere + ) + (( failed+=faildepends )) + Update tasks status 2 <<<"id = $taskid" + Update tasks status 2 <<<"requires = $taskid" + echo "SELECT COUNT(*) + FROM tasks + WHERE ( status = 0 OR status = 1 ) + AND required = $taskid;">&3 + read -u4 count + if (( count == 0 )) + then + rm -f "$cleanup" + fi + unset failedtasks[taskid] + done + for taskid in ${finishedtasks[@]} + do + gettaskinfos $taskid + Delete tasks <<<"id = $taskid" + if [ -n "$destfilename" ] + then + echo \ + "UPDATE destination_files" \ + "SET filename=\"${destfilename//\"/\"\"}\"," \ + " last_change=(" \ + " SELECT last_change" \ + " FROM source_files" \ + " WHERE id=$sourcefileid" \ + " )," \ + " old_filename=(" \ + " SELECT filename" \ + " FROM destination_files" \ + " WHERE id=$destfileid" \ + " )," \ + " rename_pattern=(" \ + " SELECT rename_pattern" \ + " FROM tasks" \ + " WHERE id=$taskid" \ + " )" \ + "WHERE id=$destfileid;" \ + >&3 + fi + echo "SELECT COUNT(*) + FROM tasks + WHERE ( status = 0 OR status = 1 ) + AND required = $taskid;">&3 + read -u4 count + if (( count == 0 )) + then + rm -f "$cleanup" + fi + unset finishedtasks[taskid] + done +} + checkworkers() { for key in ${!workers[@]} do if ! kill -0 ${workers[key]} 2>/dev/null then - if read -u4 -t 0.01 alienquery - then - alienresults=(${alienquery%%|*}) - rest="${alienquery#*|}|" - while [ -n "$rest" ] - do - alienresults+=("${rest%%|*}") - rest=${rest#*|} - done - fi - echo ' - SELECT - id, - source_file, - required, - cleanup, - fileid, - filename - FROM tasks - WHERE - id='${workertasks[key]}'; - ' >&3 - read -u4 line - taskid=${line%%|*} - rest="${line#*|}|" - sourcefileid=${rest%%|*} - rest=${rest#*|} - required=${rest%%|*} - rest=${rest#*|} - cleanup=${rest%%|*} - rest=${rest#*|} - destfileid=${rest%%|*} - rest=${rest#*|} - destfilename=${rest%%|*} - rest=${rest#*|} + taskid=${workertasks[key]} + (( ++ran )) + (( active-- )) if destroyworker $key then - (( ++ran )) - (( active-- )) || true - Delete tasks <<<"id = $taskid" - if [ -n "$destfilename" ] - then - echo \ - "UPDATE destination_files" \ - "SET filename=\"${destfilename//\"/\"\"}\"," \ - " last_change=(" \ - " SELECT last_change" \ - " FROM source_files" \ - " WHERE id=$sourcefileid" \ - " )," \ - " old_filename=(" \ - " SELECT filename" \ - " FROM destination_files" \ - " WHERE id=$destfileid" \ - " )," \ - " rename_pattern=(" \ - " SELECT rename_pattern" \ - " FROM tasks" \ - " WHERE id=$taskid" \ - " )" \ - "WHERE id=$destfileid;" \ - >&3 - fi + finishedtasks+=($taskid) else - faildepends=$( - Select tasks 'COUNT(*)' <<-EOWhere - requires = $taskid - EOWhere - ) - (( failed+=faildepends )) - Update tasks status 2 <<<"id = $taskid" - Update tasks status 2 <<<"requires = $taskid" - fi - echo "SELECT COUNT(*) - FROM tasks - WHERE ( status = 0 OR status = 1 ) - AND required = $taskid;">&3 - read -u4 count - if (( count == 0 )) - then - rm -f "$cleanup" - fi - if (( ${#alienresults[@]} )) - then - alienquery='SELECT ' - for key in ${!alienresults[@]} - do - (( key > 0 )) && alienquery+=, - expr='^[0-9]*$' - if [[ ${alienresults[key]} =~ $expr ]] - then - alienquery+=${alienresults[key]} - else - alienquery+="\"${alienresults[@]//\"/\"\"}\"" - fi - done - echo "$alienquery;" >&3 - unset alienquery alienresults + failedtasks+=($taskid) + (( ++ran )) + (( ++failed )) fi fi done @@ -2057,6 +2060,7 @@ do (( ++concurrency )) fi fi + cleaner master if ((taskcount - remaining)) then From f56801188276afc7345cec49dab390bccc41304c Mon Sep 17 00:00:00 2001 From: Vincent Riquer Date: Thu, 4 Apr 2013 15:55:18 +0200 Subject: [PATCH 11/13] small fixes --- atom | 32 ++++++++++++-------------------- 1 file changed, 12 insertions(+), 20 deletions(-) diff --git a/atom b/atom index 444137b..a477108 100755 --- a/atom +++ b/atom @@ -1269,15 +1269,9 @@ encodeFile::vorbis() { } worker() { - debug=2 - trap "kill -USR1 $masterpid" EXIT - trap - USR1 ALRM PIPE exec 2>>"$tempdir/worker$1.log" (( debug >= 2 )) && echo "${cmd_arg[@]}" >&2 "${cmd_arg[@]}" >/dev/null - status=$? - kill -USR1 $masterpid - exit $status } master() { @@ -1429,9 +1423,7 @@ master() { workerid=$(getworkerid) workertasks[workerid]=$taskid Update tasks status 1 <<<"id = $taskid" - export cmd_arg createworker $workerid - unset cmd_arg fi fi } @@ -1452,7 +1444,7 @@ getworkerid() { } createworker() { - worker & + worker $1 & workers[$1]=$! } @@ -1472,8 +1464,7 @@ gettaskinfos() { fileid, filename FROM tasks - WHERE - id='$1'; + WHERE id='$1'; ' >&3 read -u4 line taskid=${line%%|*} @@ -1491,8 +1482,9 @@ gettaskinfos() { } cleaner() { - for taskid in ${failedtasks[@]} + for key in ${!failedtasks[@]} do + taskid=${failedtasks[key]} gettaskinfos $taskid faildepends=$( Select tasks 'COUNT(*)' <<-EOWhere @@ -1511,12 +1503,12 @@ cleaner() { then rm -f "$cleanup" fi - unset failedtasks[taskid] + unset failedtasks[key] done - for taskid in ${finishedtasks[@]} + for key in ${!finishedtasks[@]} do + taskid=${finishedtasks[key]} gettaskinfos $taskid - Delete tasks <<<"id = $taskid" if [ -n "$destfilename" ] then echo \ @@ -1549,7 +1541,8 @@ cleaner() { then rm -f "$cleanup" fi - unset finishedtasks[taskid] + Delete tasks <<<"id = $taskid" + unset finishedtasks[key] done } @@ -1566,9 +1559,9 @@ checkworkers() { finishedtasks+=($taskid) else failedtasks+=($taskid) - (( ++ran )) (( ++failed )) fi + unset workertasks[key] fi done } @@ -2015,8 +2008,6 @@ done echo 'COMMIT;' >&3 echo -e "\rCreated ${count:-0} tasks for $filecount files (${copies:-0} immediate copies)" -masterpid=$$ -trap checkworkers USR1 ALRM PIPE concurrency=$(( maxload / 2 )) (( concurrency )) || concurrency=1 active=0 @@ -2031,7 +2022,7 @@ taskcount=$count failed=0 while (( ${#workers[@]} )) do - if read -n 1 -t 1 userinput + if read -n 1 -t 0.1 userinput then case $userinput in '+') @@ -2060,6 +2051,7 @@ do (( ++concurrency )) fi fi + checkworkers cleaner master if ((taskcount - remaining)) From 11cd29d8438035b1d919b18a806f87c75f0ca068 Mon Sep 17 00:00:00 2001 From: Vincent Riquer Date: Thu, 4 Apr 2013 23:06:36 +0200 Subject: [PATCH 12/13] nicer status infos --- atom | 37 ++++++++++++++++++++++++++----------- 1 file changed, 26 insertions(+), 11 deletions(-) diff --git a/atom b/atom index f2f6619..2bf93a4 100755 --- a/atom +++ b/atom @@ -2052,16 +2052,15 @@ do checkworkers cleaner master - if ((taskcount - remaining)) + if (( ran )) then currenttime=$(date +%s) - secsremaining=$(( - remaining - * - (currenttime - starttime) + avgduration=$(( + ((currenttime - starttime) * 1000) / - (taskcount - remaining) + ran )) + secsremaining=$(( remaining * avgduration / 1000 )) (( days = secsremaining / @@ -2077,12 +2076,28 @@ do / 60 )) || true + (( seconds = + secsremaining + - + ( ( ( ( days*24 + hours ) *60 ) + minutes ) *60 ) + )) || true + avgduration=$(printf %04i $avgduration) + avgdsec=${avgduration:0:-3} + avgdmsec=${avgduration#$avgdsec} fi - echo -en "\rload: $humanload / $maxload" \ - "workers: $active / $concurrency" \ - "done: $(( ran * 100 / taskcount ))%" \ - "- ${ran:-0} of $taskcount ($failed failed)" \ - "${days}d ${hours}h${minutes}m\c" + dran=$(printf %${#taskcount}i ${ran:-0}) + rtime=$( + printf '%2id %2ih%02im%02is' \ + ${days:-0} \ + ${hours:-0} \ + ${minutes:-0} \ + ${seconds:-0} + ) + percent=$(printf %3i $((ran * 100 / taskcount))) + echo -en "\rL: $humanload/$maxload" \ + "W: $active/$concurrency" \ + "T: ${dran}/$taskcount (F:$failed) $percent% $rtime" \ + "(A: ${avgdsec:--}.${avgdmsec:--}s/task)" done unset count From 98a2a132744de102eae498f39d0b93b385dbe3ba Mon Sep 17 00:00:00 2001 From: Vincent Riquer Date: Thu, 4 Apr 2013 23:34:48 +0200 Subject: [PATCH 13/13] fix file rename --- atom | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/atom b/atom index 2bf93a4..5a03d77 100755 --- a/atom +++ b/atom @@ -2140,6 +2140,8 @@ do echo ' SELECT destination_files.filename, + destination_files.id, + source_files.filename, tags.album, tags.albumartist, tags.artist, @@ -2157,11 +2159,15 @@ do INNER JOIN tags ON destination_files.source_file_id =tags.source_file + INNER JOIN source_files + ON destination_files.source_file_id + =source_files.id WHERE destinations.name="'"$destination"'" AND (destination_files.rename_pattern != "'"${destinationrenamepath[$destination]}/${destinationrename[$destination]}:${destinationfat32compat["$destination"]}"'" OR destination_files.rename_pattern is NULL) + AND destination_files.last_change > 0 ; SELECT "AtOM:NoMoreFiles"; @@ -2179,6 +2185,10 @@ do do oldfilename=${line%%|*} rest=${line#*|}'|' + destfileid=${rest%%|*} + rest=${rest#*|} + filename=${rest%%|*} + rest=${rest#*|} album=${rest%%|*} rest=${rest#*|} albumartist=${rest%%|*} @@ -2204,16 +2214,21 @@ do getDestDir getDestFile destfilename="$destdir/$destfile.$extension" - mv "$oldfilename" "$destfilename" + if [[ $oldfilename != $destfilename ]] + then + mv "$oldfilename" "$destfilename" + progressSpin + fi echo "UPDATE destination_files" \ "SET filename=\"${destfilename//\"/\"\"}\"," \ " rename_pattern=" \ "\"${destinationrenamepath[$destination]}/${destinationrename[$destination]}:${destinationfat32compat["$destination"]}\"" \ "WHERE id=$destfileid;" \ >&3 - progressSpin fi + read -u4 line done + echo $'\r'"$destination: Renamed ${count:-0} files " fi unset count done