ScriptFanix/AtOM
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Merge branch 'master' into opus

Browse Source 
* master:
  fix file rename
  nicer status infos
  small fixes
  no SQL inside the signal-handler (race-condition)
  overzealous search and replace
  get rid of pipes
  extension is stripped before passing through sanitizeFile()
  minor fixes
  typo: sanitize path
  sanitize path
  createworker() destroyworker()
  rename on fat32compat change
  fat32compat
This commit is contained in:
Vincent Riquer 2013-04-05 00:16:11 +02:00
commit 19e6b8c0f2
2 changed files with 368 additions and 357 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

463
atom
View File

@ -14,6 +14,7 @@ EFMTINVPARM=49
# config structures # config structures
declare -A \ declare -A \
destinationchannels \ destinationchannels \
destinationfat32compat \
destinationcopymime \ destinationcopymime \
destinationformat \ destinationformat \
destinationfrequency \ destinationfrequency \
@ -264,6 +265,21 @@ getConfigDestination() {
esac esac
destinationrename["$destination"]="${value##*/}" destinationrename["$destination"]="${value##*/}"
;; ;;
'fat32compat')
case $value in
'true'|'on'|'yes')
destinationfat32compat["$destination"]=1
;;
'false'|'off'|'no')
destinationfat32compat["$destination"]=0
;;
*)
echo "fat32compat takes values:" \
"'yes' ,'true' ,'on', 'no', 'false',"\
"'off'"
;;
esac
;;
'skip_mime-type') 'skip_mime-type')
destinationskipmime[$destination]="${destinationskipmime[$destination]:+${destinationskipmime[$destination]}|}$value" destinationskipmime[$destination]="${destinationskipmime[$destination]:+${destinationskipmime[$destination]}|}$value"
;; ;;
@ -759,7 +775,7 @@ gettag() {
| sed -n "/^${1}=/I{s/^${1}=//I;p;q}" | sed -n "/^${1}=/I{s/^${1}=//I;p;q}"
} }
getInfosMP3_version='ID3-1' getInfosMP3_version='ID3-2'
tagreaders+=( "$getInfosMP3_version" ) tagreaders+=( "$getInfosMP3_version" )
getInfos::MP3() { getInfos::MP3() {
tagreader="$getInfosMP3_version" tagreader="$getInfosMP3_version"
@ -773,12 +789,14 @@ getInfos::MP3() {
title=$(gettag title) title=$(gettag title)
tracknum=$(gettag tracknumber) tracknum=$(gettag tracknumber)
year=$(gettag year) year=$(gettag year)
expr='^[0-9]*$' expr='^\([0-9]*\)$'
if [[ $genre =~ $expr ]] if [[ $genre =~ $expr ]]
then then
genre=${genre%)}
genre=${genre#(}
genre="${id3genres[$genre]}" genre="${id3genres[$genre]}"
fi fi
infos="${infos//: /=}" infos="${infos/: /=}"
channels=$(gettag channels) channels=$(gettag channels)
rate=$(gettag 'sample rate') rate=$(gettag 'sample rate')
bitrate=$(gettag 'bit rate') bitrate=$(gettag 'bit rate')
@ -809,7 +827,7 @@ getInfos::Ogg() {
tracknum="$tracknum/$tracktotal" tracknum="$tracknum/$tracktotal"
fi fi
year=$(gettag date) year=$(gettag date)
infos="${infos//: /=}" infos="${infos/: /=}"
rate=$(gettag rate|head -n1) rate=$(gettag rate|head -n1)
channels=$(gettag channels|head -n1) channels=$(gettag channels|head -n1)
bitrate=$(gettag 'nominal bitrate') bitrate=$(gettag 'nominal bitrate')
@ -1088,13 +1106,14 @@ copyFile() {
" WHERE id=$destfileid" \ " WHERE id=$destfileid" \
" )," \ " )," \
" rename_pattern=" \ " rename_pattern=" \
"\"${destinationrenamepath[$destination]}/${destinationrename[$destination]}\""\ "\"${destinationrenamepath[$destination]}/${destinationrename[$destination]}:${destinationfat32compat["$destination"]}\""\
"WHERE id=$destfileid;" \ "WHERE id=$destfileid;" \
>&3 >&3
(( ++copies )) (( ++copies ))
} }
sanitizeFile() { sanitizeFile() {
shopt -s extglob
string="$1" string="$1"
# Filenames can't contain / # Filenames can't contain /
string="${string//\// }" string="${string//\// }"
@ -1112,8 +1131,7 @@ sanitizeFile() {
# Filenames can't begin or end with ' ' # Filenames can't begin or end with ' '
string=${string/#+( )/} string=${string/#+( )/}
string=${string//+( )./.} string=${string/%+( )/}
string=${string//.+( )/.}
fi fi
echo "$string" echo "$string"
} }
@ -1139,7 +1157,13 @@ getDestDir() {
replace=$(sanitizeFile "$disc") replace=$(sanitizeFile "$disc")
destdir="${destdir//%\{disc\}/$replace}" destdir="${destdir//%\{disc\}/$replace}"
else else
destdir+="${filename%/*}" destdir+=$(sanitizeFile "${filename%%/*}")
part=${filename#*/}
while [[ $part =~ / ]]
do
destdir+="/$(sanitizeFile "${part%%/*}")"
part=${part#*/}
done
fi fi
if ! [ -d "$destdir" ] if ! [ -d "$destdir" ]
then then
@ -1223,7 +1247,7 @@ encodeFile::mp3() {
cleanup $tempdir/$tmpfile.wav cleanup $tempdir/$tmpfile.wav
source_file $fileid source_file $fileid
status 0 status 0
rename_pattern "${destinationrenamepath[$destination]}/${destinationrename[$destination]}" rename_pattern ${destinationrenamepath[$destination]}/${destinationrename[$destination]}:${destinationfat32compat["$destination"]}
EOInsert EOInsert
) )
progressSpin progressSpin
@ -1297,31 +1321,88 @@ encodeFile::vorbis() {
cleanup $tempdir/$tmpfile.wav cleanup $tempdir/$tmpfile.wav
source_file $fileid source_file $fileid
status 0 status 0
rename_pattern "${destinationrenamepath[$destination]}/${destinationrename[$destination]}" rename_pattern ${destinationrenamepath[$destination]}/${destinationrename[$destination]}:${destinationfat32compat["$destination"]}
EOInsert EOInsert
) )
progressSpin progressSpin
} }
worker() { worker() {
trap "kill -USR1 $masterpid" EXIT exec 2>>"$tempdir/worker$1.log"
while : (( debug >= 2 )) && echo "${cmd_arg[@]}" >&2
do "${cmd_arg[@]}" >/dev/null
echo work }
read line
until [[ $line != AtOM:ComFail ]] master() {
do if (( active >= concurrency)) || [ -n "$quit" ]
echo work
read line
done
if [[ $line == AtOM:Die ]]
then then
break sleep 0.1
elif [[ $line == AtOM:Sleep ]] else
echo '
SELECT COUNT(*)
FROM tasks
WHERE status = 0;
SELECT COUNT(*)
FROM tasks
WHERE status = 0
AND requires is NULL;
SELECT
id,
source_file,
required,
cmd_arg0,
cmd_arg1,
cmd_arg2,
cmd_arg3,
cmd_arg4,
cmd_arg5,
cmd_arg6,
cmd_arg7,
cmd_arg8,
cmd_arg9,
cmd_arg10,
cmd_arg11,
cmd_arg12,
cmd_arg13,
cmd_arg14,
cmd_arg15,
cmd_arg16,
cmd_arg17,
cmd_arg18,
cmd_arg19,
cmd_arg20,
cmd_arg21,
cmd_arg22,
cmd_arg23,
cmd_arg24,
cmd_arg25,
cmd_arg26,
cmd_arg27,
cmd_arg28,
cmd_arg29,
cleanup,
fileid,
filename
FROM tasks
WHERE status = 0
AND requires is NULL
ORDER BY source_file
LIMIT 1;
' >&3
read -u4 remaining
read -u4 ready
if (( remaining == 0 ))
then then
sleep 1 sleep 0.1
continue continue
fi elif (( ready == 0 ))
then
sleep 0.1
else
(( ++active ))
read -u4 line
taskid=${line%%|*} taskid=${line%%|*}
rest="${line#*|}|" rest="${line#*|}|"
sourcefileid=${rest%%|*} sourcefileid=${rest%%|*}
@ -1398,166 +1479,95 @@ worker() {
do do
[ -z "${cmd_arg[key]}" ] && unset cmd_arg[key] [ -z "${cmd_arg[key]}" ] && unset cmd_arg[key]
done done
(( debug >= 2 )) && echo "${cmd_arg[@]}" >>"$tempdir/errors.log" workerid=$(getworkerid)
if "${cmd_arg[@]}" >/dev/null 2>>"$tempdir/errors.log" workertasks[workerid]=$taskid
then Update tasks status 1 <<<"id = $taskid"
echo "finished $taskid|$sourcefileid|$destfileid|$destfilename" createworker $workerid
read line
until [[ $line == AtOM:OK ]]
do
echo "finished $taskid|$sourcefileid|$destfileid|$destfilename"
read line
done
else
echo "failed $taskid"
read line
until [[ $line == AtOM:OK ]]
do
echo "failed $taskid"
read line
done
[ -n "$filename" ] \
&& eval rm -f $filename
fi
unset cmd_arg
if [ -n "$cleanup" -a -n "$required" ]
then
echo "cleanup $required"
read answer
until [[ $answer != AtOM:ComFail ]]
do
echo "cleanup $required"
read answer
done
if (( answer == 1 ))
then
eval rm -f $cleanup
fi fi
fi fi
done
return
} }
master() { getworkerid() {
for workerid in ${!workers[@]} local i
for (( i=0 ; i >= 0 ; i++ ))
do do
if read -t0.001 -u$((200+workerid)) workercommand workerquery if [ -z "${workers[i]}" ]
then then
break echo $i
return
fi fi
done done
(( ${#workers[@]} == 0 )) && break # If we reach this, we have reached the signed long limit
if [ -n "$workercommand" ] # (2^63 - 1 = 9223372036854775807 - Got a supercomputer?)
then (( concurrency-- ))
case $workercommand in }
'work')
if [ -n "$quit" ] createworker() {
then worker $1 &
dyingworker=${workers[workerid]} workers[$1]=$!
unset workers[workerid] }
eval echo AtOM:Die '>&'$((100+workerid))
destroyworker() {
dyingworker=${workers[$1]}
unset workers[$1]
wait $dyingworker wait $dyingworker
eval $((100+workerid))'>&-' }
eval $((200+workerid))'<&-'
rm "$tempdir"/worker${workerid}{in,out} gettaskinfos() {
elif (( active < concurrency ))
then
echo ' echo '
SELECT COUNT(*)
FROM tasks
WHERE status = 0;
SELECT COUNT(*)
FROM tasks
WHERE status = 0
AND requires is NULL;
SELECT SELECT
id, id,
source_file, source_file,
required, required,
cmd_arg0,
cmd_arg1,
cmd_arg2,
cmd_arg3,
cmd_arg4,
cmd_arg5,
cmd_arg6,
cmd_arg7,
cmd_arg8,
cmd_arg9,
cmd_arg10,
cmd_arg11,
cmd_arg12,
cmd_arg13,
cmd_arg14,
cmd_arg15,
cmd_arg16,
cmd_arg17,
cmd_arg18,
cmd_arg19,
cmd_arg20,
cmd_arg21,
cmd_arg22,
cmd_arg23,
cmd_arg24,
cmd_arg25,
cmd_arg26,
cmd_arg27,
cmd_arg28,
cmd_arg29,
cleanup, cleanup,
fileid, fileid,
filename filename
FROM tasks FROM tasks
WHERE status = 0 WHERE id='$1';
AND requires is NULL
ORDER BY source_file
LIMIT 1;
' >&3 ' >&3
read -u4 remaining
read -u4 ready
if (( remaining == 0 ))
then
dyingworker=${workers[workerid]}
unset workers[workerid]
eval echo AtOM:Die '>&'$((100+workerid))
wait $dyingworker
eval $((100+workerid))'>&-'
eval $((200+workerid))'<&-'
rm "$tempdir"/worker${workerid}{in,out}
continue
elif (( ready == 0 ))
then
line=AtOM:Sleep
else
(( ++active ))
read -u4 line read -u4 line
taskid=${line%%|*} taskid=${line%%|*}
Update tasks status 1 <<<"id = $taskid" rest="${line#*|}|"
fi
eval echo '"$line" >&'$((100+workerid))
else
dyingworker=${workers[workerid]}
unset workers[workerid]
eval echo AtOM:Die '>&'$((100+workerid))
wait $dyingworker
eval $((100+workerid))'>&-'
eval $((200+workerid))'<&-'
rm "$tempdir"/worker${workerid}{in,out}
fi
;;
'finished')
eval 'echo AtOM:OK >&'$((workerid+100))
(( active-- )) || true
taskid=${workerquery%%|*}
rest="${workerquery#*|}|"
sourcefileid=${rest%%|*} sourcefileid=${rest%%|*}
rest=${rest#*|} rest=${rest#*|}
required=${rest%%|*}
rest=${rest#*|}
cleanup=${rest%%|*}
rest=${rest#*|}
destfileid=${rest%%|*} destfileid=${rest%%|*}
rest=${rest#*|} rest=${rest#*|}
destfilename=${rest%%|*} destfilename=${rest%%|*}
Delete tasks <<<"id = $taskid" rest=${rest#*|}
}
cleaner() {
for key in ${!failedtasks[@]}
do
taskid=${failedtasks[key]}
gettaskinfos $taskid
faildepends=$(
Select tasks 'COUNT(*)' <<-EOWhere
requires = $taskid
EOWhere
)
(( failed+=faildepends ))
Update tasks status 2 <<<"id = $taskid"
Update tasks status 2 <<<"requires = $taskid"
echo "SELECT COUNT(*)
FROM tasks
WHERE ( status = 0 OR status = 1 )
AND required = $taskid;">&3
read -u4 count
if (( count == 0 ))
then
rm -f "$cleanup"
fi
unset failedtasks[key]
done
for key in ${!finishedtasks[@]}
do
taskid=${finishedtasks[key]}
gettaskinfos $taskid
if [ -n "$destfilename" ] if [ -n "$destfilename" ]
then then
echo \ echo \
@ -1581,55 +1591,37 @@ master() {
"WHERE id=$destfileid;" \ "WHERE id=$destfileid;" \
>&3 >&3
fi fi
;;
'failed')
eval 'echo AtOM:OK >&'$((workerid+100))
(( --active )) || true
(( ++failed ))
taskid=$workerquery
faildepends=$(
Select tasks 'COUNT(*)' <<-EOWhere
requires = taskid
EOWhere
)
(( failed+=faildepends ))
Update tasks status 2 <<<"id = $taskid"
Update tasks status 2 <<<"requires = $taskid"
;;
'cleanup')
required=$workerquery
echo "SELECT COUNT(*) echo "SELECT COUNT(*)
FROM tasks FROM tasks
WHERE ( status = 0 OR status = 1 ) WHERE ( status = 0 OR status = 1 )
AND required = $required;">&3 AND required = $taskid;">&3
read -u4 count read -u4 count
if (( count == 0 )) if (( count == 0 ))
then then
eval echo 1 '>&'$((100+workerid)) rm -f "$cleanup"
else
eval echo 0 '>&'$((100+workerid))
fi
;;
*)
eval 'echo "AtOM:ComFail" >&'$((100+workerid))
;;
esac
fi fi
Delete tasks <<<"id = $taskid"
unset finishedtasks[key]
done
} }
checkworkers() { checkworkers() {
for key in ${!workers[@]} for key in ${!workers[@]}
do do
if ! kill -0 ${workers[key]} if ! kill -0 ${workers[key]} 2>/dev/null
then then
worker $key \ taskid=${workertasks[key]}
<"$tempdir"/worker${key}in \ (( ++ran ))
>"$tempdir"/worker${key}out & (( active-- ))
workers[key]=$! if destroyworker $key
eval exec $((100+key))'>"$tempdir"/worker${key}in' then
eval exec $((200+key))'<"$tempdir"/worker${key}out' finishedtasks+=($taskid)
else
failedtasks+=($taskid)
(( ++failed )) (( ++failed ))
fi fi
unset workertasks[key]
fi
done done
} }
@ -2075,28 +2067,21 @@ done
echo 'COMMIT;' >&3 echo 'COMMIT;' >&3
echo -e "\rCreated ${count:-0} tasks for $filecount files (${copies:-0} immediate copies)" echo -e "\rCreated ${count:-0} tasks for $filecount files (${copies:-0} immediate copies)"
masterpid=$$
trap checkworkers USR1 ALRM
rm -f "$tempdir"/worker*
concurrency=$(( maxload / 2 )) concurrency=$(( maxload / 2 ))
(( concurrency )) || concurrency=1 (( concurrency )) || concurrency=1
active=0 active=0
#set -x
for (( i=0 ; i < concurrency ; i++ )) for (( i=0 ; i < concurrency ; i++ ))
do do
(( ++wnum )) master
mkfifo "$tempdir"/worker${wnum}{in,out}
worker $wnum <"$tempdir"/worker${wnum}in >"$tempdir"/worker${wnum}out &
workers[wnum]=$!
eval exec $((100+wnum))'>"$tempdir"/worker${wnum}in'
eval exec $((200+wnum))'<"$tempdir"/worker${wnum}out'
done done
concurrencychange=$(date +%s) concurrencychange=$(date +%s)
starttime=$concurrencychange starttime=$concurrencychange
taskcount=$count taskcount=$count
failed=0 failed=0
while : while (( ${#workers[@]} ))
do do
if read -n 1 -t 0.01 userinput if read -n 1 -t 0.1 userinput
then then
case $userinput in case $userinput in
'+') '+')
@ -2123,27 +2108,20 @@ do
then then
concurrencychange=$(date +%s) concurrencychange=$(date +%s)
(( ++concurrency )) (( ++concurrency ))
(( ++wnum ))
mkfifo "$tempdir"/worker${wnum}{in,out}
worker $wnum \
<"$tempdir"/worker${wnum}in \
>"$tempdir"/worker${wnum}out &
workers[wnum]=$!
eval exec $((100+wnum))'>"$tempdir"/worker${wnum}in'
eval exec $((200+wnum))'<"$tempdir"/worker${wnum}out'
fi fi
fi fi
checkworkers
cleaner
master master
if ((taskcount - remaining)) if (( ran ))
then then
currenttime=$(date +%s) currenttime=$(date +%s)
secsremaining=$(( avgduration=$((
remaining ((currenttime - starttime) * 1000)
*
(currenttime - starttime)
/ /
(taskcount - remaining) ran
)) ))
secsremaining=$(( remaining * avgduration / 1000 ))
(( days = (( days =
secsremaining secsremaining
/ /
@ -2159,12 +2137,28 @@ do
/ /
60 60
)) || true )) || true
(( seconds =
secsremaining
-
( ( ( ( days*24 + hours ) *60 ) + minutes ) *60 )
)) || true
avgduration=$(printf %04i $avgduration)
avgdsec=${avgduration:0:-3}
avgdmsec=${avgduration#$avgdsec}
fi fi
echo -en "\rload: $humanload / $maxload" \ dran=$(printf %${#taskcount}i ${ran:-0})
"workers: $active / $concurrency" \ rtime=$(
"done: $(( (taskcount - remaining ) * 100 / taskcount ))%" \ printf '%2id %2ih%02im%02is' \
"- $((taskcount - remaining)) of $taskcount ($failed failed)" \ ${days:-0} \
"${days}d ${hours}h${minutes}m " ${hours:-0} \
${minutes:-0} \
${seconds:-0}
)
percent=$(printf %3i $((ran * 100 / taskcount)))
echo -en "\rL: $humanload/$maxload" \
"W: $active/$concurrency" \
"T: ${dran}/$taskcount (F:$failed) $percent% $rtime" \
"(A: ${avgdsec:--}.${avgdmsec:--}s/task)"
done done
unset count unset count
@ -2192,7 +2186,7 @@ endtime=$(date +%s)
( ( ( ( days*24 + hours ) *60 ) + minutes ) *60 ) ( ( ( ( days*24 + hours ) *60 ) + minutes ) *60 )
)) || true )) || true
echo -e "\rRan $taskcount tasks, $failed of which failed, in $days" \ echo -e "\rRan ${ran:=0} tasks, $failed of which failed, in $days" \
"days, $hours hours, $minutes minutes and $seconds seconds." "days, $hours hours, $minutes minutes and $seconds seconds."
if [ -n "$quit" ] if [ -n "$quit" ]
@ -2207,6 +2201,8 @@ do
echo ' echo '
SELECT SELECT
destination_files.filename, destination_files.filename,
destination_files.id,
source_files.filename,
tags.album, tags.album,
tags.albumartist, tags.albumartist,
tags.artist, tags.artist,
@ -2224,11 +2220,15 @@ do
INNER JOIN tags INNER JOIN tags
ON destination_files.source_file_id ON destination_files.source_file_id
=tags.source_file =tags.source_file
INNER JOIN source_files
ON destination_files.source_file_id
=source_files.id
WHERE destinations.name="'"$destination"'" WHERE destinations.name="'"$destination"'"
AND (destination_files.rename_pattern AND (destination_files.rename_pattern
!= !=
"'"${destinationrenamepath[$destination]}/${destinationrename[$destination]}"'" "'"${destinationrenamepath[$destination]}/${destinationrename[$destination]}:${destinationfat32compat["$destination"]}"'"
OR destination_files.rename_pattern is NULL) OR destination_files.rename_pattern is NULL)
AND destination_files.last_change > 0
; ;
SELECT "AtOM:NoMoreFiles"; SELECT "AtOM:NoMoreFiles";
@ -2246,6 +2246,10 @@ do
do do
oldfilename=${line%%|*} oldfilename=${line%%|*}
rest=${line#*|}'|' rest=${line#*|}'|'
destfileid=${rest%%|*}
rest=${rest#*|}
filename=${rest%%|*}
rest=${rest#*|}
album=${rest%%|*} album=${rest%%|*}
rest=${rest#*|} rest=${rest#*|}
albumartist=${rest%%|*} albumartist=${rest%%|*}
@ -2271,16 +2275,21 @@ do
getDestDir getDestDir
getDestFile getDestFile
destfilename="$destdir/$destfile.$extension" destfilename="$destdir/$destfile.$extension"
if [[ $oldfilename != $destfilename ]]
then
mv "$oldfilename" "$destfilename" mv "$oldfilename" "$destfilename"
progressSpin
fi
echo "UPDATE destination_files" \ echo "UPDATE destination_files" \
"SET filename=\"${destfilename//\"/\"\"}\"," \ "SET filename=\"${destfilename//\"/\"\"}\"," \
" rename_pattern=" \ " rename_pattern=" \
"\"${destinationrenamepath[$destination]}/${destinationrename[$destination]}\"" \ "\"${destinationrenamepath[$destination]}/${destinationrename[$destination]}:${destinationfat32compat["$destination"]}\"" \
"WHERE id=$destfileid;" \ "WHERE id=$destfileid;" \
>&3 >&3
progressSpin
fi fi
read -u4 line
done done
echo $'\r'"$destination: Renamed ${count:-0} files "
fi fi
unset count unset count
done done

2
doc/config
View File

@ -66,6 +66,8 @@ Sections:
%{track}, %{track},
%{year}. %{year}.
Untagged files or files in unrecognized formats will not be changed. Untagged files or files in unrecognized formats will not be changed.
* fat32compat <yes>/<no>: Rename files for compatibility with FAT32
filesystems.
* skip_mime-type <mime-type>: Files with mime-type <mime-type> will not * skip_mime-type <mime-type>: Files with mime-type <mime-type> will not
be included in that destination. For more than one mime-type, use multiple be included in that destination. For more than one mime-type, use multiple
times, as needed. The '*' character is a wildcard. times, as needed. The '*' character is a wildcard.