Merge branch 'master' into toys

* master:
  fat32compat: no . at beginning or end of dirname
  copy files: fix DB update
  remove obsolete (renamed) files: read all data from sqlite first - prevents deadlock
  fix bitrate copy check
  fix filepath for copied files
  wait for last tasks
  copyFiles_action: protect '"' from SQL
  ionice
  implement -c
  add shebang on lib files
  print config
  move files copy
  copy_mime-type
  fat32compat: nodes ending with "."
This commit is contained in:
Vincent Riquer 2013-04-23 14:35:35 +02:00
commit c46f9d86cb
52 changed files with 290 additions and 63 deletions

69
atom
View File

@ -43,6 +43,8 @@ declare -r \
\
oldIFS="$IFS"
cffile="$HOME/.atom/atom.cfg"
LC_ALL=C
shopt -s extglob
@ -56,12 +58,15 @@ done
#parse arguments
OPTERR=0
while getopts ':c:l:T:F:hD' opt
while getopts ':c:Cl:T:F:hD' opt
do
case $opt in
c)
cffile="$OPTARG"
;;
C)
cfgdump=1
;;
l)
cliload="$OPTARG"
;;
@ -93,16 +98,16 @@ done
#FIXME: check sanity
if [ ! -f ~/.atom/atom.cfg ]
if [ ! -f "$cffile" ]
then
if [ ! -d ~/.atom ]
then
mkdir -p ~/.atom
fi
sed "s:%HOME%:$HOME:" "$exampleconf" > ~/.atom/atom.cfg
sed "s:%HOME%:$HOME:" "$exampleconf" > "$cffile"
cat >&2 <<-EOCfgNotice
No configuration file found!
An example file has been created as ~/.atom/atom.cfg.
An example file has been created as "${cffile/$HOME/~}".
You should change it to your likings using you favorite editor.
Bailing out.
@ -112,38 +117,8 @@ fi
getConfig
set +H
if (( debug ))
then
cat <<-EOF
General|Load|$maxload
|Load Interval|$loadinterval
|Temp Dir|$tempdir
|Database|$database
|Debug|$debug
Source|Path|$sourcepath
EOF
for prune_expression in "${skippeddirectories[@]}"
do
echo "|Skipped directory|$prune_expression"
done
for destination in ${!destinationpath[@]}
do
cat <<-EOF
$destination|Path|${destinationpath[$destination]}
|Format|${destinationformat[$destination]}
|Quality|${destinationquality[$destination]}
|Normalize|${destinationnormalize[$destination]}
|Channels|${destinationchannels[$destination]}
|Frequency|${destinationfrequency[$destination]}
|Path Change|${destinationrenamepath[$destination]}
|File Rename|${destinationrename[$destination]}
EOF
echo "|Skipped mime-type|${destinationskipmime[$destination]//\|/
|Skipped mime-type|}"
echo "|Copied mime-type|${destinationcopymime[$destination]//\|/
|Copied mime-type|}"
done
fi |column -t -s'|' -n
(( debug || cfgdump )) && printConfig
(( cfgdump )) && exit
openDatabase
@ -518,7 +493,7 @@ do
getDestFile
if (( copied ))
then
copyFile
copyFiles_matching
else
encodeFile::${destinationformat[$destination]}
fi
@ -561,7 +536,7 @@ starttime=$concurrencychange
taskcount=$count
remaining=$taskcount
failed=0
while (( remaining && ! quit ))
while (( (remaining || ${#workers[@]}) && ! quit ))
do
if read -n 1 -t 0.1 userinput
then
@ -790,6 +765,8 @@ do
unset count
done
copyFiles_action
echo '
SELECT id,
old_filename
@ -799,9 +776,16 @@ echo '
SELECT "AtOM:NoMoreFiles";
' >&3
echo "Removing obsolete files... "
echo -n 'Removing obsolete files... '
lines=()
read -u4 line
while [[ $line != AtOM:NoMoreFiles ]]
do
lines+=("$line")
read -u4 line
done
echo 'BEGIN TRANSACTION;' >&3
for line in "${lines[@]}"
do
id=${line%%|*}
filename=${line#*|}
@ -810,10 +794,11 @@ do
rm -f "$filename"
fi
Update destination_files old_filename NULL <<<"id = $id"
progressSpin
read -u4 line
(( count++ ))
printf '\b\b\b\b%3i%%' $(( (100 * count) / ${#lines[@]} ))
done
echo $'\r'"Removed ${count:-0} obsolete files."
echo 'COMMIT;' >&3
echo -e "\rRemoved ${count:-0} obsolete files.\033[K"
echo "Purging empty directories."
for path in "${destinationpath[@]}"

View File

@ -26,6 +26,8 @@ Sections:
too quickly. Set this too high, and AtOM will not adapt quickly enough to
load increase. In both cases, your hard drive will suffer. In my
experience, 30 seconds is a good value.
* ionice <class> [niceness]: IO-hungry processes will be run with ionice class
<class> and niceness [niceness] (if applicable). See man ionice for details.
* temporary-directory <directory>: String. Name speaks for itself: this is
where FIFOs (for communicating with sqlite) and temporary WAVE files will
be created. Note that debug logs (if enabled) will go there too.

View File

@ -1,4 +1,5 @@
[general]
ionice 3
max-load 6
load-interval 30
temporary-directory %HOME%/.atom/tmp

View File

@ -1,3 +1,4 @@
#!/bin/bash
getConfig() {
while read key value
do
@ -23,5 +24,5 @@ getConfig() {
getConfig$context
;;
esac
done < ~/.atom/atom.cfg
done < "$cffile"
}

View File

@ -1,3 +1,4 @@
#!/bin/bash
getConfigDestination() {
case "$key" in
'path')

View File

@ -1,3 +1,4 @@
#!/bin/bash
getConfigGeneral() {
case $key in
'max-load')
@ -22,6 +23,49 @@ getConfigGeneral() {
fi
unset expr
;;
'ionice')
read class niceness <<<"$value"
case $class in
1)
# real-time class, only root can do that
if (( UID ))
then
echo "IO class 'realtime' is"\
"not available to unprivileged"\
"users" >&2
exit $EIONICE
fi
if [ -n "$niceness" ] \
&& (( niceness >= 0 && niceness <= 7 ))
then
ionice="ionice -c1 -n$niceness "
else
echo "Invalid IO priority"\
"'$niceness'" >&2
exit $EIONICE
fi
;;
2)
if [ -n "$niceness" ] \
&& (( niceness >= 0 && niceness <= 7 ))
then
ionice="ionice -c2 -n$niceness "
else
echo "Invalid IO priority"\
"'$niceness'" >&2
exit $EIONICE
fi
;;
3)
ionice="ionice -c3 "
;;
*)
echo "Invalid ionice parameters $value"\
>&2
exit $EIONICE
;;
esac
;;
'temporary-directory')
tempdir="$value"
;;

View File

@ -1,3 +1,4 @@
#!/bin/bash
getConfigSource() {
case "$key" in
'path')

54
lib/config/print Normal file
View File

@ -0,0 +1,54 @@
#!/bin/bash
printConfig() {
{
echo "General|Config file|$cffile"
[ -n "$ionice" ] && echo "|IO Nice|$ionice"
cat <<-EOF
|Load|$maxload
|Load Interval|$loadinterval
|Temp Dir|$tempdir
|Database|$database
|Debug|$debug
Source|Path|$sourcepath
EOF
for prune_expression in "${skippeddirectories[@]}"
do
(( printed )) \
&& echo -n '||' \
|| echo -n '|Skipped directories|'
echo "$prune_expression"
printed=1
done
unset printed
for destination in ${!destinationpath[@]}
do
cat <<-EOF
$destination|Path|${destinationpath["$destination"]}
|Format|${destinationformat["$destination"]}
|Quality|${destinationquality["$destination"]}
EOF
if [[ ${destinationformat["$destination"]} == opus ]]
then
echo "|Expected loss|${destinationloss["$destination"]}"
elif [[ ${destinationformat["$destination"]} == mp3 ]]
then
echo "|Prevent resampling|${destinationnoresample["$destination"]}"
fi
cat <<-EOF
|Normalize|${destinationnormalize["$destination"]}
|Channels|${destinationchannels["$destination"]}
|Frequency|${destinationfrequency["$destination"]}
|Higher than|${destinationmaxbps["$destination"]}
|Fat32 Compat.|${destinationfat32compat["$destination"]}
|Path Change|${destinationrenamepath["$destination"]}
|File Rename|${destinationrename["$destination"]}
EOF
[ -n "${destinationskipmime["$destination"]}" ] \
&& echo "|Skipped mime-types|${destinationskipmime["$destination"]//\|/
||}"
[ -n "${destinationmskipime["$destination"]}" ] \
&& echo "|Copied mime-types|${destinationcopymime["$destination"]//\|/
||}"
done
}|column -t -s'|' -n
}

View File

@ -1,3 +1,4 @@
#!/bin/bash
checkCopy() {
(
[ -z "${destinationfrequency[$destination]}" ] \
@ -9,7 +10,7 @@ checkCopy() {
(( ${bitrate:-1000} == ${destinationquality[$destination]} )) \
|| (
[ -n "${destinationmaxbps[$destination]}" ] \
|| ((
&& ((
${bitrate:-1000}
<= ${destinationmaxbps[$destination]:-0}
))

83
lib/copy/copyFiles_action Normal file
View File

@ -0,0 +1,83 @@
#!/bin/bash
copyFiles_action() {
echo -n "Copying files... "
echo '
SELECT
source_files.filename,
source_files.last_change,
destinations.id,
destination_files.id
FROM source_files
INNER JOIN destination_files
ON source_files.id
= destination_files.source_file_id
INNER JOIN destinations
ON destination_files.destination_id=destinations.id
INNER JOIN mime_type_actions
ON mime_type_actions.id = source_files.mime_type
WHERE CAST(destination_files.last_change AS TEXT)
<> CAST(source_files.last_change AS TEXT)
AND mime_type_actions.destination_id = destinations.id
AND mime_type_actions.action = 2;
SELECT "AtOM:NoMoreFiles";' >&3
read -u4 line
while ! [[ $line = AtOM:NoMoreFiles ]]
do
copyfiles+=("$line")
read -u4 line
done
echo 'BEGIN TRANSACTION;' >&3
for copyfile in "${copyfiles[@]}"
do
sourcefilename=${copyfile%%|*}
sourcedir=${sourcefilename%/*}
rest="${copyfile#*|}|"
lastchange=${rest%%|*}
rest=${rest#*|}
destinationid=${rest%%|*}
rest=${rest#*|}
destfileid=${rest%%|*}
rest=${rest#*|}
echo 'SELECT IFNULL( (
SELECT destination_files.filename
FROM destination_files
INNER JOIN source_files
ON destination_files.source_file_id=source_files.id
INNER JOIN mime_type_actions
ON
mime_type_actions.id=source_files.mime_type
INNER JOIN destinations
ON destinations.id=destination_files.destination_id
WHERE destinations.id = '$destinationid'
AND source_files.filename LIKE
"'"${sourcedir//\"/\"\"}"'/%"
AND mime_type_actions.action = 1
LIMIT 1
),"AtOM:NotFound");
'>&3
read -u4 filename
if [[ $filename != AtOM:NotFound ]]
then
destdir=${filename%/*}
if cp -al "$sourcepath/$sourcefilename" "$destdir" 2>/dev/null\
|| cp -a "$sourcepath/$sourcefilename" "$destdir"
then
Update destination_files \
filename "$destdir/${sourcefilename##*/}"\
rename_pattern "${destinationrenamepath[$destination]}/${destinationrename[$destination]}:${destinationfat32compat["$destination"]}"\
last_change $lastchange \
<<-EOWhere
id = $destfileid
EOWhere
(( done++ ))
fi
fi
(( count++ ))
printf '\b\b\b\b%3i%%' $(( (count * 100) / ${#copyfiles[@]} ))
done
echo 'COMMIT;' >&3
echo -e "\rCopied ${done:-0} of $count files.\033[K"
unset count done
}

View File

@ -1,4 +1,5 @@
copyFile() {
#!/bin/bash
copyFiles_matching() {
extension="${filename##*.}"
cp -al \
"$sourcepath/$filename" \
@ -7,9 +8,10 @@ copyFile() {
|| cp -a \
"$sourcepath/$filename" \
"$destdir/$destfile.$extension"
echo \
echo \
"UPDATE destination_files" \
"SET filename=\"${filename//\"/\"\"}\"," \
"SET filename=" \
"\"${destdir//\"/\"\"}/${destfile//\"/\"\"}.$extension\"," \
" last_change=(" \
" SELECT last_change" \
" FROM source_files" \

View File

@ -1,3 +1,4 @@
#!/bin/bash
Delete() {
#Delete table < where_key where_operator where_value
# [where_key where_operator where_value

View File

@ -1,3 +1,4 @@
#!/bin/bash
Insert() {
#Insert table [no_id] < key value
# [key value

View File

@ -1,3 +1,4 @@
#!/bin/bash
InsertIfUnset() {
#InsertIfUnset table [no_id] < key value \n key value
local \

View File

@ -1,3 +1,4 @@
#!/bin/bash
InsertOrUpdate() {
#InsertOrUpdate table set_key set_value [set_key set_value […]] < where_key where_value
# [where_key where_value

View File

@ -1,3 +1,4 @@
#!/bin/bash
Select() {
#Select table [col1 [col2 [..]]] < WHERE_key WHERE_operator WHERE_value
# [WHERE_key WHERE_operator WHERE_value

View File

@ -1,3 +1,4 @@
#!/bin/bash
Update() {
#Update table set_key set_value [set_key set_value […]] < where_key where_operator where_value
# [where_key where_operator where_value

View File

@ -1,3 +1,4 @@
#!/bin/bash
closeDatabase() {
echo .quit >&3
(( debug )) && echo -n "Waiting for SQLite to terminate... "

View File

@ -1,3 +1,4 @@
#!/bin/bash
openDatabase() {
if [ ! -d "$tempdir" ]
then

View File

@ -1,3 +1,4 @@
#!/bin/bash
decodeFile() {
if ! decodetaskid=$(
Select tasks id <<<"key = $tmpfile"

View File

@ -1,4 +1,6 @@
#!/bin/bash
decodeMpcdec() {
tmpfile="${fileid}mpcdec"
commandline=(mpcdec "$sourcepath/$filename" "$tempdir/$tmpfile.wav")
commandline=(${ionice}mpcdec)
commandline+=("$sourcepath/$filename" "$tempdir/$tmpfile.wav")
}

View File

@ -1,4 +1,6 @@
#!/bin/bash
decodeOpusdec() {
tmpfile="${fileid}opusdec"
commandline=(opusdec "$sourcepath/$filename" "$tempdir/$tmpfile.wav")
commandline=(${ionice}opusdec)
commandline+=("$sourcepath/$filename" "$tempdir/$tmpfile.wav")
}

View File

@ -1,5 +1,6 @@
#!/bin/bash
decodeSox() {
commandline=(sox --single-threaded --temp "$tempdir")
commandline=(${ionice}sox --single-threaded --temp "$tempdir")
soxoptions_in=''
soxoptions_out=''
if (( ${destinationnormalize["$destination"]} ))

View File

@ -1,3 +1,4 @@
#!/bin/bash
createDestinations() {
for destination in ${!destinationpath[@]}
do

View File

@ -1,3 +1,4 @@
#!/bin/bash
updateMimes() {
Update mime_actions action 1 <<<"action != 1"
for destination in ${!destinationskipmime[@]}

View File

@ -1,5 +1,7 @@
#!/bin/bash
encodeFile::mp3() {
lameopts=(lame --quiet -v --abr ${destinationquality[$destination]})
lameopts=(${ionice}lame --quiet)
lameopts+=(-v --abr ${destinationquality[$destination]})
[ -n "$album" ] && lameopts+=(--tl "$album" )
[ -n "$artist" ] && lameopts+=(--ta "$artist")
[ -n "$genre" ] && lameopts+=(--tg "$genre")

View File

@ -1,5 +1,6 @@
#!/bin/bash
encodeFile::opus() {
opusencopts=(opusenc --music --quiet)
opusencopts=(${ionice}opusenc --music --quiet)
opusencopts+=(--bitrate ${destinationquality[$destination]})
[ -n "${destinationloss["$destination"]}" ] \
&& opusencopts+=(--expect-loss "${destinationloss["$destination"]}")

View File

@ -1,5 +1,6 @@
#!/bin/bash
encodeFile::vorbis() {
oggencopts=(oggenc -Q -q ${destinationquality[$destination]})
oggencopts=(${ionice}oggenc -Q -q ${destinationquality[$destination]})
[ -n "$albumartist" ] && oggencopts+=(-c "ALBUMARTIST=$albumartist")
[ -n "$album" ] && oggencopts+=(-l "$album")
[ -n "$artist" ] && oggencopts+=(-a "$artist")

View File

@ -1,29 +1,31 @@
#!/bin/bash
getDestDir() {
destdir="${destinationpath[$destination]}/"
if [ -n "${destinationrenamepath[$destination]}" ]
then
destdir+="${destinationrenamepath[$destination]//%\{album\}/$album}"
replace=$(sanitizeFile "$albumartist")
replace=$(sanitizeFile "$album" dir)
destdir+="${destinationrenamepath[$destination]//%\{album\}/$replace}"
replace=$(sanitizeFile "$albumartist" dir)
destdir="${destdir//%\{albumartist\}/$replace}"
replace=$(sanitizeFile "$artist")
replace=$(sanitizeFile "$artist" dir)
destdir="${destdir//%\{artist\}/$replace}"
replace=$(sanitizeFile "$genre")
replace=$(sanitizeFile "$genre" dir)
destdir="${destdir//%\{genre\}/$replace}"
replace=$(sanitizeFile "$title")
replace=$(sanitizeFile "$title" dir)
destdir="${destdir//%\{title\}/$replace}"
tracknumber="${track%/*}"
replace=$(sanitizeFile "$tracknumber")
replace=$(sanitizeFile "$tracknumber" dir)
destdir="${destdir//%\{track\}/$replace}"
replace=$(sanitizeFile "$year")
replace=$(sanitizeFile "$year" dir)
destdir="${destdir//%\{year\}/$replace}"
replace=$(sanitizeFile "$disc")
replace=$(sanitizeFile "$disc" dir)
destdir="${destdir//%\{disc\}/$replace}"
else
destdir+=$(sanitizeFile "${filename%%/*}")
destdir+=$(sanitizeFile "${filename%%/*}" dir)
part=${filename#*/}
while [[ $part =~ / ]]
do
destdir+="/$(sanitizeFile "${part%%/*}")"
destdir+="/$(sanitizeFile "${part%%/*}" dir)"
part=${part#*/}
done
fi

View File

@ -1,3 +1,4 @@
#!/bin/bash
getDestFile() {
if [ -n "${destinationrename[$destination]}" ]
then

View File

@ -1,3 +1,4 @@
#!/bin/bash
getFiles() {
scantime=$(date +%s)
for prune_expression in "${skippeddirectories[@]}"

View File

@ -1,3 +1,4 @@
#!/bin/bash
removeObsoleteFiles() {
Delete source_files <<-EOWhere
last_seen < $scantime

View File

@ -1,3 +1,4 @@
#!/bin/bash
sanitizeFile() {
shopt -s extglob
string="$1"
@ -18,6 +19,13 @@ sanitizeFile() {
# Filenames can't begin or end with ' '
string=${string/#+( )/}
string=${string/%+( )/}
# Directory names can't begin or end with '.'
if [[ $2 == dir ]]
then
string=${string/#+(.)/}
string=${string/%+(.)/}
fi
fi
echo "$string"
}

View File

@ -1,3 +1,4 @@
#!/bin/bash
getInfosAPE_version='APE-1'
tagreaders+=( "$getInfosAPE_version" )
getInfos::APE() {

View File

@ -1,3 +1,4 @@
#!/bin/bash
getInfosFLAC_version='FLAC-1'
tagreaders+=( "$getInfosFLAC_version" )
getInfos::FLAC() {

View File

@ -1,3 +1,4 @@
#!/bin/bash
getInfosMP3_version='ID3-2'
tagreaders+=( "$getInfosMP3_version" )
getInfos::MP3() {

View File

@ -1,3 +1,4 @@
#!/bin/bash
getInfosOgg_version='Ogg-1'
tagreaders+=( "$getInfosOgg_version" )
getInfos::Ogg() {

View File

@ -1,3 +1,4 @@
#!/bin/bash
getInfosOpus_version='Opus-1'
tagreaders+=( "$getInfosOpus_version" )
getInfos::Opus() {

View File

@ -1,3 +1,4 @@
#!/bin/bash
getRateChannelMPC() {
while read key value garbage
do

View File

@ -1,3 +1,4 @@
#!/bin/bash
getRateChannelSoxi() {
rate=$(soxi -r "$sourcepath/$filename" 2>/dev/null)
channels=$(soxi -c "$sourcepath/$filename" 2>/dev/null)

View File

@ -1,3 +1,4 @@
#!/bin/bash
getTags_version='unknown-2'
tagreaders+=( "$getTags_version" )
getTags() {

View File

@ -1,3 +1,4 @@
#!/bin/bash
gettag() {
echo -e "$infos" \
| sed -n "/^${1}=/I{s/^${1}=//I;p;q}"

View File

@ -1,3 +1,4 @@
#!/bin/bash
tryAPE() {
grep -q 'APETAGEX' \
"$sourcepath/$filename" \

View File

@ -1,3 +1,4 @@
#!/bin/bash
gettaskinfos() {
echo '
SELECT

View File

@ -1,3 +1,4 @@
#!/bin/bash
progressSpin() {
case $(( ++count % 40 )) in
0) echo -ne '\b|' ;;

View File

@ -1,3 +1,4 @@
#!/bin/bash
checkworkers() {
for key in ${!workers[@]}
do

View File

@ -1,3 +1,4 @@
#!/bin/bash
cleaner() {
for key in ${!failedtasks[@]}
do

View File

@ -1,3 +1,4 @@
#!/bin/bash
createworker() {
worker $1 &
workers[$1]=$!

View File

@ -1,3 +1,4 @@
#!/bin/bash
destroyworker() {
dyingworker=${workers[$1]}
unset workers[$1]

View File

@ -1,3 +1,4 @@
#!/bin/bash
getworkerid() {
local i
for (( i=0 ; i >= 0 ; i++ ))

View File

@ -1,3 +1,4 @@
#!/bin/bash
master() {
if (( active >= concurrency)) || [ -n "$quit" ]
then

View File

@ -1,3 +1,4 @@
#!/bin/bash
worker() {
exec 2>>"$tempdir/worker$1.log"
(( debug >= 2 )) && echo "${cmd_arg[@]}" >&2