AtOM/atom
2013-06-27 18:34:30 +02:00

844 lines
18 KiB
Bash
Executable File

#!/bin/bash
## Define exit codes
# General config errors [10-19]
ELOAD=10
EINTERVAL=11
ENOCFG=19
# Source cofig errors [20-29]
# Destination config errors [30-49]
EFORMAT=30
ECHANNEL=31
EFMTINVPARM=49
# config structures
declare -A \
destinationchannels \
destinationfat32compat \
destinationcopymime \
destinationformat \
destinationfrequency \
destinationid \
destinationloss \
destinationmaxbps \
destinationnormalize \
destinationpath \
destinationquality \
destinationrename \
destinationnoresample \
destinationrenamepath \
destinationskipmime \
|| {
echo "Check your Bash version. You need >= 4.0" >&2
exit $EBASHVERS
}
declare -r \
DOCDIR=./doc \
LIBDIR=./lib \
SHAREDIR=./share
declare -r \
exampleconf=$DOCDIR/example.cfg \
schema=$SHAREDIR/schema.sql \
\
oldIFS="$IFS"
cffile="$HOME/.atom/atom.cfg"
LC_ALL=C
shopt -s extglob
source $SHAREDIR/id3genres
for function in "$LIBDIR"/*/*
do
source "$function"
done
help() {
cat <<-EOF
Options:
-c <file> Load configuration file <file>
-C Dump configuration and exit
-l <load> Override max-load
-T <seconds> override load-interval
-F <destination> Force re-generation of all files in
<destination>
-S Run setup
-h Show this text
-D Increase debug
EOF
}
#parse arguments
OPTERR=0
while getopts ':c:Cl:T:F:ShD' opt
do
case $opt in
c)
cffile="$OPTARG"
;;
C)
cfgdump=1
;;
l)
cliload="$OPTARG"
;;
T)
cliltimer="$OPTARG"
;;
F)
forceall+=("$OPTARG")
;;
S)
forcesetup=1
;;
h)
help
exit 0
;;
D)
(( debug++ ))
;;
:)
echo "-$OPTARG requires an argument"
help
exit 127
;;
*)
echo "Unrecognized option: -$OPTARG"
help
exit 127
;;
esac
done
askconf() {
read -p"Create one now? [Y/n/q] " createconf
case $createconf in
''|[yY])
setup
;;
[nNqQ])
echo "You need a configuration file. If you" \
"want to create it yourself, please" \
"read doc/config and doc/example.cfg." >&2
exit $ENOCFG
;;
*)
echo "Come again?" >&2
askconf
;;
esac
}
if [ ! -f "$cffile" ]
then
if [ ! -d "${cffile%/*}" ]
then
mkdir -p "${cffile%/*}"
fi
echo "No configuration file found!" >&2
askconf
fi
getConfig
(( forcesetup )) && setup
set +H
# Apply CLI overrides
[ -n "$cliload" ] && maxload=$cliload
[ -n "$cliltimer" ] && loadinterval=$cliltimer
(( debug || cfgdump )) && printConfig
(( cfgdump )) && exit
# check sanity
if [ ! -d "$tempdir" ] && ! mkdir -p "$tempdir"
then
echo "[FATAL] Could not create temp directory $tempdir" >&2
(( sanityfail++ ))
fi
if [ ! -f "$database" ] && [ ! -d "${database%/*}" ] && ! mkdir -p "${database%/*}"
then
echo "[FATAL] Directory holding database file does not exist and could" \
"not be created" >&2
(( sanityfail++ ))
fi
if [ ! -d "$sourcepath" ]
then
echo "[FATAL] Source path $sourcepath does not exist or is not a directory" >&2
(( sanityfail++ ))
fi
if ! which sed >/dev/null
then
echo "[FATAL] Required tool sed is not installed or not in PATH
I never thought this would actually hit someone..." >&2
(( sanityfail++ ))
fi
if ! which sox >/dev/null
then
echo "[FATAL] Required tool sox is not installed or not in PATH" >&2
(( sanityfail++ ))
fi
if ! which ogginfo >/dev/null
then
echo "[WARNING] Tool ogginfo (from vorbis-tools) is not" \
"installed or not in PATH
Vorbis metadata disabled
WebM metadata disabled" >&2
disableogginfo=1
(( sanitywarn++ ))
fi
if (( oggencneeded )) && ! which oggenc >/dev/null
then
echo "[WARNING] Tool oggenc (from vorbis-tools) is not" \
"installed or not in PATH
Vorbis targets disabled" >&2
disableoggenc=1
(( sanitywarn++ ))
fi
if ! which opusinfo >/dev/null
then
echo "[WARNING] Tool opusinfo (from opus-tools) is not" \
"installed or not in PATH
Opus metadata disabled" >&2
disableopusinfo=1
(( sanitywarn++ ))
fi
if (( opusencneeded )) && ! which opusenc >/dev/null
then
echo "[WARNING] Tool opusenc (from opus-tools) is not" \
"installed or not in PATH
Opus targets disabled" >&2
disableopusenc=1
(( sanitywarn++ ))
fi
if ! which opusdec >/dev/null
then
echo "[WARNING] Tool opusdec (from opus-tools) is not" \
"installed or not in PATH
Opus support disabled" >&2
disableopusdec=1
(( sanitywarn++ ))
fi
if (( lameneeded )) && ! which lame >/dev/null
then
echo "[WARNING] Tool lame is not installed or not in PATH
MP3 targets disabled" >&2
disablelame=1
(( sanitywarn++ ))
fi
if ! which metaflac >/dev/null
then
echo "[WARNING] Tool metaflac (from FLAC) is not installed" \
"or not in PATH
FLAC metadata disabled" >&2
disableflac=1
(( sanitywarn++ ))
fi
if ! which mpcdec >/dev/null
then
echo "[WARNING] Tool mpcdec (from Musepack) is not" \
"installed or not in PATH
Musepack support disabled" >&2
disablempcdec=1
(( sanitywarn++ ))
fi
if ! which mkvextract >/dev/null
then
echo "[WARNING] Tool mkvextract (from MKVToolNix) is not" \
"installed or not in PATH
WebM metadata disabled
WebM support disabled" >&2
disablemkvextract=1
(( sanitywarn++ ))
fi
if ! which ffprobe >/dev/null
then
echo "[WARNING] Tool ffprobe (from FFmpeg) is not installed or not in PATH
Video metadata disabled
MPEG metadata disabled
MusePack metadata disabled
Unknown format metadata disabled" >&2
disableffprobe=1
(( sanitywarn++ ))
fi
if ! which ffmpeg >/dev/null
then
echo "[WARNING] Tool ffmpeg is not installed or not in PATH
Video support disabled" >&2
disablevideo=1
(( sanitywarn++ ))
fi
if (( sanityfail ))
then
echo "
Sanity checks raised ${sanitywarn:-0} warnings, $sanityfail failures. Dying now." >&2
exit $ESANITY
elif (( sanitywarn ))
then
echo "
Sanity checks raised $sanitywarn warnings... Hit Control-C to abort." >&2
timeout=$(( sanitywarn * 10 ))
echo -n "Starting in $(printf %3i $timeout)" \
$'seconds...\b\b\b\b\b\b\b\b\b\b\b' >&2
while (( timeout ))
do
echo -n $'\b\b\b'"$(printf %3i $timeout)" >&2
sleep 1
(( timeout-- ))
done
echo -en "\r\033[K"
fi
openDatabase
createDestinations
getFiles
updateMimes
removeObsoleteFiles
echo '
SELECT id,
filename
FROM destination_files
WHERE source_file_id is NULL;
SELECT "AtOM:NoMoreFiles";
' >&3
deleted=0
removed=0
read -u4 line
until [[ $line == AtOM:NoMoreFiles ]]
do
id=${line%::AtOM:SQL:Sep::*}
filename=${line#*::AtOM:SQL:Sep::}
if [ -n "$filename" ]
then
if rm -f "$filename"
then
Delete destination_files <<<"id = $id"
(( ++deleted ))
fi
else
Delete destination_files <<<"id = $id"
(( ++removed ))
fi
read -u4 line
done
echo "Suppressed $deleted files, $removed removed from database"
updateTags
for forcedest in "${forceall[@]}"
do
if forcedestid=$(Select destinations id <<<"name = $forcedest")
then
echo "Resetting destination files timestamps on" \
"$forcedest ($forcedestid)..."
Update destination_files last_change 1 \
<<<"destination_id = $forcedestid"
else
echo "Destination $forcedest does not exist!" >&2
fi
done
echo '
CREATE TEMPORARY TABLE tasks(
id INTEGER PRIMARY KEY,
key TEXT UNIQUE,
rename_pattern TEXT,
source_file INTEGER,
fileid INTEGER,
filename TEXT,
cmd_arg0 TEXT,
cmd_arg1 TEXT,
cmd_arg2 TEXT,
cmd_arg3 TEXT,
cmd_arg4 TEXT,
cmd_arg5 TEXT,
cmd_arg6 TEXT,
cmd_arg7 TEXT,
cmd_arg8 TEXT,
cmd_arg9 TEXT,
cmd_arg10 TEXT,
cmd_arg11 TEXT,
cmd_arg12 TEXT,
cmd_arg13 TEXT,
cmd_arg14 TEXT,
cmd_arg15 TEXT,
cmd_arg16 TEXT,
cmd_arg17 TEXT,
cmd_arg18 TEXT,
cmd_arg19 TEXT,
cmd_arg20 TEXT,
cmd_arg21 TEXT,
cmd_arg22 TEXT,
cmd_arg23 TEXT,
cmd_arg24 TEXT,
cmd_arg25 TEXT,
cmd_arg26 TEXT,
cmd_arg27 TEXT,
cmd_arg28 TEXT,
cmd_arg29 TEXT,
requires INTEGER,
required INTEGER,
status INTEGER NOT NULL,
cleanup TEXT,
FOREIGN KEY(requires) REFERENCES tasks(id)
ON DELETE SET NULL
);
CREATE INDEX tasks_by_key ON tasks ( key );
CREATE INDEX tasks_by_sourcefile ON tasks ( source_file );
' >&3
echo '
SELECT COUNT(source_files.id)
FROM source_files
INNER JOIN destination_files
ON source_files.id
= destination_files.source_file_id
INNER JOIN destinations
ON destination_files.destination_id=destinations.id
INNER JOIN mime_type_actions
ON mime_type_actions.id = source_files.mime_type
INNER JOIN tags
ON source_files.id = tags.source_file
WHERE CAST(destination_files.last_change AS TEXT)
<> CAST(source_files.last_change AS TEXT)
AND mime_type_actions.destination_id = destinations.id
AND mime_type_actions.action = 1;' >&3
read -u4 filecount
echo '
SELECT
source_files.id,
source_files.filename,
mime_type_actions.mime_text,
destinations.name,
destination_files.id,
tags.rate,
tags.channels,
tags.bitrate,
tags.genre,
tags.albumartist,
tags.year,
tags.album,
tags.disc,
tags.artist,
tags.track,
tags.title,
tags.composer,
tags.performer
FROM source_files
INNER JOIN destination_files
ON source_files.id
= destination_files.source_file_id
INNER JOIN destinations
ON destination_files.destination_id=destinations.id
INNER JOIN mime_type_actions
ON mime_type_actions.id = source_files.mime_type
INNER JOIN tags
ON source_files.id = tags.source_file
WHERE CAST(destination_files.last_change AS TEXT)
<> CAST(source_files.last_change AS TEXT)
AND mime_type_actions.destination_id = destinations.id
AND mime_type_actions.action = 1;
SELECT "AtOM:NoMoreFiles";' >&3
read -u4 line
while ! [[ $line = AtOM:NoMoreFiles ]]
do
decodefiles+=("$line::AtOM:SQL:Sep::")
read -u4 line
done
echo -n 'Creating tasks... '
echo 'BEGIN TRANSACTION;' >&3
for line in "${decodefiles[@]}"
do
fileid=${line%%::AtOM:SQL:Sep::*}
rest=${line#*::AtOM:SQL:Sep::}
filename=${rest%%::AtOM:SQL:Sep::*}
rest=${rest#*::AtOM:SQL:Sep::}
mimetype=${rest%%::AtOM:SQL:Sep::*}
rest=${rest#*::AtOM:SQL:Sep::}
destination=${rest%%::AtOM:SQL:Sep::*}
rest=${rest#*::AtOM:SQL:Sep::}
destfileid=${rest%%::AtOM:SQL:Sep::*}
rest=${rest#*::AtOM:SQL:Sep::}
rate=${rest%%::AtOM:SQL:Sep::*}
rest=${rest#*::AtOM:SQL:Sep::}
channels=${rest%%::AtOM:SQL:Sep::*}
rest=${rest#*::AtOM:SQL:Sep::}
bitrate=${rest%%::AtOM:SQL:Sep::*}
rest=${rest#*::AtOM:SQL:Sep::}
genre=${rest%%::AtOM:SQL:Sep::*}
rest=${rest#*::AtOM:SQL:Sep::}
albumartist=${rest%%::AtOM:SQL:Sep::*}
rest=${rest#*::AtOM:SQL:Sep::}
year=${rest%%::AtOM:SQL:Sep::*}
rest=${rest#*::AtOM:SQL:Sep::}
album=${rest%%::AtOM:SQL:Sep::*}
rest=${rest#*::AtOM:SQL:Sep::}
disc=${rest%%::AtOM:SQL:Sep::*}
rest=${rest#*::AtOM:SQL:Sep::}
artist=${rest%%::AtOM:SQL:Sep::*}
rest=${rest#*::AtOM:SQL:Sep::}
track=${rest%%::AtOM:SQL:Sep::*}
rest=${rest#*::AtOM:SQL:Sep::}
title=${rest%%::AtOM:SQL:Sep::*}
rest=${rest#*::AtOM:SQL:Sep::}
composer=${rest%%::AtOM:SQL:Sep::*}
rest=${rest#*::AtOM:SQL:Sep::}
performer=${rest%%::AtOM:SQL:Sep::*}
unset rest
case ${destinationformat["$destination"]} in
vorbis) (( disableoggenc )) && continue ;;
opus) (( disableopusenc )) && continue ;;
mp3) (( disablelame )) && continue ;;
esac
decodeFile
getDestDir
getDestFile
if (( copied ))
then
copyFiles_matching
else
encodeFile::${destinationformat[$destination]}
fi
unset \
album \
albumartist \
artist \
bitrate \
channels \
commandline \
composer \
copied \
decodetaskid \
destfileid \
destination \
disc \
fileid \
filename \
mimetype \
performer \
rate \
rest \
sox_needed \
soxoptions_in \
soxoptions_out \
soxtaskid \
title \
track \
year \
tmpfile
done
echo 'COMMIT;' >&3
echo -e "\rCreated ${count:-0} tasks for $filecount files (${copies:-0} immediate copies)"
concurrency=$(( maxload / 2 ))
(( concurrency )) || concurrency=1
active=0
concurrencychange=$(date +%s)
starttime=$concurrencychange
taskcount=$count
remaining=$taskcount
failed=0
while (( (remaining || ${#workers[@]}) && ! quit ))
do
if read -n 1 -t 0.1 userinput
then
case $userinput in
'+')
((maxload++))
;;
'-')
((--maxload)) || ((maxload=1))
;;
[qQ])
quit=1
;;
esac
fi
read humanload garbage < /proc/loadavg
load=${humanload%.*}
if [ -z "$quit" ] && (( $(date +%s)-concurrencychange >= loadinterval ))
then
if (( concurrency > 1 )) \
&& (( load > maxload ))
then
concurrencychange=$(date +%s)
(( --concurrency ))
elif (( load < maxload )) && (( active > concurrency - 1 ))
then
concurrencychange=$(date +%s)
(( ++concurrency ))
fi
fi
checkworkers
cleaner
master
if (( ran - failed ))
then
currenttime=$(date +%s)
avgduration=$((
((currenttime - starttime) * 1000)
/
( ran - failed )
))
secsremaining=$(( remaining * avgduration / 1000 ))
(( days =
secsremaining
/
( 24*60*60 )
)) || true
(( hours =
( secsremaining - ( days*24*60*60 ) )
/
( 60*60 )
)) || true
(( minutes =
( secsremaining - ( ( days*24 + hours ) *60*60 ) )
/
60
)) || true
(( seconds =
secsremaining
-
( ( ( ( days*24 + hours ) *60 ) + minutes ) *60 )
)) || true
avgduration=$(printf %04i $avgduration)
avgdsec=${avgduration:0:-3}
avgdmsec=${avgduration#$avgdsec}
fi
fmtload='L:%4.1f/%i'
fmtworkers='W:%i/%i'
fmtprogress="T:%${#taskcount}i/%i (F:%i) %3i%%"
fmttime='%2id %2ih%02im%02is (A:%4.1fs/task)'
eta="ETA:$(
date -d "${days:-0} days
${hours:-0} hours
${minutes:-0} minutes
${seconds:-0} seconds" \
+'%d/%m %H:%M:%S'
)"
printf \
"\r$fmtload $fmtworkers $fmtprogress $fmttime $eta\033[K"\
$humanload \
$maxload \
${active:-0} \
${concurrency:-0} \
${ran:-0} \
${taskcount:-0} \
${failed:-0} \
$(( ran * 100 / taskcount )) \
${days:-0} \
${hours:-0} \
${minutes:-0} \
${seconds:-0} \
${avgdsec:-0}.${avgdmsec:-0}
done
unset count
endtime=$(date +%s)
(( elapsedseconds = endtime - starttime ))
(( days =
elapsedseconds
/
( 24*60*60 )
)) || true
(( hours =
( elapsedseconds - ( days*24*60*60 ) )
/
( 60*60 )
)) || true
(( minutes =
( elapsedseconds - ( ( days*24 + hours ) *60*60 ) )
/
60
)) || true
(( seconds =
elapsedseconds
-
( ( ( ( days*24 + hours ) *60 ) + minutes ) *60 )
)) || true
echo -e "\rRan ${ran:=0} tasks, $failed of which failed, in $days" \
"days, $hours hours, $minutes minutes and $seconds seconds.\033[K"
if [ -n "$quit" ]
then
closeDatabase
exit
fi
#set -x
for destination in "${!destinationpath[@]}"
do
echo '
SELECT
destination_files.filename,
destination_files.id,
source_files.filename,
tags.album,
tags.albumartist,
tags.artist,
tags.composer,
tags.disc,
tags.genre,
tags.performer,
tags.title,
tags.track,
tags.year
FROM destination_files
INNER JOIN destinations
ON destination_files.destination_id
=destinations.id
INNER JOIN tags
ON destination_files.source_file_id
=tags.source_file
INNER JOIN source_files
ON destination_files.source_file_id
=source_files.id
WHERE destinations.name="'"$destination"'"
AND (destination_files.rename_pattern
!=
"'"${destinationrenamepath[$destination]}/${destinationrename[$destination]}:${destinationfat32compat["$destination"]}"'"
OR destination_files.rename_pattern is NULL)
AND destination_files.last_change > 0
;
SELECT "AtOM:NoMoreFiles";
' >&3
read -u4 line
while [[ $line != AtOM:NoMoreFiles ]]
do
renamefiles+=("$line")
read -u4 line
done
if (( ${#renamefiles[@]} ))
then
case "${destinationformat[$destination]}" in
'mp3') extension=mp3 ;;
'opus') extension=opus ;;
'vorbis') extension=ogg ;;
esac
echo -n "$destination: rename pattern changed, renaming files... "
echo 'BEGIN TRANSACTION;' >&3
for line in "${renamefiles[@]}"
do
oldfilename=${line%%::AtOM:SQL:Sep::*}
rest=${line#*::AtOM:SQL:Sep::}'::AtOM:SQL:Sep::'
destfileid=${rest%%::AtOM:SQL:Sep::*}
rest=${rest#*::AtOM:SQL:Sep::}
filename=${rest%%::AtOM:SQL:Sep::*}
rest=${rest#*::AtOM:SQL:Sep::}
album=${rest%%::AtOM:SQL:Sep::*}
rest=${rest#*::AtOM:SQL:Sep::}
albumartist=${rest%%::AtOM:SQL:Sep::*}
rest=${rest#*::AtOM:SQL:Sep::}
artist=${rest%%::AtOM:SQL:Sep::*}
rest=${rest#*::AtOM:SQL:Sep::}
composer=${rest%%::AtOM:SQL:Sep::*}
rest=${rest#*::AtOM:SQL:Sep::}
disc=${rest%%::AtOM:SQL:Sep::*}
rest=${rest#*::AtOM:SQL:Sep::}
genre=${rest%%::AtOM:SQL:Sep::*}
rest=${rest#*::AtOM:SQL:Sep::}
performer=${rest%%::AtOM:SQL:Sep::*}
rest=${rest#*::AtOM:SQL:Sep::}
title=${rest%%::AtOM:SQL:Sep::*}
rest=${rest#*::AtOM:SQL:Sep::}
track=${rest%%::AtOM:SQL:Sep::*}
rest=${rest#*::AtOM:SQL:Sep::}
year=${rest%%::AtOM:SQL:Sep::*}
rest=${rest#*::AtOM:SQL:Sep::}
if [ -n "$oldfilename" -a -f "$oldfilename" ]
then
getDestDir
getDestFile
destfilename="$destdir/$destfile.$extension"
progressSpin
if [[ "$oldfilename" != "$destfilename" ]]
then
mv "$oldfilename" "$destfilename"
(( changedcount++ ))
commit=1
fi
echo "UPDATE destination_files" \
"SET filename=\"${destfilename//\"/\"\"}\"," \
" rename_pattern=" \
"\"${destinationrenamepath[$destination]}/${destinationrename[$destination]}:${destinationfat32compat["$destination"]}\"" \
"WHERE id=$destfileid;" \
>&3
if (( commit ))
then
echo $'COMMIT;\nBEGIN TRANSACTION;' >&3
unset commit
fi
fi
done
echo 'COMMIT;' >&3
echo -e $'\r'"$destination: Renamed ${changedcount:-0} files\033[K"
fi
unset count changedcount renamefiles
done
copyFiles_action
echo '
SELECT id,
filename,
old_filename
FROM destination_files
WHERE old_filename IS NOT NULL;
SELECT "AtOM:NoMoreFiles";
' >&3
echo -n 'Removing obsolete files... '
lines=()
read -u4 line
while [[ $line != AtOM:NoMoreFiles ]]
do
lines+=("$line")
read -u4 line
done
echo 'BEGIN TRANSACTION;' >&3
for line in "${lines[@]}"
do
id=${line%%::AtOM:SQL:Sep::*}
rest=${line#*::AtOM:SQL:Sep::}
filename=${rest%%::AtOM:SQL:Sep::*}
oldfilename=${rest#*::AtOM:SQL:Sep::}
if [[ $oldfilename != "$filename" ]] && [ -f "$oldfilename" ]
then
rm -f "$oldfilename"
fi
Update destination_files old_filename NULL <<<"id = $id"
(( count++ ))
printf '\b\b\b\b%3i%%' $(( (100 * count) / ${#lines[@]} ))
done
echo 'COMMIT;' >&3
echo -e "\rRemoved ${count:-0} obsolete files.\033[K"
echo "Purging empty directories."
for path in "${destinationpath[@]}"
do
find "$path" -type d -empty -delete
done
closeDatabase
# vim:set ts=8 sw=8: