missing space between read args in Insert()

Resolve "Script is uselessly verbose"

See merge request atom/AtOM!36

CHANGELOG.md

@@ -1,3 +1,26 @@
+# DEV
+### BUGS (Minor)
+* `toys/createindex`: handle empty channel count, bitdepth and sampling rate cleanly
+
+### Enhancements
+* Allow ignoring microsecond precision in timestamps
+* Don't print useless information (stuff that handled 0 files and such)
+* Store transcoded file paths relative to their destination's root
+* Add missing error codes (used but not declared)
+* Support for newline character in filenames
+
+# 1.0.4
+## `BREAKING CHANGES`
+* ffmpeg parser was fixed. All files previously parsed by it and parsers depending on it will be reprocessed. Running in batches (`-B <batchsize>`) is recommended.
+
+### BUGS
+* `ffmpeg` output parsed incorrectly
+* `toys/createindex`: incorrect path for ID3v1 tag index file
+
+### Enhancements
+* Use bash builtin `printf` instead of command `date`
+* Commit to database more frequently when reading tags
+
 # 1.0.3
 ## `BREAKING CHANGES`
 * Implementing replaygain copy meant bumping versions of every tag parser. All file tags will be read again. Running in batches (`-B <batchsize>`) is recommended.

README.md

@@ -21,12 +21,12 @@ have AtOM do that!
 
 Here's what I have for my tests:
 | Directory | Format | Sample rate | Bitrate   | Channels | FAT32 compat. | ASCII | Size |
-| --------- | ------ | ----------- | ------- | -------- | ------------- | ----- | ---- |
-| 0-Full    | Mixed  | Mixed       | Mixed   | Mixed    | No            | No    | 508G |
-| 1-High    | Opus   | Same        | 128     | Same     | Yes           | No    | 101G |
-| 2-Medium  | Opus   | Same        | 64      | Same     | Yes           | No    | 59G  |
-| 3-Small   | Opus   | Same        | 32      | Same     | Yes           | No    | 30G  |
-| 4-MP3     | MP3    | 44100       | 128     | 2        | Yes           | Yes   | 114G |
+| --------- | ------ | ----------- | --------- | -------- | ------------- | ----- | ---- |
+| 0-Full    | Mixed  | Mixed       | Mixed     | Mixed    | No            | No    | 568G |
+| 1-High    | Vorbis | Same        | Quality 5 | Same     | Yes           | No    | 143G |
+| 2-Medium  | Opus   | Same        | 64        | Same     | Yes           | No    | 60G  |
+| 3-Small   | Opus   | Same        | 32        | Same     | Yes           | No    | 31G  |
+| 4-MP3     | MP3    | 44100       | 128       | 2        | Yes           | Yes   | 119G |
 
 
 

atom

@@ -1,15 +1,17 @@
 #!/usr/bin/env bash
 
+declare -r				\
+	DOCDIR=%DOCDIR%			\
+	LIBDIR=%LIBDIR%			\
+	SHAREDIR=%SHAREDIR%
+declare -r				\
+	exampleconf=$DOCDIR/example.cfg	\
+	schema=$SHAREDIR/schema.sql	\
+					\
+	oldIFS="$IFS"
+
 ## Define exit codes
-# General config errors [10-19]
-ELOAD=10
-EINTERVAL=11
-ENOCFG=19
-# Source cofig errors [20-29]
-# Destination config errors [30-49]
-EFORMAT=30
-ECHANNEL=31
-EFMTINVPARM=49
+source "$SHAREDIR"/errorcodes
 
 # config structures
 declare -A			\
@@ -36,21 +38,11 @@ declare -A			\
 	exit $EBASHVERS
 }
 
-declare -r				\
-	DOCDIR=%DOCDIR%			\
-	LIBDIR=%LIBDIR%			\
-	SHAREDIR=%SHAREDIR%
-declare -r				\
-	exampleconf=$DOCDIR/example.cfg	\
-	schema=$SHAREDIR/schema.sql	\
-					\
-	oldIFS="$IFS"
-
 LC_ALL=C
 
 shopt -s extglob
 
-source $SHAREDIR/id3genres
+source "$SHAREDIR"/id3genres
 
 for function in "$LIBDIR"/*/*
 do
@@ -125,12 +117,12 @@ do
 		:)
 			echo "-$OPTARG requires an argument"
 			help
-			exit 127
+			exit $EINVARG
 		;;
 		*)
 			echo "Unrecognized option: -$OPTARG"
 			help
-			exit 127
+			exit $EINVARG
 		;;
 	esac
 done
@@ -210,29 +202,37 @@ echo '
 	FROM	destination_files
 	WHERE	source_file_id is NULL;' >&3
 
-read -u4 removecount
+read -u4 -r -d $'\0' removecount
 until (( ${#removefile[@]} == removecount ))
 do
 	echo '
-		SELECT	id,
-			filename
+		SELECT	destination_files.id,
+			destinations.name,
+			destination_files.filename
 		FROM	destination_files
+		INNER JOIN	destinations
+			ON	destination_files.destination_id
+					= destinations.id
 		WHERE	source_file_id is NULL
 		LIMIT	500 OFFSET '${#removefile[@]}';
 
 		SELECT "AtOM:NoMoreFiles";
 	' >&3
 
-	read -u4 line
+	read -u4 -r -d $'\0' line
 	until [[ $line == AtOM:NoMoreFiles ]]
 	do
-		removefile[${line%::AtOM:SQL:Sep::*}]="${line#*::AtOM:SQL:Sep::}"
-		read -u4 line
+		removeFileId=${line%%::AtOM:SQL:Sep::*}
+		rest=${line#*::AtOM:SQL:Sep::}
+		removeFileDestName=${line%%::AtOM:SQL:Sep::*}
+		rest=${line#*::AtOM:SQL:Sep::}
+		removefile[$removeFileId]="${destinationpath["$removeFileDestName"]}/${rest%%::AtOM:SQL:Sep::*}"
+		read -u4 -r -d $'\0' line
 	done
 done
 
-deleted=0
-removed=0
+unset deleted
+unset removed
 echo 'BEGIN TRANSACTION;' >&3
 for id in ${!removefile[@]}
 do
@@ -256,9 +256,9 @@ do
 done
 echo 'COMMIT;' >&3
 (( cron )) || echo -n $'\r'
-echo -n "Suppressed $deleted files, $removed removed from database"
+echo -n "${deleted+$deleted files deleted${removed:+, }}${removed:+$removed removed from database}"
 (( cron )) || echo -ne "\033[K"
-echo
+(( deleted || removed )) && echo
 unset removecount deleted removed removefile
 
 updateTags
@@ -272,7 +272,8 @@ do
 		Update destination_files last_change 0	\
 			<<<"destination_id = $forcedestid"
 	else
-		echo "Destination $forcedest does not exist!" >&2
+		echo "Full rebuild of destination $forcedest was requested,"  \
+			"but it does not exist!" >&2
 	fi
 done
 
@@ -288,6 +289,7 @@ echo '
 		ascii		INTEGER,
 		source_file	INTEGER,
 		fileid		INTEGER,
+		destdir		TEXT,
 		filename	TEXT,
 		cmd_arg0	TEXT,
 		cmd_arg1	TEXT,
@@ -385,7 +387,7 @@ echo '
 			<> CAST(source_files.last_change AS TEXT)
 	AND	mime_type_actions.destination_id = destinations.id
 	AND	mime_type_actions.action = 1;' >&3
-read -u4 filecount
+read -u4 -r -d $'\0' filecount
 if [ -n "$maxbatch" ] && (( maxbatch < filecount ))
 then
 	(( togo = filecount - maxbatch ))
@@ -434,13 +436,13 @@ echo '
 (( maxbatch )) && echo "LIMIT $maxbatch" >&3
 echo ';
 	SELECT "AtOM:NoMoreFiles";' >&3
-read -u4 line
+read -u4 -r -d $'\0' line
 while ! [[ $line = AtOM:NoMoreFiles ]]
 do
 	decodefiles+=("$line::AtOM:SQL:Sep::")
-	read -u4 line
+	read -u4 -r -d $'\0' line
 done
-(( cron )) || echo -n 'Creating tasks...  '
+(( cron )) || echo -n $'Creating tasks...\033[K'
 
 (( textunidecodeneeded )) && ascii
 
@@ -546,8 +548,11 @@ do
 		tmpfile
 done
 echo 'COMMIT;' >&3
-(( cron )) || echo -n $'\r'
-echo "Created ${count:-0} tasks for $filecount files ${togo:+($togo left) }(${copies:-0} immediate copies)"
+(( cron )) || echo -n $'\r\033[K'
+(( count ))						\
+&& echo "Created $count tasks for $filecount files	\
+	${togo:+($togo left) }				\
+	${copies:+($copies immediate copies)}"
 
 # remove perl unicode to ascii coprocess
 (( textunidecodeneeded )) && eval exec "${toascii[1]}>&-"
@@ -562,7 +567,7 @@ remaining=$taskcount
 failed=0
 echo 'BEGIN TRANSACTION;' >&3
 committime=$EPOCHSECONDS
-while (( (remaining || ${#workers[@]}) && ! quit ))
+while (( remaining || ${#workers[@]} ))
 do
 	timestamp=$EPOCHSECONDS
 	if (( $timestamp - committime >= 60 ))
@@ -576,16 +581,15 @@ do
 	then
 		concurrency="$fixed_workers"
 	else
-		if [ -z "$quit" ]					\
-		&& (( ! pause ))					\
-		&& (( timestamp - concurrencychange >= loadinterval ))
+		if (( timestamp - concurrencychange >= loadinterval ))
 		then
-			if (( concurrency > 1 ))	\
-			&& (( load > maxload ))
+			if (( concurrency > 1 || allow_zero_running ))	\
+			&& (( load > maxload && concurrency ))
 			then
 				concurrencychange=$timestamp
 				(( --concurrency ))
-			elif (( load < maxload )) && (( active > concurrency - 1 ))
+			elif (( load < maxload ))	\
+			&& (( active > concurrency - 1 ))
 			then
 				concurrencychange=$timestamp
 				(( ++concurrency ))
@@ -594,16 +598,11 @@ do
 	fi
 	checkworkers
 	cleaner
-	(( pause )) || master
+	master
 	if (( ran - failed ))
 	then
 		currenttime=$timestamp
-		if (( pause ))
-		then
-			(( runtime = pausestart - starttime - pausedtime ))
-		else
-			(( runtime = currenttime - starttime - pausedtime ))
-		fi
+		(( runtime = currenttime - starttime ))
 		avgduration=$((
 			( runtime * 1000)
 			/
@@ -639,11 +638,7 @@ do
 	fmtprogress="T:%${#taskcount}i/%i (F:%i) %3i%%"
 	fmttime='%2id %2ih%02im%02is (A:%4.1fs/task)'
 	eta="ETA:$(
-		date -d	"${days:-0} days
-			${hours:-0} hours
-			${minutes:-0} minutes
-			${seconds:-0} seconds"	\
-			+'%d/%m %H:%M:%S'
+		printf "%(%c)T" "$(( currenttime + secsremaining ))"
 	)"
 	(( cron )) || printf						\
 		"\r$fmtload $fmtworkers $fmtprogress $fmttime $eta\033[K"\
@@ -660,7 +655,7 @@ do
 		${minutes:-0}						\
 		${seconds:-0}   					\
 		${avgdsec:-0}.${avgdmsec:-0}
-	if (( pause ))
+	if ! (( concurrency )) && ! (( cron ))
 	then
 		if (( active ))
 		then
@@ -675,22 +670,25 @@ unset count
 
 endtime=$EPOCHSECONDS
 
-(( elapsedseconds = endtime - starttime - pausedtime ))
+(( elapsedseconds = endtime - starttime ))
 (( days =
 	elapsedseconds
 	/
 	( 24*60*60 )
 )) || true
+(( days )) || unset days
 (( hours =
 	( elapsedseconds - ( days*24*60*60 ) )
 	/
 	( 60*60 )
 )) || true
+(( days && hours )) || unset hours
 (( minutes =
 	( elapsedseconds - ( ( days*24 + hours ) *60*60 ) )
 	/
 	60
 )) || true
+(( days && hours && minutes )) || unset minutes
 (( seconds =
 	elapsedseconds
 	-
@@ -698,10 +696,15 @@ endtime=$EPOCHSECONDS
 )) || true
 
 (( cron )) || echo -n $'\r'
-echo -n	"Ran ${ran:=0} tasks, $failed of which failed, in $days"	\
-	"days, $hours hours, $minutes minutes and $seconds seconds."
+(( ran ))	\
+&& echo -n	"Ran $ran tasks${failed:+, $failed of which failed,}	\
+		in ${days:+$days days,}					\
+		${hours:+$hours hours,}					\
+		${minutes:+$minutes minutes and}			\
+		$seconds seconds."
 (( cron )) || echo -en "\033[K"
-echo
+(( ran )) && echo
+
 if (( failed ))
 then
 	echo $'\nFailed tasks:\n'
@@ -773,28 +776,21 @@ then
 		WHERE		tasks.status = 2;
 
 		SELECT "AtOM:NoMoreFiles";' >&3
-	read -u4 line
+	read -u4 -r -d $'\0' line
 	while ! [[ $line = AtOM:NoMoreFiles ]]
 	do
 		failedtasks+=("$line")
-		read -u4 line
+		read -u4 -r -d $'\0' line
 	done
 	for line in "${failedtasks[@]}"
 	do
 		echo "${line%%::AtOM:SQL:Sep::*}"
 		line="${line#*::AtOM:SQL:Sep::}"
 		line="${line//::AtOM:SQL:Sep::/ }"
-		echo $'\t'"${line/+( )$/}"
-		echo
+		echo $'\t'"${line/+( )$/}"$'\n'
 	done
 fi
 
-if [ -n "$quit" ]
-then
-	closeDatabase
-	exit
-fi
-
 for destination in "${!destinationpath[@]}"
 do
 	echo '
@@ -840,11 +836,11 @@ do
 	SELECT	"AtOM:NoMoreFiles";
 ' >&3
 
-	read -u4 line
+	read -u4 -r -d $'\0' line
 	while [[ $line != AtOM:NoMoreFiles ]]
 	do
 		renamefiles+=("$line")
-		read -u4 line
+		read -u4 -r -d $'\0' line
 	done
 	if (( ${#renamefiles[@]} ))
 	then
@@ -894,7 +890,9 @@ do
 				progressSpin
 				if [[ "$oldfilename" != "$destfilename" ]]
 				then
-					mv "$oldfilename" "$destfilename"
+					mv				\
+					"${destinationpath[$destination]}/$oldfilename"	\
+					"${destinationpath[$destination]}/$destfilename"
 					(( changedcount++ ))
 					commit=1
 				fi
@@ -919,9 +917,10 @@ do
 		(( textunidecodeneeded )) && eval exec "${toascii[1]}>&-"
 		echo 'COMMIT;' >&3
 		(( cron )) || echo -n $'\r'
-		echo -n "$destination: Renamed ${changedcount:-0} files"
+		(( changedcount ))					\
+		&& echo -n "$destination: Renamed $changedcount files"
 		(( cron )) || echo -en "\033[K"
-		echo
+		(( changedcount )) && echo
 	fi
 	unset count changedcount renamefiles
 done
@@ -929,22 +928,26 @@ done
 copyFiles_action
 
 echo '
-	SELECT	id,
-		filename,
-		old_filename
+	SELECT	destination_files.id,
+		destination_files.filename,
+		destination_files.old_filename,
+		destinations.name
 	FROM	destination_files
+	INNER JOIN	destinations
+		ON	destination_files.destination_id
+				= destinations.id
 	WHERE	old_filename IS NOT NULL;
 
 	SELECT	"AtOM:NoMoreFiles";
 ' >&3
 
-(( cron )) || echo -n 'Removing obsolete files...     '
+(( cron )) || echo -n 'Removing obsolete files...'$'\033[K'
 lines=()
-read -u4 line
+read -u4 -r -d $'\0' line
 while [[ $line != AtOM:NoMoreFiles ]]
 do
 	lines+=("$line")
-	read -u4 line
+	read -u4 -r -d $'\0' line
 done
 echo 'BEGIN TRANSACTION;' >&3
 for line in "${lines[@]}"
@@ -952,10 +955,12 @@ do
 	id=${line%%::AtOM:SQL:Sep::*}
 	rest=${line#*::AtOM:SQL:Sep::}
 	filename=${rest%%::AtOM:SQL:Sep::*}
-	oldfilename=${rest#*::AtOM:SQL:Sep::}
+	rest=${line#*::AtOM:SQL:Sep::}
+	oldfilename=${rest%%::AtOM:SQL:Sep::*}
+	destination=${rest#*::AtOM:SQL:Sep::}
 	if [[ $oldfilename != "$filename" ]] && [ -f "$oldfilename" ]
 	then
-		rm -f "$oldfilename"
+		rm -f "${destinationpath[$destination]}/$oldfilename"
 	fi
 	Update destination_files old_filename NULL <<<"id = $id"
 	(( count++ ))
@@ -963,11 +968,12 @@ do
 done
 echo 'COMMIT;' >&3
 (( cron )) || echo -n $'\r'
-echo -n "Removed ${count:-0} obsolete files."
+(( count ))					\
+&& echo -n "Removed $count obsolete files."
 (( cron )) || echo -en "\033[K"
-echo
+(( count )) && echo
 
-echo "Purging empty directories."
+(( debug )) && echo "Purging empty directories..."
 for path in "${destinationpath[@]}"
 do
 	find "$path" -type d -empty -delete

doc/config

@@ -39,6 +39,11 @@ Sections:
   * debug <level>: Integer. Currently defined values:
       * 1: few additional status informations.
       * 3: log SQL queries.
+  * skip-timestamp-microsec <bool>: Ignore microsecond precision in timestamps.
+      Microsec precise timestamps are still stored as-is in the DB, this
+      setting just impacts comparisons when determining if a file has been
+      changed.
+
 
 [source]
   This section defines where are the files you want transcoded.

doc/example.cfg

@@ -34,6 +34,9 @@ database		/home/user/.local/share/AtOM/atom.db
 #    3: log SQL queries to <temporary-directory>/debug.log
 #debug			1
 
+# * skip-timestamp-microsec: Ignore microsecond precision in timestamps.
+skip-timestamp-microsec	0
+
 
 [source]
 # This section defines where are the files you want transcoded.

lib/config/getDestination

@@ -5,7 +5,7 @@ getConfigDestination() {
 			destinationenabled["$destination"]="$value"
 		;;
 		'path')
-			destinationpath["$destination"]="$value"
+			destinationpath["$destination"]="${value%/}"
 		;;
 		'format')
 			case "$value" in

lib/config/getGeneral

@@ -75,6 +75,9 @@ getConfigGeneral() {
 		'database')
 			database="$value"
 		;;
+		'skip-timestamp-microsec')
+			skip_us_timestamp="$value"
+		;;
 		debug)
 			(( value > debug )) && debug=$value
 		;;

lib/config/write

@@ -33,6 +33,9 @@ database		$database
 # * debug <level>: Integer.
 #debug			1
 
+# * skip-timestamp-microsec: Ignore microsecond precision in timestamps.
+skip-timestamp-microsec	${skip_us_timestamp:-0}
+
 
 [source]
 # This section defines where are the files you want transcoded.

lib/copy/action

@@ -1,6 +1,6 @@
 #!/bin/bash
 copyFiles_action() {
-	(( cron )) || echo -n "Copying files...     "
+	(( cron )) || echo -n $'Copying files...\033[K'
 	echo '
 		SELECT
 			source_files.filename,
@@ -22,11 +22,11 @@ copyFiles_action() {
 		AND	mime_type_actions.action = 2;
 
 		SELECT "AtOM:NoMoreFiles";' >&3
-	read -u4 line
+	read -u4 -r -d $'\0' line
 	while ! [[ $line = AtOM:NoMoreFiles ]]
 	do
 		copyfiles+=("$line")
-		read -u4 line
+		read -u4 -r -d $'\0' line
 	done
 
 	echo 'BEGIN TRANSACTION;' >&3
@@ -79,15 +79,29 @@ copyFiles_action() {
 				fi
 			fi
 		fi
-		if cp -al "$sourcepath/$sourcefilename" "$destdir" 2>/dev/null\
-		|| cp -a "$sourcepath/$sourcefilename" "$destdir"
+		if cp -a --reflink=always		\
+			"$sourcepath/$sourcefilename"	\
+			"$destdir"			\
+			2>/dev/null			\
+		|| cp -al				\
+			"$sourcepath/$sourcefilename"	\
+			"$destdir"			\
+			2>/dev/null			\
+		|| cp -a				\
+			"$sourcepath/$sourcefilename"	\
+			"$destdir"
 		then
 			Update destination_files			\
-				filename "$destdir/${sourcefilename##*/}"\
-				rename_pattern "${destinationrenamepath[$destination]}/${destinationrename[$destination]}"\
-				fat32compat	${destinationfat32compat["$destination"]}\
-				ascii		${destinationascii["$destination"]}\
-				last_change $lastchange			\
+				filename				\
+					"$destdir/${sourcefilename##*/}"\
+				rename_pattern 				\
+					"${destinationrenamepath[$destination]}/${destinationrename[$destination]}"\
+				fat32compat				\
+					${destinationfat32compat["$destination"]}\
+				ascii					\
+					${destinationascii["$destination"]}\
+				last_change				\
+					$lastchange			\
 			 <<-EOWhere
 				id = $destfileid
 			EOWhere
@@ -103,7 +117,7 @@ copyFiles_action() {
 		(( cron )) || echo -en "\033[K"
 		echo
 	else
-		(( cron )) || echo -e "\rNothing to copy.\033[K"
+		(( cron )) || echo -n $'\r\033[K'
 	fi
 	unset count done
 }

lib/copy/guessPath

@@ -21,7 +21,7 @@ guessPath() {
 		LIMIT 1
 	),"0.0");
 	'>&3
-	read -u4 timestamp
+	read -u4 -r -d $'\0' timestamp
 	if (( ${timestamp/./} == 0 ))
 	then
 		return 2
@@ -46,7 +46,7 @@ guessPath() {
 		LIMIT 1
 	),"AtOM:NotFound");
 	'>&3
-	read -u4 filename
+	read -u4 -r -d $'\0' filename
 	if [[ $filename != AtOM:NotFound ]]
 	then
 		echo "${filename%/*}"

lib/database/Insert

@@ -36,7 +36,7 @@ Insert() {
 				"( $insert_values );" >&3
 	(( no_id )) || {
 		echo 'SELECT LAST_INSERT_ROWID();' >&3
-		read -u 4 results
+		read -u4 -r -d $'\0' results
 		echo "$results"
 	}
 }

lib/database/Select

@@ -26,7 +26,7 @@ Select() {
 				"WHERE ${where_statement[@]})"	\
 			",'SQL::Select:not found'"		\
 		");" >&3
-	read -u 4 results
+	read -u 4 -r -d $'\0' results
 	if ! [[ $results == "SQL::Select:not found" ]]
 	then
 		echo "$results"

lib/database/checkVersion

@@ -1,5 +1,5 @@
 #!/usr/bin/env bash
-currentdbversion=6
+currentdbversion=7
 checkDatabaseVersion() {
 	local dbversion
 	if dbversion=$(Select atom version <<<"\"1\" = 1")
@@ -18,7 +18,7 @@ checkDatabaseVersion() {
 			echo "Database schema version $dbversion is higher than
 				that of this version of AtOM
 				($currentdbversion). Bailing out." >&2
-			exit 1
+			exit $EDBVERSION
 		fi
 	else
 		Insert atom 1 <<<"version $currentdbversion"

lib/database/open

@@ -3,12 +3,16 @@ openDatabase() {
 	[[ -f "$database" ]] || populate_db=1
 	rm -f "$tempdir"/sqlite.{in,out}
 	mkfifo "$tempdir"/sqlite.{in,out}
-	sqlite3 -bail "$database"	\
+	stdbuf -o0 sqlite3 -bail 				\
+		-newline $'::AtOM:SQL:EOL::\n'			\
+		"$database"					\
 		< "$tempdir/sqlite.in"				\
+		| stdbuf -o0					\
+		sed 's/::AtOM:SQL:EOL::/\x0/g;s/\(\x0\)\xA/\1/g'	\
 		> "$tempdir/sqlite.out"	&
 	exec 3> "$tempdir"/sqlite.in
 	exec 4< "$tempdir"/sqlite.out
-	rm "$tempdir"/sqlite.in "$tempdir"/sqlite.out
+	rm "$tempdir"/sqlite.{in,out}
 	if (( debug > 2 ))
 	then
 		exec 5>&3
@@ -20,7 +24,7 @@ openDatabase() {
 	echo 'PRAGMA recursive_triggers = ON;' >&3
 	echo 'PRAGMA temp_store = 2;' >&3
 	echo 'PRAGMA locking_mode = EXCLUSIVE;' >&3
-	read -u4
+	read -u4 -r -d $'\0'
 	unset REPLY
 	checkDatabaseVersion
 }

lib/database/upgradedatabase_1_2

@@ -18,11 +18,11 @@ FROM		destination_files;
 
 SELECT	"AtOM::NoMoreData";' >&3
 
-	read -u4 data
+	read -u4 -r -d $'\0' data
 	while [[ $data != AtOM::NoMoreData ]]
 	do
 		datas+=( "$data" )
-		read -u4 data
+		read -u4 -r -d $'\0' data
 	done
 	echo 'BEGIN TRANSACTION;' >&3
 	for data in "${datas[@]}"

lib/database/upgradedatabase_6_7

@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+
+upgradedatabase_6_7() {
+	echo "Upgrading database to version 7... (backup is $database.bak_v6)"
+        cp "$database" "$database.bak_v6"
+        for destination in "${destinations[@]}"
+        do
+                echo "UPDATE destination_files SET filename = REPLACE(filename,'${destinationpath[$destination]}/','') WHERE filename LIKE '${destinationpath[$destination]}/%';" >&3
+        done
+        Update atom version 7 <<<"1 = 1"
+}

lib/encode/mp3

@@ -53,7 +53,7 @@ encodeFile::mp3() {
 			esac
 		fi
 	fi
-	lameopts+=("$tempdir/$tmpfile.wav" "$destdir/$destfile.mp3")
+	lameopts+=("$tempdir/$tmpfile.wav" "${destinationpath[$destination]}/$destdir/$destfile.mp3")
 	encodetaskid=$(
 		Insert tasks <<-EOInsert
 			key		${fileid}lame$destination

lib/encode/opus

@@ -23,7 +23,7 @@ encodeFile::opus() {
 	[ -n "$track"	] && opusencopts+=(--comment "TRACKNUMBER=${track%/*}")
 	[ -n "${track#*/}" ] && opusencopts+=(--comment "TRACKTOTAL=${track#*/}")
 	[ -n "$year"	] && opusencopts+=(--comment "DATE=$year")
-	opusencopts+=("$tempdir/$tmpfile".wav "$destdir/$destfile.opus")
+	opusencopts+=("$tempdir/$tmpfile".wav "${destinationpath[$destination]}/$destdir/$destfile.opus")
 	encodetaskid=$(
 		Insert tasks <<-EOInsert
 			key		${fileid}opusenc$destination

lib/encode/vorbis

@@ -17,7 +17,7 @@ encodeFile::vorbis() {
 	[ -n "$title"	] && oggencopts+=(-t "$title")
 	[ -n "$track"	] && oggencopts+=(-N "$track")
 	[ -n "$year"	] && oggencopts+=(-d "$year")
-	oggencopts+=(-o "$destdir/$destfile.ogg" "$tempdir/$tmpfile.wav")
+	oggencopts+=(-o "${destinationpath[$destination]}/$destdir/$destfile.ogg" "$tempdir/$tmpfile.wav")
 	encodetaskid=$(
 		Insert tasks <<-EOInsert
 			key		${fileid}oggenc$destination

lib/files/getDestDir

@@ -41,7 +41,7 @@ getDestDir() {
 		)
 	)
 	then
-		destdir="${destinationpath[$destination]}/"
+		destdir=""
 		if (( ${destinationascii["$destination"]} ))
 		then
 			echo "$album" >&${toascii[1]}
@@ -83,8 +83,7 @@ getDestDir() {
 		replace=$(sanitizeFile "$disc" dir)
 		destdir="${destdir//?(\[)%\{disc\}?(\])/$replace}"
 	else
-		destdir="${destinationpath[$destination]}/"
-		destdir+=$(sanitizeFile "${filename%%/*}" dir)
+		destdir=$(sanitizeFile "${filename%%/*}" dir)
 		part=${filename#*/}
 		while [[ $part =~ / ]]
 		do
@@ -101,7 +100,7 @@ getDestDir() {
 	fi
 	if ! [ -d "$destdir" ]
 	then
-		mkdir -p "$destdir"
+		mkdir -p "${destinationpath[$destination]}/$destdir"
 	fi
 	destdir="${destdir//+(\/)//}"
 }

lib/files/getFiles

@@ -8,12 +8,19 @@ getFiles() {
 	(( cron )) || echo -n "Scanning $sourcepath...  "
 	# We probably have thousands of files, don't waste time on disk writes
 	echo 'BEGIN TRANSACTION;' >&3
-	while read time size filename
+	while read -d $'\0' time size filename
 	do
+		if (( skip_us_timestamp ))
+		then
+			compare_time=${time%.*}.%
+		else
+			compare_time=$time
+		fi
 		if ! Select source_files id >/dev/null <<-EOWhere
 				filename = $filename
 				mime_type > 0
-				last_change = $time
+				last_change LIKE $compare_time
+				size = $size
 			EOWhere
 		then
 			mimetype=$(file -b --mime-type "$sourcepath/$filename")
@@ -56,10 +63,14 @@ getFiles() {
 		fi
 		progressSpin
 	done < <(
-		find "$sourcepath" "${prunes[@]}" -type f -not -name '.*' -printf "%T@ %s %P\n"
+		find "$sourcepath" "${prunes[@]}" -type f -not -name '.*' -printf "%T@ %s %P\0"
 	)
 	echo 'COMMIT;' >&3
 	(( cron )) || echo -n $'\r'
-	echo "${count:-0} files found, ${new:=0} new or changed."$'\033[K'
+	if (( count ))
+	then
+		echo "$count files found${new:+, $new new or changed}."	\
+			$'\033[K'
+	fi
 	unset count
 }

lib/tags/gettags

@@ -1,7 +1,7 @@
 #!/bin/bash
 getTags_version='unknown-4'
 getTags() {
-	unset type
+	local type
 	case "$mimetype" in
 		audio/mpeg)
 			type=ffmpeg

lib/tags/update

@@ -52,12 +52,12 @@ echo '
 ;
 
 		SELECT "AtOM:NoMoreFiles";' >&3
-	read -u4 line
+	read -u4 -r -d $'\0' line
 	while ! [[ $line = AtOM:NoMoreFiles ]]
 	do
 		tagfiles+=("$line")
 		(( filecount++ ))
-		read -u4 line
+		read -u4 -r -d $'\0' line
 	done
 	echo 'BEGIN TRANSACTION;' >&3
 	for line in "${tagfiles[@]}"
@@ -206,8 +206,8 @@ echo '
 	done
 	echo 'COMMIT;' >&3
 	(( cron )) || echo -n $'\r'
-	echo -n "Read tags from ${count:-0} files."
-	(( cron )) || echo -ne "\033[K"
-	echo
+	(( count )) && echo -n "Read tags from $count files."
+	(( cron )) || echo -n $'\033[K'
+	(( count )) && echo
 	unset count tagfiles
 }

lib/tasks/gettaskinfos

@@ -10,7 +10,7 @@ gettaskinfos() {
 		FROM	tasks
 		WHERE	id='$1';
 	' >&3
-	read -u4 line
+	read -u4 -r -d $'\0' line
 	taskid=${line%%::AtOM:SQL:Sep::*}
 	rest="${line#*::AtOM:SQL:Sep::}::AtOM:SQL:Sep::"
 	sourcefileid=${rest%%::AtOM:SQL:Sep::*}

lib/workers/create

@@ -2,7 +2,7 @@
 
 createworker() {
         (( ++active ))
-        read -u4 line
+        read -u4 -r -d $'\0' line
         taskid=${line%%::AtOM:SQL:Sep::*}
         rest="${line#*::AtOM:SQL:Sep::}::AtOM:SQL:Sep::"
         sourcefileid=${rest%%::AtOM:SQL:Sep::*}

lib/workers/master

@@ -10,7 +10,7 @@ master() {
 			WHERE	status = 0;
 		'>&3
 
-		read -u4 remaining
+		read -u4 -r -d $'\0' remaining
 		if (( remaining == 0 ))
 		then
 			sleep 0.1
@@ -108,7 +108,7 @@ master() {
 				LIMIT 1;
 			'>&3
 
-			read -u4 ready
+			read -u4 -r -d $'\0' ready
 			if (( ready > 0 ))
 			then
 				createworker
@@ -194,7 +194,7 @@ master() {
 				LIMIT 1;
 			' >&3
 
-			read -u4 ready
+			read -u4 -r -d $'\0' ready
 
 			if (( active == 0 && ready == 0 ))
 			then
@@ -220,7 +220,7 @@ master() {
 				echo "Waiting for children to come back home..."
 				wait
 				echo $'\nGood luck!'
-				exit 1
+				exit $ETASKLEFT
 			elif (( ready == 0 ))
 			then
 				sleep 0.1
@@ -234,7 +234,7 @@ master() {
 				WHERE	status = 0;
 			'>&3
 
-			read -u4 remaining
+			read -u4 -r -d $'\0' remaining
 		done
 	fi
 }

share/errorcodes

@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+
+## Define exit codes
+# General config errors [10-19]
+EDBVERSION=10
+ELOAD=11
+EINTERVAL=12
+ELOAD=13
+EIONICE=14
+ENOCFG=19
+# Source cofig errors [20-29]
+# Destination config errors [30-49]
+EFORMAT=30
+ECHANNEL=31
+EQUALITY=32
+EMAXBPS=33
+EINVDEST=34
+EFMTINVPARM=49
+# Tasks
+ETASKLEFT=50
+# Sanity
+ESANITY=60
+# Invalid arguments
+EINVARG=127

toys/checkextensions

@@ -1,5 +1,26 @@
 #!/usr/bin/env bash
 
+cat <<-EOWarn
+	This script is unmaintained and provided as-is. It may or may not work.
+
+	Use at your own risk!
+EOWarn
+
+read -p "Press Enter to continue or Ctrl-C to abort"
+
+declare -r				\
+	DOCDIR=%DOCDIR%			\
+	LIBDIR=%LIBDIR%			\
+	SHAREDIR=%SHAREDIR%
+declare -r				\
+	exampleconf=$DOCDIR/example.cfg	\
+	schema=$SHAREDIR/schema.sql	\
+					\
+	oldIFS="$IFS"
+
+## Define exit codes
+source "$SHAREDIR"/errorcodes
+
 # config structures
 declare -A			\
 	destinationchannels	\
@@ -22,16 +43,6 @@ declare -A			\
 	exit $EBASHVERS
 }
 
-declare -r				\
-	DOCDIR=%DOCDIR%			\
-	LIBDIR=%LIBDIR%			\
-	SHAREDIR=%SHAREDIR%
-declare -r				\
-	exampleconf=$DOCDIR/example.cfg	\
-	schema=$SHAREDIR/schema.sql	\
-					\
-	oldIFS="$IFS"
-
 LC_ALL=C
 
 shopt -s extglob
@@ -110,7 +121,7 @@ getdstfiles() {
 			;
 			SELECT	"AtOM:NoMoreFiles";
 		'>&3
-		while read -u4 line
+		while read -u4 -r -d $'\0' line
 		do
 			if [[ $line == AtOM:NoMoreFiles ]]
 			then
@@ -148,7 +159,7 @@ renameFile() {
 	fi
 }
 
-while read -u4 line
+while read -u4 -r -d $'\0' line
 do
 	if [[ $line == AtOM:NoMoreFiles ]]
 	then

toys/checkgenre

@@ -1,5 +1,18 @@
 #!/usr/bin/env bash
 
+declare -r				\
+	DOCDIR=%DOCDIR%			\
+	LIBDIR=%LIBDIR%			\
+	SHAREDIR=%SHAREDIR%
+declare -r				\
+	exampleconf=$DOCDIR/example.cfg	\
+	schema=$SHAREDIR/schema.sql	\
+					\
+	oldIFS="$IFS"
+
+## Define exit codes
+source "$SHAREDIR"/errorcodes
+
 # config structures
 declare -A			\
 	destinationchannels	\
@@ -22,16 +35,6 @@ declare -A			\
 	exit $EBASHVERS
 }
 
-declare -r				\
-	DOCDIR=%DOCDIR%			\
-	LIBDIR=%LIBDIR%			\
-	SHAREDIR=%SHAREDIR%
-declare -r				\
-	exampleconf=$DOCDIR/example.cfg	\
-	schema=$SHAREDIR/schema.sql	\
-					\
-	oldIFS="$IFS"
-
 LC_ALL=C
 
 shopt -s extglob

toys/checkmissing

@@ -1,5 +1,18 @@
 #!/usr/bin/env bash
 
+declare -r				\
+	DOCDIR=%DOCDIR%			\
+	LIBDIR=%LIBDIR%			\
+	SHAREDIR=%SHAREDIR%
+declare -r				\
+	exampleconf=$DOCDIR/example.cfg	\
+	schema=$SHAREDIR/schema.sql	\
+					\
+	oldIFS="$IFS"
+
+## Define exit codes
+source "$SHAREDIR"/errorcodes
+
 # config structures
 declare -A			\
 	destinationchannels	\
@@ -22,16 +35,6 @@ declare -A			\
 	exit $EBASHVERS
 }
 
-declare -r				\
-	DOCDIR=%DOCDIR%			\
-	LIBDIR=%LIBDIR%			\
-	SHAREDIR=%SHAREDIR%
-declare -r				\
-	exampleconf=$DOCDIR/example.cfg	\
-	schema=$SHAREDIR/schema.sql	\
-					\
-	oldIFS="$IFS"
-
 LC_ALL=C
 
 shopt -s extglob
@@ -62,27 +65,42 @@ getConfig
 sanityCheck
 openDatabase
 
-echo 'SELECT id,filename FROM destination_files WHERE filename IS NOT NULL;' >&3
+echo '
+	SELECT 
+		destination_files.id,
+		destinations.name,
+		destination_files.filename
+	FROM destinations
+	INNER JOIN destination_files 
+		ON destinations.id=destination_files.destination_id
+	WHERE filename IS NOT NULL;' >&3
 echo 'SELECT "AtOM:NoMoreFiles";' >&3
 
-read -u4 filename
-until [[ $filename == AtOM:NoMoreFiles ]]
+declare -a 			\
+	destination_names	\
+	files
+read -u4 -r -d $'\0' line
+until [[ $line == AtOM:NoMoreFiles ]]
 do
-	files+=("$filename")
-	read -u4 filename
+	id=${line%%::AtOM:SQL:Sep::*}
+	rest=${line#*::AtOM:SQL:Sep::}
+	destination_names[id]=${rest%%::AtOM:SQL:Sep::*}
+	rest=${rest#*::AtOM:SQL:Sep::}
+	files[id]=${rest}
+	read -u4 line
 done
 
 echo 'BEGIN TRANSACTION;' >&3
 
 echo -n "Checking for missing files...  "
-for filename in "${files[@]}"
+for index in "${!files[@]}"
 do
-	id=${filename%%::AtOM:SQL:Sep::*}
-	filename=${filename#*::AtOM:SQL:Sep::}
+	destination=${destination_names[index]}
+	filename="${destinationpath[$destination]}/${files[index]}"
 	if ! [ -f "$filename" ]
 	then
 		echo -e "\r$filename\033[K"
-		((regen))&&Update destination_files last_change 0 <<<"id = $id"
+		((regen))&&Update destination_files last_change 0 <<<"id = $index"
 		echo -n "Checking for missing files...  "
 		(( missing++ ))
 	fi
@@ -91,6 +109,6 @@ done
 
 echo 'COMMIT;' >&3
 
-echo -e "\r${missing:-0} missing files\033[K"
+echo -e "\r${missing:-No} missing files\033[K"
 
 closeDatabase

toys/cleandestinations

@@ -1,5 +1,18 @@
 #!/usr/bin/env bash
 
+declare -r				\
+	DOCDIR=%DOCDIR%			\
+	LIBDIR=%LIBDIR%			\
+	SHAREDIR=%SHAREDIR%
+declare -r				\
+	exampleconf=$DOCDIR/example.cfg	\
+	schema=$SHAREDIR/schema.sql	\
+					\
+	oldIFS="$IFS"
+
+## Define exit codes
+source "$SHAREDIR"/errorcodes
+
 # config structures
 declare -A			\
 	destinationchannels	\
@@ -22,16 +35,6 @@ declare -A			\
 	exit $EBASHVERS
 }
 
-declare -r				\
-	DOCDIR=%DOCDIR%			\
-	LIBDIR=%LIBDIR%			\
-	SHAREDIR=%SHAREDIR%
-declare -r				\
-	exampleconf=$DOCDIR/example.cfg	\
-	schema=$SHAREDIR/schema.sql	\
-					\
-	oldIFS="$IFS"
-
 LC_ALL=C
 
 shopt -s extglob
@@ -73,7 +76,7 @@ do
 	do
 		if ! Select destination_files id	\
 			>/dev/null			\
-			<<<"filename = $filename"
+			<<<"filename = ${filename#${destinationpath["$destination"]}/}"
 		then
 			echo -e $'\r'"$filename\033[K"
 			(( remove )) && rm -f "$filename"

toys/createindex

@@ -1,6 +1,17 @@
 #!/usr/bin/env bash
 
-#!/bin/bash
+declare -r				\
+	DOCDIR=%DOCDIR%			\
+	LIBDIR=%LIBDIR%			\
+	SHAREDIR=%SHAREDIR%
+declare -r				\
+	exampleconf=$DOCDIR/example.cfg	\
+	schema=$SHAREDIR/schema.sql	\
+					\
+	oldIFS="$IFS"
+
+## Define exit codes
+source "$SHAREDIR"/errorcodes
 
 # config structures
 declare -A			\
@@ -24,16 +35,6 @@ declare -A			\
 	exit $EBASHVERS
 }
 
-declare -r				\
-	DOCDIR=%DOCDIR%			\
-	LIBDIR=%LIBDIR%			\
-	SHAREDIR=%SHAREDIR%
-declare -r				\
-	exampleconf=$DOCDIR/example.cfg	\
-	schema=$SHAREDIR/schema.sql	\
-					\
-	oldIFS="$IFS"
-
 LC_ALL=C
 
 shopt -s extglob
@@ -273,7 +274,7 @@ fi
 echo 'SELECT IFNULL(
 	(SELECT last_seen FROM source_files ORDER BY last_seen DESC LIMIT 1),
 	0);' >&3
-read -u4 lastupdate
+read -u4 -r -d $'\0' lastupdate
 
 if ! [[ "$output" == - ]]
 then
@@ -368,11 +369,11 @@ COLLATE NOCASE;
 
 SELECT	"AtOM:NoMoreFiles";' >&3
 
-read -u4 line
+read -u4 -r -d $'\0' line
 until [[ $line == AtOM:NoMoreFiles ]]
 do
 	files+=("$line")
-	read -u4 line
+	read -u4 -r -d $'\0' line
 done
 
 for line in "${files[@]}"
@@ -452,7 +453,10 @@ do
 		expr2='(,|$)'
 		if ! [[ $channelss =~ $expr1"$channels"$expr2 ]]
 		then
-			if [ -n "$channelss" ]			\
+		 	if [[ -n "$channels" ]]
+			then
+				:
+			elif [ -n "$channelss" ]		\
 			&& (( channels < ${channelss%%,*} ))
 			then
 				channelss="$channels,$channelss"
@@ -462,7 +466,10 @@ do
 		fi
 		if ! [[ $rates =~ $expr1"$rate"$expr2 ]]
 		then
-			if [ -n "$rates" ]			\
+			if [[ -n "$rate" ]]
+			then
+				:
+			elif [ -n "$rates" ]			\
 			&& (( rate < ${rates%%,*} ))
 			then
 				rates="$rate,$rates"
@@ -472,7 +479,10 @@ do
 		fi
 		if [ -n "$depth" ] && ! [[ $depths =~ $expr1"$depth"$expr2 ]]
 		then
-			if [ -n "$depths" ]			\
+		 	if [[ -n "$depth" ]]
+			then
+				:
+			elif [ -n "$depths" ]			\
 			&& (( depth < ${depths%%,*} ))
 			then
 				depths="$depth,$depths"
@@ -665,7 +675,7 @@ echo '
 	
 SELECT	"AtOM:NoMoreFiles";' >&3
 
-read -u4 line
+read -u4 -r -d $'\0' line
 until [[ $line == AtOM:NoMoreFiles ]]
 do
 	artist="${line%%::AtOM:SQL:Sep::*}"
@@ -675,7 +685,7 @@ do
 	artists+=( "$artist" )
 	maxcountlen=$(( ${#count} > maxcountlen ? ${#count} : maxcountlen ))
 	maxartistlen=$(( ${#artist} > maxartistlen ? ${#artist} : maxartistlen ))
-	read -u4 line
+	read -u4 -r -d $'\0' line
 done
 head=$(
 	printf "|  # | %'${maxcoutlen}s | %-${maxartistlen}s |"	\
@@ -704,7 +714,7 @@ echo '
 	FROM		source_files
 	INNER JOIN	mime_types
 		ON	source_files.mime_type=mime_types.id;' >&3
-read -u4 line
+read -u4 -r -d $'\0' line
 totalcount="${line%%::AtOM:SQL:Sep::*}"
 maxcountlen=$(printf "%'i" $totalcount)
 maxcountlen=${#maxcountlen}
@@ -733,7 +743,7 @@ do
 		INNER JOIN	mime_types
 			ON	source_files.mime_type=mime_types.id
 		WHERE		mime_text LIKE "'"$format"'";' >&3
-	read -u4 line
+	read -u4 -r -d $'\0' line
 	count="${line%%::AtOM:SQL:Sep::*}"
 	rest="${line#*::AtOM:SQL:Sep::}::AtOM:SQL:Sep::"
 	size="${rest%%::AtOM:SQL:Sep::*}"

toys/lowquality

@@ -1,5 +1,18 @@
 #!/usr/bin/env bash
 
+declare -r				\
+	DOCDIR=%DOCDIR%			\
+	LIBDIR=%LIBDIR%			\
+	SHAREDIR=%SHAREDIR%
+declare -r				\
+	exampleconf=$DOCDIR/example.cfg	\
+	schema=$SHAREDIR/schema.sql	\
+					\
+	oldIFS="$IFS"
+
+## Define exit codes
+source "$SHAREDIR"/errorcodes
+
 # config structures
 declare -A			\
 	destinationchannels	\
@@ -22,16 +35,6 @@ declare -A			\
 	exit $EBASHVERS
 }
 
-declare -r				\
-	DOCDIR=%DOCDIR%			\
-	LIBDIR=%LIBDIR%			\
-	SHAREDIR=%SHAREDIR%
-declare -r				\
-	exampleconf=$DOCDIR/example.cfg	\
-	schema=$SHAREDIR/schema.sql	\
-					\
-	oldIFS="$IFS"
-
 LC_ALL=C
 
 shopt -s extglob
@@ -128,11 +131,11 @@ echo ') ORDER BY bitrate;' >&3
 
 echo 'SELECT "AtOM:NoMoreFiles";' >&3
 
-read -u4 line
+read -u4 -r -d $'\0' line
 until [[ $line == AtOM:NoMoreFiles ]]
 do
 	echo "${line//::AtOM:SQL:Sep::/$'\t'}"
-	read -u4 line
+	read -u4 -r -d $'\0' line
 done
 
 closeDatabase

toys/missingtags

@@ -1,5 +1,18 @@
 #!/usr/bin/env bash
 
+declare -r				\
+	DOCDIR=%DOCDIR%			\
+	LIBDIR=%LIBDIR%			\
+	SHAREDIR=%SHAREDIR%
+declare -r				\
+	exampleconf=$DOCDIR/example.cfg	\
+	schema=$SHAREDIR/schema.sql	\
+					\
+	oldIFS="$IFS"
+
+## Define exit codes
+source "$SHAREDIR"/errorcodes
+
 # config structures
 declare -A			\
 	destinationchannels	\
@@ -23,16 +36,6 @@ declare -A			\
 	exit $EBASHVERS
 }
 
-declare -r				\
-	DOCDIR=%DOCDIR%			\
-	LIBDIR=%LIBDIR%			\
-	SHAREDIR=%SHAREDIR%
-declare -r				\
-	exampleconf=$DOCDIR/example.cfg	\
-	schema=$SHAREDIR/schema.sql	\
-					\
-	oldIFS="$IFS"
-
 LC_ALL=C
 
 shopt -s extglob
@@ -146,11 +149,11 @@ cat >&3 <<-EOSelect
 	SELECT	"AtOM:NoMoreFiles";
 EOSelect
 
-read -u4 line
+read -u4 -r -d $'\0' line
 until [[ $line == AtOM:NoMoreFiles ]]
 do
 	lines+=( "$line" )
-	read -u4 line
+	read -u4 -r -d $'\0' line
 done
 
 for line in "${lines[@]}"