From 2b87c4159a016af5a9f384c50cfeac9723bdbf4e Mon Sep 17 00:00:00 2001 From: Vincent Riquer Date: Tue, 30 Apr 2013 00:38:57 +0200 Subject: [PATCH 01/35] use ffmpeg for MP3, MPC, and unknown file-type tags --- lib/tags/getInfos::APE | 55 ------------------------------------- lib/tags/getInfos::MP3 | 27 ------------------ lib/tags/getRateChannelMPC | 16 ----------- lib/tags/getRateChannelSoxi | 5 ---- lib/tags/getTags | 19 ++----------- 5 files changed, 3 insertions(+), 119 deletions(-) delete mode 100644 lib/tags/getInfos::APE delete mode 100644 lib/tags/getInfos::MP3 delete mode 100644 lib/tags/getRateChannelMPC delete mode 100644 lib/tags/getRateChannelSoxi diff --git a/lib/tags/getInfos::APE b/lib/tags/getInfos::APE deleted file mode 100644 index 66f2a29..0000000 --- a/lib/tags/getInfos::APE +++ /dev/null @@ -1,55 +0,0 @@ -#!/bin/bash -getInfosAPE_version='APE-1' -tagreaders+=( "$getInfosAPE_version" ) -getInfos::APE() { - # I was not able to find a decent cli tool to read APE tags. - # This is raw but works for the very few MusePack files I got. - # - # Please tell me if you know of any good tool. - tagreader="$getInfosAPE_version" - IFS='=' - while read tag value - do - IFS="$oldIFS" - case $tag in - [Aa][Ll][Bb][Uu][Mm]' '[Aa][Rr][Tt][Ii][Ss][Tt]) - albumartist="$value" - ;; - [Aa][Rr][Tt][Ii][Ss][Tt]) - artist="$value" - ;; - [Yy][Ee][Aa][Rr]) - year="$value" - ;; - [Aa][Ll][Bb][Uu][Mm]) - album="$value" - ;; - [Tt][Ii][Tt][Ll][Ee]) - title="$value" - ;; - [Tt][Rr][Aa][Cc][Kk]) - tracknum="$value" - ;; - [Gg][Ee][Nn][Rr][Ee]) - genre="$value" - ;; - [Cc][Oo][Mm][Pp][Oo][Ss][Ee][Rr]) - composer="$value" - ;; - [Pp][Ee][Rr][Ff][Oo][Rr][Mm][Ee][Rr]) - performer="$value" - ;; - *) - ;; - esac - IFS='=' - done < <( - IFS="$oldIFS" - sed \ - 's/APETAGEX/\n/;s/[\x00\-\x1F]\x00\+/\n/g;s/\x00/=/g' \ - "$sourcepath/$filename" \ - | egrep -i \ - '^(Album Artist|Artist|Year|Album|Title|Track|Genre|Composer|Performer)=' - ) - IFS="$oldIFS" -} diff --git a/lib/tags/getInfos::MP3 b/lib/tags/getInfos::MP3 deleted file mode 100644 index 87c4d4f..0000000 --- a/lib/tags/getInfos::MP3 +++ /dev/null @@ -1,27 +0,0 @@ -#!/bin/bash -getInfosMP3_version='ID3-3' -tagreaders+=( "$getInfosMP3_version" ) -getInfos::MP3() { - tagreader="$getInfosMP3_version" - infos=$( - soxi "$sourcepath/$filename" 2>/dev/null \ - | sed 's/ *: /=/' - ) - album=$(gettag album) - artist=$(gettag artist) - genre=$(gettag genre) - title=$(gettag title) - tracknum=$(gettag tracknumber) - year=$(gettag year) - expr='^[0-9]*$' - if [[ $genre =~ $expr ]] - then - genre="${id3genres[$genre]}" - fi - infos="${infos/: /=}" - channels=$(gettag channels) - rate=$(gettag 'sample rate') - bitrate=$(gettag 'bit rate') - bitrate=${bitrate%%.*} - bitrate=${bitrate%k} -} diff --git a/lib/tags/getRateChannelMPC b/lib/tags/getRateChannelMPC deleted file mode 100644 index 14b28ba..0000000 --- a/lib/tags/getRateChannelMPC +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/bash -getRateChannelMPC() { - while read key value garbage - do - case $key in - 'samplerate:') - rate=$value - ;; - 'channels:') - channels=$value - ;; - esac - done < <( - mpcdec "$sourcepath/$filename" -i 2>&1 - ) -} diff --git a/lib/tags/getRateChannelSoxi b/lib/tags/getRateChannelSoxi deleted file mode 100644 index 91db5fe..0000000 --- a/lib/tags/getRateChannelSoxi +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash -getRateChannelSoxi() { - rate=$(soxi -r "$sourcepath/$filename" 2>/dev/null) - channels=$(soxi -c "$sourcepath/$filename" 2>/dev/null) -} diff --git a/lib/tags/getTags b/lib/tags/getTags index 0774d68..3f6d9e1 100644 --- a/lib/tags/getTags +++ b/lib/tags/getTags @@ -1,11 +1,11 @@ #!/bin/bash -getTags_version='unknown-3' +getTags_version='unknown-4' tagreaders+=( "$getTags_version" ) getTags() { unset type case "$mimetype" in audio/mpeg) - type=MP3 + type=ffmpeg ;; 'application/ogg opus') type=Opus @@ -20,20 +20,7 @@ getTags() { type=ffmpeg ;; *) - extendedtype=$(file -b "$sourcepath/$filename") - case "$extendedtype" in - *' ID3 '*) - type=MP3 - ;; - *'Musepack '*) - getRateChannelMPC - tryAPE - ;; - *) - getRateChannelSoxi - tryAPE - ;; - esac + type=ffmpeg ;; esac if [ -n "$type" ] From 45685f8a7cd5b294cf7941d904337ebdc87eda35 Mon Sep 17 00:00:00 2001 From: Vincent Riquer Date: Tue, 30 Apr 2013 00:52:18 +0200 Subject: [PATCH 02/35] Force update of destinations after a tag update Also force an update of the DB schema on each run. --- lib/database/openDatabase | 2 +- share/schema.sql | 34 +++++++++++++++++++++------------- 2 files changed, 22 insertions(+), 14 deletions(-) diff --git a/lib/database/openDatabase b/lib/database/openDatabase index 74f4745..df05d84 100644 --- a/lib/database/openDatabase +++ b/lib/database/openDatabase @@ -12,7 +12,6 @@ openDatabase() { then mkdir -p "${database%/*}" fi - sqlite3 "$database" < $schema fi sqlite3 -bail "$database" \ < "$tempdir/sqlite.in" \ @@ -24,5 +23,6 @@ openDatabase() { exec 5>&3 exec 3> >(tee -a $tempdir/debug.log >&5) fi + cat $schema >&3 echo 'PRAGMA foreign_keys = ON;' >&3 } diff --git a/share/schema.sql b/share/schema.sql index 9e82275..2a915e4 100644 --- a/share/schema.sql +++ b/share/schema.sql @@ -1,5 +1,5 @@ BEGIN TRANSACTION; -CREATE TABLE source_files ( +CREATE TABLE IF NOT EXISTS source_files ( id INTEGER PRIMARY KEY, filename TEXT UNIQUE NOT NULL, size INTEGER NOT NULL, @@ -10,11 +10,11 @@ CREATE TABLE source_files ( FOREIGN KEY (mime_type) REFERENCES mime_types(id) ON DELETE SET NULL ); -CREATE TABLE destinations ( +CREATE TABLE IF NOT EXISTS destinations ( id INTEGER PRIMARY KEY, name TEXT UNIQUE NOT NULL ); -CREATE TABLE destination_files ( +CREATE TABLE IF NOT EXISTS destination_files ( id INTEGER PRIMARY KEY, filename TEXT, old_filename TEXT, @@ -27,11 +27,11 @@ CREATE TABLE destination_files ( FOREIGN KEY (destination_id) REFERENCES destinations(id) ON DELETE CASCADE ); -CREATE TABLE mime_types ( +CREATE TABLE IF NOT EXISTS mime_types ( id INTEGER PRIMARY KEY, mime_text TEXT UNIQUE NOT NULL ); -CREATE TABLE mime_actions ( +CREATE TABLE IF NOT EXISTS mime_actions ( id INTEGER PRIMARY KEY, mime_type INTEGER, destination_id INTEGER, @@ -40,7 +40,7 @@ CREATE TABLE mime_actions ( FOREIGN KEY (destination_id) REFERENCES destinations(id) ON DELETE CASCADE ); -CREATE TABLE tags ( +CREATE TABLE IF NOT EXISTS tags ( source_file INTEGER PRIMARY KEY, genre TEXT, albumartist TEXT, @@ -61,13 +61,14 @@ CREATE TABLE tags ( ON DELETE CASCADE ); -CREATE VIEW mime_type_actions AS +CREATE VIEW IF NOT EXISTS mime_type_actions AS SELECT mime_types.id,mime_types.mime_text, mime_actions.destination_id,mime_actions.action FROM mime_types INNER JOIN mime_actions ON mime_actions.mime_type = mime_types.id; -CREATE TRIGGER update_mime_actions INSTEAD OF UPDATE OF action ON mime_type_actions +CREATE TRIGGER IF NOT EXISTS update_mime_actions + INSTEAD OF UPDATE OF action ON mime_type_actions BEGIN UPDATE mime_actions SET action=new.action @@ -75,7 +76,8 @@ BEGIN AND destination_id=old.destination_id; END; -CREATE TRIGGER create_dest_files_and_mime_actions AFTER INSERT ON destinations +CREATE TRIGGER IF NOT EXISTS create_dest_files_and_mime_actions + AFTER INSERT ON destinations BEGIN INSERT INTO mime_actions (mime_type,destination_id) @@ -87,7 +89,7 @@ BEGIN FROM source_files; END; -CREATE TRIGGER create_mime_actions AFTER INSERT ON mime_types +CREATE TRIGGER IF NOT EXISTS create_mime_actions AFTER INSERT ON mime_types BEGIN INSERT INTO mime_actions (mime_type,destination_id) SELECT mime_types.id,destinations.id @@ -95,18 +97,24 @@ BEGIN WHERE mime_types.id=new.id; END; -CREATE INDEX sourcefiles_by_name ON source_files (filename,id); +CREATE INDEX IF NOT EXISTS sourcefiles_by_name ON source_files (filename,id); -CREATE TRIGGER create_destinations AFTER INSERT ON source_files +CREATE TRIGGER IF NOT EXISTS create_destinations AFTER INSERT ON source_files BEGIN INSERT INTO destination_files (source_file_id,destination_id) SELECT source_files.id,destinations.id FROM source_files INNER JOIN destinations WHERE source_files.id=new.id; END; -CREATE TRIGGER create_tags AFTER INSERT ON source_files +CREATE TRIGGER IF NOT EXISTS create_tags AFTER INSERT ON source_files BEGIN INSERT INTO tags (source_file,last_change) VALUES (new.id,0); END; +CREATE TRIGGER IF NOT EXISTS force_destination_update_on_tag_update + AFTER UPDATE ON tags +BEGIN + UPDATE destination_files SET last_change=0 + WHERE source_file_id=old.source_file; +END; COMMIT; From 360ab376df87ca6f45ba6642f4f4e387ec3d4774 Mon Sep 17 00:00:00 2001 From: Vincent Riquer Date: Tue, 30 Apr 2013 02:33:33 +0200 Subject: [PATCH 03/35] update README --- README | 23 +++++++++++++++++------ atom | 2 +- 2 files changed, 18 insertions(+), 7 deletions(-) diff --git a/README b/README index efa2414..3f5f6ed 100644 --- a/README +++ b/README @@ -19,21 +19,25 @@ Required: Optional: * vorbis-tools http://www.vorbis.com/ - * ogginfo (Ogg Vorbis tags support) - * oggenc (Ogg Vorbis destination) + * ogginfo (Ogg Vorbis metadata) + * oggenc (Ogg Vorbis encoding) * opus-tools http://opus-codec.org/ - * opusinfo (Opus tags support) - * opusenc (Opus destination) + * opusinfo (Opus metadata) + * opusenc (Opus encoding) * LAME MP3 Encoder http://lame.sourceforge.net/ - * lame (MP3 destination) + * lame (MP3 encoding) * FLAC http://flac.sourceforge.net/ - * metaflac (FLAC tags support) + * metaflac (FLAC metadata) * Musepack http://www.musepack.net/ * mpcdec (Musepack decoding) +* FFmpeg + http://ffmpeg.org/ + * ffprobe (ID3v2, Musepack, Windows Media and video metadata) + * ffmpeg (Windows Media and video decoding) ================== Using the software @@ -41,3 +45,10 @@ Using the software Configuration: Please read doc/config before anything else. +==== +Toys +---- +AtOM requires a database to function. Now that we have a database containing +various information about our media files, why not use it? +AtOM comes with a small set of tools in the toys/ directory. These are +documented in toys/README. diff --git a/atom b/atom index c338785..83a3906 100755 --- a/atom +++ b/atom @@ -268,7 +268,7 @@ do fi done echo 'COMMIT;' >&3 -echo -e "\rRead tags from ${count:-0} files." +echo -e "\rRead tags from ${count:-0} files.\033[K" unset count tagfiles echo ' From 0ae39ecab7e7a27c2bc044c8e012f91dbfd7fb94 Mon Sep 17 00:00:00 2001 From: Vincent Riquer Date: Tue, 30 Apr 2013 02:37:59 +0200 Subject: [PATCH 04/35] Album artist tag for ID3v2 now supported --- toys/missingtags | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/toys/missingtags b/toys/missingtags index 4c6c6de..c64f5ee 100755 --- a/toys/missingtags +++ b/toys/missingtags @@ -98,19 +98,6 @@ fi for check in ${!checks[@]} do case $check in - albumartist) - cat >&2 <<-EONotice - Album artist is not supported for ID3. Files in this format will be ignored by - this check (other checks still apply). - EONotice - mimemp3=$( - Select mime_types id <<<"mime_text = audio/mpeg" - ) - whereclause+="${whereclause+ OR }(" - whereclause+='tags.albumartist IS NULL ' - whereclause+="AND NOT tags.tagreader LIKE \"ID3-%\"" - whereclause+=')' - ;; tracktotal) whereclause+="${whereclause+ OR }NOT tags.track LIKE \"%/%\"" ;; From 1764b8afeaee31e1ac01dafbd6dedc554c72ae03 Mon Sep 17 00:00:00 2001 From: Vincent Riquer Date: Tue, 30 Apr 2013 03:18:42 +0200 Subject: [PATCH 05/35] I have no shame --- README | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/README b/README index 3f5f6ed..b87cabe 100644 --- a/README +++ b/README @@ -52,3 +52,26 @@ AtOM requires a database to function. Now that we have a database containing various information about our media files, why not use it? AtOM comes with a small set of tools in the toys/ directory. These are documented in toys/README. + +======================== +Shameless Self Promotion +------------------------ +I am the author of free (Creative Commons CC-By-SA) music which you can stream +for free, or buy to get high quality and bonuses from Bandcamp +(http://djblackred.bandcamp.com). If you like electronic music taking its +inspiration from Trance, Drum & Bass, Ambient and (rarely) Free Jazz, please +check it out! +Downloads are available in FLAC, Ogg, MP3, and more, and includes the "source +code" (sequencer files and the likes) for most tracks. +I am receiving 85% of the money you'll spend, so you won't be feeding some +greedy BigCorp producer or distributor. +And if you don't like it, you can still spread the word to friends who may like. +You can see this as a way to thank me for this piece of code. + +===== +Legal +----- +Some of the format and/or tool names cited above are trademarks belonging to +their rightful owners. AtOM and its authors are not linked in any way to +those companies or individuals. Said companies do not endorse nor support +AtOM in any way. From 048693ec8c1ef69b965701f3f98cd304b8c0a0d8 Mon Sep 17 00:00:00 2001 From: Vincent Riquer Date: Tue, 30 Apr 2013 13:32:47 +0200 Subject: [PATCH 06/35] don't rename if tags missing --- lib/files/getDestDir | 29 ++++++++++++++++++++++++++++- lib/files/getDestFile | 29 ++++++++++++++++++++++++++++- 2 files changed, 56 insertions(+), 2 deletions(-) diff --git a/lib/files/getDestDir b/lib/files/getDestDir index 81b6839..787adaf 100644 --- a/lib/files/getDestDir +++ b/lib/files/getDestDir @@ -1,7 +1,34 @@ #!/bin/bash getDestDir() { destdir="${destinationpath[$destination]}/" - if [ -n "${destinationrenamepath[$destination]}" ] + if [ -n "${destinationrenamepath[$destination]}" ] \ + && ( + ( + ! [[ ${destinationrenamepath[$destination]} =~ %{album} ]] \ + || [ -n "$album" ] + ) && ( + ! [[ ${destinationrenamepath[$destination]} =~ %{albumartist} ]] \ + || [ -n "$albumartist" ] + ) && ( + ! [[ ${destinationrenamepath[$destination]} =~ %{artist} ]] \ + || [ -n "$artist" ] + ) && ( + ! [[ ${destinationrenamepath[$destination]} =~ %{genre} ]] \ + || [ -n "$genre" ] + ) && ( + ! [[ ${destinationrenamepath[$destination]} =~ %{title} ]] \ + || [ -n "$title" ] + ) && ( + ! [[ ${destinationrenamepath[$destination]} =~ %{track} ]] \ + || [ -n "$track" ] + ) && ( + ! [[ ${destinationrenamepath[$destination]} =~ %{year} ]] \ + || [ -n "$year" ] + ) && ( + ! [[ ${destinationrenamepath[$destination]} =~ %{disc} ]] \ + || [ -n "$disc" ] + ) + ) then replace=$(sanitizeFile "$album" dir) destdir+="${destinationrenamepath[$destination]//%\{album\}/$replace}" diff --git a/lib/files/getDestFile b/lib/files/getDestFile index 7730b48..e943b41 100644 --- a/lib/files/getDestFile +++ b/lib/files/getDestFile @@ -1,6 +1,33 @@ #!/bin/bash getDestFile() { - if [ -n "${destinationrename[$destination]}" ] + if [ -n "${destinationrename[$destination]}" ] \ + && ( + ( + ! [[ ${destinationrename[$destination]} =~ %{album} ]] \ + || [ -n "$album" ] + ) && ( + ! [[ ${destinationrename[$destination]} =~ %{albumartist} ]] \ + || [ -n "$albumartist" ] + ) && ( + ! [[ ${destinationrename[$destination]} =~ %{artist} ]] \ + || [ -n "$artist" ] + ) && ( + ! [[ ${destinationrename[$destination]} =~ %{genre} ]] \ + || [ -n "$genre" ] + ) && ( + ! [[ ${destinationrename[$destination]} =~ %{title} ]] \ + || [ -n "$title" ] + ) && ( + ! [[ ${destinationrename[$destination]} =~ %{track} ]] \ + || [ -n "$track" ] + ) && ( + ! [[ ${destinationrename[$destination]} =~ %{year} ]] \ + || [ -n "$year" ] + ) && ( + ! [[ ${destinationrename[$destination]} =~ %{disc} ]] \ + || [ -n "$disc" ] + ) + ) then destfile="${destinationrename[$destination]//%\{album\}/$album}" destfile="${destfile//%\{albumartist\}/$albumartist}" From 28b11a77eaf2ee936c1773e80bff24450bb9d853 Mon Sep 17 00:00:00 2001 From: Vincent Riquer Date: Wed, 1 May 2013 03:40:43 +0200 Subject: [PATCH 07/35] ffmpeg: fix tracknumber tag --- lib/tags/getInfos::ffmpeg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/tags/getInfos::ffmpeg b/lib/tags/getInfos::ffmpeg index d855139..5f129f3 100644 --- a/lib/tags/getInfos::ffmpeg +++ b/lib/tags/getInfos::ffmpeg @@ -31,7 +31,7 @@ getInfos::ffmpeg() { genre=$(gettag genre) performer=$(gettag TOPE) title=$(gettag title) - tracknum=$(gettag tracknumber) + tracknum=$(gettag track) year=$(gettag year) expr='^[0-9]*$' if [ -n "$genre" ] && [[ $genre =~ $expr ]] From 1fa431cc5f90a8d64132848f9d5ee80801515f77 Mon Sep 17 00:00:00 2001 From: Vincent Riquer Date: Thu, 2 May 2013 02:23:24 +0200 Subject: [PATCH 08/35] filenames and/or tags may contain | --- atom | 148 +++++++++++++++++++------------------- lib/copy/copyFiles_action | 16 ++--- lib/database/openDatabase | 1 + lib/files/sanitizeFile | 7 ++ lib/tasks/gettaskinfos | 24 +++---- lib/workers/master | 144 ++++++++++++++++++------------------- toys/checkextensions | 16 ++--- toys/missingtags | 44 ++++++------ 8 files changed, 204 insertions(+), 196 deletions(-) diff --git a/atom b/atom index 83a3906..dff6205 100755 --- a/atom +++ b/atom @@ -144,8 +144,8 @@ removed=0 read -u4 line until [[ $line == AtOM:NoMoreFiles ]] do - id=${line%|*} - filename=${line#*|} + id=${line%::AtOM:SQL:Sep::*} + filename=${line#*::AtOM:SQL:Sep::} if [ -n "$filename" ] then if rm -f "$filename" @@ -220,12 +220,12 @@ done echo 'BEGIN TRANSACTION;' >&3 for line in "${tagfiles[@]}" do - sourcefileid=${line%%|*} - rest=${line#*|} - lastchange=${rest%%|*} - rest=${rest#*|} - mimetype=${rest%%|*} - filename=${rest#*|} + sourcefileid=${line%%::AtOM:SQL:Sep::*} + rest=${line#*::AtOM:SQL:Sep::} + lastchange=${rest%%::AtOM:SQL:Sep::*} + rest=${rest#*::AtOM:SQL:Sep::} + mimetype=${rest%%::AtOM:SQL:Sep::*} + filename=${rest#*::AtOM:SQL:Sep::} echo -en "\rTags: $((++count*100/filecount))%" if (( count % 1000 == 0 )) then @@ -376,7 +376,7 @@ echo ' read -u4 line while ! [[ $line = AtOM:NoMoreFiles ]] do - decodefiles+=("$line|") + decodefiles+=("$line::AtOM:SQL:Sep::") read -u4 line done echo -n 'Creating tasks... ' @@ -384,41 +384,41 @@ echo -n 'Creating tasks... ' echo 'BEGIN TRANSACTION;' >&3 for line in "${decodefiles[@]}" do - fileid=${line%%|*} - rest=${line#*|} - filename=${rest%%|*} - rest=${rest#*|} - mimetype=${rest%%|*} - rest=${rest#*|} - destination=${rest%%|*} - rest=${rest#*|} - destfileid=${rest%%|*} - rest=${rest#*|} - rate=${rest%%|*} - rest=${rest#*|} - channels=${rest%%|*} - rest=${rest#*|} - bitrate=${rest%%|*} - rest=${rest#*|} - genre=${rest%%|*} - rest=${rest#*|} - albumartist=${rest%%|*} - rest=${rest#*|} - year=${rest%%|*} - rest=${rest#*|} - album=${rest%%|*} - rest=${rest#*|} - disc=${rest%%|*} - rest=${rest#*|} - artist=${rest%%|*} - rest=${rest#*|} - track=${rest%%|*} - rest=${rest#*|} - title=${rest%%|*} - rest=${rest#*|} - composer=${rest%%|*} - rest=${rest#*|} - performer=${rest%%|*} + fileid=${line%%::AtOM:SQL:Sep::*} + rest=${line#*::AtOM:SQL:Sep::} + filename=${rest%%::AtOM:SQL:Sep::*} + rest=${rest#*::AtOM:SQL:Sep::} + mimetype=${rest%%::AtOM:SQL:Sep::*} + rest=${rest#*::AtOM:SQL:Sep::} + destination=${rest%%::AtOM:SQL:Sep::*} + rest=${rest#*::AtOM:SQL:Sep::} + destfileid=${rest%%::AtOM:SQL:Sep::*} + rest=${rest#*::AtOM:SQL:Sep::} + rate=${rest%%::AtOM:SQL:Sep::*} + rest=${rest#*::AtOM:SQL:Sep::} + channels=${rest%%::AtOM:SQL:Sep::*} + rest=${rest#*::AtOM:SQL:Sep::} + bitrate=${rest%%::AtOM:SQL:Sep::*} + rest=${rest#*::AtOM:SQL:Sep::} + genre=${rest%%::AtOM:SQL:Sep::*} + rest=${rest#*::AtOM:SQL:Sep::} + albumartist=${rest%%::AtOM:SQL:Sep::*} + rest=${rest#*::AtOM:SQL:Sep::} + year=${rest%%::AtOM:SQL:Sep::*} + rest=${rest#*::AtOM:SQL:Sep::} + album=${rest%%::AtOM:SQL:Sep::*} + rest=${rest#*::AtOM:SQL:Sep::} + disc=${rest%%::AtOM:SQL:Sep::*} + rest=${rest#*::AtOM:SQL:Sep::} + artist=${rest%%::AtOM:SQL:Sep::*} + rest=${rest#*::AtOM:SQL:Sep::} + track=${rest%%::AtOM:SQL:Sep::*} + rest=${rest#*::AtOM:SQL:Sep::} + title=${rest%%::AtOM:SQL:Sep::*} + rest=${rest#*::AtOM:SQL:Sep::} + composer=${rest%%::AtOM:SQL:Sep::*} + rest=${rest#*::AtOM:SQL:Sep::} + performer=${rest%%::AtOM:SQL:Sep::*} unset rest decodeFile getDestDir @@ -647,32 +647,32 @@ do echo -n "$destination: rename pattern changed, renaming files... " while [[ $line != AtOM:NoMoreFiles ]] do - oldfilename=${line%%|*} - rest=${line#*|}'|' - destfileid=${rest%%|*} - rest=${rest#*|} - filename=${rest%%|*} - rest=${rest#*|} - album=${rest%%|*} - rest=${rest#*|} - albumartist=${rest%%|*} - rest=${rest#*|} - artist=${rest%%|*} - rest=${rest#*|} - composer=${rest%%|*} - rest=${rest#*|} - disc=${rest%%|*} - rest=${rest#*|} - genre=${rest%%|*} - rest=${rest#*|} - performer=${rest%%|*} - rest=${rest#*|} - title=${rest%%|*} - rest=${rest#*|} - track=${rest%%|*} - rest=${rest#*|} - year=${rest%%|*} - rest=${rest#*|} + oldfilename=${line%%::AtOM:SQL:Sep::*} + rest=${line#*::AtOM:SQL:Sep::}'::AtOM:SQL:Sep::' + destfileid=${rest%%::AtOM:SQL:Sep::*} + rest=${rest#*::AtOM:SQL:Sep::} + filename=${rest%%::AtOM:SQL:Sep::*} + rest=${rest#*::AtOM:SQL:Sep::} + album=${rest%%::AtOM:SQL:Sep::*} + rest=${rest#*::AtOM:SQL:Sep::} + albumartist=${rest%%::AtOM:SQL:Sep::*} + rest=${rest#*::AtOM:SQL:Sep::} + artist=${rest%%::AtOM:SQL:Sep::*} + rest=${rest#*::AtOM:SQL:Sep::} + composer=${rest%%::AtOM:SQL:Sep::*} + rest=${rest#*::AtOM:SQL:Sep::} + disc=${rest%%::AtOM:SQL:Sep::*} + rest=${rest#*::AtOM:SQL:Sep::} + genre=${rest%%::AtOM:SQL:Sep::*} + rest=${rest#*::AtOM:SQL:Sep::} + performer=${rest%%::AtOM:SQL:Sep::*} + rest=${rest#*::AtOM:SQL:Sep::} + title=${rest%%::AtOM:SQL:Sep::*} + rest=${rest#*::AtOM:SQL:Sep::} + track=${rest%%::AtOM:SQL:Sep::*} + rest=${rest#*::AtOM:SQL:Sep::} + year=${rest%%::AtOM:SQL:Sep::*} + rest=${rest#*::AtOM:SQL:Sep::} if [ -n "$oldfilename" -a -f "$oldfilename" ] then getDestDir @@ -720,10 +720,10 @@ done echo 'BEGIN TRANSACTION;' >&3 for line in "${lines[@]}" do - id=${line%%|*} - rest=${line#*|} - filename=${rest%%|*} - oldfilename=${rest#*|} + id=${line%%::AtOM:SQL:Sep::*} + rest=${line#*::AtOM:SQL:Sep::} + filename=${rest%%::AtOM:SQL:Sep::*} + oldfilename=${rest#*::AtOM:SQL:Sep::} if [[ $oldfilename != $filename ]] && [ -f "$oldfilename" ] then rm -f "$oldfilename" diff --git a/lib/copy/copyFiles_action b/lib/copy/copyFiles_action index 86820cc..c0c352f 100644 --- a/lib/copy/copyFiles_action +++ b/lib/copy/copyFiles_action @@ -31,15 +31,15 @@ copyFiles_action() { echo 'BEGIN TRANSACTION;' >&3 for copyfile in "${copyfiles[@]}" do - sourcefilename=${copyfile%%|*} + sourcefilename=${copyfile%%::AtOM:SQL:Sep::*} sourcedir=${sourcefilename%/*} - rest="${copyfile#*|}|" - lastchange=${rest%%|*} - rest=${rest#*|} - destinationid=${rest%%|*} - rest=${rest#*|} - destfileid=${rest%%|*} - rest=${rest#*|} + rest="${copyfile#*::AtOM:SQL:Sep::}::AtOM:SQL:Sep::" + lastchange=${rest%%::AtOM:SQL:Sep::*} + rest=${rest#*::AtOM:SQL:Sep::} + destinationid=${rest%%::AtOM:SQL:Sep::*} + rest=${rest#*::AtOM:SQL:Sep::} + destfileid=${rest%%::AtOM:SQL:Sep::*} + rest=${rest#*::AtOM:SQL:Sep::} echo 'SELECT IFNULL( ( SELECT destination_files.filename FROM destination_files diff --git a/lib/database/openDatabase b/lib/database/openDatabase index df05d84..b75cef4 100644 --- a/lib/database/openDatabase +++ b/lib/database/openDatabase @@ -24,5 +24,6 @@ openDatabase() { exec 3> >(tee -a $tempdir/debug.log >&5) fi cat $schema >&3 + echo '.separator ::AtOM:SQL:Sep::' >&3 echo 'PRAGMA foreign_keys = ON;' >&3 } diff --git a/lib/files/sanitizeFile b/lib/files/sanitizeFile index 07e8e50..2968f5e 100644 --- a/lib/files/sanitizeFile +++ b/lib/files/sanitizeFile @@ -27,5 +27,12 @@ sanitizeFile() { string=${string/%+(.)/} fi fi + # Not exactly filename sanitity, but these will make the script fail in + # a decorative way.. + string="${string//&/\\&}" + strint="${string//;/\\;}" + # And these would exhibit strange behaviors + string="${string//\[/\\\[}" + string="${string//\]/\\\]}" echo "$string" } diff --git a/lib/tasks/gettaskinfos b/lib/tasks/gettaskinfos index f0023bd..5a51657 100644 --- a/lib/tasks/gettaskinfos +++ b/lib/tasks/gettaskinfos @@ -12,16 +12,16 @@ gettaskinfos() { WHERE id='$1'; ' >&3 read -u4 line - taskid=${line%%|*} - rest="${line#*|}|" - sourcefileid=${rest%%|*} - rest=${rest#*|} - required=${rest%%|*} - rest=${rest#*|} - cleanup=${rest%%|*} - rest=${rest#*|} - destfileid=${rest%%|*} - rest=${rest#*|} - destfilename=${rest%%|*} - rest=${rest#*|} + taskid=${line%%::AtOM:SQL:Sep::*} + rest="${line#*::AtOM:SQL:Sep::}::AtOM:SQL:Sep::" + sourcefileid=${rest%%::AtOM:SQL:Sep::*} + rest=${rest#*::AtOM:SQL:Sep::} + required=${rest%%::AtOM:SQL:Sep::*} + rest=${rest#*::AtOM:SQL:Sep::} + cleanup=${rest%%::AtOM:SQL:Sep::*} + rest=${rest#*::AtOM:SQL:Sep::} + destfileid=${rest%%::AtOM:SQL:Sep::*} + rest=${rest#*::AtOM:SQL:Sep::} + destfilename=${rest%%::AtOM:SQL:Sep::*} + rest=${rest#*::AtOM:SQL:Sep::} } diff --git a/lib/workers/master b/lib/workers/master index 4c22abe..a6d0c99 100644 --- a/lib/workers/master +++ b/lib/workers/master @@ -69,78 +69,78 @@ master() { else (( ++active )) read -u4 line - taskid=${line%%|*} - rest="${line#*|}|" - sourcefileid=${rest%%|*} - rest=${rest#*|} - required=${rest%%|*} - rest=${rest#*|} - cmd_arg=("${rest%%|*}") - rest=${rest#*|} - cmd_arg+=("${rest%%|*}") - rest=${rest#*|} - cmd_arg+=("${rest%%|*}") - rest=${rest#*|} - cmd_arg+=("${rest%%|*}") - rest=${rest#*|} - cmd_arg+=("${rest%%|*}") - rest=${rest#*|} - cmd_arg+=("${rest%%|*}") - rest=${rest#*|} - cmd_arg+=("${rest%%|*}") - rest=${rest#*|} - cmd_arg+=("${rest%%|*}") - rest=${rest#*|} - cmd_arg+=("${rest%%|*}") - rest=${rest#*|} - cmd_arg+=("${rest%%|*}") - rest=${rest#*|} - cmd_arg+=("${rest%%|*}") - rest=${rest#*|} - cmd_arg+=("${rest%%|*}") - rest=${rest#*|} - cmd_arg+=("${rest%%|*}") - rest=${rest#*|} - cmd_arg+=("${rest%%|*}") - rest=${rest#*|} - cmd_arg+=("${rest%%|*}") - rest=${rest#*|} - cmd_arg+=("${rest%%|*}") - rest=${rest#*|} - cmd_arg+=("${rest%%|*}") - rest=${rest#*|} - cmd_arg+=("${rest%%|*}") - rest=${rest#*|} - cmd_arg+=("${rest%%|*}") - rest=${rest#*|} - cmd_arg+=("${rest%%|*}") - rest=${rest#*|} - cmd_arg+=("${rest%%|*}") - rest=${rest#*|} - cmd_arg+=("${rest%%|*}") - rest=${rest#*|} - cmd_arg+=("${rest%%|*}") - rest=${rest#*|} - cmd_arg+=("${rest%%|*}") - rest=${rest#*|} - cmd_arg+=("${rest%%|*}") - rest=${rest#*|} - cmd_arg+=("${rest%%|*}") - rest=${rest#*|} - cmd_arg+=("${rest%%|*}") - rest=${rest#*|} - cmd_arg+=("${rest%%|*}") - rest=${rest#*|} - cmd_arg+=("${rest%%|*}") - rest=${rest#*|} - cmd_arg+=("${rest%%|*}") - rest=${rest#*|} - cleanup=${rest%%|*} - rest=${rest#*|} - destfileid=${rest%%|*} - rest=${rest#*|} - destfilename=${rest%%|*} - rest=${rest#*|} + taskid=${line%%::AtOM:SQL:Sep::*} + rest="${line#*::AtOM:SQL:Sep::}::AtOM:SQL:Sep::" + sourcefileid=${rest%%::AtOM:SQL:Sep::*} + rest=${rest#*::AtOM:SQL:Sep::} + required=${rest%%::AtOM:SQL:Sep::*} + rest=${rest#*::AtOM:SQL:Sep::} + cmd_arg=("${rest%%::AtOM:SQL:Sep::*}") + rest=${rest#*::AtOM:SQL:Sep::} + cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}") + rest=${rest#*::AtOM:SQL:Sep::} + cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}") + rest=${rest#*::AtOM:SQL:Sep::} + cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}") + rest=${rest#*::AtOM:SQL:Sep::} + cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}") + rest=${rest#*::AtOM:SQL:Sep::} + cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}") + rest=${rest#*::AtOM:SQL:Sep::} + cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}") + rest=${rest#*::AtOM:SQL:Sep::} + cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}") + rest=${rest#*::AtOM:SQL:Sep::} + cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}") + rest=${rest#*::AtOM:SQL:Sep::} + cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}") + rest=${rest#*::AtOM:SQL:Sep::} + cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}") + rest=${rest#*::AtOM:SQL:Sep::} + cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}") + rest=${rest#*::AtOM:SQL:Sep::} + cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}") + rest=${rest#*::AtOM:SQL:Sep::} + cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}") + rest=${rest#*::AtOM:SQL:Sep::} + cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}") + rest=${rest#*::AtOM:SQL:Sep::} + cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}") + rest=${rest#*::AtOM:SQL:Sep::} + cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}") + rest=${rest#*::AtOM:SQL:Sep::} + cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}") + rest=${rest#*::AtOM:SQL:Sep::} + cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}") + rest=${rest#*::AtOM:SQL:Sep::} + cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}") + rest=${rest#*::AtOM:SQL:Sep::} + cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}") + rest=${rest#*::AtOM:SQL:Sep::} + cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}") + rest=${rest#*::AtOM:SQL:Sep::} + cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}") + rest=${rest#*::AtOM:SQL:Sep::} + cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}") + rest=${rest#*::AtOM:SQL:Sep::} + cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}") + rest=${rest#*::AtOM:SQL:Sep::} + cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}") + rest=${rest#*::AtOM:SQL:Sep::} + cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}") + rest=${rest#*::AtOM:SQL:Sep::} + cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}") + rest=${rest#*::AtOM:SQL:Sep::} + cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}") + rest=${rest#*::AtOM:SQL:Sep::} + cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}") + rest=${rest#*::AtOM:SQL:Sep::} + cleanup=${rest%%::AtOM:SQL:Sep::*} + rest=${rest#*::AtOM:SQL:Sep::} + destfileid=${rest%%::AtOM:SQL:Sep::*} + rest=${rest#*::AtOM:SQL:Sep::} + destfilename=${rest%%::AtOM:SQL:Sep::*} + rest=${rest#*::AtOM:SQL:Sep::} for key in ${!cmd_arg[@]} do [ -z "${cmd_arg[key]}" ] && unset cmd_arg[key] diff --git a/toys/checkextensions b/toys/checkextensions index a390bc1..3a427ff 100755 --- a/toys/checkextensions +++ b/toys/checkextensions @@ -112,8 +112,8 @@ getdstfiles() { done for line in "${lines[@]}" do - fileid=${line%|*} - filename=${line#*|} + fileid=${line%::AtOM:SQL:Sep::*} + filename=${line#*::AtOM:SQL:Sep::} echo $'\t'"$filename" (( rename )) && echo -n $'\t' (( rename )) && renameFile @@ -149,12 +149,12 @@ do done for line in "${lines[@]}" do - fileid=${line%%|*} - rest="${line#*|}|" - filename=${rest%%|*} - rest=${rest#*|} - mimetype=${rest%%|*} - rest=${rest#*|} + fileid=${line%%::AtOM:SQL:Sep::*} + rest="${line#*::AtOM:SQL:Sep::}::AtOM:SQL:Sep::" + filename=${rest%%::AtOM:SQL:Sep::*} + rest=${rest#*::AtOM:SQL:Sep::} + mimetype=${rest%%::AtOM:SQL:Sep::*} + rest=${rest#*::AtOM:SQL:Sep::} dest=$sourcepath case "$mimetype" in 'audio/mpeg') diff --git a/toys/missingtags b/toys/missingtags index c64f5ee..ab73d1b 100755 --- a/toys/missingtags +++ b/toys/missingtags @@ -140,33 +140,33 @@ done for line in "${lines[@]}" do missing=() - filename=${line%%|*} - rest="${line#*|}|" - (( ${checks[genre]} )) && [ -z "${rest%%|*}" ] && missing+=( "Genre" ) - rest=${rest#*|} - (( ${checks[albumartist]} )) && [ -z "${rest%%|*}" ] && missing+=( "AlbumArtist" ) - rest=${rest#*|} - (( ${checks[year]} )) && [ -z "${rest%%|*}" ] && missing+=( "Year" ) - rest=${rest#*|} - (( ${checks[album]} )) && [ -z "${rest%%|*}" ] && missing+=( "Album" ) - rest=${rest#*|} - (( ${checks[disc]} )) && [ -z "${rest%%|*}" ] && missing+=( "Disc" ) - rest=${rest#*|} - (( ${checks[artist]} )) && [ -z "${rest%%|*}" ] && missing+=( "Artist" ) - rest=${rest#*|} - track=${rest%%|*} + filename=${line%%::AtOM:SQL:Sep::*} + rest="${line#*::AtOM:SQL:Sep::}::AtOM:SQL:Sep::" + (( ${checks[genre]} )) && [ -z "${rest%%::AtOM:SQL:Sep::*}" ] && missing+=( "Genre" ) + rest=${rest#*::AtOM:SQL:Sep::} + (( ${checks[albumartist]} )) && [ -z "${rest%%::AtOM:SQL:Sep::*}" ] && missing+=( "AlbumArtist" ) + rest=${rest#*::AtOM:SQL:Sep::} + (( ${checks[year]} )) && [ -z "${rest%%::AtOM:SQL:Sep::*}" ] && missing+=( "Year" ) + rest=${rest#*::AtOM:SQL:Sep::} + (( ${checks[album]} )) && [ -z "${rest%%::AtOM:SQL:Sep::*}" ] && missing+=( "Album" ) + rest=${rest#*::AtOM:SQL:Sep::} + (( ${checks[disc]} )) && [ -z "${rest%%::AtOM:SQL:Sep::*}" ] && missing+=( "Disc" ) + rest=${rest#*::AtOM:SQL:Sep::} + (( ${checks[artist]} )) && [ -z "${rest%%::AtOM:SQL:Sep::*}" ] && missing+=( "Artist" ) + rest=${rest#*::AtOM:SQL:Sep::} + track=${rest%%::AtOM:SQL:Sep::*} (( ${checks[track]} )) && [ -z "$track" ] && missing+=( "Track" ) if (( ${checks[tracktotal]} )) && ! [[ $track =~ / ]] then missing+=( TrackTotal ) fi - rest=${rest#*|} - (( ${checks[title]} )) && [ -z "${rest%%|*}" ] && missing+=( "Title" ) - rest=${rest#*|} - (( ${checks[composer]} )) && [ -z "${rest%%|*}" ] && missing+=( "Composer" ) - rest=${rest#*|} - (( ${checks[performer]} )) && [ -z "${rest%%|*}" ] && missing+=( "Performer" ) - rest=${rest#*|} + rest=${rest#*::AtOM:SQL:Sep::} + (( ${checks[title]} )) && [ -z "${rest%%::AtOM:SQL:Sep::*}" ] && missing+=( "Title" ) + rest=${rest#*::AtOM:SQL:Sep::} + (( ${checks[composer]} )) && [ -z "${rest%%::AtOM:SQL:Sep::*}" ] && missing+=( "Composer" ) + rest=${rest#*::AtOM:SQL:Sep::} + (( ${checks[performer]} )) && [ -z "${rest%%::AtOM:SQL:Sep::*}" ] && missing+=( "Performer" ) + rest=${rest#*::AtOM:SQL:Sep::} echo "$filename: ${missing[@]}" done From 9c2f765f5c95d67e6f2f1c20a1079df4c1873b31 Mon Sep 17 00:00:00 2001 From: Vincent Riquer Date: Thu, 2 May 2013 03:40:24 +0200 Subject: [PATCH 09/35] toys/missingtags: skip text and image files --- toys/missingtags | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/toys/missingtags b/toys/missingtags index ab73d1b..780b475 100755 --- a/toys/missingtags +++ b/toys/missingtags @@ -99,10 +99,10 @@ for check in ${!checks[@]} do case $check in tracktotal) - whereclause+="${whereclause+ OR }NOT tags.track LIKE \"%/%\"" + whereclause+="${whereclause+OR }NOT tags.track LIKE \"%/%\"" ;; *) - whereclause+="${whereclause+ OR }tags.$check IS NULL" + whereclause+="${whereclause+OR }tags.$check IS NULL" ;; esac whereclause+=' @@ -124,8 +124,15 @@ cat >&3 <<-EOSelect FROM tags INNER JOIN source_files ON tags.source_file=source_files.id - WHERE $whereclause - AND NOT tags.tagreader LIKE "unknown-%"; + INNER JOIN mime_types + ON source_files.mime_type=mime_types.id + WHERE ( + $whereclause + ) + AND NOT tags.tagreader LIKE "unknown-%" + AND NOT mime_types.mime_text LIKE "text/%" + AND NOT mime_types.mime_text LIKE "image/%" + ; SELECT "AtOM:NoMoreFiles"; EOSelect From 7b34e0eb11993dd3ffd6a17a9b0ebe5791f827aa Mon Sep 17 00:00:00 2001 From: Vincent Riquer Date: Thu, 2 May 2013 03:49:21 +0200 Subject: [PATCH 10/35] ffmpag: fix "year" tag --- lib/tags/getInfos::ffmpeg | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/tags/getInfos::ffmpeg b/lib/tags/getInfos::ffmpeg index 5f129f3..e03e879 100644 --- a/lib/tags/getInfos::ffmpeg +++ b/lib/tags/getInfos::ffmpeg @@ -1,5 +1,5 @@ #!/bin/bash -getInfosffmpeg_version='ffmpeg-1' +getInfosffmpeg_version='ffmpeg-2' tagreaders+=( "$getInfosffmpeg_version" ) getInfos::ffmpeg() { tagreader="$getInfosffmpeg_version" @@ -32,7 +32,7 @@ getInfos::ffmpeg() { performer=$(gettag TOPE) title=$(gettag title) tracknum=$(gettag track) - year=$(gettag year) + year=$(gettag date) expr='^[0-9]*$' if [ -n "$genre" ] && [[ $genre =~ $expr ]] then From 7c8617d713cafe4f70ec81ad6ad0421107bfdeef Mon Sep 17 00:00:00 2001 From: Vincent Riquer Date: Thu, 2 May 2013 03:58:15 +0200 Subject: [PATCH 11/35] create function updateTags() * move code from atom to updateTags() * toys/missingtags: update tags --- atom | 110 +------------------------------------------ lib/tags/updateTags | 111 ++++++++++++++++++++++++++++++++++++++++++++ toys/missingtags | 1 + 3 files changed, 113 insertions(+), 109 deletions(-) create mode 100644 lib/tags/updateTags diff --git a/atom b/atom index dff6205..5c35308 100755 --- a/atom +++ b/atom @@ -161,115 +161,7 @@ do done echo "Suppressed $deleted files, $removed removed from database" -# get files -for reader in "${tagreaders[@]}" -do - tagreaderclause+="${tagreaderclause:+ AND }NOT tags.tagreader = \"$reader\"" -done -echo ' - SELECT COUNT(DISTINCT source_files.filename) - FROM source_files - INNER JOIN destination_files - ON destination_files.source_file_id=source_files.id - INNER JOIN destinations - ON destination_files.destination_id=destinations.id - INNER JOIN mime_type_actions - ON destinations.id=mime_type_actions.destination_id - INNER JOIN tags - ON source_files.id=tags.source_file - WHERE mime_type_actions.id = source_files.mime_type - AND ( - CAST(tags.last_change AS TEXT) - <> - CAST(source_files.last_change AS TEXT) - OR ('"$tagreaderclause"') - ) - AND mime_type_actions.action = 1;' >&3 -read -u4 filecount -echo ' - SELECT DISTINCT - source_files.id, - source_files.last_change, - mime_type_actions.mime_text, - source_files.filename - FROM source_files - INNER JOIN destination_files - ON destination_files.source_file_id=source_files.id - INNER JOIN destinations - ON destination_files.destination_id=destinations.id - INNER JOIN mime_type_actions - ON destinations.id=mime_type_actions.destination_id - INNER JOIN tags - ON source_files.id=tags.source_file - WHERE mime_type_actions.id = source_files.mime_type - AND ( - CAST(tags.last_change AS TEXT) - <> - CAST(source_files.last_change AS TEXT) - OR ('"$tagreaderclause"') - ) - AND mime_type_actions.action = 1; - - SELECT "AtOM:NoMoreFiles";' >&3 -read -u4 line -while ! [[ $line = AtOM:NoMoreFiles ]] -do - tagfiles+=("$line") - read -u4 line -done -echo 'BEGIN TRANSACTION;' >&3 -for line in "${tagfiles[@]}" -do - sourcefileid=${line%%::AtOM:SQL:Sep::*} - rest=${line#*::AtOM:SQL:Sep::} - lastchange=${rest%%::AtOM:SQL:Sep::*} - rest=${rest#*::AtOM:SQL:Sep::} - mimetype=${rest%%::AtOM:SQL:Sep::*} - filename=${rest#*::AtOM:SQL:Sep::} - echo -en "\rTags: $((++count*100/filecount))%" - if (( count % 1000 == 0 )) - then - echo 'COMMIT;BEGIN TRANSACTION;' >&3 - (( debug )) \ - && echo -n " $count files read, committing..." - fi - if getTags - then - Update tags \ - album "${album:-NULL}" \ - albumartist "${albumartist:-NULL}" \ - artist "${artist:-NULL}" \ - composer "${composer:-NULL}" \ - disc "${disc:-NULL}" \ - genre "${genre:-NULL}" \ - performer "${performer:-NULL}" \ - title "${title:-NULL}" \ - track "${tracknum:-NULL}" \ - year "${year:-NULL}" \ - last_change "$lastchange" \ - rate "${rate:-NULL}" \ - channels "${channels:-NULL}" \ - bitrate "${bitrate:-NULL}" \ - tagreader "$tagreader" \ - >/dev/null <<<"source_file = $sourcefileid" - unset genre \ - albumartist \ - year \ - album \ - disc \ - artist \ - tracknum \ - title \ - composer \ - performer \ - rate \ - bitrate \ - channels - fi -done -echo 'COMMIT;' >&3 -echo -e "\rRead tags from ${count:-0} files.\033[K" -unset count tagfiles +updateTags echo ' CREATE TEMPORARY TABLE tasks( diff --git a/lib/tags/updateTags b/lib/tags/updateTags new file mode 100644 index 0000000..d317c1a --- /dev/null +++ b/lib/tags/updateTags @@ -0,0 +1,111 @@ +#!/bin/bash +updateTags() { + for reader in "${tagreaders[@]}" + do + tagreaderclause+="${tagreaderclause:+ AND }NOT tags.tagreader = \"$reader\"" + done + echo ' + SELECT COUNT(DISTINCT source_files.filename) + FROM source_files + INNER JOIN destination_files + ON destination_files.source_file_id=source_files.id + INNER JOIN destinations + ON destination_files.destination_id=destinations.id + INNER JOIN mime_type_actions + ON destinations.id=mime_type_actions.destination_id + INNER JOIN tags + ON source_files.id=tags.source_file + WHERE mime_type_actions.id = source_files.mime_type + AND ( + CAST(tags.last_change AS TEXT) + <> + CAST(source_files.last_change AS TEXT) + OR ('"$tagreaderclause"') + ) + AND mime_type_actions.action = 1;' >&3 + read -u4 filecount + echo ' + SELECT DISTINCT + source_files.id, + source_files.last_change, + mime_type_actions.mime_text, + source_files.filename + FROM source_files + INNER JOIN destination_files + ON destination_files.source_file_id=source_files.id + INNER JOIN destinations + ON destination_files.destination_id=destinations.id + INNER JOIN mime_type_actions + ON destinations.id=mime_type_actions.destination_id + INNER JOIN tags + ON source_files.id=tags.source_file + WHERE mime_type_actions.id = source_files.mime_type + AND ( + CAST(tags.last_change AS TEXT) + <> + CAST(source_files.last_change AS TEXT) + OR ('"$tagreaderclause"') + ) + AND mime_type_actions.action = 1; + + SELECT "AtOM:NoMoreFiles";' >&3 + read -u4 line + while ! [[ $line = AtOM:NoMoreFiles ]] + do + tagfiles+=("$line") + read -u4 line + done + echo 'BEGIN TRANSACTION;' >&3 + for line in "${tagfiles[@]}" + do + sourcefileid=${line%%::AtOM:SQL:Sep::*} + rest=${line#*::AtOM:SQL:Sep::} + lastchange=${rest%%::AtOM:SQL:Sep::*} + rest=${rest#*::AtOM:SQL:Sep::} + mimetype=${rest%%::AtOM:SQL:Sep::*} + filename=${rest#*::AtOM:SQL:Sep::} + echo -en "\rTags: $((++count*100/filecount))%" + if (( count % 1000 == 0 )) + then + echo 'COMMIT;BEGIN TRANSACTION;' >&3 + (( debug )) \ + && echo -n " $count files read, committing..." + fi + if getTags + then + Update tags \ + album "${album:-NULL}" \ + albumartist "${albumartist:-NULL}" \ + artist "${artist:-NULL}" \ + composer "${composer:-NULL}" \ + disc "${disc:-NULL}" \ + genre "${genre:-NULL}" \ + performer "${performer:-NULL}" \ + title "${title:-NULL}" \ + track "${tracknum:-NULL}" \ + year "${year:-NULL}" \ + last_change "$lastchange" \ + rate "${rate:-NULL}" \ + channels "${channels:-NULL}" \ + bitrate "${bitrate:-NULL}" \ + tagreader "$tagreader" \ + >/dev/null <<<"source_file = $sourcefileid" + unset genre \ + albumartist \ + year \ + album \ + disc \ + artist \ + tracknum \ + title \ + composer \ + performer \ + rate \ + bitrate \ + channels + fi + done + echo 'COMMIT;' >&3 + echo -e "\rRead tags from ${count:-0} files.\033[K" + unset count tagfiles +} diff --git a/toys/missingtags b/toys/missingtags index 780b475..72f723b 100755 --- a/toys/missingtags +++ b/toys/missingtags @@ -93,6 +93,7 @@ if (( update )) then getFiles updateMimes + updateTags fi for check in ${!checks[@]} From 8b3f121179d5f8226f18c36df614b6e7e6f087ae Mon Sep 17 00:00:00 2001 From: Vincent Riquer Date: Fri, 3 May 2013 17:25:07 +0200 Subject: [PATCH 12/35] Fix file renaming --- lib/copy/copyFiles_action | 3 +++ lib/files/getDestDir | 16 ++++++++-------- lib/files/getDestFile | 16 ++++++++-------- 3 files changed, 19 insertions(+), 16 deletions(-) diff --git a/lib/copy/copyFiles_action b/lib/copy/copyFiles_action index c0c352f..a944ef8 100644 --- a/lib/copy/copyFiles_action +++ b/lib/copy/copyFiles_action @@ -6,6 +6,7 @@ copyFiles_action() { source_files.filename, source_files.last_change, destinations.id, + destinations.name, destination_files.id FROM source_files INNER JOIN destination_files @@ -38,6 +39,8 @@ copyFiles_action() { rest=${rest#*::AtOM:SQL:Sep::} destinationid=${rest%%::AtOM:SQL:Sep::*} rest=${rest#*::AtOM:SQL:Sep::} + destination=${rest%%::AtOM:SQL:Sep::*} + rest=${rest#*::AtOM:SQL:Sep::} destfileid=${rest%%::AtOM:SQL:Sep::*} rest=${rest#*::AtOM:SQL:Sep::} echo 'SELECT IFNULL( ( diff --git a/lib/files/getDestDir b/lib/files/getDestDir index 787adaf..ff4f26b 100644 --- a/lib/files/getDestDir +++ b/lib/files/getDestDir @@ -4,28 +4,28 @@ getDestDir() { if [ -n "${destinationrenamepath[$destination]}" ] \ && ( ( - ! [[ ${destinationrenamepath[$destination]} =~ %{album} ]] \ + ! [[ ${destinationrenamepath[$destination]} =~ %\{album\} ]] \ || [ -n "$album" ] ) && ( - ! [[ ${destinationrenamepath[$destination]} =~ %{albumartist} ]] \ + ! [[ ${destinationrenamepath[$destination]} =~ %\{albumartist\} ]] \ || [ -n "$albumartist" ] ) && ( - ! [[ ${destinationrenamepath[$destination]} =~ %{artist} ]] \ + ! [[ ${destinationrenamepath[$destination]} =~ %\{artist\} ]] \ || [ -n "$artist" ] ) && ( - ! [[ ${destinationrenamepath[$destination]} =~ %{genre} ]] \ + ! [[ ${destinationrenamepath[$destination]} =~ %\{genre\} ]] \ || [ -n "$genre" ] ) && ( - ! [[ ${destinationrenamepath[$destination]} =~ %{title} ]] \ + ! [[ ${destinationrenamepath[$destination]} =~ %\{title\} ]] \ || [ -n "$title" ] ) && ( - ! [[ ${destinationrenamepath[$destination]} =~ %{track} ]] \ + ! [[ ${destinationrenamepath[$destination]} =~ %\{track\} ]] \ || [ -n "$track" ] ) && ( - ! [[ ${destinationrenamepath[$destination]} =~ %{year} ]] \ + ! [[ ${destinationrenamepath[$destination]} =~ %\{year\} ]] \ || [ -n "$year" ] ) && ( - ! [[ ${destinationrenamepath[$destination]} =~ %{disc} ]] \ + ! [[ ${destinationrenamepath[$destination]} =~ %\{disc\} ]] \ || [ -n "$disc" ] ) ) diff --git a/lib/files/getDestFile b/lib/files/getDestFile index e943b41..1ec9087 100644 --- a/lib/files/getDestFile +++ b/lib/files/getDestFile @@ -3,28 +3,28 @@ getDestFile() { if [ -n "${destinationrename[$destination]}" ] \ && ( ( - ! [[ ${destinationrename[$destination]} =~ %{album} ]] \ + ! [[ ${destinationrename[$destination]} =~ %\{album\} ]] \ || [ -n "$album" ] ) && ( - ! [[ ${destinationrename[$destination]} =~ %{albumartist} ]] \ + ! [[ ${destinationrename[$destination]} =~ %\{albumartist\} ]] \ || [ -n "$albumartist" ] ) && ( - ! [[ ${destinationrename[$destination]} =~ %{artist} ]] \ + ! [[ ${destinationrename[$destination]} =~ %\{artist\} ]] \ || [ -n "$artist" ] ) && ( - ! [[ ${destinationrename[$destination]} =~ %{genre} ]] \ + ! [[ ${destinationrename[$destination]} =~ %\{genre\} ]] \ || [ -n "$genre" ] ) && ( - ! [[ ${destinationrename[$destination]} =~ %{title} ]] \ + ! [[ ${destinationrename[$destination]} =~ %\{title\} ]] \ || [ -n "$title" ] ) && ( - ! [[ ${destinationrename[$destination]} =~ %{track} ]] \ + ! [[ ${destinationrename[$destination]} =~ %\{track\} ]] \ || [ -n "$track" ] ) && ( - ! [[ ${destinationrename[$destination]} =~ %{year} ]] \ + ! [[ ${destinationrename[$destination]} =~ %\{year\} ]] \ || [ -n "$year" ] ) && ( - ! [[ ${destinationrename[$destination]} =~ %{disc} ]] \ + ! [[ ${destinationrename[$destination]} =~ %\{disc\} ]] \ || [ -n "$disc" ] ) ) From 0d511b26745cc8e93e1420792be2f7301cf97900 Mon Sep 17 00:00:00 2001 From: Vincent Riquer Date: Sat, 4 May 2013 01:38:48 +0200 Subject: [PATCH 13/35] toys/cleandestinations: handle special chars correctly --- toys/cleandestinations | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/toys/cleandestinations b/toys/cleandestinations index a2e9845..b80e767 100755 --- a/toys/cleandestinations +++ b/toys/cleandestinations @@ -61,7 +61,7 @@ checkwanted() { for destination in "${!destinationpath[@]}" do echo -ne "\rScanning destination $destination... \033[K" - while read filename + while read -r filename do if ! Select destination_files id \ >/dev/null \ From 2b55e7ef66fd10dfeccc142928823beffa45c43c Mon Sep 17 00:00:00 2001 From: Vincent Riquer Date: Sat, 4 May 2013 03:14:26 +0200 Subject: [PATCH 14/35] string "vorbis" may not appear on line 1 --- lib/files/getFiles | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/files/getFiles b/lib/files/getFiles index 1b53e6f..08a145f 100644 --- a/lib/files/getFiles +++ b/lib/files/getFiles @@ -19,7 +19,7 @@ getFiles() { mimetype=$(file -b --mime-type "$sourcepath/$filename") if [[ $mimetype == application/ogg ]] then - case "$(head -n1 "$sourcepath/$filename")" in + case "$(head -n5 "$sourcepath/$filename")" in *'vorbis'*) mimetype+=' vorbis' ;; From b3d39239ca5e900ad67888a3221f5095d1b6fac5 Mon Sep 17 00:00:00 2001 From: Vincent Riquer Date: Sun, 5 May 2013 04:10:07 +0200 Subject: [PATCH 15/35] Ogg/Opus: Fix rate/channels/bitrate --- lib/tags/getInfos::Ogg | 6 +++--- lib/tags/getInfos::Opus | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/lib/tags/getInfos::Ogg b/lib/tags/getInfos::Ogg index c376496..658c3ab 100644 --- a/lib/tags/getInfos::Ogg +++ b/lib/tags/getInfos::Ogg @@ -1,5 +1,5 @@ #!/bin/bash -getInfosOgg_version='Ogg-1' +getInfosOgg_version='Ogg-2' tagreaders+=( "$getInfosOgg_version" ) getInfos::Ogg() { tagreader="$getInfosOgg_version" @@ -22,9 +22,9 @@ getInfos::Ogg() { tracknum="$tracknum/$tracktotal" fi year=$(gettag date) - infos="${infos/: /=}" + infos="${infos//: /=}" rate=$(gettag rate|head -n1) channels=$(gettag channels|head -n1) - bitrate=$(gettag 'nominal bitrate') + bitrate=$(gettag 'average bitrate') bitrate=${bitrate%%,*} } diff --git a/lib/tags/getInfos::Opus b/lib/tags/getInfos::Opus index 2f99289..bed6242 100644 --- a/lib/tags/getInfos::Opus +++ b/lib/tags/getInfos::Opus @@ -1,5 +1,5 @@ #!/bin/bash -getInfosOpus_version='Opus-1' +getInfosOpus_version='Opus-2' tagreaders+=( "$getInfosOpus_version" ) getInfos::Opus() { tagreader="$getInfosOpus_version" @@ -22,7 +22,7 @@ getInfos::Opus() { tracknum="$tracknum/$tracktotal" fi year=$(gettag date) - infos="${infos/: /=}" + infos="${infos//: /=}" rate=$(gettag 'original sample rate'|head -n1) channels=$(gettag channels|head -n1) bitrate=$(gettag 'average bitrate') From fb2ec4bd9cd15f1f7fd9fbb1e91d33331a6b6f28 Mon Sep 17 00:00:00 2001 From: Vincent Riquer Date: Sat, 4 May 2013 03:04:40 +0200 Subject: [PATCH 16/35] createindex DRAFT --- toys/createindex | 402 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 402 insertions(+) create mode 100755 toys/createindex diff --git a/toys/createindex b/toys/createindex new file mode 100755 index 0000000..d51a458 --- /dev/null +++ b/toys/createindex @@ -0,0 +1,402 @@ +#!/bin/bash + +#!/bin/bash + +# config structures +declare -A \ + destinationchannels \ + destinationfat32compat \ + destinationcopymime \ + destinationformat \ + destinationfrequency \ + destinationid \ + destinationloss \ + destinationmaxbps \ + destinationnormalize \ + destinationpath \ + destinationquality \ + destinationrename \ + destinationnoresample \ + destinationrenamepath \ + destinationskipmime \ +|| { + echo "Check your Bash version. You need >= 4.0" >&2 + exit $EBASHVERS +} + +declare -r \ + DOCDIR=./doc \ + LIBDIR=./lib \ + SHAREDIR=./share +declare -r \ + exampleconf=$DOCDIR/example.cfg \ + schema=$SHAREDIR/schema.sql \ + \ + oldIFS="$IFS" + +cffile="$HOME/.atom/atom.cfg" +LC_ALL=C + +shopt -s extglob + +for function in "$LIBDIR"/*/* +do + source "$function" +done + +while [ -n "$1" ] +do + opt="$1" + shift + case $opt in + '-A') show+=(albumartists) ;; + '-l') show+=(albums) ;; + '-a') show+=(artists) ;; + '-b') show+=(bitrates) ;; + '-c') show+=(composers) ;; + '-C') show+=(channelss) ;; + '-d') show+=(discs) ;; + '-f') show+=(path) ;; + '-g') show+=(genres) ;; + '-m') show+=(oldtimestamp) ;; + '-M') show+=(types) ;; + '-p') show+=(performers) ;; + '-s') show+=(rates) ;; + '-t') show+=(titles) ;; + '-y') show+=(years) ;; + + '-T') timeformat="$OPTARG" ;; + '-o') output="$1" + continue ;; + '-u') update=1 ;; + '-D') (( debug++ )) ;; + [0-9]*) length[count-1]=$opt + continue ;; + esac + (( count++ )) +done + +[ -z "$output" ] && { + cat <<-EOHelp + No output specified! + -f Path + -b Average bitrate + -C Channels + -s Sample rate + -T Mofification time + + -A Album artist + -l Album + -a Artist + -c Composer + -d Disc + -g Genre + -p Performer + -t Title + -y Year + + -d : date-time format (see 'man date' for possible values) + -o : output file (path relative to Source) + + -u : update database first + -D : debug + EOHelp + exit 1 +} + +getConfig + +openDatabase + +columns="${show[@]//*/-}" + +if [[ "$output" == - ]] +then + exec > >(column -x -c ${#show[@]}) +else + exec > >(column -x -c ${#show[@]} > "$output") +fi + +printline() { + for index in ${!show[@]} + do + info="${show[index]}" + locallength="${length[index]}" + path="$olddir" + case $info in + 'bitrates') + info=$((bitrates/count)) + (( info )) || unset info + ;; + 'oldtimestamp') + info=$(printDate ${!info}) + ;; + *) + info="${!info}" + ;; + esac + if [ -n "$locallength" ] + then + echo "${info:0:$locallength}" + else + echo "$info" + fi + done +} + +if (( update )) +then + getFiles + updateMimes + updateTags +fi + +printDate() { + date -d"@$1" +"${timeformat:-%x %X}" +} + +for index in ${!show[@]} +do + info="${show[index]}" + locallength="${length[index]}" + case $info in + albumartists) info="Album artist" ;; + albums) info="Album" ;; + artists) info="Artist" ;; + bitrates) info="Bitrate" ;; + composers) info="Composer" ;; + channelss) info="Channels" ;; + discs) info="Disc" ;; + path) info="Directory name" ;; + genres) info="Genre" ;; + oldtimestamp) info="Last modified" ;; + types) info="Format" ;; + performers) info="Performer" ;; + rates) info="Sample rate" ;; + titles) info="Title" ;; + years) info="Date" ;; + esac + if [ -n "$locallength" ] + then + line+="${line+ +}${info:0:$locallength}" + else + line+="${line+ +}$info" + fi +done +echo "$line" +#echo "${line//[^\n]/-}" +unset line + +echo ' +SELECT + source_files.filename, + tags.bitrate, + tags.channels, + tags.rate, + source_files.last_change, + mime_types.mime_text, + tags.albumartist, + tags.album, + tags.artist, + tags.composer, + tags.disc, + tags.genre, + tags.performer, + tags.title, + tags.year +FROM source_files + INNER JOIN mime_types + ON source_files.mime_type=mime_types.id + INNER JOIN tags + ON source_files.id=tags.source_file +WHERE + NOT mime_types.mime_text LIKE "text/%" +AND NOT mime_types.mime_text LIKE "image/%" +ORDER BY source_files.filename +COLLATE NOCASE; + +SELECT "AtOM:NoMoreFiles";' >&3 + +read -u4 line +until [[ $line == AtOM:NoMoreFiles ]] +do + files+=("$line") + read -u4 line +done + +for line in "${files[@]}" +do + filename="${line%%::AtOM:SQL:Sep::*}" + dir="${filename%/*}" + rest="${line#*::AtOM:SQL:Sep::}::AtOM:SQL:Sep::" + bitrate="${rest%%::AtOM:SQL:Sep::*}" + rest="${rest#*::AtOM:SQL:Sep::}" + channels="${rest%%::AtOM:SQL:Sep::*}" + rest="${rest#*::AtOM:SQL:Sep::}" + rate="${rest%%::AtOM:SQL:Sep::*}" + rest="${rest#*::AtOM:SQL:Sep::}" + timestamp="${rest%%::AtOM:SQL:Sep::*}" + timestamp="${timestamp%%.*}" + rest="${rest#*::AtOM:SQL:Sep::}" + mimetype="${rest%%::AtOM:SQL:Sep::*}" + rest="${rest#*::AtOM:SQL:Sep::}" + albumartist="${rest%%::AtOM:SQL:Sep::*}" + rest="${rest#*::AtOM:SQL:Sep::}" + album="${rest%%::AtOM:SQL:Sep::*}" + rest="${rest#*::AtOM:SQL:Sep::}" + artist="${rest%%::AtOM:SQL:Sep::*}" + rest="${rest#*::AtOM:SQL:Sep::}" + composer="${rest%%::AtOM:SQL:Sep::*}" + rest="${rest#*::AtOM:SQL:Sep::}" + disc="${rest%%::AtOM:SQL:Sep::*}" + rest="${rest#*::AtOM:SQL:Sep::}" + genre="${rest%%::AtOM:SQL:Sep::*}" + rest="${rest#*::AtOM:SQL:Sep::}" + performer="${rest%%::AtOM:SQL:Sep::*}" + rest="${rest#*::AtOM:SQL:Sep::}" + title="${rest%%::AtOM:SQL:Sep::*}" + rest="${rest#*::AtOM:SQL:Sep::}" + year="${rest%%::AtOM:SQL:Sep::*}" + rest="${rest#*::AtOM:SQL:Sep::}" + case $mimetype in + application/ogg\ opus) type=Opus ;; + application/ogg\ vorbis) type=Vorbis ;; + audio/mp4) type=MPEG4\ Audio;; + audio/mpeg) type=MPEG\ Audio;; + audio/x-flac) type=FLAC ;; + video/mpeg) type=MPEG\ Video;; + video/webm) type=WebM ;; + audio/*) type=Other\ Audio;; + video/*) type=Other\ Video;; + *) type=Other ;; + esac + if [[ $dir == $olddir ]] + then + (( $bitrate )) && (( count++ , bitrates+=bitrate )) + (( + oldtimestamp = ( + timestamp > oldtimestamp + ? timestamp + : oldtimestamp + ) + )) + expr1='(^|,)' + expr2='(,|$)' + if ! [[ $channelss =~ $expr1"$channels"$expr2 ]] + then + if [ -n "$channelss" ] \ + && (( channels < ${channelss%%,*} )) + then + channelss="$channels,$channelss" + else + channelss+="${channelss+,}$channels" + fi + fi + if ! [[ $rates =~ $expr1"$rate"$expr2 ]] + then + if [ -n "$rates" ] \ + && (( rate < ${rates%%,*} )) + then + rates="$rate,$rates" + else + rates+="${rates+,}$rate" + fi + fi + if ! [[ $types =~ $expr1"$type"$expr2 ]] + then + [ -z "$types" ] \ + && unset types + [ -n "$type" ] \ + && types+="${types+,}$type" + fi + if ! [[ $albumartists =~ $expr1"$albumartist"$expr2 ]] + then + [ -z "$albumartists" ] \ + && unset albumartists + [ -n "$albumartist" ] \ + && albumartists+="${albumartists+,}$albumartist" + fi + if ! [[ $albums =~ $expr1"$album"$expr2 ]] + then + [ -z "$albums" ] \ + && unset albums + [ -n "$album" ] \ + && albums+="${albums+,}$album" + fi + if ! [[ $artists =~ $expr1"$artist"$expr2 ]] + then + [ -z "$artists" ] \ + && unset artists + [ -n "$artist" ] \ + && artists+="${artists+,}$artist" + fi + if ! [[ $composers =~ $expr1"$composer"$expr2 ]] + then + [ -z "$composers" ] \ + && unset composers + [ -n "$composer" ] \ + && composers+="${composers+,}$composer" + fi + if ! [[ $discs =~ $expr1"$disc"$expr2 ]] + then + [ -z "$discs" ] \ + && unset discs + [ -n "$disc" ] \ + && discs+="${discs+,}$disc" + fi + if ! [[ $genres =~ $expr1"$genre"$expr2 ]] + then + [ -z "$genres" ] \ + && unset genres + [ -n "$genre" ] \ + && genres+="${genres+,}$genre" + fi + if ! [[ $performers =~ $expr1"$performer"$expr2 ]] + then + [ -z "$performers" ] \ + && unset performers + [ -n "$performer" ] \ + && performers+="${performers+,}$performer" + fi + if ! [[ $titles =~ $expr1"$title"$expr2 ]] + then + [ -z "$titles" ] \ + && unset titles + [ -n "$title" ] \ + && titles+="${titles+,}$title" + fi + if ! [[ $years =~ $expr1"$year"$expr2 ]] + then + [ -z "$years" ] \ + && unset years + [ -n "$year" ] \ + && years+="${years+,}$year" + fi + else + if [ -n "$olddir" ] + then + printline + fi + unset bitrates + channelss="$channels" + rates="$rate" + types="$type" + albumartists="$albumartist" + albums="$album" + artists="$artist" + composers="$composer" + discs="$disc" + genres="$genre" + performers="$performer" + titles="$title" + years="$year" + (( bitrate )) && (( count=1 , bitrates=bitrate )) + oldmimetype=$mimetype + oldrate=$rate + oldtimestamp=$timestamp + fi + olddir="$dir" +done +printline From 0ae3252665302ed713235c57ab2c42cf10de4bee Mon Sep 17 00:00:00 2001 From: Vincent Riquer Date: Wed, 8 May 2013 14:30:26 +0200 Subject: [PATCH 17/35] fix "&" and special characters in filename --- atom | 2 +- lib/encode/encodeFile::mp3 | 7 ++++++- lib/encode/encodeFile::opus | 7 ++++++- lib/encode/encodeFile::vorbis | 5 +++++ lib/files/sanitizeFile | 7 ------- 5 files changed, 18 insertions(+), 10 deletions(-) diff --git a/atom b/atom index 5c35308..a3f86b7 100755 --- a/atom +++ b/atom @@ -616,7 +616,7 @@ do rest=${line#*::AtOM:SQL:Sep::} filename=${rest%%::AtOM:SQL:Sep::*} oldfilename=${rest#*::AtOM:SQL:Sep::} - if [[ $oldfilename != $filename ]] && [ -f "$oldfilename" ] + if [[ $oldfilename != "$filename" ]] && [ -f "$oldfilename" ] then rm -f "$oldfilename" fi diff --git a/lib/encode/encodeFile::mp3 b/lib/encode/encodeFile::mp3 index c958f2a..1496570 100644 --- a/lib/encode/encodeFile::mp3 +++ b/lib/encode/encodeFile::mp3 @@ -51,7 +51,12 @@ encodeFile::mp3() { $( for key in ${!lameopts[@]} do - echo "cmd_arg$key ${lameopts[key]}" + cleanedopts="${lameopts[key]//\&/\\\&}" + cleanedopts="${cleanedopts//\[/\\[}" + cleanedopts="${cleanedopts//\]/\\]}" + cleanedopts="${cleanedopts//\{/\\{}" + cleanedopts="${cleanedopts//\}/\\\}}" + echo "cmd_arg$key $cleanedopts" done ) cleanup $tempdir/$tmpfile.wav diff --git a/lib/encode/encodeFile::opus b/lib/encode/encodeFile::opus index 2794a05..8e9bbcd 100644 --- a/lib/encode/encodeFile::opus +++ b/lib/encode/encodeFile::opus @@ -26,7 +26,12 @@ encodeFile::opus() { $( for key in ${!opusencopts[@]} do - echo "cmd_arg$key ${opusencopts[key]}" + cleanedopts="${opusencopts[key]//\&/\\\&}" + cleanedopts="${cleanedopts//\[/\\[}" + cleanedopts="${cleanedopts//\]/\\]}" + cleanedopts="${cleanedopts//\{/\\{}" + cleanedopts="${cleanedopts//\}/\\\}}" + echo "cmd_arg$key $cleanedopts" done ) cleanup $tempdir/$tmpfile.wav diff --git a/lib/encode/encodeFile::vorbis b/lib/encode/encodeFile::vorbis index c3a72e3..7c32538 100644 --- a/lib/encode/encodeFile::vorbis +++ b/lib/encode/encodeFile::vorbis @@ -22,6 +22,11 @@ encodeFile::vorbis() { $( for key in ${!oggencopts[@]} do + cleanedopts="${oggencopts[key]//\&/\\\&}" + cleanedopts="${cleanedopts//\[/\\[}" + cleanedopts="${cleanedopts//\]/\\]}" + cleanedopts="${cleanedopts//\{/\\{}" + cleanedopts="${cleanedopts//\}/\\\}}" echo "cmd_arg$key ${oggencopts[key]}" done ) diff --git a/lib/files/sanitizeFile b/lib/files/sanitizeFile index 2968f5e..07e8e50 100644 --- a/lib/files/sanitizeFile +++ b/lib/files/sanitizeFile @@ -27,12 +27,5 @@ sanitizeFile() { string=${string/%+(.)/} fi fi - # Not exactly filename sanitity, but these will make the script fail in - # a decorative way.. - string="${string//&/\\&}" - strint="${string//;/\\;}" - # And these would exhibit strange behaviors - string="${string//\[/\\\[}" - string="${string//\]/\\\]}" echo "$string" } From a51d00b2e95d7dae4dc8d9adb8947d5f99898b8b Mon Sep 17 00:00:00 2001 From: Vincent Riquer Date: Mon, 13 May 2013 10:19:25 +0200 Subject: [PATCH 18/35] createindex --- toys/createindex | 45 +++++++++++++++++++++------------------------ 1 file changed, 21 insertions(+), 24 deletions(-) diff --git a/toys/createindex b/toys/createindex index d51a458..b2c15ac 100755 --- a/toys/createindex +++ b/toys/createindex @@ -35,7 +35,6 @@ declare -r \ oldIFS="$IFS" cffile="$HOME/.atom/atom.cfg" -LC_ALL=C shopt -s extglob @@ -110,18 +109,17 @@ openDatabase columns="${show[@]//*/-}" -if [[ "$output" == - ]] +if ! [[ "$output" == - ]] then - exec > >(column -x -c ${#show[@]}) -else - exec > >(column -x -c ${#show[@]} > "$output") + exec > "$output" fi printline() { + local print for index in ${!show[@]} do info="${show[index]}" - locallength="${length[index]}" + locallength="${length[index]:=50}" path="$olddir" case $info in 'bitrates') @@ -135,13 +133,14 @@ printline() { info="${!info}" ;; esac - if [ -n "$locallength" ] - then - echo "${info:0:$locallength}" - else - echo "$info" - fi + printtmp="${info:0:$locallength}" + until (( ${#printtmp} == locallength )) + do + printtmp+=' ' + done + print+=(${print+|} "$printtmp") done + echo "${print[@]}" } if (( update )) @@ -158,7 +157,7 @@ printDate() { for index in ${!show[@]} do info="${show[index]}" - locallength="${length[index]}" + locallength="${length[index]:=50}" case $info in albumartists) info="Album artist" ;; albums) info="Album" ;; @@ -176,18 +175,16 @@ do titles) info="Title" ;; years) info="Date" ;; esac - if [ -n "$locallength" ] - then - line+="${line+ -}${info:0:$locallength}" - else - line+="${line+ -}$info" - fi + printtmp="${info:0:$locallength}" + until (( ${#printtmp} == locallength )) + do + printtmp+=' ' + done + print+=(${print+|} "$printtmp") done -echo "$line" -#echo "${line//[^\n]/-}" -unset line +echo "${print[@]}" +echo "${print[@]//[^|]/=}" +unset print echo ' SELECT From 5365eb11891bfeafe297759d8804b79341d7f445 Mon Sep 17 00:00:00 2001 From: Vincent Riquer Date: Thu, 16 May 2013 11:55:01 +0200 Subject: [PATCH 19/35] nicer output --- toys/createindex | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/toys/createindex b/toys/createindex index b2c15ac..1291dce 100755 --- a/toys/createindex +++ b/toys/createindex @@ -114,6 +114,18 @@ then exec > "$output" fi +printPath() { + for key in ${!pathparts[@]} + do + if [[ ${pathparts[key]} == ${oldpathparts[key]} ]] + then + echo -n " ${pathparts[key]//?/ }" + else + echo -n "${pathparts[key]}/" + fi + done +} + printline() { local print for index in ${!show[@]} @@ -129,11 +141,34 @@ printline() { 'oldtimestamp') info=$(printDate ${!info}) ;; + 'path') + while [[ $path =~ / ]] + do + pathparts+=("${path%%/*}") + path=${path#*/} + done + pathparts+=("$path") + info=$(printPath) + unset oldpathparts + for key in ${!pathparts[@]} + do + oldpathparts[key]=${pathparts[key]} + done + unset pathparts + ;; *) info="${!info}" ;; esac printtmp="${info:0:$locallength}" + if [ -z "$printtmp" ] + then + until (( ${#printtmp} == locallength/2)) + do + printtmp+=' ' + done + printtmp+='-' + fi until (( ${#printtmp} == locallength )) do printtmp+=' ' From 8c26d11db2960c40256e4882e345ef1a5b8fa1c2 Mon Sep 17 00:00:00 2001 From: Vincent Riquer Date: Thu, 16 May 2013 14:23:03 +0200 Subject: [PATCH 20/35] Document toys/createindex --- toys/README | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/toys/README b/toys/README index d3d7af9..8a57574 100644 --- a/toys/README +++ b/toys/README @@ -1,3 +1,36 @@ +createindex +=========== +Creates a nice index of your collection, similar to what oidua +(http://oidua.suxbad.com/) does, but much faster, as it is using AtOM's DB +instead of rescanning your whole collection. + +Options define what will be shown and may optionnally be followed by the column +width (default: 50). + +Options: + -f: Path + -b: Average bitrate + -C: Channels + -s: Sample rate + -T: Mofification time + + -A: Album artist + -l: Album + -a: Artist + -c: Composer + -d: Disc + -g: Genre + -p: Performer + -t: Title + -y: Year + + -d : date-time format (see 'man date' for possible values) + -o -|: output file (path relative to Source) - mandatory - must + appear last. + + -u: update database first + -D: debug + checkextensions =============== Reports files whose extension does not match the (detected) mime-type. From bb484454a6fde81bec06b9ce25b3275738ea9eb7 Mon Sep 17 00:00:00 2001 From: Vincent Riquer Date: Thu, 16 May 2013 14:31:45 +0200 Subject: [PATCH 21/35] toys/createindex: fix doc and usage --- toys/README | 5 +++-- toys/createindex | 5 +++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/toys/README b/toys/README index 8a57574..f9f57ab 100644 --- a/toys/README +++ b/toys/README @@ -12,7 +12,8 @@ Options: -b: Average bitrate -C: Channels -s: Sample rate - -T: Mofification time + -m: Mofification time + -M: Format -A: Album artist -l: Album @@ -24,7 +25,7 @@ Options: -t: Title -y: Year - -d : date-time format (see 'man date' for possible values) + -T : date-time format (see 'man date' for possible values) -o -|: output file (path relative to Source) - mandatory - must appear last. diff --git a/toys/createindex b/toys/createindex index 1291dce..4fc6034 100755 --- a/toys/createindex +++ b/toys/createindex @@ -82,7 +82,8 @@ done -b Average bitrate -C Channels -s Sample rate - -T Mofification time + -m Mofification time + -M Format -A Album artist -l Album @@ -94,7 +95,7 @@ done -t Title -y Year - -d : date-time format (see 'man date' for possible values) + -T : date-time format (see 'man date' for possible values) -o : output file (path relative to Source) -u : update database first From 24aef12e33f8a3d56b2a4a5fcdcb8d02ff29f030 Mon Sep 17 00:00:00 2001 From: Vincent Riquer Date: Sat, 25 May 2013 15:03:50 +0200 Subject: [PATCH 22/35] smarter tags update trigger --- share/schema.sql | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/share/schema.sql b/share/schema.sql index 2a915e4..1859f55 100644 --- a/share/schema.sql +++ b/share/schema.sql @@ -110,8 +110,23 @@ CREATE TRIGGER IF NOT EXISTS create_tags AFTER INSERT ON source_files BEGIN INSERT INTO tags (source_file,last_change) VALUES (new.id,0); END; -CREATE TRIGGER IF NOT EXISTS force_destination_update_on_tag_update - AFTER UPDATE ON tags +DROP TRIGGER IF EXISTS force_destination_update_on_tag_update; +CREATE TRIGGER force_destination_update_on_tag_update + AFTER UPDATE OF + genre, + albumartist, + year, + album, + disc, + artist, + track, + title, + composer, + performer, + rate, + channels, + bitrate + ON tags BEGIN UPDATE destination_files SET last_change=0 WHERE source_file_id=old.source_file; From 64b14b67f7f7219b59f8e522719463756e51e0ba Mon Sep 17 00:00:00 2001 From: Vincent Riquer Date: Sat, 25 May 2013 15:04:50 +0200 Subject: [PATCH 23/35] updateTags(): read old data --- lib/tags/updateTags | 44 ++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 42 insertions(+), 2 deletions(-) diff --git a/lib/tags/updateTags b/lib/tags/updateTags index d317c1a..c473ee4 100644 --- a/lib/tags/updateTags +++ b/lib/tags/updateTags @@ -29,7 +29,20 @@ updateTags() { source_files.id, source_files.last_change, mime_type_actions.mime_text, - source_files.filename + source_files.filename, + tags.album, + tags.albumartist, + tags.artist, + tags.composer, + tags.disc, + tags.genre, + tags.performer, + tags.title, + tags.track, + tags.year, + tags.rate, + tags.channels, + tags.bitrate FROM source_files INNER JOIN destination_files ON destination_files.source_file_id=source_files.id @@ -63,7 +76,34 @@ updateTags() { lastchange=${rest%%::AtOM:SQL:Sep::*} rest=${rest#*::AtOM:SQL:Sep::} mimetype=${rest%%::AtOM:SQL:Sep::*} - filename=${rest#*::AtOM:SQL:Sep::} + rest=${rest#*::AtOM:SQL:Sep::} + filename=${rest%%::AtOM:SQL:Sep::*} + rest=${rest#*::AtOM:SQL:Sep::} + oldalbum=${rest%%::AtOM:SQL:Sep::*} + rest=${rest#*::AtOM:SQL:Sep::} + oldalbumartist=${rest%%::AtOM:SQL:Sep::*} + rest=${rest#*::AtOM:SQL:Sep::} + oldartist=${rest%%::AtOM:SQL:Sep::*} + rest=${rest#*::AtOM:SQL:Sep::} + oldcomposer=${rest%%::AtOM:SQL:Sep::*} + rest=${rest#*::AtOM:SQL:Sep::} + olddisc=${rest%%::AtOM:SQL:Sep::*} + rest=${rest#*::AtOM:SQL:Sep::} + oldgenre=${rest%%::AtOM:SQL:Sep::*} + rest=${rest#*::AtOM:SQL:Sep::} + oldperformer=${rest%%::AtOM:SQL:Sep::*} + rest=${rest#*::AtOM:SQL:Sep::} + oldtitle=${rest%%::AtOM:SQL:Sep::*} + rest=${rest#*::AtOM:SQL:Sep::} + oldtrack=${rest%%::AtOM:SQL:Sep::*} + rest=${rest#*::AtOM:SQL:Sep::} + oldyear=${rest%%::AtOM:SQL:Sep::*} + rest=${rest#*::AtOM:SQL:Sep::} + oldrate=${rest%%::AtOM:SQL:Sep::*} + rest=${rest#*::AtOM:SQL:Sep::} + oldchannels=${rest%%::AtOM:SQL:Sep::*} + rest=${rest#*::AtOM:SQL:Sep::} + oldbitrate=${rest%%::AtOM:SQL:Sep::*} echo -en "\rTags: $((++count*100/filecount))%" if (( count % 1000 == 0 )) then From 055ba6dec09e499b6a2f8bbcd5f88858b95d8374 Mon Sep 17 00:00:00 2001 From: Vincent Riquer Date: Sat, 25 May 2013 15:35:04 +0200 Subject: [PATCH 24/35] Copy more reliably when path change is not set --- lib/copy/copyFiles_action | 63 ++++++++++++++++++--------------------- lib/copy/guessPath | 28 +++++++++++++++++ 2 files changed, 57 insertions(+), 34 deletions(-) create mode 100644 lib/copy/guessPath diff --git a/lib/copy/copyFiles_action b/lib/copy/copyFiles_action index a944ef8..0134eb5 100644 --- a/lib/copy/copyFiles_action +++ b/lib/copy/copyFiles_action @@ -43,42 +43,37 @@ copyFiles_action() { rest=${rest#*::AtOM:SQL:Sep::} destfileid=${rest%%::AtOM:SQL:Sep::*} rest=${rest#*::AtOM:SQL:Sep::} - echo 'SELECT IFNULL( ( - SELECT destination_files.filename - FROM destination_files - INNER JOIN source_files - ON destination_files.source_file_id=source_files.id - INNER JOIN mime_type_actions - ON - mime_type_actions.id=source_files.mime_type - INNER JOIN destinations - ON destinations.id=destination_files.destination_id - WHERE destinations.id = '$destinationid' - AND source_files.filename LIKE - "'"${sourcedir//\"/\"\"}"'/%" - AND mime_type_actions.action = 1 - LIMIT 1 - ),"AtOM:NotFound"); - '>&3 - read -u4 filename - if [[ $filename != AtOM:NotFound ]] - then - destdir=${filename%/*} - if cp -al "$sourcepath/$sourcefilename" "$destdir" 2>/dev/null\ - || cp -a "$sourcepath/$sourcefilename" "$destdir" - then - Update destination_files \ - filename "$destdir/${sourcefilename##*/}"\ - rename_pattern "${destinationrenamepath[$destination]}/${destinationrename[$destination]}:${destinationfat32compat["$destination"]}"\ - last_change $lastchange \ - <<-EOWhere - id = $destfileid - EOWhere - (( done++ )) - fi - fi (( count++ )) printf '\b\b\b\b%3i%%' $(( (count * 100) / ${#copyfiles[@]} )) + if [ -n "${renamepath["$destination"]}" ] + then + destdir="$(guessPath)" || continue + else + destdir="${destinationpath["$destination"]}/" + destdir+=$(sanitizeFile "${sourcefilename%%/*}" dir) + part=${sourcefilename#*/} + while [[ $part =~ / ]] + do + destdir+="/$(sanitizeFile "${part%%/*}" dir)" + part=${part#*/} + done + if ! [ -d "$destdir" ] + then + mkdir -p "$destdir" + fi + fi + if cp -al "$sourcepath/$sourcefilename" "$destdir" 2>/dev/null\ + || cp -a "$sourcepath/$sourcefilename" "$destdir" + then + Update destination_files \ + filename "$destdir/${sourcefilename##*/}"\ + rename_pattern "${destinationrenamepath[$destination]}/${destinationrename[$destination]}:${destinationfat32compat["$destination"]}"\ + last_change $lastchange \ + <<-EOWhere + id = $destfileid + EOWhere + (( done++ )) + fi done echo 'COMMIT;' >&3 echo -e "\rCopied ${done:-0} of $count files.\033[K" diff --git a/lib/copy/guessPath b/lib/copy/guessPath new file mode 100644 index 0000000..ccb0e4b --- /dev/null +++ b/lib/copy/guessPath @@ -0,0 +1,28 @@ +#!/bin/bash + +guessPath() { + echo 'SELECT IFNULL( ( + SELECT destination_files.filename + FROM destination_files + INNER JOIN source_files + ON destination_files.source_file_id=source_files.id + INNER JOIN mime_type_actions + ON + mime_type_actions.id=source_files.mime_type + INNER JOIN destinations + ON destinations.id=destination_files.destination_id + WHERE destinations.id = '$destinationid' + AND source_files.filename LIKE + "'"${sourcedir//\"/\"\"}"'/%" + AND mime_type_actions.action = 1 + LIMIT 1 + ),"AtOM:NotFound"); + '>&3 + read -u4 filename + if [[ $filename != AtOM:NotFound ]] + then + echo "${filename%/*}" + else + return 1 + fi +} From 822a266c7cb31be872124abb422ccce16b9875b5 Mon Sep 17 00:00:00 2001 From: Vincent Riquer Date: Sat, 25 May 2013 23:46:51 +0200 Subject: [PATCH 25/35] only update changed tags --- lib/tags/updateTags | 80 +++++++++++++++++++++++++-------------------- 1 file changed, 44 insertions(+), 36 deletions(-) diff --git a/lib/tags/updateTags b/lib/tags/updateTags index c473ee4..d2d00ac 100644 --- a/lib/tags/updateTags +++ b/lib/tags/updateTags @@ -1,29 +1,23 @@ #!/bin/bash updateTags() { + local \ + ual \ + uaa \ + uar \ + uco \ + udi \ + uge \ + upe \ + uti \ + utr \ + uye \ + ura \ + uch \ + ubi for reader in "${tagreaders[@]}" do tagreaderclause+="${tagreaderclause:+ AND }NOT tags.tagreader = \"$reader\"" done - echo ' - SELECT COUNT(DISTINCT source_files.filename) - FROM source_files - INNER JOIN destination_files - ON destination_files.source_file_id=source_files.id - INNER JOIN destinations - ON destination_files.destination_id=destinations.id - INNER JOIN mime_type_actions - ON destinations.id=mime_type_actions.destination_id - INNER JOIN tags - ON source_files.id=tags.source_file - WHERE mime_type_actions.id = source_files.mime_type - AND ( - CAST(tags.last_change AS TEXT) - <> - CAST(source_files.last_change AS TEXT) - OR ('"$tagreaderclause"') - ) - AND mime_type_actions.action = 1;' >&3 - read -u4 filecount echo ' SELECT DISTINCT source_files.id, @@ -66,6 +60,7 @@ updateTags() { while ! [[ $line = AtOM:NoMoreFiles ]] do tagfiles+=("$line") + (( filecount++ )) read -u4 line done echo 'BEGIN TRANSACTION;' >&3 @@ -113,22 +108,35 @@ updateTags() { fi if getTags then - Update tags \ - album "${album:-NULL}" \ - albumartist "${albumartist:-NULL}" \ - artist "${artist:-NULL}" \ - composer "${composer:-NULL}" \ - disc "${disc:-NULL}" \ - genre "${genre:-NULL}" \ - performer "${performer:-NULL}" \ - title "${title:-NULL}" \ - track "${tracknum:-NULL}" \ - year "${year:-NULL}" \ - last_change "$lastchange" \ - rate "${rate:-NULL}" \ - channels "${channels:-NULL}" \ - bitrate "${bitrate:-NULL}" \ - tagreader "$tagreader" \ + [[ $oldalbum != "$album" ]]&& ual=1 + [[ $oldalbumartist != "$albumartist" ]]&&uaa=1 + [[ $oldartist != "$artist" ]]&& uar=1 + [[ $oldcomposer != "$composer" ]]&& uco=1 + [[ $olddisc != "$disc" ]]&& udi=1 + [[ $oldgenre != "$genre" ]]&& uge=1 + [[ $oldperformer != "$performer" ]]&& upe=1 + [[ $oldtitle != "$title" ]]&& uti=1 + [[ $oldtrack != "$track" ]]&& utr=1 + [[ $oldyear != "$year" ]]&& uye=1 + [[ $oldrate != "$rate" ]]&& ura=1 + [[ $oldchannels != "$channels" ]]&& uch=1 + [[ $oldbitrate != "$bitrate" ]]&& ubi=1 + Update tags \ + ${ual:+album "${album:-NULL}"} \ + ${uaa:+albumartist "${albumartist:-NULL}"} \ + ${uar:+artist "${artist:-NULL}"} \ + ${uco:+composer "${composer:-NULL}"} \ + ${udi:+disc "${disc:-NULL}"} \ + ${uge:+genre "${genre:-NULL}"} \ + ${upe:+performer "${performer:-NULL}"} \ + ${uti:+title "${title:-NULL}"} \ + ${utr:+track "${tracknum:-NULL}"} \ + ${uye:+year "${year:-NULL}"} \ + last_change "$lastchange" \ + ${ura:+rate "${rate:-NULL}"} \ + ${uch:+channels "${channels:-NULL}"} \ + ${ubi:+bitrate "${bitrate:-NULL}"} \ + tagreader "$tagreader" \ >/dev/null <<<"source_file = $sourcefileid" unset genre \ albumartist \ From d92460f3bf98c028796de4eb1e279af4789ebb8e Mon Sep 17 00:00:00 2001 From: Vincent Riquer Date: Sun, 26 May 2013 01:41:55 +0200 Subject: [PATCH 26/35] fix tracknumber handling --- lib/tags/updateTags | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/tags/updateTags b/lib/tags/updateTags index d2d00ac..ff2351a 100644 --- a/lib/tags/updateTags +++ b/lib/tags/updateTags @@ -116,7 +116,7 @@ updateTags() { [[ $oldgenre != "$genre" ]]&& uge=1 [[ $oldperformer != "$performer" ]]&& upe=1 [[ $oldtitle != "$title" ]]&& uti=1 - [[ $oldtrack != "$track" ]]&& utr=1 + [[ $oldtrack != "$tracknum" ]]&& utr=1 [[ $oldyear != "$year" ]]&& uye=1 [[ $oldrate != "$rate" ]]&& ura=1 [[ $oldchannels != "$channels" ]]&& uch=1 From 7efd2bc513285c56a94011b0aa9fa96ba8939735 Mon Sep 17 00:00:00 2001 From: Vincent Riquer Date: Sun, 26 May 2013 22:50:20 +0200 Subject: [PATCH 27/35] Reset tag-update flags --- lib/tags/updateTags | 29 ++++++++++++++--------------- 1 file changed, 14 insertions(+), 15 deletions(-) diff --git a/lib/tags/updateTags b/lib/tags/updateTags index ff2351a..9036869 100644 --- a/lib/tags/updateTags +++ b/lib/tags/updateTags @@ -1,19 +1,5 @@ #!/bin/bash updateTags() { - local \ - ual \ - uaa \ - uar \ - uco \ - udi \ - uge \ - upe \ - uti \ - utr \ - uye \ - ura \ - uch \ - ubi for reader in "${tagreaders[@]}" do tagreaderclause+="${tagreaderclause:+ AND }NOT tags.tagreader = \"$reader\"" @@ -150,7 +136,20 @@ updateTags() { performer \ rate \ bitrate \ - channels + channels \ + ual \ + uaa \ + uar \ + uco \ + udi \ + uge \ + upe \ + uti \ + utr \ + uye \ + ura \ + uch \ + ubi fi done echo 'COMMIT;' >&3 From 47e8f1dc22397aedbea7a45ff44c5b657911ff22 Mon Sep 17 00:00:00 2001 From: Vincent Riquer Date: Mon, 27 May 2013 12:44:39 +0200 Subject: [PATCH 28/35] force some fields as text --- .gitignore | 3 +++ lib/database/Insert | 4 ++++ lib/database/InsertIfUnset | 3 ++- lib/database/Update | 4 ++++ lib/tags/updateTags | 14 +++++++------- 5 files changed, 20 insertions(+), 8 deletions(-) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..21963fd --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +*.ex +*.EX +trace.log diff --git a/lib/database/Insert b/lib/database/Insert index 77f7ec2..32d09be 100644 --- a/lib/database/Insert +++ b/lib/database/Insert @@ -15,6 +15,10 @@ Insert() { insert_keys+='`'"$key"'`' (( ${#insert_values} )) && insert_values+="," case $value in + '::AtOM:FT::'*) + value="${value//::AtOM:FT::/}" + insert_values+='"'"${value//\"/\"\"}"'"' + ;; 'NULL') insert_values+="NULL" ;; diff --git a/lib/database/InsertIfUnset b/lib/database/InsertIfUnset index 4277a62..5f53e7a 100644 --- a/lib/database/InsertIfUnset +++ b/lib/database/InsertIfUnset @@ -25,7 +25,8 @@ InsertIfUnset() { Select "$table" "$column" < <( for key in ${!keys[@]} do - echo "${keys[$key]}" = "${values[$key]}" + echo "${keys[$key]}" = \ + "${values[$key]//::AtOM:FT::}" done ) ) diff --git a/lib/database/Update b/lib/database/Update index 0fe5db5..015eac6 100644 --- a/lib/database/Update +++ b/lib/database/Update @@ -26,6 +26,10 @@ Update() { ;; value) case $argument in + '::AtOM:FT::'*) + argument="${argument//::AtOM:FT::/}" + set_statement+=" = "'"'"${argument//\"/\"\"}"'"' + ;; 'NULL') set_statement+=" = NULL" ;; diff --git a/lib/tags/updateTags b/lib/tags/updateTags index 9036869..9881843 100644 --- a/lib/tags/updateTags +++ b/lib/tags/updateTags @@ -108,15 +108,15 @@ updateTags() { [[ $oldchannels != "$channels" ]]&& uch=1 [[ $oldbitrate != "$bitrate" ]]&& ubi=1 Update tags \ - ${ual:+album "${album:-NULL}"} \ - ${uaa:+albumartist "${albumartist:-NULL}"} \ - ${uar:+artist "${artist:-NULL}"} \ - ${uco:+composer "${composer:-NULL}"} \ + ${ual:+album "::AtOM:FT::${album:-NULL}"}\ + ${uaa:+albumartist "::AtOM:FT::${albumartist:-NULL}"}\ + ${uar:+artist "::AtOM:FT::${artist:-NULL}"}\ + ${uco:+composer "::AtOM:FT::${composer:-NULL}"}\ ${udi:+disc "${disc:-NULL}"} \ ${uge:+genre "${genre:-NULL}"} \ - ${upe:+performer "${performer:-NULL}"} \ - ${uti:+title "${title:-NULL}"} \ - ${utr:+track "${tracknum:-NULL}"} \ + ${upe:+performer "::AtOM:FT::${performer:-NULL}"}\ + ${uti:+title "::AtOM:FT::${title:-NULL}"}\ + ${utr:+track "::AtOM:FT::${tracknum:-NULL}"}\ ${uye:+year "${year:-NULL}"} \ last_change "$lastchange" \ ${ura:+rate "${rate:-NULL}"} \ From 2d345047f75475f4d29a62fe0f1cb5dec20eef46 Mon Sep 17 00:00:00 2001 From: Vincent Riquer Date: Tue, 28 May 2013 13:38:00 +0200 Subject: [PATCH 29/35] toys/lowquality Get file names by format and quality constraints --- toys/lowquality | 130 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 130 insertions(+) create mode 100755 toys/lowquality diff --git a/toys/lowquality b/toys/lowquality new file mode 100755 index 0000000..0e73163 --- /dev/null +++ b/toys/lowquality @@ -0,0 +1,130 @@ +#!/bin/bash + +# config structures +declare -A \ + destinationchannels \ + destinationfat32compat \ + destinationcopymime \ + destinationformat \ + destinationfrequency \ + destinationid \ + destinationloss \ + destinationmaxbps \ + destinationnormalize \ + destinationpath \ + destinationquality \ + destinationrename \ + destinationnoresample \ + destinationrenamepath \ + destinationskipmime \ +|| { + echo "Check your Bash version. You need >= 4.0" >&2 + exit $EBASHVERS +} + +declare -r \ + DOCDIR=./doc \ + LIBDIR=./lib \ + SHAREDIR=./share +declare -r \ + exampleconf=$DOCDIR/example.cfg \ + schema=$SHAREDIR/schema.sql \ + \ + oldIFS="$IFS" + +cffile="$HOME/.atom/atom.cfg" +LC_ALL=C + +shopt -s extglob + +for function in "$LIBDIR"/*/* +do + source "$function" +done + +while getopts 'fm:o:p:*:uD' opt +do + case $opt in + f) mimetypes+=("audio/x-flac") + bitrates+=("") + ;; + m) mimetypes+=("audio/mpeg") + bitrates+=("$OPTARG") + ;; + o) mimetypes+=("application/ogg vorbis") + bitrates+=("$OPTARG") + ;; + p) mimetypes+=("application/ogg opus") + bitrates+=("$OPTARG") + ;; + \*) mimetypes+=("*") + bitrates+=("$OPTARG") + ;; + + u) update=1 ;; + D) (( debug++ )) ;; + esac +done + +getConfig + +openDatabase + +if (( update )) +then + getFiles + updateMimes + updateTags +fi + +echo ' + SELECT + mime_type_actions.mime_text, + tags.bitrate, + source_files.filename + FROM source_files + INNER JOIN tags + ON source_files.id=tags.source_file + INNER JOIN mime_type_actions + ON mime_type_actions.id=source_files.mime_type + WHERE mime_type_actions.action=1 + AND ( +' >&3 + +for indice in ${!mimetypes[@]} +do + (( notfirst )) && echo OR >&3 + case ${mimetypes[indice]} in + 'audio/x-flac') + echo 'mime_type_actions.mime_text="audio/x-flac"'>&3 + ;; + '*') + echo '( ( + NOT mime_type_actions.mime_text="audio/x-flac" + AND NOT mime_type_actions.mime_text="audio/mpeg" + AND NOT + mime_type_actions.mime_text="application/ogg vorbis" + AND NOT + mime_type_actions.mime_text="application/ogg opus" + ) AND tags.bitrate < '${bitrates[indice]}' )' >&3 + ;; + *) + echo '( + mime_type_actions.mime_text="'"${mimetypes[indice]}"'" + AND tags.bitrate < '${bitrates[indice]}' )' >&3 + ;; + esac + notfirst=1 +done +echo ') ORDER BY bitrate;' >&3 + +echo 'SELECT "AtOM:NoMoreFiles";' >&3 + +read -u4 line +until [[ $line == AtOM:NoMoreFiles ]] +do + echo "${line//::AtOM:SQL:Sep::/$'\t'}" + read -u4 line +done + +closeDatabase From d3fdf154c1a98f150ce0e398b1e257f1a8621fd1 Mon Sep 17 00:00:00 2001 From: Vincent Riquer Date: Tue, 28 May 2013 13:49:52 +0200 Subject: [PATCH 30/35] DISTINCT --- toys/lowquality | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/toys/lowquality b/toys/lowquality index 0e73163..0e3395b 100755 --- a/toys/lowquality +++ b/toys/lowquality @@ -78,7 +78,7 @@ then fi echo ' - SELECT + SELECT DISTINCT mime_type_actions.mime_text, tags.bitrate, source_files.filename From a0ce3925a84c97d1f0c7983aeaed2fbdf557cfed Mon Sep 17 00:00:00 2001 From: Vincent Riquer Date: Wed, 29 May 2013 01:47:01 +0200 Subject: [PATCH 31/35] Sanity checks --- README | 1 + atom | 136 +++++++++++++++++++++++++++++++- lib/config/getConfigDestination | 3 + lib/database/openDatabase | 11 --- lib/decode/decodeFile | 3 + lib/tags/getTags | 7 +- 6 files changed, 147 insertions(+), 14 deletions(-) diff --git a/README b/README index b87cabe..be1e3e4 100644 --- a/README +++ b/README @@ -25,6 +25,7 @@ Optional: http://opus-codec.org/ * opusinfo (Opus metadata) * opusenc (Opus encoding) + * opusdec (Opus decoding) * LAME MP3 Encoder http://lame.sourceforge.net/ * lame (MP3 encoding) diff --git a/atom b/atom index a3f86b7..ed283b9 100755 --- a/atom +++ b/atom @@ -96,8 +96,6 @@ do esac done -#FIXME: check sanity - if [ ! -f "$cffile" ] then if [ ! -d ~/.atom ] @@ -120,6 +118,135 @@ set +H (( debug || cfgdump )) && printConfig (( cfgdump )) && exit +# check sanity +if [ ! -d "$tempdir" ] && ! mkdir -p "$tempdir" +then + echo "[FATAL] Could not create temp directory $tempdir" >&2 + (( sanityfail++ )) +fi +if [ ! -f "$database" ] && [ ! -d "${database%/*}" ] && ! mkdir -p "${database%/*}" +then + echo "[FATAL] Directory holding database file does not exist and could" \ + "not be created" >&2 + (( sanityfail++ )) +fi +if [ ! -d "$sourcepath" ] +then + echo "[FATAL] Source path $sourcepath does not exist or is not a directory" >&2 + (( sanityfail++ )) +fi +if ! which sed >/dev/null +then + echo "[FATAL] Required tool sed is not installed or not in PATH + I never thought this would actually hit someone..." >&2 + (( sanityfail++ )) +fi +if ! which sox >/dev/null +then + echo "[FATAL] Required tool sox is not installed or not in PATH" >&2 + (( sanityfail++ )) +fi +if ! which ogginfo >/dev/null +then + echo "[WARNING] Tool ogginfo (from vorbis-tools) is not" \ + "installed or not in PATH + Vorbis metadata disabled" >&2 + disableogginfo=1 + (( sanitywarn++ )) +fi +if (( oggencneeded )) && ! which oggenc >/dev/null +then + echo "[WARNING] Tool oggenc (from vorbis-tools) is not" \ + "installed or not in PATH + Vorbis targets disabled" >&2 + disableoggenc=1 + (( sanitywarn++ )) +fi +if ! which opusinfo >/dev/null +then + echo "[WARNING] Tool opusinfo (from opus-tools) is not" \ + "installed or not in PATH + Opus metadata disabled" >&2 + disableopusinfo=1 + (( sanitywarn++ )) +fi +if (( opusencneeded )) && ! which opusenc >/dev/null +then + echo "[WARNING] Tool opusenc (from opus-tools) is not" \ + "installed or not in PATH + Opus targets disabled" >&2 + disableopusenc=1 + (( sanitywarn++ )) +fi +if ! which opusdec >/dev/null +then + echo "[WARNING] Tool opusdec (from opus-tools) is not" \ + "installed or not in PATH + Opus support disabled" >&2 + disableopusdec=1 + (( sanitywarn++ )) +fi +if (( lameneeded )) && ! which lame >/dev/null +then + echo "[WARNING] Tool lame is not installed or not in PATH + MP3 targets disabled" >&2 + disablelame=1 + (( sanitywarn++ )) +fi +if ! which metaflac >/dev/null +then + echo "[WARNING] Tool metaflac (from FLAC) is not installed" \ + "or not in PATH + FLAC metadata disabled" >&2 + disableflac=1 + (( sanitywarn++ )) +fi +if ! which mpcdec >/dev/null +then + echo "[WARNING] Tool mpcdec (from Musepack) is not" \ + "installed or not in PATH + Musepack support disabled" >&2 + disablempcdec=1 + (( sanitywarn++ )) +fi +if ! which ffprobe >/dev/null +then + echo "[WARNING] Tool ffprobe (from FFmpeg) is not installed or not in PATH + Video metadata disabled + MPEG metadata disabled + MusePack metadata disabled + Unknown format metadata disabled" >&2 + disableffprobe=1 + (( sanitywarn++ )) +fi +if ! which ffmpeg >/dev/null +then + echo "[WARNING] Tool ffmpeg is not installed or not in PATH + Video support disabled" >&2 + disablevideo=1 + (( sanitywarn++ )) +fi +if (( sanityfail )) +then + echo " +Sanity checks raised ${sanitywarn:-0} warnings, $sanityfail failures. Dying now." >&2 + exit $ESANITY +elif (( sanitywarn )) +then + echo " +Sanity checks raised $sanitywarn warnings... Hit Control-C to abort." >&2 + timeout=$(( sanitywarn * 10 )) + echo -n "Starting in $(printf %3i $timeout)" \ + $'seconds...\b\b\b\b\b\b\b\b\b\b\b' >&2 + while (( timeout )) + do + echo -n $'\b\b\b'"$(printf %3i $timeout)" >&2 + sleep 1 + (( timeout-- )) + done + echo -en "\r\033[K" +fi + openDatabase createDestinations @@ -312,6 +439,11 @@ do rest=${rest#*::AtOM:SQL:Sep::} performer=${rest%%::AtOM:SQL:Sep::*} unset rest + case ${destinationformat["$destination"]} in + vorbis) (( disableoggenc )) && continue ;; + opus) (( disableopusenc )) && continue ;; + mp3) (( disablelame )) && continue ;; + esac decodeFile getDestDir getDestFile diff --git a/lib/config/getConfigDestination b/lib/config/getConfigDestination index a6fa6a8..8fa63c6 100644 --- a/lib/config/getConfigDestination +++ b/lib/config/getConfigDestination @@ -8,12 +8,15 @@ getConfigDestination() { case "$value" in 'mp3') destinationformat["$destination"]=mp3 + lameneeded=1 ;; 'opus') destinationformat["$destination"]=opus + opusencneeded=1 ;; 'vorbis') destinationformat["$destination"]=vorbis + oggencneeded=1 ;; *) echo "Unsupported destination format: $value" >2& diff --git a/lib/database/openDatabase b/lib/database/openDatabase index b75cef4..ff829fe 100644 --- a/lib/database/openDatabase +++ b/lib/database/openDatabase @@ -1,18 +1,7 @@ #!/bin/bash openDatabase() { - if [ ! -d "$tempdir" ] - then - mkdir -p "$tempdir" - fi rm -f "$tempdir"/sqlite.{in,out} mkfifo "$tempdir"/sqlite.{in,out} - if [ ! -f "$database" ] - then - if [ ! -d "${database%/*}" ] - then - mkdir -p "${database%/*}" - fi - fi sqlite3 -bail "$database" \ < "$tempdir/sqlite.in" \ > "$tempdir/sqlite.out" & diff --git a/lib/decode/decodeFile b/lib/decode/decodeFile index 09c2dc3..1571222 100644 --- a/lib/decode/decodeFile +++ b/lib/decode/decodeFile @@ -2,6 +2,7 @@ decodeFile() { case "$mimetype" in 'video/'*) + (( disablevideo )) && continue extractAudio if (( ${destinationnormalize["$destination"]}))\ || ( @@ -30,6 +31,7 @@ decodeFile() { then copied=1 else + (( disableopusdec )) && continue decodeOpusdec if (( ${destinationnormalize["$destination"]}))\ || ( @@ -60,6 +62,7 @@ decodeFile() { extendedtype=$(file -b "$sourcepath/$filename") case "$extendedtype" in *'Musepack '*) + (( disablempcdec )) && continue decodeMpcdec if (( ${destinationnormalize["$destination"]}))\ || ( diff --git a/lib/tags/getTags b/lib/tags/getTags index 3f6d9e1..ecd996c 100644 --- a/lib/tags/getTags +++ b/lib/tags/getTags @@ -1,26 +1,31 @@ #!/bin/bash getTags_version='unknown-4' -tagreaders+=( "$getTags_version" ) getTags() { unset type case "$mimetype" in audio/mpeg) type=ffmpeg + (( disableffprobe )) && unset type ;; 'application/ogg opus') type=Opus + (( disableopusinfo )) && unset type ;; application/ogg*) type=Ogg + (( disableogginfo )) && unset type ;; audio/x-flac) type=FLAC + (( disableflac )) && unset type ;; video/*) type=ffmpeg + (( disableffprobe )) && unset type ;; *) type=ffmpeg + (( disableffprobe )) && unset type ;; esac if [ -n "$type" ] From ab92e8b5f68895bb966ec33573305b7c8ca1914c Mon Sep 17 00:00:00 2001 From: Vincent Riquer Date: Wed, 29 May 2013 13:51:21 +0200 Subject: [PATCH 32/35] Apply CLI arguments --- atom | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/atom b/atom index ed283b9..7d21def 100755 --- a/atom +++ b/atom @@ -115,6 +115,11 @@ fi getConfig set +H + +# Apply CLI overrides +[ -n "$cliload" ] && maxload=$cliload +[ -n "$cliltimer" ] && loadinterval=$cliltimer + (( debug || cfgdump )) && printConfig (( cfgdump )) && exit @@ -290,6 +295,19 @@ echo "Suppressed $deleted files, $removed removed from database" updateTags +for forcedest in "${forceall[@]}" +do + if forcedestid=$(Select destinations id <<<"name = $forcedest") + then + echo "Resetting destination files timestamps on" \ + "$forcedest ($forcedestid)..." + Update destination_files last_change 1 \ + <<<"destination_id = $forcedestid" + else + echo "Destination $forcedest does not exist!" >&2 + fi +done + echo ' CREATE TEMPORARY TABLE tasks( id INTEGER PRIMARY KEY, From d511bda10625e48ab8e48f82a8f3bd52890b5cae Mon Sep 17 00:00:00 2001 From: Vincent Riquer Date: Thu, 30 May 2013 12:44:28 +0200 Subject: [PATCH 33/35] Fix file copy for files not in subdirectory --- lib/copy/copyFiles_action | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/lib/copy/copyFiles_action b/lib/copy/copyFiles_action index 0134eb5..0174f76 100644 --- a/lib/copy/copyFiles_action +++ b/lib/copy/copyFiles_action @@ -50,16 +50,23 @@ copyFiles_action() { destdir="$(guessPath)" || continue else destdir="${destinationpath["$destination"]}/" - destdir+=$(sanitizeFile "${sourcefilename%%/*}" dir) - part=${sourcefilename#*/} - while [[ $part =~ / ]] - do - destdir+="/$(sanitizeFile "${part%%/*}" dir)" - part=${part#*/} - done - if ! [ -d "$destdir" ] + if [[ $sourcefilename =~ / ]] then - mkdir -p "$destdir" + destdir+=$( + sanitizeFile "${sourcefilename%%/*}" dir + ) + part=${sourcefilename#*/} + while [[ $part =~ / ]] + do + destdir+="/$( + sanitizeFile "${part%%/*}" dir + )" + part=${part#*/} + done + if ! [ -d "$destdir" ] + then + mkdir -p "$destdir" + fi fi fi if cp -al "$sourcepath/$sourcefilename" "$destdir" 2>/dev/null\ From d6130ee0444e6f56ee757c6f42fe4728da9f2a04 Mon Sep 17 00:00:00 2001 From: Vincent Riquer Date: Fri, 31 May 2013 11:54:31 +0200 Subject: [PATCH 34/35] Better display when no files are to be copied --- lib/copy/copyFiles_action | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/lib/copy/copyFiles_action b/lib/copy/copyFiles_action index 0174f76..42a9823 100644 --- a/lib/copy/copyFiles_action +++ b/lib/copy/copyFiles_action @@ -83,6 +83,11 @@ copyFiles_action() { fi done echo 'COMMIT;' >&3 - echo -e "\rCopied ${done:-0} of $count files.\033[K" + if (( count )) + then + echo -e "\rCopied ${done:-0} of $count files.\033[K" + else + echo -e "\rNothing to copy.\033[K" + fi unset count done } From 9f44a1f6d224c0647df17d533e3a500cac2913ba Mon Sep 17 00:00:00 2001 From: Vincent Riquer Date: Fri, 31 May 2013 13:54:10 +0200 Subject: [PATCH 35/35] Better progress and time estimation --- atom | 4 ++-- lib/workers/cleaner | 1 + lib/workers/worker | 9 --------- 3 files changed, 3 insertions(+), 11 deletions(-) diff --git a/atom b/atom index 7d21def..31775ae 100755 --- a/atom +++ b/atom @@ -544,13 +544,13 @@ do checkworkers cleaner master - if (( ran )) + if (( ran - failed )) then currenttime=$(date +%s) avgduration=$(( ((currenttime - starttime) * 1000) / - ran + ( ran - failed ) )) secsremaining=$(( remaining * avgduration / 1000 )) (( days = diff --git a/lib/workers/cleaner b/lib/workers/cleaner index 3db4e0a..56071ea 100644 --- a/lib/workers/cleaner +++ b/lib/workers/cleaner @@ -10,6 +10,7 @@ cleaner() { EOWhere ) (( failed+=faildepends )) + (( ran+=faildepends )) Update tasks status 2 <<<"id = $taskid" Update tasks status 2 <<<"requires = $taskid" echo "SELECT COUNT(*) diff --git a/lib/workers/worker b/lib/workers/worker index c4b5954..5df3376 100644 --- a/lib/workers/worker +++ b/lib/workers/worker @@ -4,12 +4,3 @@ worker() { (( debug >= 2 )) && echo "${cmd_arg[@]}" >&2 "${cmd_arg[@]}" >/dev/null } -createworker() { - worker $1 & - workers[$1]=$! -} -destroyworker() { - dyingworker=${workers[$1]} - unset workers[$1] - wait $dyingworker -}