From a0ce3925a84c97d1f0c7983aeaed2fbdf557cfed Mon Sep 17 00:00:00 2001 From: Vincent Riquer Date: Wed, 29 May 2013 01:47:01 +0200 Subject: [PATCH 001/112] Sanity checks --- README | 1 + atom | 136 +++++++++++++++++++++++++++++++- lib/config/getConfigDestination | 3 + lib/database/openDatabase | 11 --- lib/decode/decodeFile | 3 + lib/tags/getTags | 7 +- 6 files changed, 147 insertions(+), 14 deletions(-) diff --git a/README b/README index b87cabe..be1e3e4 100644 --- a/README +++ b/README @@ -25,6 +25,7 @@ Optional: http://opus-codec.org/ * opusinfo (Opus metadata) * opusenc (Opus encoding) + * opusdec (Opus decoding) * LAME MP3 Encoder http://lame.sourceforge.net/ * lame (MP3 encoding) diff --git a/atom b/atom index a3f86b7..ed283b9 100755 --- a/atom +++ b/atom @@ -96,8 +96,6 @@ do esac done -#FIXME: check sanity - if [ ! -f "$cffile" ] then if [ ! -d ~/.atom ] @@ -120,6 +118,135 @@ set +H (( debug || cfgdump )) && printConfig (( cfgdump )) && exit +# check sanity +if [ ! -d "$tempdir" ] && ! mkdir -p "$tempdir" +then + echo "[FATAL] Could not create temp directory $tempdir" >&2 + (( sanityfail++ )) +fi +if [ ! -f "$database" ] && [ ! -d "${database%/*}" ] && ! mkdir -p "${database%/*}" +then + echo "[FATAL] Directory holding database file does not exist and could" \ + "not be created" >&2 + (( sanityfail++ )) +fi +if [ ! -d "$sourcepath" ] +then + echo "[FATAL] Source path $sourcepath does not exist or is not a directory" >&2 + (( sanityfail++ )) +fi +if ! which sed >/dev/null +then + echo "[FATAL] Required tool sed is not installed or not in PATH + I never thought this would actually hit someone..." >&2 + (( sanityfail++ )) +fi +if ! which sox >/dev/null +then + echo "[FATAL] Required tool sox is not installed or not in PATH" >&2 + (( sanityfail++ )) +fi +if ! which ogginfo >/dev/null +then + echo "[WARNING] Tool ogginfo (from vorbis-tools) is not" \ + "installed or not in PATH + Vorbis metadata disabled" >&2 + disableogginfo=1 + (( sanitywarn++ )) +fi +if (( oggencneeded )) && ! which oggenc >/dev/null +then + echo "[WARNING] Tool oggenc (from vorbis-tools) is not" \ + "installed or not in PATH + Vorbis targets disabled" >&2 + disableoggenc=1 + (( sanitywarn++ )) +fi +if ! which opusinfo >/dev/null +then + echo "[WARNING] Tool opusinfo (from opus-tools) is not" \ + "installed or not in PATH + Opus metadata disabled" >&2 + disableopusinfo=1 + (( sanitywarn++ )) +fi +if (( opusencneeded )) && ! which opusenc >/dev/null +then + echo "[WARNING] Tool opusenc (from opus-tools) is not" \ + "installed or not in PATH + Opus targets disabled" >&2 + disableopusenc=1 + (( sanitywarn++ )) +fi +if ! which opusdec >/dev/null +then + echo "[WARNING] Tool opusdec (from opus-tools) is not" \ + "installed or not in PATH + Opus support disabled" >&2 + disableopusdec=1 + (( sanitywarn++ )) +fi +if (( lameneeded )) && ! which lame >/dev/null +then + echo "[WARNING] Tool lame is not installed or not in PATH + MP3 targets disabled" >&2 + disablelame=1 + (( sanitywarn++ )) +fi +if ! which metaflac >/dev/null +then + echo "[WARNING] Tool metaflac (from FLAC) is not installed" \ + "or not in PATH + FLAC metadata disabled" >&2 + disableflac=1 + (( sanitywarn++ )) +fi +if ! which mpcdec >/dev/null +then + echo "[WARNING] Tool mpcdec (from Musepack) is not" \ + "installed or not in PATH + Musepack support disabled" >&2 + disablempcdec=1 + (( sanitywarn++ )) +fi +if ! which ffprobe >/dev/null +then + echo "[WARNING] Tool ffprobe (from FFmpeg) is not installed or not in PATH + Video metadata disabled + MPEG metadata disabled + MusePack metadata disabled + Unknown format metadata disabled" >&2 + disableffprobe=1 + (( sanitywarn++ )) +fi +if ! which ffmpeg >/dev/null +then + echo "[WARNING] Tool ffmpeg is not installed or not in PATH + Video support disabled" >&2 + disablevideo=1 + (( sanitywarn++ )) +fi +if (( sanityfail )) +then + echo " +Sanity checks raised ${sanitywarn:-0} warnings, $sanityfail failures. Dying now." >&2 + exit $ESANITY +elif (( sanitywarn )) +then + echo " +Sanity checks raised $sanitywarn warnings... Hit Control-C to abort." >&2 + timeout=$(( sanitywarn * 10 )) + echo -n "Starting in $(printf %3i $timeout)" \ + $'seconds...\b\b\b\b\b\b\b\b\b\b\b' >&2 + while (( timeout )) + do + echo -n $'\b\b\b'"$(printf %3i $timeout)" >&2 + sleep 1 + (( timeout-- )) + done + echo -en "\r\033[K" +fi + openDatabase createDestinations @@ -312,6 +439,11 @@ do rest=${rest#*::AtOM:SQL:Sep::} performer=${rest%%::AtOM:SQL:Sep::*} unset rest + case ${destinationformat["$destination"]} in + vorbis) (( disableoggenc )) && continue ;; + opus) (( disableopusenc )) && continue ;; + mp3) (( disablelame )) && continue ;; + esac decodeFile getDestDir getDestFile diff --git a/lib/config/getConfigDestination b/lib/config/getConfigDestination index a6fa6a8..8fa63c6 100644 --- a/lib/config/getConfigDestination +++ b/lib/config/getConfigDestination @@ -8,12 +8,15 @@ getConfigDestination() { case "$value" in 'mp3') destinationformat["$destination"]=mp3 + lameneeded=1 ;; 'opus') destinationformat["$destination"]=opus + opusencneeded=1 ;; 'vorbis') destinationformat["$destination"]=vorbis + oggencneeded=1 ;; *) echo "Unsupported destination format: $value" >2& diff --git a/lib/database/openDatabase b/lib/database/openDatabase index b75cef4..ff829fe 100644 --- a/lib/database/openDatabase +++ b/lib/database/openDatabase @@ -1,18 +1,7 @@ #!/bin/bash openDatabase() { - if [ ! -d "$tempdir" ] - then - mkdir -p "$tempdir" - fi rm -f "$tempdir"/sqlite.{in,out} mkfifo "$tempdir"/sqlite.{in,out} - if [ ! -f "$database" ] - then - if [ ! -d "${database%/*}" ] - then - mkdir -p "${database%/*}" - fi - fi sqlite3 -bail "$database" \ < "$tempdir/sqlite.in" \ > "$tempdir/sqlite.out" & diff --git a/lib/decode/decodeFile b/lib/decode/decodeFile index 09c2dc3..1571222 100644 --- a/lib/decode/decodeFile +++ b/lib/decode/decodeFile @@ -2,6 +2,7 @@ decodeFile() { case "$mimetype" in 'video/'*) + (( disablevideo )) && continue extractAudio if (( ${destinationnormalize["$destination"]}))\ || ( @@ -30,6 +31,7 @@ decodeFile() { then copied=1 else + (( disableopusdec )) && continue decodeOpusdec if (( ${destinationnormalize["$destination"]}))\ || ( @@ -60,6 +62,7 @@ decodeFile() { extendedtype=$(file -b "$sourcepath/$filename") case "$extendedtype" in *'Musepack '*) + (( disablempcdec )) && continue decodeMpcdec if (( ${destinationnormalize["$destination"]}))\ || ( diff --git a/lib/tags/getTags b/lib/tags/getTags index 3f6d9e1..ecd996c 100644 --- a/lib/tags/getTags +++ b/lib/tags/getTags @@ -1,26 +1,31 @@ #!/bin/bash getTags_version='unknown-4' -tagreaders+=( "$getTags_version" ) getTags() { unset type case "$mimetype" in audio/mpeg) type=ffmpeg + (( disableffprobe )) && unset type ;; 'application/ogg opus') type=Opus + (( disableopusinfo )) && unset type ;; application/ogg*) type=Ogg + (( disableogginfo )) && unset type ;; audio/x-flac) type=FLAC + (( disableflac )) && unset type ;; video/*) type=ffmpeg + (( disableffprobe )) && unset type ;; *) type=ffmpeg + (( disableffprobe )) && unset type ;; esac if [ -n "$type" ] From ab92e8b5f68895bb966ec33573305b7c8ca1914c Mon Sep 17 00:00:00 2001 From: Vincent Riquer Date: Wed, 29 May 2013 13:51:21 +0200 Subject: [PATCH 002/112] Apply CLI arguments --- atom | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/atom b/atom index ed283b9..7d21def 100755 --- a/atom +++ b/atom @@ -115,6 +115,11 @@ fi getConfig set +H + +# Apply CLI overrides +[ -n "$cliload" ] && maxload=$cliload +[ -n "$cliltimer" ] && loadinterval=$cliltimer + (( debug || cfgdump )) && printConfig (( cfgdump )) && exit @@ -290,6 +295,19 @@ echo "Suppressed $deleted files, $removed removed from database" updateTags +for forcedest in "${forceall[@]}" +do + if forcedestid=$(Select destinations id <<<"name = $forcedest") + then + echo "Resetting destination files timestamps on" \ + "$forcedest ($forcedestid)..." + Update destination_files last_change 1 \ + <<<"destination_id = $forcedestid" + else + echo "Destination $forcedest does not exist!" >&2 + fi +done + echo ' CREATE TEMPORARY TABLE tasks( id INTEGER PRIMARY KEY, From d511bda10625e48ab8e48f82a8f3bd52890b5cae Mon Sep 17 00:00:00 2001 From: Vincent Riquer Date: Thu, 30 May 2013 12:44:28 +0200 Subject: [PATCH 003/112] Fix file copy for files not in subdirectory --- lib/copy/copyFiles_action | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/lib/copy/copyFiles_action b/lib/copy/copyFiles_action index 0134eb5..0174f76 100644 --- a/lib/copy/copyFiles_action +++ b/lib/copy/copyFiles_action @@ -50,16 +50,23 @@ copyFiles_action() { destdir="$(guessPath)" || continue else destdir="${destinationpath["$destination"]}/" - destdir+=$(sanitizeFile "${sourcefilename%%/*}" dir) - part=${sourcefilename#*/} - while [[ $part =~ / ]] - do - destdir+="/$(sanitizeFile "${part%%/*}" dir)" - part=${part#*/} - done - if ! [ -d "$destdir" ] + if [[ $sourcefilename =~ / ]] then - mkdir -p "$destdir" + destdir+=$( + sanitizeFile "${sourcefilename%%/*}" dir + ) + part=${sourcefilename#*/} + while [[ $part =~ / ]] + do + destdir+="/$( + sanitizeFile "${part%%/*}" dir + )" + part=${part#*/} + done + if ! [ -d "$destdir" ] + then + mkdir -p "$destdir" + fi fi fi if cp -al "$sourcepath/$sourcefilename" "$destdir" 2>/dev/null\ From d6130ee0444e6f56ee757c6f42fe4728da9f2a04 Mon Sep 17 00:00:00 2001 From: Vincent Riquer Date: Fri, 31 May 2013 11:54:31 +0200 Subject: [PATCH 004/112] Better display when no files are to be copied --- lib/copy/copyFiles_action | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/lib/copy/copyFiles_action b/lib/copy/copyFiles_action index 0174f76..42a9823 100644 --- a/lib/copy/copyFiles_action +++ b/lib/copy/copyFiles_action @@ -83,6 +83,11 @@ copyFiles_action() { fi done echo 'COMMIT;' >&3 - echo -e "\rCopied ${done:-0} of $count files.\033[K" + if (( count )) + then + echo -e "\rCopied ${done:-0} of $count files.\033[K" + else + echo -e "\rNothing to copy.\033[K" + fi unset count done } From 9f44a1f6d224c0647df17d533e3a500cac2913ba Mon Sep 17 00:00:00 2001 From: Vincent Riquer Date: Fri, 31 May 2013 13:54:10 +0200 Subject: [PATCH 005/112] Better progress and time estimation --- atom | 4 ++-- lib/workers/cleaner | 1 + lib/workers/worker | 9 --------- 3 files changed, 3 insertions(+), 11 deletions(-) diff --git a/atom b/atom index 7d21def..31775ae 100755 --- a/atom +++ b/atom @@ -544,13 +544,13 @@ do checkworkers cleaner master - if (( ran )) + if (( ran - failed )) then currenttime=$(date +%s) avgduration=$(( ((currenttime - starttime) * 1000) / - ran + ( ran - failed ) )) secsremaining=$(( remaining * avgduration / 1000 )) (( days = diff --git a/lib/workers/cleaner b/lib/workers/cleaner index 3db4e0a..56071ea 100644 --- a/lib/workers/cleaner +++ b/lib/workers/cleaner @@ -10,6 +10,7 @@ cleaner() { EOWhere ) (( failed+=faildepends )) + (( ran+=faildepends )) Update tasks status 2 <<<"id = $taskid" Update tasks status 2 <<<"requires = $taskid" echo "SELECT COUNT(*) diff --git a/lib/workers/worker b/lib/workers/worker index c4b5954..5df3376 100644 --- a/lib/workers/worker +++ b/lib/workers/worker @@ -4,12 +4,3 @@ worker() { (( debug >= 2 )) && echo "${cmd_arg[@]}" >&2 "${cmd_arg[@]}" >/dev/null } -createworker() { - worker $1 & - workers[$1]=$! -} -destroyworker() { - dyingworker=${workers[$1]} - unset workers[$1] - wait $dyingworker -} From 5790b04b9cd06d0798ea34071a26721686caf6bc Mon Sep 17 00:00:00 2001 From: Vincent Riquer Date: Sat, 1 Jun 2013 00:12:19 +0200 Subject: [PATCH 006/112] WebM information --- atom | 12 +++++++++++- lib/tags/getInfos::WebM | 17 +++++++++++++++++ lib/tags/getInfos::ffmpeg | 2 +- lib/tags/getInfos::ffmpeg_other | 7 +++++++ lib/tags/getInfos::ffmpeg_video | 7 +++++++ lib/tags/getTags | 8 ++++++-- 6 files changed, 49 insertions(+), 4 deletions(-) create mode 100644 lib/tags/getInfos::WebM create mode 100644 lib/tags/getInfos::ffmpeg_other create mode 100644 lib/tags/getInfos::ffmpeg_video diff --git a/atom b/atom index 31775ae..e177d24 100755 --- a/atom +++ b/atom @@ -155,7 +155,8 @@ if ! which ogginfo >/dev/null then echo "[WARNING] Tool ogginfo (from vorbis-tools) is not" \ "installed or not in PATH - Vorbis metadata disabled" >&2 + Vorbis metadata disabled + WebM metadata disabled" >&2 disableogginfo=1 (( sanitywarn++ )) fi @@ -214,6 +215,15 @@ then disablempcdec=1 (( sanitywarn++ )) fi +if ! which mkvextract >/dev/null +then + echo "[WARNING] Tool mkvextract (from MKVToolNix) is not" \ + "installed or not in PATH + WebM metadata disabled + WebM support disabled" >&2 + disablemkvextract=1 + (( sanitywarn++ )) +fi if ! which ffprobe >/dev/null then echo "[WARNING] Tool ffprobe (from FFmpeg) is not installed or not in PATH diff --git a/lib/tags/getInfos::WebM b/lib/tags/getInfos::WebM new file mode 100644 index 0000000..2303f67 --- /dev/null +++ b/lib/tags/getInfos::WebM @@ -0,0 +1,17 @@ +#!/bin/bash +getInfoswebm_version='webm-1' +tagreaders+=( "$getInfoswebm_version" ) +getInfos::WebM() { + getInfos::ffmpeg_video + tagreader="$getInfoswebm_version" + local infos=$( + mkvextract tracks \ + "$sourcepath/$filename" \ + 1:>(ogginfo /dev/stdin) \ + | sed 's/\t//;s/: /=/g' + ) + rate=$(gettag rate|head -n1) + channels=$(gettag channels|head -n1) + bitrate=$(gettag 'average bitrate') + bitrate=${bitrate%%,*} +} diff --git a/lib/tags/getInfos::ffmpeg b/lib/tags/getInfos::ffmpeg index e03e879..fbfc59d 100644 --- a/lib/tags/getInfos::ffmpeg +++ b/lib/tags/getInfos::ffmpeg @@ -1,5 +1,5 @@ #!/bin/bash -getInfosffmpeg_version='ffmpeg-2' +getInfosffmpeg_version='ffmpeg-3' tagreaders+=( "$getInfosffmpeg_version" ) getInfos::ffmpeg() { tagreader="$getInfosffmpeg_version" diff --git a/lib/tags/getInfos::ffmpeg_other b/lib/tags/getInfos::ffmpeg_other new file mode 100644 index 0000000..608e733 --- /dev/null +++ b/lib/tags/getInfos::ffmpeg_other @@ -0,0 +1,7 @@ +#!/bin/bash +getInfosffmpeg_other_version='ffmpeg_other-1' +tagreaders+=( "$getInfosffmpeg_other_version" ) +getInfos::ffmpeg_other() { + getInfos::ffmpeg + tagreader="$getInfosffmpeg_other_version" +} diff --git a/lib/tags/getInfos::ffmpeg_video b/lib/tags/getInfos::ffmpeg_video new file mode 100644 index 0000000..ef31d51 --- /dev/null +++ b/lib/tags/getInfos::ffmpeg_video @@ -0,0 +1,7 @@ +#!/bin/bash +getInfosffmpeg_video_version='ffmpeg_video-1' +tagreaders+=( "$getInfosffmpeg_video_version" ) +getInfos::ffmpeg_video() { + getInfos::ffmpeg + tagreader="$getInfosffmpeg_video_version" +} diff --git a/lib/tags/getTags b/lib/tags/getTags index ecd996c..dcaa0e9 100644 --- a/lib/tags/getTags +++ b/lib/tags/getTags @@ -19,12 +19,16 @@ getTags() { type=FLAC (( disableflac )) && unset type ;; + video/webm) + type=WebM + (( disablemkvextract || disableogginfo )) && unset type + ;; video/*) - type=ffmpeg + type=ffmpeg_video (( disableffprobe )) && unset type ;; *) - type=ffmpeg + type=ffmpeg_other (( disableffprobe )) && unset type ;; esac From 1feb3b0dcc679d3d7cc7c35144ce5b8177e2db0a Mon Sep 17 00:00:00 2001 From: Vincent Riquer Date: Mon, 3 Jun 2013 03:30:30 +0200 Subject: [PATCH 007/112] help() --- atom | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/atom b/atom index 31775ae..a89f4ba 100755 --- a/atom +++ b/atom @@ -56,6 +56,20 @@ do source "$function" done +help() { + cat <<-EOF + Options: + -c Load configuration file + -C Dump configuration and exit + -l Override max-load + -T override load-interval + -F Force re-generation of all files in + + -h Show this text + -D Increase debug + EOF +} + #parse arguments OPTERR=0 while getopts ':c:Cl:T:F:hD' opt From f1619ee328c267e4ddbd813e0245239eba6def2f Mon Sep 17 00:00:00 2001 From: Vincent Riquer Date: Mon, 3 Jun 2013 03:30:51 +0200 Subject: [PATCH 008/112] Running, part I, II, III --- README | 68 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 68 insertions(+) diff --git a/README b/README index be1e3e4..f41bc5f 100644 --- a/README +++ b/README @@ -43,9 +43,77 @@ Optional: ================== Using the software ------------------ + Configuration: +-------------- Please read doc/config before anything else. +Preparing data: +--------------- +Nothing specific needs to be done. You can edit ypur tags, rename files, move +them around how you see fit. However, make sure you setup your tag editor +to *do* update the files' timestamps: though it was initially plan to make this +optional, using checksums or tags, it was abandoned due to the huge amount of +IO required. + +Running: +-------- +Make sure your configuration is correct by running +$ atom -C +This will produce a human-readable dump of your current configuration. +If all settings are correct, simply run atom with no argument. Go get a beer. +Meet some friends. Go to bed. Depending on the size of your collection, the +first run can take hours, even days. +After adding/tagging/renaming/deleting files, just re-run atom. It should be +much faster this time, as only changed data will be treated. + +If, for whatever reason, you need to force the regeneration of a destination, +after changing the quality settings for example, run +$ atom -F + +================= +Technical details +----------------- +I. Source scan +-------------- +After reading its configuration file, AtOM uses find to get a list of all files +in the source directory. +Each file is checked against the database. If it's already there, and its last +modification time is unchanged, the last_seen field is updated, and that's all. +If its mtime has changed, mime-type scan is attempted. It is updated in the +database, along with last_seen. +If the file is new, its mime-type is scanned, and it is added to the database. + +II. Obsolete files +------------------ +Using the last_seen field, AtOM removes from destinations each files which are +not present anymore in the source directory. AtOM never touches files not +present in its database (unless there is a filename conflict, in which case your +file *WILL* be overwritten). If you wish to clear unknown files from your +destinations, have a look at toys/cleandestinations. + +III. Reading metadata +--------------------- +AtOM then tries to read metadata from each new or changed file. It also re-reads +metadata from files scanned with an older version of AtOM, if the parser for +that format has changed. The actual data read depends on the format, but at the +very least, AtOM should identify the sampling rate, bitrate and number of +channels. Unknown file types are scanned with ffprobe, so you may still have +some luck, depending on your FFmpeg setup. + +IV. Task creation +----------------- + +V. Transcoding +-------------- + +VI. Copies +---------- + +VII. Obsolete files 2 +------------------- + + ==== Toys ---- From 779449c19a72598d8499be6f1348c92ceb911ec0 Mon Sep 17 00:00:00 2001 From: Vincent Riquer Date: Mon, 3 Jun 2013 12:15:58 +0200 Subject: [PATCH 009/112] Running, part IV --- README | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/README b/README index f41bc5f..b6658e5 100644 --- a/README +++ b/README @@ -103,9 +103,26 @@ some luck, depending on your FFmpeg setup. IV. Task creation ----------------- +For every destination files having their last change field different from their +corresponding source file entry, we create one or more tasks, to generate or +update (overwrite) teh destination file. AtOM tries to generate as few tasks as +possible, by reusing intermediate files wherever feasible. E.g. if you define +two destinations, only differing by the encoding format, we will create only one +decoding task, and two encoding tasks. On average, the number of generated tasks +shall always be less than 2*n (where n is destinations*file count), unless each +of them uses different sampling rate/normalization parameters. + +Files matching the format, sampling-rate, channel count and bitrate constraints +are copied (symlinked where possible) during that stage ("immediate copies"). + +The steps required for each file depend on the format and destination parameters +(resampling, aso.). Basically, one destination file requires 1 (if reusing +decoded/resampled file) to 3 (if format can't be decoded using sox and +resampling/normalization is required) tasks. + +V. Running tasks +---------------- -V. Transcoding --------------- VI. Copies ---------- From 2f8aeedf434b608094a42d5c238598534db9142d Mon Sep 17 00:00:00 2001 From: Vincent Riquer Date: Mon, 3 Jun 2013 14:49:08 +0200 Subject: [PATCH 010/112] Running, Part V, VI, VII --- README | 29 ++++++++++++++++++++++++++--- 1 file changed, 26 insertions(+), 3 deletions(-) diff --git a/README b/README index b6658e5..2f1b836 100644 --- a/README +++ b/README @@ -120,16 +120,39 @@ The steps required for each file depend on the format and destination parameters decoded/resampled file) to 3 (if format can't be decoded using sox and resampling/normalization is required) tasks. -V. Running tasks ----------------- +V.1 Running tasks +----------------- +While running tasks, AtOM responds to the following keypresses: ++/- Increase/decrease max-load +q Quit (will skip all following steps) +Progress display: +L:/ +W:/ +T:/ (F:) +% (A:s/task) +ETA: + +V.2 Renaming files +------------------ +If rename pattern (or FAT32 compatibility) for one or more destinations has +changed, files already transcoded will be renamed. Otherwise, this step is +skipped. VI. Copies ---------- +During that stage, files which mime-types matched copy_mime-type directives are +copied (symlinked where possible) to the destination. +When a rename pattern is defined and impacts path, files that are not in the +same directories as files which have been successfully transcoded are ignored, +as AtOM cannot guess what the destination path should be. Otherwise, files are +copied with their name and path unchanged. VII. Obsolete files 2 ------------------- - +Whenever a file is transcoded, if it was already present in the database but its +name changed, following a rename pattern change, the old file is removed during +that stage. ==== Toys From 854578e65740f7a8936f45efb5864a238d9e6b3a Mon Sep 17 00:00:00 2001 From: Vincent Riquer Date: Tue, 4 Jun 2013 13:41:56 +0200 Subject: [PATCH 011/112] Create TODO --- TODO | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 TODO diff --git a/TODO b/TODO new file mode 100644 index 0000000..e5e27d4 --- /dev/null +++ b/TODO @@ -0,0 +1,5 @@ +Tag Guessing +------------ +From a user-defined pattern, guess tags for unsupported formats/untagged files +from the file path and file name. + From 5c6151570b5989af1b65d03de5d98c78d791ef29 Mon Sep 17 00:00:00 2001 From: Vincent Riquer Date: Tue, 4 Jun 2013 13:56:58 +0200 Subject: [PATCH 012/112] document known bugs --- BUGS | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 BUGS diff --git a/BUGS b/BUGS new file mode 100644 index 0000000..d6e5d6f --- /dev/null +++ b/BUGS @@ -0,0 +1,24 @@ +Known bugs: + +1. Video formats suck +--------------------- +One would expect video formats to allow for a wide range of tags. Things like +writer, director, main actors, original author, etc. But most of them only +provide ONE tag: title... +Planned workaround: tag guessing (see TODO). + +2. Random hang +-------------- +Something I encountered and don't understand: sometimes, seemingly randomly, +the call to wait, to retrieve the exit status of a worker, does not return, +though the worker *has* exited. This happens rarely, but it will probably hit +you if you have more than 20.000 tasks to run. When this happens, just hit +Control-C and re-run AtOM to resume transcoding. + +Symptoms: +- Progress does not update anymore +- a pstree only show one child process: sqlite3 +- strace()ing shows the script process stuck on waitpid() + +Please let me know if this hits you, or if you have any idea as to why this is +happening and how to avoid it. From 9bb66ba76cf78fa7aa40429198a8f348940e94a8 Mon Sep 17 00:00:00 2001 From: Vincent Riquer Date: Mon, 17 Jun 2013 13:37:24 +0200 Subject: [PATCH 013/112] Typo --- README | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README b/README index 2f1b836..a8da963 100644 --- a/README +++ b/README @@ -105,7 +105,7 @@ IV. Task creation ----------------- For every destination files having their last change field different from their corresponding source file entry, we create one or more tasks, to generate or -update (overwrite) teh destination file. AtOM tries to generate as few tasks as +update (overwrite) the destination file. AtOM tries to generate as few tasks as possible, by reusing intermediate files wherever feasible. E.g. if you define two destinations, only differing by the encoding format, we will create only one decoding task, and two encoding tasks. On average, the number of generated tasks @@ -114,6 +114,7 @@ of them uses different sampling rate/normalization parameters. Files matching the format, sampling-rate, channel count and bitrate constraints are copied (symlinked where possible) during that stage ("immediate copies"). +(also see higher-than setting). The steps required for each file depend on the format and destination parameters (resampling, aso.). Basically, one destination file requires 1 (if reusing From 9cde5ba27228a10df64402effd550f132d5108ae Mon Sep 17 00:00:00 2001 From: Vincent Riquer Date: Mon, 17 Jun 2013 13:39:28 +0200 Subject: [PATCH 014/112] Setup * -S option * call setup when no conf file present * [general] setup --- atom | 41 +++++++--- lib/config/getConfig | 1 + lib/config/getConfigDestination | 2 +- lib/setup/setup | 15 ++++ lib/setup/setupGeneral | 132 ++++++++++++++++++++++++++++++++ 5 files changed, 178 insertions(+), 13 deletions(-) create mode 100644 lib/setup/setup create mode 100644 lib/setup/setupGeneral diff --git a/atom b/atom index a89f4ba..4844961 100755 --- a/atom +++ b/atom @@ -72,7 +72,7 @@ help() { #parse arguments OPTERR=0 -while getopts ':c:Cl:T:F:hD' opt +while getopts ':c:Cl:T:F:ShD' opt do case $opt in c) @@ -90,6 +90,9 @@ do F) forceall+=("$OPTARG") ;; + S) + forcesetup=1 + ;; h) help exit 0 @@ -110,24 +113,38 @@ do esac done +askconf() { + read -p"Create one now? [Y/n/q] " createconf + case $createconf in + ''|[yY]) + setup + ;; + [nNqQ]) + echo "You need a configuration file. If you" \ + "want to create it yourself, please" \ + "read doc/config and doc/example.cfg." >&2 + exit $ENOCFG + ;; + *) + echo "Come again?" >&2 + askconf + ;; + esac +} + if [ ! -f "$cffile" ] then - if [ ! -d ~/.atom ] + if [ ! -d "${cffile%/*}" ] then - mkdir -p ~/.atom + mkdir -p "${cffile%/*}" fi - sed "s:%HOME%:$HOME:" "$exampleconf" > "$cffile" - cat >&2 <<-EOCfgNotice - No configuration file found! - An example file has been created as "${cffile/$HOME/~}". - You should change it to your likings using you favorite editor. - - Bailing out. - EOCfgNotice - exit $ENOCFG + echo "No configuration file found!" >&2 + askconf fi getConfig +(( forcesetup )) && setup + set +H # Apply CLI overrides diff --git a/lib/config/getConfig b/lib/config/getConfig index cbc9b5f..63e0e50 100644 --- a/lib/config/getConfig +++ b/lib/config/getConfig @@ -19,6 +19,7 @@ getConfig() { context=Destination destination="${key#[}" destination="${destination%]}" + destinations+=("${destination%]}") ;; *) getConfig$context diff --git a/lib/config/getConfigDestination b/lib/config/getConfigDestination index 8fa63c6..6620dff 100644 --- a/lib/config/getConfigDestination +++ b/lib/config/getConfigDestination @@ -19,7 +19,7 @@ getConfigDestination() { oggencneeded=1 ;; *) - echo "Unsupported destination format: $value" >2& + echo "Unsupported destination format: $value" >&2 exit $EFORMAT ;; esac diff --git a/lib/setup/setup b/lib/setup/setup new file mode 100644 index 0000000..c814d31 --- /dev/null +++ b/lib/setup/setup @@ -0,0 +1,15 @@ +#!/bin/bash + +setup() { + cat <<-EOStartConf +You will now be asked (hopefully) simple questions to help you configure AtOM's +behavior. + +Completion is available for prompts asking for a paths or filenames. + EOStartConf + setupGeneral + setupSource + setupDestinations + unset expr + exit +} diff --git a/lib/setup/setupGeneral b/lib/setup/setupGeneral new file mode 100644 index 0000000..3d6d0c3 --- /dev/null +++ b/lib/setup/setupGeneral @@ -0,0 +1,132 @@ +#!/bin/bash + +setupGeneral() { + cat <<-EODesc + +[General] + We will start by setting the general parameters defining the program's + behavior. + EODesc + cat <<-EODesc + + Target load (integer): + Defines how parallel processing will behave. AtOM will try to keep the + 1 minute load average between and +1 by adjusting + concurrency. Initial concurrency will be set to half of that value. + EODesc + expr='^[0-9]*$' + comeagain() { + read \ + -e \ + -p'Target load: (integer) ' \ + ${maxload+-i"$maxload"} \ + value + if [ -n "$value" ] && [[ $value =~ $expr ]] + then + maxload="$value" + else + echo "Invalid max-load value: $value" >&2 + comeagain + fi + } + comeagain + cat <<-EODesc + + Load interval (seconds, integer): + How often should we check the load average and adjust concurrency. Set + this too low, and concurrency may be increased too quickly. Set this + too high, and AtOM will not adapt quickly enough to load increase. In + both cases, your hard drive will suffer. In my experience, 30 seconds + is a good value. + EODesc + comeagain() { + read \ + -e \ + -p'Load interval: (integer) ' \ + -i${loadinterval:-30} \ + value + if [[ $value =~ $expr ]] + then + loadinterval="$value" + else + echo "Invalid load-interval value: $value" >&2 + comeagain + fi + } + comeagain + cat <<-EODesc + + Ionice [niceness]: + IO-hungry processes will be run with ionice class and niceness + [niceness] (if applicable). See man ionice for details. + EODesc + comeagain() { + read \ + -e \ + -p'Ionice: <1-3> [0-7] ' \ + -i"${class:-3} ${niceness}" \ + class niceness + case $class in + 1) + # real-time class, only root can do that + if (( UID )) + then + echo "IO class 'realtime' is"\ + "not available to unprivileged"\ + "users" >&2 + comeagain + fi + if [ -n "$niceness" ] \ + && (( niceness >= 0 && niceness <= 7 )) + then + ionice="ionice -c1 -n$niceness " + else + echo "Invalid IO priority"\ + "'$niceness'" >&2 + comeagain + fi + ;; + 2) + if [ -n "$niceness" ] \ + && (( niceness >= 0 && niceness <= 7 )) + then + ionice="ionice -c2 -n$niceness " + else + echo "Invalid IO priority"\ + "'$niceness'" >&2 + comeagain + fi + ;; + 3) + ionice="ionice -c3 " + ;; + *) + echo "Invalid ionice parameters $value"\ + >&2 + comeagain + ;; + esac + } + comeagain + cat <<-EODesc + + Temporary Directory (path): + Name speaks for itself: this is where FIFOs (for communicating with + sqlite) and temporary WAVE files will be created. Note that debug logs + (if enabled) will go there too. + EODesc + read \ + -e \ + -i"${tempdir:-$HOME/.atom/tmp}" \ + -p'Temporary directory ( for completion): '\ + tempdir + cat <<-EODesc + + Database (filename): + EODesc + read \ + -e \ + -i"${database:-$HOME/.atom/atom.db}" \ + -p'Database file ( for completion): ' \ + database +} From da49eb7086661f1002d5f3d58e56ee068056f3cc Mon Sep 17 00:00:00 2001 From: Vincent Riquer Date: Mon, 17 Jun 2013 13:40:57 +0200 Subject: [PATCH 015/112] [source] setup --- lib/setup/setupSource | 53 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) create mode 100644 lib/setup/setupSource diff --git a/lib/setup/setupSource b/lib/setup/setupSource new file mode 100644 index 0000000..a94d410 --- /dev/null +++ b/lib/setup/setupSource @@ -0,0 +1,53 @@ +#!/bin/bash + +setupSource() { + cat <<-EODesc + +[Source] + Here we will define which directory AtOM should look for media files, and + what it should completely ignore. + EODesc + cat <<-EODesc + + Path (path): + Which directory to scan for new media files. + EODesc + comeagain() { + read \ + -e \ + -i"${sourcepath:-/var/lib/mpd/music}" \ + -p'Music collection ( for completion): ' \ + sourcepath + if ! [ -d "$sourcepath" ] + then + echo "$sourcepath does not exist or is not a" \ + "directory!" >&2 + comeagain + fi + } + comeagain + cat <<-EODesc + + Skip (path): + Files in these directories will be ignored. + Path is relative to $sourcepath. + + This prompt will loop until an empty string is encountered. + EODesc + cd "$sourcepath" + for (( i=0 ; 1 ; i++ )) + do + read \ + -e \ + ${skippeddirectories[i]+-i"${skippeddirectories[i]}"}\ + -p'Skip: ' \ + value + if [ -n "$value" ] + then + skippeddirectories[i]="$value" + else + break + fi + done + cd - >/dev/null +} From 0176483217ad8ab18232f31dfd51d19d28d801d2 Mon Sep 17 00:00:00 2001 From: Vincent Riquer Date: Mon, 17 Jun 2013 13:41:25 +0200 Subject: [PATCH 016/112] [destination] setup (mandatory parameters) --- lib/setup/setupDestination | 227 ++++++++++++++++++++++++++++++++++++ lib/setup/setupDestinations | 80 +++++++++++++ 2 files changed, 307 insertions(+) create mode 100644 lib/setup/setupDestination create mode 100644 lib/setup/setupDestinations diff --git a/lib/setup/setupDestination b/lib/setup/setupDestination new file mode 100644 index 0000000..8c6b994 --- /dev/null +++ b/lib/setup/setupDestination @@ -0,0 +1,227 @@ +#!/bin/bash + +setupDestination() { + cat <<-EODesc + + Format: + vorbis, opus or mp3. Other formats may appear in the future. + EODesc + comeagain() { + read \ + -e \ + ${destinationformat["$destination"]+-i"${destinationformat["$destination"]}"}\ + -p 'Format: ' \ + value + case "$value" in + 'mp3') + destinationformat["$destination"]=mp3 + lameneeded=1 + ;; + 'opus') + destinationformat["$destination"]=opus + opusencneeded=1 + ;; + 'vorbis'|'ogg') + destinationformat["$destination"]=vorbis + oggencneeded=1 + ;; + *) + echo "Unsupported destination format: $value" >&2 + comeagain + ;; + esac + } + comeagain + cat <<-EODesc + + Path (path): + Where to store transcoded files (will be created if it does not + exist). + EODesc + read \ + -e \ + -p'Path: ' \ + ${destinationpath["$destination"]+-i"${destinationpath["$destination"]}"}\ + destinationpath["$destination"] + case ${destinationformat["$destination"]} in + vorbis) + cat <<-EODesc + + Quality (integer): + The quality parameter of oggenc. See man oggenc for more info. + EODesc + expr='^[0-9]*$' + comeagain() { + read \ + -p'Quality: ' \ + -e \ + -i \ + ${destinationquality["$destination"]:-3}\ + value + if ! [[ $value =~ $expr ]] + then + echo "Invalid quality value: $value" >&2 + comeagain + fi + } + comeagain + destinationquality["$destination"]=$value + ;; + opus) + cat <<-EODesc + + Bitrate (kbps, integer): + Set (VBR) bitrate to . Note that while Opus allows for + decimal values, AtOM does not. The reason for this is simple: we do + numeric comparisons, and Bash only manipulates integers. + EODesc + expr='^[0-9]*$' + comeagain() { + read \ + -e \ + -i \ + ${destinationquality["$destination"]:-128}\ + -p 'Bitrate: ' + value + if ! [[ $value =~ $expr ]] + then + echo "Invalid bitrate value: $value" >&2 + comeagain + fi + } + comeagain + destinationquality["$destination"]=$value + cat <<-EODesc + + Loss (percent, integer): + If you intend to stream the resulting files over an unreliable + protocol, you may want to make use of Opus' Forward Error + Correction algorythm. See the Opus-codec.org website for details. + EODesc + comeagain() { + read \ + -e \ + -i \ + ${destinationloss["$destination"]:-0}\ + value + if ! [[ $value =~ $expr ]] + then + echo "Invalid loss value: $value" >&2 + comeagain + fi + } + comeagain + destinationloss["$destination"]=$value + ;; + mp3) + cat <<-EODesc + + Bitrate (kbps, integer): + Set (ABR) bitrate to . + EODesc + expr='^[0-9]*$' + comeagain() { + read \ + -e \ + -i \ + ${destinationquality["$destination"]:-128}\ + value + if ! [[ $value =~ $expr ]] + then + echo "Invalid bitrate value: $value" >&2 + comeagain + fi + } + comeagain + destinationquality["$destination"]=$value + cat <<-EODesc + + Prevent resampling (boolean): + LAME may decide to encode your file to a lower sampling-rate if you + use a low bitrate. Setting this to yes will append --resample + , preventing any resampling from happening. + EODesc + case ${destinationnoresample["$destination"]} in + 0) initialvalue=n ;; + 1) initialvalue=y ;; + *) unset initialvalue ;; + esac + comeagain() { + read \ + -e \ + ${initialvalue+-i $initialvalue}\ + -p'Prevent resampling (y/N): ' \ + value + case $value in + [yY]) + destinationnoresample["$destination"]=1 + ;; + ''|[nN]) + destinationnoresample["$destination"]=0 + ;; + *) + comeagain + ;; + esac + } + comeagain + ;; + esac + cat <<-EODesc + + [Optional parameters] + Now you will have the opportunity to configure "advanced" parameters + for $destination. You may leave any of these fields blank. + EODesc + cat <<-EODesc + + Normalize (boolean): + Normalize output files. + EODesc + comeagain() { + : + } + comeagain + cat <<-EODesc + EODesc + comeagain() { + : + } + comeagain + cat <<-EODesc + EODesc + comeagain() { + : + } + comeagain + cat <<-EODesc + EODesc + comeagain() { + : + } + comeagain + cat <<-EODesc + EODesc + comeagain() { + : + } + comeagain + cat <<-EODesc + EODesc + comeagain() { + : + } + comeagain + cat <<-EODesc + EODesc + comeagain() { + : + } + comeagain + cat <<-EODesc + EODesc + comeagain() { + : + } + comeagain +} diff --git a/lib/setup/setupDestinations b/lib/setup/setupDestinations new file mode 100644 index 0000000..305852b --- /dev/null +++ b/lib/setup/setupDestinations @@ -0,0 +1,80 @@ +#!/bin/bash + +setupDestinations() { + cat <<-EODesc + + +[Destinations] + Finally, we'll setup your destination(s). + EODesc + if (( ${#destinations[@]} )) + then + cat <<-EODesc + + [Existing destinations] + We will review your currently configured destinations. Clear the 'Name' + field to remove one. + EODesc + for destination in "${destinations[@]}" + do + cat <<-EODesc + + Name (string): + A simple name for this destination. Clear to remove this destination. + EODesc + expr='^[A-z0-9]*$' + comeagain() { + read -p'Name: ' -e -i"$destination" value + if [ -z "$value" ] + then + read \ + -p"Really remove destination $destination? [y/N]" + if [[ $REPLY == y ]] + then + removeDestination "$destination" + continue + else + value="$destination" + fi + elif ! [[ $value =~ $expr ]] + then + echo "Invalid name $value." \ + 'Please use only' \ + 'alphanumeric characters.' >&2 + comeagain + fi + } + comeagain + destination="$value" + setupDestination + done + fi + cat <<-EODesc + + [New destinations] + This section will loop until you enter an empty 'Name'. + EODesc + for (( i=0 ; 1 ; i++ )) + do + cat <<-EODesc + + Name (string): + A simple name for this destination. Empty string to end this + configuration loop. + EODesc + expr='^[A-z0-9]*$' + comeagain() { + read -p'Name: ' value + [ -z "$value" ] && break + if ! [[ $value =~ $expr ]] + then + echo "Invalid name $value. Please use" \ + 'only alphanumeric characters.' >&2 + comeagain + fi + } + comeagain + destination="$value" + setupDestination + done +} From 618e086799f68f953348be127d2ae4d96d009395 Mon Sep 17 00:00:00 2001 From: Vincent Riquer Date: Tue, 18 Jun 2013 12:30:37 +0200 Subject: [PATCH 017/112] typo --- doc/config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/config b/doc/config index 4aa2f32..f434082 100644 --- a/doc/config +++ b/doc/config @@ -74,7 +74,7 @@ Sections: be included in that destination. For more than one mime-type, use multiple times, as needed. The '*' character is a wildcard. * copy_mime-type : Same as skip_mime-type, except that files - matching will be copied as-is to tha destination. E.g. image/* will copy + matching will be copied as-is to the destination. E.g. image/* will copy covers and other images to the destination. In fact, AtOM will try to use hard links instead of copies. * channels : Files with more than channels will be From 8f7c907fb9b55239d21ef875579cbd88e6efc3d3 Mon Sep 17 00:00:00 2001 From: Vincent Riquer Date: Tue, 18 Jun 2013 12:30:53 +0200 Subject: [PATCH 018/112] Setup: * normalize * rename * fat32compat * skip_mime-type * copy_mime-type --- lib/setup/setupDestination | 157 +++++++++++++++++++++++++++++++++---- 1 file changed, 141 insertions(+), 16 deletions(-) diff --git a/lib/setup/setupDestination b/lib/setup/setupDestination index 8c6b994..3239834 100644 --- a/lib/setup/setupDestination +++ b/lib/setup/setupDestination @@ -178,34 +178,159 @@ setupDestination() { Normalize (boolean): Normalize output files. EODesc + case ${destinationnormalize["$destination"]} in + 0) initialvalue=n ;; + 1) initialvalue=y ;; + *) unset initialvalue ;; + esac comeagain() { - : + read \ + -e \ + ${initialvalue+-i $initialvalue}\ + -p'Normalize (y/N): ' \ + value + case $value in + [yY]) + destinationnormalize["$destination"]=1 + ;; + ''|[nN]) + destinationnormalize["$destination"]=0 + ;; + *) + comeagain + ;; + esac } comeagain cat <<-EODesc + + Rename (string): + Destination files will be named according to , after + expansion of special strings: + %{album}, + %{albumartist}, + %{artist}, + %{disc}, + %{genre}, + %{title}, + %{track}, + %{year}. + Untagged files or files in unrecognized formats will not be changed. + + Leave blank if you don't want file renaming. EODesc + initialvalue="${destinationrenamepath["$destination"]}" + initialvalue+="${initialvalue+/}" + initialvalue+="${destinationrename["$destination"]}" + read \ + -e \ + ${initialvalue+-i"$initialvalue"} \ + -p'Rename pattern: ' \ + value + case "$value" in + */*) + destinationrenamepath["$destination"]="${value%/*}" + ;; + esac + destinationrename["$destination"]="${value##*/}" + cat <<-EODesc + + FAT32 Compatibility (boolean): + Rename files for compatibility with FAT32 filesystems. + EODesc + case ${destinationfat32compat["$destination"]} in + 0) initialvalue=n ;; + 1) initialvalue=y ;; + *) unset initialvalue ;; + esac comeagain() { - : + read \ + -e \ + ${initialvalue+-i $initialvalue}\ + -p'FAT32 Compatibility (y/N): ' \ + value + case $value in + [yY]) + destinationfat32compat["$destination"]=1 + ;; + ''|[nN]) + destinationfat32compat["$destination"]=0 + ;; + *) + comeagain + ;; + esac } comeagain cat <<-EODesc + + Skip mime-type (mime-type, string): + Files with mime-type will not be included in that + destination. The '*' character is a wildcard. + + This prompt will loop until an empty string is encountered. EODesc - comeagain() { - : - } - comeagain + destinationskipmime["$destination"]="${destinationskipmime["$destination"]}|" + while [[ ${destinationskipmime["$destination"]} =~ / ]] + do + skippedmimes+=("${destinationskipmime["$destination"]%%|*}") + destinationskipmime["$destination"]="${destinationskipmime["$destination"]#*|}" + done + count=${#skippedmimes[@]} + unset destinationskipmime["$destination"] + for (( i=0 ; 1 ; i++ )) + do + read \ + -e \ + ${skippedmimes[i]+-i"${skippedmimes[i]}"} \ + -p 'Skip mime-type: ' \ + value + if [ -n "$value" ] + then + destinationskipmime[$destination]="${destinationskipmime[$destination]:+${destinationskipmime[$destination]}|}$value" + elif (( i < count )) + then + continue + else + break + fi + done + unset skippedmimes cat <<-EODesc + + Copy mime-type (mime-type, string): + Files with mime-type will be copied as-is to the + destination. E.g. image/* will copy covers and other images to the + destination. The '*' character is a wildcard. + + This prompt will loop until an empty string is encountered. EODesc - comeagain() { - : - } - comeagain - cat <<-EODesc - EODesc - comeagain() { - : - } - comeagain + destinationcopymime["$destination"]="${destinationcopymime["$destination"]}|" + while [[ ${destinationcopymime["$destination"]} =~ / ]] + do + copiedmimes+=("${destinationcopymime["$destination"]%%|*}") + destinationcopymime["$destination"]="${destinationcopymime["$destination"]#*|}" + done + count=${#copiedmimes[@]} + unset destinationcopymime["$destination"] + for (( i=0 ; 1 ; i++ )) + do + read \ + -e \ + ${copiedmimes[i]+-i"${copiedmimes[i]}"} \ + -p 'Copy mime-type: ' \ + value + if [ -n "$value" ] + then + destinationcopymime[$destination]="${destinationcopymime[$destination]:+${destinationcopymime[$destination]}|}$value" + elif (( i < count )) + then + continue + else + break + fi + done + unset copiedmimes cat <<-EODesc EODesc comeagain() { From 6bfbc024a30c8d28cc3ea86679c563b7f9a08e56 Mon Sep 17 00:00:00 2001 From: Vincent Riquer Date: Wed, 19 Jun 2013 02:07:07 +0200 Subject: [PATCH 019/112] setup/source: allow user to remove one skipped dir --- lib/setup/setupSource | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/lib/setup/setupSource b/lib/setup/setupSource index a94d410..03c2c5a 100644 --- a/lib/setup/setupSource +++ b/lib/setup/setupSource @@ -35,6 +35,7 @@ setupSource() { This prompt will loop until an empty string is encountered. EODesc cd "$sourcepath" + count=${#skippeddirectories[@]} for (( i=0 ; 1 ; i++ )) do read \ @@ -45,9 +46,13 @@ setupSource() { if [ -n "$value" ] then skippeddirectories[i]="$value" + elif (( i < count )) + then + unset skippeddirectories[i] else break fi done + unset count cd - >/dev/null } From 374a59921b9461b164a3ae43d8befd8f21f743e5 Mon Sep 17 00:00:00 2001 From: Vincent Riquer Date: Wed, 19 Jun 2013 02:33:21 +0200 Subject: [PATCH 020/112] fix bitrate prompts --- lib/setup/setupDestination | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/setup/setupDestination b/lib/setup/setupDestination index 3239834..ab624c8 100644 --- a/lib/setup/setupDestination +++ b/lib/setup/setupDestination @@ -81,7 +81,7 @@ setupDestination() { -e \ -i \ ${destinationquality["$destination"]:-128}\ - -p 'Bitrate: ' + -p 'Bitrate: ' \ value if ! [[ $value =~ $expr ]] then @@ -103,6 +103,7 @@ setupDestination() { -e \ -i \ ${destinationloss["$destination"]:-0}\ + -p 'Loss: ' \ value if ! [[ $value =~ $expr ]] then @@ -125,6 +126,7 @@ setupDestination() { -e \ -i \ ${destinationquality["$destination"]:-128}\ + -p 'Bitrate: ' \ value if ! [[ $value =~ $expr ]] then From 81b7f8b247aeb2c04b771e1527657945f859df0f Mon Sep 17 00:00:00 2001 From: Vincent Riquer Date: Wed, 19 Jun 2013 02:34:47 +0200 Subject: [PATCH 021/112] setup/destination: channels --- lib/setup/setupDestination | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/lib/setup/setupDestination b/lib/setup/setupDestination index ab624c8..0697a77 100644 --- a/lib/setup/setupDestination +++ b/lib/setup/setupDestination @@ -334,11 +334,25 @@ setupDestination() { done unset copiedmimes cat <<-EODesc + + Channels (integer): + Produced files should have this many channels, no more, no less. EODesc + expr='^[0-9]*$' comeagain() { - : + read \ + -e \ + ${destinationchannels["$destination"]+-i${destinationchannels["$destination"]}}\ + -p'Channel count: ' \ + value + if ! [[ $value =~ $expr ]] + then + echo "Invalid channel count: $value" >&2 + comeagain + fi } comeagain + destinationchannels["$destination"]=$value cat <<-EODesc EODesc comeagain() { From b4d7c9d9f0ea0bb0ed759479f7baca9e9e30f34b Mon Sep 17 00:00:00 2001 From: Vincent Riquer Date: Wed, 19 Jun 2013 02:35:10 +0200 Subject: [PATCH 022/112] setup/destination: frequency --- lib/setup/setupDestination | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/lib/setup/setupDestination b/lib/setup/setupDestination index 0697a77..e618841 100644 --- a/lib/setup/setupDestination +++ b/lib/setup/setupDestination @@ -354,11 +354,37 @@ setupDestination() { comeagain destinationchannels["$destination"]=$value cat <<-EODesc + + Sampling rate (Hertz, integer): + Files will be resampled as needed to the specified sampling-rate. + Shoutcast/Icecast streams require a constant sampling-rate. + Telephony systems often require a sampling-rate of 8000Hz. EODesc + if [[ ${destinationformat["$destination"]} == opus ]] + then + cat <<-EODesc + + Please note that Opus supports only the following sample-rates: + 8000, 12000, 16000, 24000, and 48000 Hz. So don't set + resampling on an Opus destination to any other value or files + will be resampled twice. + EODesc + fi comeagain() { - : + read \ + -e \ + ${destinationfrequency["$destination"]+-i${destinationfrequency["$destination"]}}\ + -p'Sampling-rate: ' \ + value + if ! [[ $value =~ $expr ]] + then + echo "Invalid frequency value: $value" >&2 + comeagain + fi + unset expr } comeagain + destinationfrequency["$destination"]=$value cat <<-EODesc EODesc comeagain() { From 46a43828e4922ceb461addba3eac22ac583c864d Mon Sep 17 00:00:00 2001 From: Vincent Riquer Date: Wed, 19 Jun 2013 02:35:27 +0200 Subject: [PATCH 023/112] setup/destination: higher-than --- lib/setup/setupDestination | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/lib/setup/setupDestination b/lib/setup/setupDestination index e618841..bd3c152 100644 --- a/lib/setup/setupDestination +++ b/lib/setup/setupDestination @@ -386,9 +386,29 @@ setupDestination() { comeagain destinationfrequency["$destination"]=$value cat <<-EODesc + + Higher-Than (bitrate, integer): + Only reencode files with bitrates higher then kbps. This + only applies if sample-rate, channel count and of course format are + equal. If unset, only files with bitrates equal to that of the + target will be copied (actually, hardlinking will be attempted + first). + + As Ogg Vorbis target quality is not defined by its bitrate, Ogg + Vorbis files will always be reencoded if unset. EODesc comeagain() { - : + read \ + -e \ + ${destinationmaxbps["$destination"]+-i${destinationmaxbps["$destination"]}}\ + -p'Higher-Than: ' \ + value + if ! [[ $value =~ $expr ]] + then + echo "Invalid higher-than bitrate value: $value" >&2 + comeagain + fi } comeagain + destinationmaxbps[$destination]="$value" } From 4822d95760e190f41e196d90fb882cdccf82170a Mon Sep 17 00:00:00 2001 From: Vincent Riquer Date: Wed, 19 Jun 2013 11:07:39 +0200 Subject: [PATCH 024/112] Move files around --- lib/config/{getConfig => get} | 0 lib/config/{getConfigDestination => getDestination} | 0 lib/config/{getConfigGeneral => getGeneral} | 0 lib/config/{getConfigSource => getSource} | 0 lib/copy/{copyFiles_action => action} | 0 lib/copy/{checkCopy => check} | 0 lib/copy/{copyFiles_matching => matching} | 0 lib/database/{closeDatabase => close} | 0 lib/database/{openDatabase => open} | 0 lib/decode/{decodeFile => file} | 0 lib/decode/{decodeMpcdec => mpcdec} | 0 lib/decode/{decodeOpusdec => opusdec} | 0 lib/decode/{decodeSox => sox} | 0 lib/destinations/{createDestinations => create} | 0 lib/encode/{encodeFile::mp3 => mp3} | 0 lib/encode/{encodeFile::opus => opus} | 0 lib/encode/{encodeFile::vorbis => vorbis} | 0 lib/setup/{setupDestination => destination} | 0 lib/setup/{setupDestinations => destinations} | 0 lib/setup/{setupGeneral => general} | 0 lib/setup/{setupSource => source} | 0 lib/tags/{getInfos::ffmpeg => ffmpeg} | 0 lib/tags/{getInfos::FLAC => flac} | 0 lib/tags/{getTags => gettags} | 0 lib/tags/{getInfos::Ogg => ogg} | 0 lib/tags/{getInfos::Opus => opus} | 0 lib/tags/{updateTags => update} | 0 lib/workers/{checkworkers => check} | 0 lib/workers/{createworker => create} | 0 lib/workers/{destroyworker => destroy} | 0 lib/workers/{getworkerid => getid} | 0 31 files changed, 0 insertions(+), 0 deletions(-) rename lib/config/{getConfig => get} (100%) rename lib/config/{getConfigDestination => getDestination} (100%) rename lib/config/{getConfigGeneral => getGeneral} (100%) rename lib/config/{getConfigSource => getSource} (100%) rename lib/copy/{copyFiles_action => action} (100%) rename lib/copy/{checkCopy => check} (100%) rename lib/copy/{copyFiles_matching => matching} (100%) rename lib/database/{closeDatabase => close} (100%) rename lib/database/{openDatabase => open} (100%) rename lib/decode/{decodeFile => file} (100%) rename lib/decode/{decodeMpcdec => mpcdec} (100%) rename lib/decode/{decodeOpusdec => opusdec} (100%) rename lib/decode/{decodeSox => sox} (100%) rename lib/destinations/{createDestinations => create} (100%) rename lib/encode/{encodeFile::mp3 => mp3} (100%) rename lib/encode/{encodeFile::opus => opus} (100%) rename lib/encode/{encodeFile::vorbis => vorbis} (100%) rename lib/setup/{setupDestination => destination} (100%) rename lib/setup/{setupDestinations => destinations} (100%) rename lib/setup/{setupGeneral => general} (100%) rename lib/setup/{setupSource => source} (100%) rename lib/tags/{getInfos::ffmpeg => ffmpeg} (100%) rename lib/tags/{getInfos::FLAC => flac} (100%) rename lib/tags/{getTags => gettags} (100%) rename lib/tags/{getInfos::Ogg => ogg} (100%) rename lib/tags/{getInfos::Opus => opus} (100%) rename lib/tags/{updateTags => update} (100%) rename lib/workers/{checkworkers => check} (100%) rename lib/workers/{createworker => create} (100%) rename lib/workers/{destroyworker => destroy} (100%) rename lib/workers/{getworkerid => getid} (100%) diff --git a/lib/config/getConfig b/lib/config/get similarity index 100% rename from lib/config/getConfig rename to lib/config/get diff --git a/lib/config/getConfigDestination b/lib/config/getDestination similarity index 100% rename from lib/config/getConfigDestination rename to lib/config/getDestination diff --git a/lib/config/getConfigGeneral b/lib/config/getGeneral similarity index 100% rename from lib/config/getConfigGeneral rename to lib/config/getGeneral diff --git a/lib/config/getConfigSource b/lib/config/getSource similarity index 100% rename from lib/config/getConfigSource rename to lib/config/getSource diff --git a/lib/copy/copyFiles_action b/lib/copy/action similarity index 100% rename from lib/copy/copyFiles_action rename to lib/copy/action diff --git a/lib/copy/checkCopy b/lib/copy/check similarity index 100% rename from lib/copy/checkCopy rename to lib/copy/check diff --git a/lib/copy/copyFiles_matching b/lib/copy/matching similarity index 100% rename from lib/copy/copyFiles_matching rename to lib/copy/matching diff --git a/lib/database/closeDatabase b/lib/database/close similarity index 100% rename from lib/database/closeDatabase rename to lib/database/close diff --git a/lib/database/openDatabase b/lib/database/open similarity index 100% rename from lib/database/openDatabase rename to lib/database/open diff --git a/lib/decode/decodeFile b/lib/decode/file similarity index 100% rename from lib/decode/decodeFile rename to lib/decode/file diff --git a/lib/decode/decodeMpcdec b/lib/decode/mpcdec similarity index 100% rename from lib/decode/decodeMpcdec rename to lib/decode/mpcdec diff --git a/lib/decode/decodeOpusdec b/lib/decode/opusdec similarity index 100% rename from lib/decode/decodeOpusdec rename to lib/decode/opusdec diff --git a/lib/decode/decodeSox b/lib/decode/sox similarity index 100% rename from lib/decode/decodeSox rename to lib/decode/sox diff --git a/lib/destinations/createDestinations b/lib/destinations/create similarity index 100% rename from lib/destinations/createDestinations rename to lib/destinations/create diff --git a/lib/encode/encodeFile::mp3 b/lib/encode/mp3 similarity index 100% rename from lib/encode/encodeFile::mp3 rename to lib/encode/mp3 diff --git a/lib/encode/encodeFile::opus b/lib/encode/opus similarity index 100% rename from lib/encode/encodeFile::opus rename to lib/encode/opus diff --git a/lib/encode/encodeFile::vorbis b/lib/encode/vorbis similarity index 100% rename from lib/encode/encodeFile::vorbis rename to lib/encode/vorbis diff --git a/lib/setup/setupDestination b/lib/setup/destination similarity index 100% rename from lib/setup/setupDestination rename to lib/setup/destination diff --git a/lib/setup/setupDestinations b/lib/setup/destinations similarity index 100% rename from lib/setup/setupDestinations rename to lib/setup/destinations diff --git a/lib/setup/setupGeneral b/lib/setup/general similarity index 100% rename from lib/setup/setupGeneral rename to lib/setup/general diff --git a/lib/setup/setupSource b/lib/setup/source similarity index 100% rename from lib/setup/setupSource rename to lib/setup/source diff --git a/lib/tags/getInfos::ffmpeg b/lib/tags/ffmpeg similarity index 100% rename from lib/tags/getInfos::ffmpeg rename to lib/tags/ffmpeg diff --git a/lib/tags/getInfos::FLAC b/lib/tags/flac similarity index 100% rename from lib/tags/getInfos::FLAC rename to lib/tags/flac diff --git a/lib/tags/getTags b/lib/tags/gettags similarity index 100% rename from lib/tags/getTags rename to lib/tags/gettags diff --git a/lib/tags/getInfos::Ogg b/lib/tags/ogg similarity index 100% rename from lib/tags/getInfos::Ogg rename to lib/tags/ogg diff --git a/lib/tags/getInfos::Opus b/lib/tags/opus similarity index 100% rename from lib/tags/getInfos::Opus rename to lib/tags/opus diff --git a/lib/tags/updateTags b/lib/tags/update similarity index 100% rename from lib/tags/updateTags rename to lib/tags/update diff --git a/lib/workers/checkworkers b/lib/workers/check similarity index 100% rename from lib/workers/checkworkers rename to lib/workers/check diff --git a/lib/workers/createworker b/lib/workers/create similarity index 100% rename from lib/workers/createworker rename to lib/workers/create diff --git a/lib/workers/destroyworker b/lib/workers/destroy similarity index 100% rename from lib/workers/destroyworker rename to lib/workers/destroy diff --git a/lib/workers/getworkerid b/lib/workers/getid similarity index 100% rename from lib/workers/getworkerid rename to lib/workers/getid From c81f1879fcf4ee72b9e2f638de1f4634c720902c Mon Sep 17 00:00:00 2001 From: Vincent Riquer Date: Wed, 19 Jun 2013 12:54:05 +0200 Subject: [PATCH 025/112] setup/destination: tiny fixes --- lib/setup/destination | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/lib/setup/destination b/lib/setup/destination index bd3c152..9c4925a 100644 --- a/lib/setup/destination +++ b/lib/setup/destination @@ -222,18 +222,19 @@ setupDestination() { Leave blank if you don't want file renaming. EODesc initialvalue="${destinationrenamepath["$destination"]}" - initialvalue+="${initialvalue+/}" + initialvalue+=/ + [[ $initialvalue == / ]] && unset initialvalue initialvalue+="${destinationrename["$destination"]}" + [ -z "$initialvalue" ] && unset initialvalue read \ -e \ ${initialvalue+-i"$initialvalue"} \ -p'Rename pattern: ' \ value - case "$value" in - */*) - destinationrenamepath["$destination"]="${value%/*}" - ;; - esac + if [[ $value =~ / ]] + then + destinationrenamepath["$destination"]="${value%/*}" + fi destinationrename["$destination"]="${value##*/}" cat <<-EODesc @@ -272,12 +273,13 @@ setupDestination() { This prompt will loop until an empty string is encountered. EODesc - destinationskipmime["$destination"]="${destinationskipmime["$destination"]}|" - while [[ ${destinationskipmime["$destination"]} =~ / ]] + while [[ ${destinationskipmime["$destination"]} =~ \| ]] do skippedmimes+=("${destinationskipmime["$destination"]%%|*}") destinationskipmime["$destination"]="${destinationskipmime["$destination"]#*|}" done + [ -n "${destinationskipmime["$destination"]}" ] \ + && skippedmimes+=("${destinationskipmime["$destination"]}") count=${#skippedmimes[@]} unset destinationskipmime["$destination"] for (( i=0 ; 1 ; i++ )) @@ -307,12 +309,13 @@ setupDestination() { This prompt will loop until an empty string is encountered. EODesc - destinationcopymime["$destination"]="${destinationcopymime["$destination"]}|" - while [[ ${destinationcopymime["$destination"]} =~ / ]] + while [[ ${destinationcopymime["$destination"]} =~ \| ]] do copiedmimes+=("${destinationcopymime["$destination"]%%|*}") destinationcopymime["$destination"]="${destinationcopymime["$destination"]#*|}" done + [ -n "${destinationcopymime["$destination"]}" ] \ + && copiedmimes+=("${destinationcopymime["$destination"]}") count=${#copiedmimes[@]} unset destinationcopymime["$destination"] for (( i=0 ; 1 ; i++ )) From daac6e00e560452f2d4ee3f3218e3d7b65478ea8 Mon Sep 17 00:00:00 2001 From: Vincent Riquer Date: Fri, 21 Jun 2013 02:32:39 +0200 Subject: [PATCH 026/112] writeConfig() --- lib/config/write | 237 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 237 insertions(+) create mode 100644 lib/config/write diff --git a/lib/config/write b/lib/config/write new file mode 100644 index 0000000..f376731 --- /dev/null +++ b/lib/config/write @@ -0,0 +1,237 @@ +#!/bin/bash + +writeConfig() { + cat <<-EOCfg +[general] +# This section contains parameters of the program itself. + +# * max-load : Integer. Defines how parallel processing will behave. AtOM +# will try to keep the 1 minute load average between and +1 by +# adjusting concurrency. +# Initial concurrency will be set to half of that value. +max-load $maxload + +# * load-interval : Integer. How often should we check the load average +# and adjust concurrency. Set this too low, and concurrency may be increased +# too quickly. Set this too high, and AtOM will not adapt quickly enough to +# load increase. In both cases, your hard drive will suffer. In my +# experience, 30 seconds is a good value. +load-interval $loadinterval + +# * ionice [niceness]: IO-hungry processes will be run with ionice class +# and niceness [niceness] (if applicable). See man ionice for details. +ionice $class $niceness + +# * temporary-directory : String. Name speaks for itself: this is +# where FIFOs (for communicating with sqlite) and temporary WAVE files will +# be created. Note that debug logs (if enabled) will go there too. +temporary-directory $tempdir + +# * database : String. Where the SQLite database should be stored. +database $database + +# * debug : Integer. +#debug 1 + + +[source] +# This section defines where are the files you want transcoded. + +# * path : String. The root of your collection. +# Default: /var/lib/mpd/music +path $sourcepath + +# * skip : String. Files in will be ignored. Note that +# can be any expression accepted by find. + EOCfg + for dir in "${skippeddirectories[@]}" + do + echo $'skip\t\t\t'"$dir" + done + cat <<-EOCfg + + + EOCfg + for destination in "${!destinationpath[@]}" + do + cat <<-EOCfg +[$destination] +# Each section not named 'general' or 'source' will define a new destination. + +# Common parameters: +# Mandatory parameters: +# * path: Where files will be written +path ${destinationpath["$destination"]} + +# * format: ogg, opus or mp3. Other formats may appear in the future - feel +# free to implement your preferred format. +format ${destinationformat["$destination"]} + + EOCfg + case "${destinationformat[@]}" in + vorbis) + cat <<-EOCfg +# Ogg parameters: +# * quality : The quality parameter of oggenc. See man oggenc for +# more info. This is the only mode supported and planned. Still, if you want +# to be able to use bitrate settings, feel free to fork and file a pull +# request. +quality ${destinationquality["$destination"]} + + + EOCfg + ;; + opus) + cat <<-EOCfg +# Opus parameters: +# * bitrate : Set (VBR) bitrate to . Note that while Opus +# allows for decimal values, AtOM does not. The reason for this is simple: +# we do numeric comparisons, and Bash only manipulates integers. +bitrate ${destinationquality["$destination"]} + +# * loss : If you intend to stream the resulting files over an +# unreliable protocol, you may want to make use of Opus' Forward Error +# Correction algorythm. See the Opus-codec.org website for details. +# Default: 0 +loss ${destinationloss["$destination"]:-0} + + + EOCfg + ;; + mp3) + cat <<-EOCfg +# MP3 parameters: +# * bitrate : Set ABR to . Again, if you want CBR or any +# other mode supported by lame, please fork and file a pull request. +bitrate ${destinationquality["$destination"]} + +# * noresample /: LAME may decide to encode your file to a lower +# sampling-rate if you use a low bitrate. Setting this to yes will +# append --resample , preventing any resampling from +# happening. + EOCfg + if (( ${destinationnoresample["$destination"]} )) + then + echo $'noresample\t\tyes' + else + echo $'noresample\t\tno' + fi + cat <<-EOCfg + + + EOCfg + ;; + esac + cat <<-EOCfg +# Optional parameters: +# * normalize /: Normalize output files. + EOCfg + if (( ${destinationnormalize["$destination"]} )) + then + echo $'normalize\t\tyes' + else + echo $'normalize\t\tno' + fi + cat <<-EOCfg + +# * rename : Destination files will be named according to , +# after expansion of special strings: +# %{album}, +# %{albumartist}, +# %{artist}, +# %{disc}, +# %{genre}, +# %{title}, +# %{track}, +# %{year}. +# Untagged files or files in unrecognized formats will not be changed. + EOCfg + if [[ ${destinationrenamepath["$destination"]}/${destinationrename["$destination"]} == / ]] + then + echo $'#rename\t\t\t' + elif [[ ${destinationrenamepath["$destination"]}/${destinationrename["$destination"]} == /* ]] + then + echo $'rename\t\t\t'"${destinationrename["$destination"]}" + else + echo $'rename\t\t\t'"${destinationrenamepath["$destination"]}/${destinationrename["$destination"]}" + fi + cat <<-EOCfg + +# * fat32compat /: Rename files for compatibility with FAT32 +# filesystems. + EOCfg + if (( ${destinationfat32compat["$destination"]} )) + then + echo $'fat32compat\t\tyes' + else + echo $'fat32compat\t\tno' + fi + cat <<-EOCfg + +# * skip_mime-type : Files with mime-type will not +# be included in that destination. For more than one mime-type, use multiple +# times, as needed. The '*' character is a wildcard. + EOCfg + destinationskipmime["$destination"]="${destinationskipmime["$destination"]}|" + while [[ ${destinationskipmime["$destination"]} =~ \| ]] + do + echo $'skip_mime-type\t\t'"${destinationskipmime["$destination"]%%|*}" + destinationskipmime["$destination"]="${destinationskipmime["$destination"]#*|}" + done + cat <<-EOCfg + +# * copy_mime-type : Same as skip_mime-type, except that files +# matching will be copied as-is to the destination. E.g. image/* will copy +# covers and other images to the destination. In fact, AtOM will try to use +# hard links instead of copies. + EOCfg + destinationcopymime["$destination"]="${destinationcopymime["$destination"]}|" + while [[ ${destinationcopymime["$destination"]} =~ \| ]] + do + echo $'copy_mime-type\t\t'"${destinationcopymime["$destination"]%%|*}" + destinationcopymime["$destination"]="${destinationcopymime["$destination"]#*|}" + done + cat <<-EOCfg + +# * channels : Files with more than channels will be +# downmixed. Useful if you create files for telephony music-on-hold. + EOCfg + if (( ${destinationchannels["$destination"]} )) + then + echo $'channels\t\t'${destinationchannels["$destination"]} + else + echo $'#channels\t\t2' + fi + cat <<-EOCfg + +# * frequency : Files will be resampled as needed to Hz +# sampling-rate. Shoutcast/Icecast streams require a constant sampling-rate. +# Telephony systems often require a sample rate of 8000Hz. + EOCfg + if (( ${destinationfrequency["$destination"]} )) + then + echo $'frequency\t\t'${destinationfrequency["$destination"]} + else + echo $'#frequency\t\t44100' + fi + cat <<-EOCfg + +# * higher-than : Integer. Only reencode files with bitrates higher +# then kbps. This only applies if sample-rate, channel count and of +# course format are equal. If unset, only files with bitrates equal to that +# of the target will be copied (actually, hardlinking will be attempted +# first). As Ogg Vorbis target quality is not defined by its bitrate, Ogg +# Vorbis files will always be reencoded if unset. + EOCfg + if (( ${destinationmaxbps["$destination"]} )) + then + echo $'higher-than\t\t'${destinationmaxbps["$destination"]} + else + echo $'#higher-than\t\t128' + fi + cat <<-EOCfg + + + EOCfg + done +} From a2ee934149fa779789d80733ad456bb5f96910d8 Mon Sep 17 00:00:00 2001 From: Vincent Riquer Date: Fri, 21 Jun 2013 09:46:44 +0200 Subject: [PATCH 027/112] writeConfig(): fix format-specific parameters loop --- lib/config/write | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/config/write b/lib/config/write index f376731..6564832 100644 --- a/lib/config/write +++ b/lib/config/write @@ -68,7 +68,7 @@ path ${destinationpath["$destination"]} format ${destinationformat["$destination"]} EOCfg - case "${destinationformat[@]}" in + case "${destinationformat["$destination"]}" in vorbis) cat <<-EOCfg # Ogg parameters: From 4a4c6a966d97e2b64b980540c50955b024befa58 Mon Sep 17 00:00:00 2001 From: Vincent Riquer Date: Fri, 21 Jun 2013 09:47:58 +0200 Subject: [PATCH 028/112] printConfig(): cosmetic --- lib/config/print | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/config/print b/lib/config/print index 0a4fea4..bd296e6 100644 --- a/lib/config/print +++ b/lib/config/print @@ -9,6 +9,7 @@ printConfig() { |Temp Dir|$tempdir |Database|$database |Debug|$debug + Source|Path|$sourcepath EOF for prune_expression in "${skippeddirectories[@]}" @@ -23,6 +24,7 @@ printConfig() { for destination in ${!destinationpath[@]} do cat <<-EOF + $destination|Path|${destinationpath["$destination"]} |Format|${destinationformat["$destination"]} |Quality|${destinationquality["$destination"]} From e4ca4d63d86ca80f722846b58aaf74b47c902a60 Mon Sep 17 00:00:00 2001 From: Vincent Riquer Date: Fri, 21 Jun 2013 09:48:36 +0200 Subject: [PATCH 029/112] setup: * user confirmation * write config --- lib/setup/setup | 69 ++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 68 insertions(+), 1 deletion(-) diff --git a/lib/setup/setup b/lib/setup/setup index c814d31..a41601f 100644 --- a/lib/setup/setup +++ b/lib/setup/setup @@ -11,5 +11,72 @@ Completion is available for prompts asking for a paths or filenames. setupSource setupDestinations unset expr - exit + writeConfig >"$cffile".tmp + unset \ + sourcepath \ + skippeddirectories \ + maxload \ + loadinterval \ + ionice \ + tempdir \ + database \ + debug \ + destinationchannels \ + destinationfat32compat \ + destinationcopymime \ + destinationformat \ + destinationfrequency \ + destinationid \ + destinationloss \ + destinationmaxbps \ + destinationnormalize \ + destinationpath \ + destinationquality \ + destinationrename \ + destinationnoresample \ + destinationrenamepath \ + destinationskipmime + declare -A \ + destinationchannels \ + destinationfat32compat \ + destinationcopymime \ + destinationformat \ + destinationfrequency \ + destinationid \ + destinationloss \ + destinationmaxbps \ + destinationnormalize \ + destinationpath \ + destinationquality \ + destinationrename \ + destinationnoresample \ + destinationrenamepath \ + destinationskipmime + oldcffile="$cffile" + cffile="$cffile".tmp + getConfig + { + echo $'Please review your new configuration:\n' + printConfig + }| less -F -e + cffile="$oldcffile" + read -p'Write config file? [y/N] ' do_write + case $do_write in + y) + mv -f "$cffile".tmp "$cffile" + ;; + *) + rm "$cffile".tmp + read -p'Re-run (s)etup, (q)uit [s/Q] ' do_rerun + case $do_rerun in + s) setup ;; + *) exit ;; + esac + ;; + esac + read -p'Run now? [Y/n] ' do_run + case $do_run in + n) exit ;; + *) ;; + esac } From c5b50565c558c4ac641a337e991813607ae3b4c0 Mon Sep 17 00:00:00 2001 From: Vincent Riquer Date: Fri, 21 Jun 2013 14:06:00 +0200 Subject: [PATCH 030/112] Rename pattern change: fix hang, DB optimisation --- atom | 27 ++++++++++++++++++++------- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/atom b/atom index e1a22a4..b9b529f 100755 --- a/atom +++ b/atom @@ -720,7 +720,12 @@ do ' >&3 read -u4 line - if [[ $line != AtOM:NoMoreFiles ]] + while [[ $line != AtOM:NoMoreFiles ]] + do + renamefiles+=("$line") + read -u4 line + done + if (( ${#renamefiles[@]} )) then case "${destinationformat[$destination]}" in 'mp3') extension=mp3 ;; @@ -728,7 +733,8 @@ do 'vorbis') extension=ogg ;; esac echo -n "$destination: rename pattern changed, renaming files... " - while [[ $line != AtOM:NoMoreFiles ]] + echo 'BEGIN TRANSACTION;' >&3 + for line in "${renamefiles[@]}" do oldfilename=${line%%::AtOM:SQL:Sep::*} rest=${line#*::AtOM:SQL:Sep::}'::AtOM:SQL:Sep::' @@ -761,10 +767,12 @@ do getDestDir getDestFile destfilename="$destdir/$destfile.$extension" - if [[ $oldfilename != $destfilename ]] + progressSpin + if [[ "$oldfilename" != "$destfilename" ]] then mv "$oldfilename" "$destfilename" - progressSpin + (( changedcount++ )) + commit=1 fi echo "UPDATE destination_files" \ "SET filename=\"${destfilename//\"/\"\"}\"," \ @@ -772,12 +780,17 @@ do "\"${destinationrenamepath[$destination]}/${destinationrename[$destination]}:${destinationfat32compat["$destination"]}\"" \ "WHERE id=$destfileid;" \ >&3 + if (( commit )) + then + echo $'COMMIT;\nBEGIN TRANSACTION;' >&3 + unset commit + fi fi - read -u4 line done - echo -e $'\r'"$destination: Renamed ${count:-0} files\033[K" + echo 'COMMIT;' >&3 + echo -e $'\r'"$destination: Renamed ${changedcount:-0} files\033[K" fi - unset count + unset count changedcount renamefiles done copyFiles_action From 2fbeba32950cefa2b4089bcac0dac9d117044191 Mon Sep 17 00:00:00 2001 From: Vincent Riquer Date: Thu, 27 Jun 2013 18:34:30 +0200 Subject: [PATCH 031/112] Document setup --- README | 8 +++++++- atom | 1 + doc/config | 6 ++++-- 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/README b/README index a8da963..96ac15e 100644 --- a/README +++ b/README @@ -46,7 +46,13 @@ Using the software Configuration: -------------- -Please read doc/config before anything else. +On first run, AtOM will ask a set of questions to help you create a +configuration file. +You can run atom -S at any time to re-run the setup. It will be prefilled with +your current configuration. + +If, however, you still want to make changes manually, please read doc/config. +There are a lot of comments in the generated config file too. Preparing data: --------------- diff --git a/atom b/atom index b9b529f..1eb4ced 100755 --- a/atom +++ b/atom @@ -65,6 +65,7 @@ help() { -T override load-interval -F Force re-generation of all files in + -S Run setup -h Show this text -D Increase debug EOF diff --git a/doc/config b/doc/config index f434082..fbe078e 100644 --- a/doc/config +++ b/doc/config @@ -1,5 +1,7 @@ -On first launch, AtOM will create an example configuration file for you. This is -only an example, suitable to *my* needs (I'm lazy). +On first launch, AtOM will create a configuration file for you. The recommended +way of changing your configuration is by running +$ atom -S. + Default config file is ~/.atom/atom.cfg. The file is divided in sections, beginning with []. The section From 13734a0f1d92591dbf1c82cce525fe350f03cab5 Mon Sep 17 00:00:00 2001 From: Vincent Riquer Date: Thu, 27 Jun 2013 21:08:30 +0200 Subject: [PATCH 032/112] Regen files on key parameters' change --- atom | 2 +- lib/setup/destination | 44 +++++++++++++++++++++++++++++++++++++++++++ lib/setup/regen | 28 +++++++++++++++++++++++++++ 3 files changed, 73 insertions(+), 1 deletion(-) create mode 100644 lib/setup/regen diff --git a/atom b/atom index 1eb4ced..3e42f01 100755 --- a/atom +++ b/atom @@ -343,7 +343,7 @@ do then echo "Resetting destination files timestamps on" \ "$forcedest ($forcedestid)..." - Update destination_files last_change 1 \ + Update destination_files last_change 0 \ <<<"destination_id = $forcedestid" else echo "Destination $forcedest does not exist!" >&2 diff --git a/lib/setup/destination b/lib/setup/destination index 9c4925a..40f4ed1 100644 --- a/lib/setup/destination +++ b/lib/setup/destination @@ -65,6 +65,11 @@ setupDestination() { fi } comeagain + if [ -n "${destinationquality["$destination"]}" ] \ + && (( value != ${destinationquality["$destination"]} )) + then + setupRegen quality + fi destinationquality["$destination"]=$value ;; opus) @@ -90,6 +95,11 @@ setupDestination() { fi } comeagain + if [ -n "${destinationquality["$destination"]}" ] \ + && (( value != ${destinationquality["$destination"]} )) + then + setupRegen bitrate + fi destinationquality["$destination"]=$value cat <<-EODesc @@ -112,6 +122,11 @@ setupDestination() { fi } comeagain + if [ -n "${destinationloss["$destination"]}" ] \ + && (( value != ${destinationloss["$destination"]} )) + then + setupRegen loss + fi destinationloss["$destination"]=$value ;; mp3) @@ -135,6 +150,11 @@ setupDestination() { fi } comeagain + if [ -n "${destinationquality["$destination"]}" ] \ + && (( value != ${destinationquality["$destination"]} )) + then + setupRegen bitrate + fi destinationquality["$destination"]=$value cat <<-EODesc @@ -156,9 +176,13 @@ setupDestination() { value case $value in [yY]) + [[ $initialvalue == n ]]\ + && setupRegen noresample destinationnoresample["$destination"]=1 ;; ''|[nN]) + [[ $initialvalue == y ]]\ + && setupRegen noresample destinationnoresample["$destination"]=0 ;; *) @@ -193,9 +217,13 @@ setupDestination() { value case $value in [yY]) + [[ $initialvalue == n ]] \ + && setupRegen normalize destinationnormalize["$destination"]=1 ;; ''|[nN]) + [[ $initialvalue == y ]] \ + && setupRegen normalize destinationnormalize["$destination"]=0 ;; *) @@ -355,6 +383,11 @@ setupDestination() { fi } comeagain + if [ -n "${destinationchannels["$destination"]}" ] \ + && (( value != ${destinationchannels["$destination"]} )) + then + setupRegen channels + fi destinationchannels["$destination"]=$value cat <<-EODesc @@ -387,6 +420,11 @@ setupDestination() { unset expr } comeagain + if [ -n "${destinationfrequency["$destination"]}" ] \ + && (( value != ${destinationfrequency["$destination"]} )) + then + setupRegen frequency + fi destinationfrequency["$destination"]=$value cat <<-EODesc @@ -413,5 +451,11 @@ setupDestination() { fi } comeagain + if [ -n "${destinationmaxbps["$destination"]}" ] \ + && (( value != ${destinationmaxbps["$destination"]} )) + then + setupRegen maxbps + fi destinationmaxbps[$destination]="$value" + unset regen } diff --git a/lib/setup/regen b/lib/setup/regen new file mode 100644 index 0000000..c6c9fe8 --- /dev/null +++ b/lib/setup/regen @@ -0,0 +1,28 @@ +#!/bin/bash + +setupRegen() { + (( regen )) && return 0 + echo "Parameter $1 for destination $destination changed." + read -p'Regenerate all files? [y/n] ' + case $REPLY in + y) + regen=1 + openDatabase + if forcedestid=$(Select destinations id <<<"name = $destination") + then + echo "Resetting destination files timestamps on" \ + "$destination ($forcedestid)..." + Update destination_files last_change 0 \ + <<<"destination_id = $forcedestid" + else + echo "Destination $destination does not exist!" >&2 + fi + closeDatabase + ;; + n) + ;; + *) + setupRegen $1 + ;; + esac +} From cfc94d0a47281157939d0a29f9c32cb2de9f3501 Mon Sep 17 00:00:00 2001 From: Vincent Riquer Date: Mon, 9 Sep 2013 01:16:27 +0200 Subject: [PATCH 033/112] Target formats can't handle 24bps * read bit depth fron FLAC files * downsample to 16 --- atom | 4 ++++ lib/decode/sox | 5 +++++ lib/tags/flac | 4 +++- share/schema.sql | 1 + 4 files changed, 13 insertions(+), 1 deletion(-) diff --git a/atom b/atom index 3e42f01..df88835 100755 --- a/atom +++ b/atom @@ -423,6 +423,7 @@ echo ' mime_type_actions.mime_text, destinations.name, destination_files.id, + tags.depth, tags.rate, tags.channels, tags.bitrate, @@ -473,6 +474,8 @@ do rest=${rest#*::AtOM:SQL:Sep::} destfileid=${rest%%::AtOM:SQL:Sep::*} rest=${rest#*::AtOM:SQL:Sep::} + bitdepth=${rest%%::AtOM:SQL:Sep::*} + rest=${rest#*::AtOM:SQL:Sep::} rate=${rest%%::AtOM:SQL:Sep::*} rest=${rest#*::AtOM:SQL:Sep::} channels=${rest%%::AtOM:SQL:Sep::*} @@ -517,6 +520,7 @@ do album \ albumartist \ artist \ + bitdepth \ bitrate \ channels \ commandline \ diff --git a/lib/decode/sox b/lib/decode/sox index 5d7e6df..ef4c7f6 100644 --- a/lib/decode/sox +++ b/lib/decode/sox @@ -26,6 +26,11 @@ decodeSox() { commandline+=(-c ${destinationchannels["$destination"]}) soxoptions_out+=" -c ${destinationchannels["$destination"]}" fi + if (( ${bitdepth:-0} > 16 )) + then + commandline+=(-b 16) + soxoptions_out+=" -b 16" + fi tmpfile="$fileid${soxoptions_in// /}${soxoptions_out// /}" commandline+=("$tempdir/$tmpfile.wav") } diff --git a/lib/tags/flac b/lib/tags/flac index 288eac7..32d3f4a 100644 --- a/lib/tags/flac +++ b/lib/tags/flac @@ -1,5 +1,5 @@ #!/bin/bash -getInfosFLAC_version='FLAC-1' +getInfosFLAC_version='FLAC-2' tagreaders+=( "$getInfosFLAC_version" ) getInfos::FLAC() { tagreader="$getInfosFLAC_version" @@ -36,10 +36,12 @@ getInfos::FLAC() { { read rate read channels + read bitdepth } < <( metaflac \ --show-sample-rate \ --show-channels \ + --show-bps \ "$sourcepath/$filename" ) } diff --git a/share/schema.sql b/share/schema.sql index 1859f55..9de9a43 100644 --- a/share/schema.sql +++ b/share/schema.sql @@ -52,6 +52,7 @@ CREATE TABLE IF NOT EXISTS tags ( title TEXT, composer TEXT, performer TEXT, + depth INTEGER, rate INTEGER, channels INTEGER, bitrate INTEGER, From 178f6a82af1936b38ec4e07b27af2899f56d6ef9 Mon Sep 17 00:00:00 2001 From: Vincent Riquer Date: Mon, 9 Sep 2013 01:48:10 +0200 Subject: [PATCH 034/112] MP3 can't handle sample rates above 48k --- lib/encode/mp3 | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/encode/mp3 b/lib/encode/mp3 index 1496570..7474ab2 100644 --- a/lib/encode/mp3 +++ b/lib/encode/mp3 @@ -26,6 +26,9 @@ encodeFile::mp3() { 11025) lameopts+=(--resample 11.025) ;; 8000) lameopts+=(--resample 8) ;; esac + elif (( rate > 48000 )) + then + lameopts+=(--resample 48) else case $rate in 48000) lameopts+=(--resample 48) ;; From 5e006ae7cc8a74c1416183e8bb5809bd58abe4f2 Mon Sep 17 00:00:00 2001 From: Vincent Riquer Date: Sat, 28 Sep 2013 00:09:02 +0200 Subject: [PATCH 035/112] Add "cron" mode (suppress most progress info) --- atom | 62 ++++++++++++++++++++++++++++-------------- lib/copy/action | 9 ++++-- lib/files/getFiles | 5 ++-- lib/tags/update | 8 ++++-- lib/tools/progressSpin | 19 ++++++++----- 5 files changed, 69 insertions(+), 34 deletions(-) diff --git a/atom b/atom index df88835..d1c7316 100755 --- a/atom +++ b/atom @@ -68,6 +68,7 @@ help() { -S Run setup -h Show this text -D Increase debug + -q Cron mode EOF } @@ -101,6 +102,9 @@ do D) (( debug++ )) ;; + q) + cron=1 + ;; :) echo "-$OPTARG requires an argument" help @@ -115,7 +119,13 @@ do done askconf() { - read -p"Create one now? [Y/n/q] " createconf + if (( cron )) + then + echo 'Non-interactive, not running setup. Please run atom -S.' >&2 + createconf=n + else + read -p"Create one now? [Y/n/q] " createconf + fi case $createconf in ''|[yY]) setup @@ -282,16 +292,19 @@ elif (( sanitywarn )) then echo " Sanity checks raised $sanitywarn warnings... Hit Control-C to abort." >&2 - timeout=$(( sanitywarn * 10 )) - echo -n "Starting in $(printf %3i $timeout)" \ - $'seconds...\b\b\b\b\b\b\b\b\b\b\b' >&2 - while (( timeout )) - do - echo -n $'\b\b\b'"$(printf %3i $timeout)" >&2 - sleep 1 - (( timeout-- )) - done - echo -en "\r\033[K" + if ! (( cron )) + then + timeout=$(( sanitywarn * 10 )) + echo -n "Starting in $(printf %3i $timeout)" \ + $'seconds...\b\b\b\b\b\b\b\b\b\b\b' >&2 + while (( timeout )) + do + echo -n $'\b\b\b'"$(printf %3i $timeout)" >&2 + sleep 1 + (( timeout-- )) + done + echo -en "\r\033[K" + fi fi openDatabase @@ -457,9 +470,9 @@ read -u4 line while ! [[ $line = AtOM:NoMoreFiles ]] do decodefiles+=("$line::AtOM:SQL:Sep::") +(( cron )) || echo -n 'Creating tasks... ' read -u4 line done -echo -n 'Creating tasks... ' echo 'BEGIN TRANSACTION;' >&3 for line in "${decodefiles[@]}" @@ -634,7 +647,7 @@ do ${seconds:-0} seconds" \ +'%d/%m %H:%M:%S' )" - printf \ + (( cron )) || printf \ "\r$fmtload $fmtworkers $fmtprogress $fmttime $eta\033[K"\ $humanload \ $maxload \ @@ -676,8 +689,11 @@ endtime=$(date +%s) ( ( ( ( days*24 + hours ) *60 ) + minutes ) *60 ) )) || true -echo -e "\rRan ${ran:=0} tasks, $failed of which failed, in $days" \ - "days, $hours hours, $minutes minutes and $seconds seconds.\033[K" +(( cron )) || echo -n $'\r' +echo -n "Ran ${ran:=0} tasks, $failed of which failed, in $days" \ + "days, $hours hours, $minutes minutes and $seconds seconds." +(( cron )) || echo -en "\033[K" +echo if [ -n "$quit" ] then @@ -737,7 +753,7 @@ do 'opus') extension=opus ;; 'vorbis') extension=ogg ;; esac - echo -n "$destination: rename pattern changed, renaming files... " + (( cron )) || echo -n "$destination: rename pattern changed, renaming files... " echo 'BEGIN TRANSACTION;' >&3 for line in "${renamefiles[@]}" do @@ -793,7 +809,10 @@ do fi done echo 'COMMIT;' >&3 - echo -e $'\r'"$destination: Renamed ${changedcount:-0} files\033[K" + (( cron )) || echo -n $'\r' + echo -n "$destination: Renamed ${changedcount:-0} files" + (( cron )) || echo -en "\033[K" + echo fi unset count changedcount renamefiles done @@ -810,7 +829,7 @@ echo ' SELECT "AtOM:NoMoreFiles"; ' >&3 -echo -n 'Removing obsolete files... ' +(( cron )) || echo -n 'Removing obsolete files... ' lines=() read -u4 line while [[ $line != AtOM:NoMoreFiles ]] @@ -831,10 +850,13 @@ do fi Update destination_files old_filename NULL <<<"id = $id" (( count++ )) - printf '\b\b\b\b%3i%%' $(( (100 * count) / ${#lines[@]} )) + (( cron )) || printf '\b\b\b\b%3i%%' $(( (100 * count) / ${#lines[@]} )) done echo 'COMMIT;' >&3 -echo -e "\rRemoved ${count:-0} obsolete files.\033[K" +(( cron )) || echo -n $'\r' +echo -n "Removed ${count:-0} obsolete files." +(( cron )) || echo -en "\033[K" +echo echo "Purging empty directories." for path in "${destinationpath[@]}" diff --git a/lib/copy/action b/lib/copy/action index 42a9823..95bbb4a 100644 --- a/lib/copy/action +++ b/lib/copy/action @@ -1,6 +1,6 @@ #!/bin/bash copyFiles_action() { - echo -n "Copying files... " + (( cron )) || echo -n "Copying files... " echo ' SELECT source_files.filename, @@ -85,9 +85,12 @@ copyFiles_action() { echo 'COMMIT;' >&3 if (( count )) then - echo -e "\rCopied ${done:-0} of $count files.\033[K" + (( cron )) || echo -n $'\r' + echo -n "Copied ${done:-0} of $count files." + (( cron )) || echo -en "\033[K" + echo else - echo -e "\rNothing to copy.\033[K" + (( cron )) || echo -e "\rNothing to copy.\033[K" fi unset count done } diff --git a/lib/files/getFiles b/lib/files/getFiles index 08a145f..cd79f5f 100644 --- a/lib/files/getFiles +++ b/lib/files/getFiles @@ -5,7 +5,7 @@ getFiles() { do prunes+="-path $sourcepath$prune_expression -prune -o " done - echo -n "Scanning $sourcepath... " + (( cron )) || echo -n "Scanning $sourcepath... " # We probably have thousands of files, don't waste time on disk writes echo 'BEGIN TRANSACTION;' >&3 while read time size filename @@ -59,6 +59,7 @@ getFiles() { find "$sourcepath" $prunes -type f -printf "%T@ %s %P\n" ) echo 'COMMIT;' >&3 - echo -e "\r${count:-0} files found, ${new:=0} new or changed." + (( cron )) || echo $'\r' + echo "${count:-0} files found, ${new:=0} new or changed." unset count } diff --git a/lib/tags/update b/lib/tags/update index 9881843..09af6e7 100644 --- a/lib/tags/update +++ b/lib/tags/update @@ -85,7 +85,8 @@ updateTags() { oldchannels=${rest%%::AtOM:SQL:Sep::*} rest=${rest#*::AtOM:SQL:Sep::} oldbitrate=${rest%%::AtOM:SQL:Sep::*} - echo -en "\rTags: $((++count*100/filecount))%" + ((++count)) + (( cron )) || echo -en "\rTags: $((count*100/filecount))%" if (( count % 1000 == 0 )) then echo 'COMMIT;BEGIN TRANSACTION;' >&3 @@ -153,6 +154,9 @@ updateTags() { fi done echo 'COMMIT;' >&3 - echo -e "\rRead tags from ${count:-0} files.\033[K" + (( cron )) || echo -n $'\r' + echo -n "Read tags from ${count:-0} files." + (( cron )) || echo -ne "\033[K" + echo unset count tagfiles } diff --git a/lib/tools/progressSpin b/lib/tools/progressSpin index cecff55..3cd54b8 100644 --- a/lib/tools/progressSpin +++ b/lib/tools/progressSpin @@ -1,10 +1,15 @@ #!/bin/bash progressSpin() { - case $(( ++count % 40 )) in - 0) echo -ne '\b|' ;; - 10) echo -ne '\b/' ;; - 20) echo -en '\b-' ;; - 30) echo -ne '\b\\' ;; - *) ;; - esac + if (( cron )) + then + (( ++count )) + else + case $(( ++count % 40 )) in + 0) echo -ne '\b|' ;; + 10) echo -ne '\b/' ;; + 20) echo -en '\b-' ;; + 30) echo -ne '\b\\' ;; + *) ;; + esac + fi } From 00dd038f7b1fbedaf166631a1a8ba8bbe3415293 Mon Sep 17 00:00:00 2001 From: Vincent Riquer Date: Sat, 28 Sep 2013 00:11:31 +0200 Subject: [PATCH 036/112] Add max batch size CLI option --- atom | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/atom b/atom index d1c7316..fc6ef9b 100755 --- a/atom +++ b/atom @@ -65,6 +65,7 @@ help() { -T override load-interval -F Force re-generation of all files in + -B Create/update no more than files -S Run setup -h Show this text -D Increase debug @@ -74,7 +75,7 @@ help() { #parse arguments OPTERR=0 -while getopts ':c:Cl:T:F:ShD' opt +while getopts ':c:Cl:T:F:B:ShDq' opt do case $opt in c) @@ -92,6 +93,9 @@ do F) forceall+=("$OPTARG") ;; + B) + maxbatch="$OPTARG" + ;; S) forcesetup=1 ;; @@ -429,6 +433,11 @@ echo ' AND mime_type_actions.destination_id = destinations.id AND mime_type_actions.action = 1;' >&3 read -u4 filecount +if [ -n "$maxbatch" ] && (( maxbatch < filecount )) +then + (( togo = filecount - maxbatch )) + filecount=$maxbatch +fi echo ' SELECT source_files.id, @@ -463,16 +472,17 @@ echo ' WHERE CAST(destination_files.last_change AS TEXT) <> CAST(source_files.last_change AS TEXT) AND mime_type_actions.destination_id = destinations.id - AND mime_type_actions.action = 1; - + AND mime_type_actions.action = 1' >&3 +(( maxbatch )) && echo "LIMIT $maxbatch" >&3 +echo '; SELECT "AtOM:NoMoreFiles";' >&3 read -u4 line while ! [[ $line = AtOM:NoMoreFiles ]] do decodefiles+=("$line::AtOM:SQL:Sep::") -(( cron )) || echo -n 'Creating tasks... ' read -u4 line done +(( cron )) || echo -n 'Creating tasks... ' echo 'BEGIN TRANSACTION;' >&3 for line in "${decodefiles[@]}" @@ -559,7 +569,8 @@ do tmpfile done echo 'COMMIT;' >&3 -echo -e "\rCreated ${count:-0} tasks for $filecount files (${copies:-0} immediate copies)" +(( cron )) || echo -n $'\r' +echo "Created ${count:-0} tasks for $filecount files ${togo:+($togo left) }(${copies:-0} immediate copies)" concurrency=$(( maxload / 2 )) (( concurrency )) || concurrency=1 From a3ef61551ba1a0b7e497559965db7da68eeb8575 Mon Sep 17 00:00:00 2001 From: Vincent Riquer Date: Sat, 28 Sep 2013 00:12:38 +0200 Subject: [PATCH 037/112] Pause: temporarily stop processing --- atom | 41 +++++++++++++++++++++++++++++++++++++---- 1 file changed, 37 insertions(+), 4 deletions(-) diff --git a/atom b/atom index fc6ef9b..8fcf787 100755 --- a/atom +++ b/atom @@ -594,11 +594,29 @@ do [qQ]) quit=1 ;; + [pP]) + if (( pause )) + then + pause=0 + (( + pausedtime += + $(date +%s) - pausestart + )) + concurrencychange=$(date +%s) + else + pause=1 + pausestart=$(date +%s) + (( concurrency = maxload / 2 )) + (( concurrency )) || concurrency=1 + fi + ;; esac fi read humanload garbage < /proc/loadavg load=${humanload%.*} - if [ -z "$quit" ] && (( $(date +%s)-concurrencychange >= loadinterval )) + if [ -z "$quit" ] \ + && (( ! pause )) \ + && (( $(date +%s)-concurrencychange >= loadinterval )) then if (( concurrency > 1 )) \ && (( load > maxload )) @@ -613,12 +631,18 @@ do fi checkworkers cleaner - master + (( pause )) || master if (( ran - failed )) then currenttime=$(date +%s) + if (( pause )) + then + (( runtime = pausestart - starttime - pausedtime )) + else + (( runtime = currenttime - starttime - pausedtime )) + fi avgduration=$(( - ((currenttime - starttime) * 1000) + ( runtime * 1000) / ( ran - failed ) )) @@ -673,12 +697,21 @@ do ${minutes:-0} \ ${seconds:-0} \ ${avgdsec:-0}.${avgdmsec:-0} + if (( pause )) + then + if (( active )) + then + echo -n ' | (pause)' + else + echo -n ' | PAUSED' + fi + fi done unset count endtime=$(date +%s) -(( elapsedseconds = endtime - starttime )) +(( elapsedseconds = endtime - starttime - pausedtime )) (( days = elapsedseconds / From 17b01fd308c41d1d45add0d49418831b8aba1930 Mon Sep 17 00:00:00 2001 From: Vincent Riquer Date: Sat, 28 Sep 2013 00:14:16 +0200 Subject: [PATCH 038/112] FFMpeg; get bit depth bump version of ffmpeg dependent parsers --- lib/tags/ffmpeg | 6 ++++-- lib/tags/getInfos::WebM | 2 +- lib/tags/getInfos::ffmpeg_other | 2 +- lib/tags/getInfos::ffmpeg_video | 2 +- 4 files changed, 7 insertions(+), 5 deletions(-) diff --git a/lib/tags/ffmpeg b/lib/tags/ffmpeg index fbfc59d..6b836a3 100644 --- a/lib/tags/ffmpeg +++ b/lib/tags/ffmpeg @@ -1,5 +1,5 @@ #!/bin/bash -getInfosffmpeg_version='ffmpeg-3' +getInfosffmpeg_version='ffmpeg-4' tagreaders+=( "$getInfosffmpeg_version" ) getInfos::ffmpeg() { tagreader="$getInfosffmpeg_version" @@ -18,7 +18,7 @@ getInfos::ffmpeg() { echo -e "$allinfos" \ |sed -n \ '/codec_type=audio/,/\[STREAM\]/{ - /^\(sample_rate\|bit_rate\|channels\)=/{ + /^\(sample_fmt\|sample_rate\|bit_rate\|channels\)=/{ p } }' @@ -42,6 +42,8 @@ getInfos::ffmpeg() { channels=$(gettag channels) rate=$(gettag 'sample_rate') bitrate=$(gettag 'bit_rate') + bitdepth=$(gettag 'sample_fmt') + bitdepth=${bitdepth//[A-z]/} if [[ $bitrate == N/A ]] then unset bitrate diff --git a/lib/tags/getInfos::WebM b/lib/tags/getInfos::WebM index 2303f67..03a0f75 100644 --- a/lib/tags/getInfos::WebM +++ b/lib/tags/getInfos::WebM @@ -1,5 +1,5 @@ #!/bin/bash -getInfoswebm_version='webm-1' +getInfoswebm_version='webm-2' tagreaders+=( "$getInfoswebm_version" ) getInfos::WebM() { getInfos::ffmpeg_video diff --git a/lib/tags/getInfos::ffmpeg_other b/lib/tags/getInfos::ffmpeg_other index 608e733..1ac3528 100644 --- a/lib/tags/getInfos::ffmpeg_other +++ b/lib/tags/getInfos::ffmpeg_other @@ -1,5 +1,5 @@ #!/bin/bash -getInfosffmpeg_other_version='ffmpeg_other-1' +getInfosffmpeg_other_version='ffmpeg_other-2' tagreaders+=( "$getInfosffmpeg_other_version" ) getInfos::ffmpeg_other() { getInfos::ffmpeg diff --git a/lib/tags/getInfos::ffmpeg_video b/lib/tags/getInfos::ffmpeg_video index ef31d51..9004d7f 100644 --- a/lib/tags/getInfos::ffmpeg_video +++ b/lib/tags/getInfos::ffmpeg_video @@ -1,5 +1,5 @@ #!/bin/bash -getInfosffmpeg_video_version='ffmpeg_video-1' +getInfosffmpeg_video_version='ffmpeg_video-2' tagreaders+=( "$getInfosffmpeg_video_version" ) getInfos::ffmpeg_video() { getInfos::ffmpeg From 444e0b58f1962de6774fc7a2faa2b539f1f7b139 Mon Sep 17 00:00:00 2001 From: Vincent Riquer Date: Sat, 28 Sep 2013 00:19:38 +0200 Subject: [PATCH 039/112] updateTags: don't quote NULL --- lib/tags/update | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/lib/tags/update b/lib/tags/update index 09af6e7..652bdd2 100644 --- a/lib/tags/update +++ b/lib/tags/update @@ -109,15 +109,15 @@ updateTags() { [[ $oldchannels != "$channels" ]]&& uch=1 [[ $oldbitrate != "$bitrate" ]]&& ubi=1 Update tags \ - ${ual:+album "::AtOM:FT::${album:-NULL}"}\ - ${uaa:+albumartist "::AtOM:FT::${albumartist:-NULL}"}\ - ${uar:+artist "::AtOM:FT::${artist:-NULL}"}\ - ${uco:+composer "::AtOM:FT::${composer:-NULL}"}\ + ${ual:+album "${album:+::AtOM:FT::}${album:-NULL}"}\ + ${uaa:+albumartist "${albumartist:+::AtOM:FT::}${albumartist:-NULL}"}\ + ${uar:+artist "${artist:+::AtOM:FT::}${artist:-NULL}"}\ + ${uco:+composer "${composer:+::AtOM:FT::}${composer:-NULL}"}\ ${udi:+disc "${disc:-NULL}"} \ ${uge:+genre "${genre:-NULL}"} \ - ${upe:+performer "::AtOM:FT::${performer:-NULL}"}\ - ${uti:+title "::AtOM:FT::${title:-NULL}"}\ - ${utr:+track "::AtOM:FT::${tracknum:-NULL}"}\ + ${upe:+performer "${performer:+::AtOM:FT::}${performer:-NULL}"}\ + ${uti:+title "${title:+::AtOM:FT::}${title:-NULL}"}\ + ${utr:+track "${track:+::AtOM:FT::}${tracknum:-NULL}"}\ ${uye:+year "${year:-NULL}"} \ last_change "$lastchange" \ ${ura:+rate "${rate:-NULL}"} \ From cb6047f33fbd760da602c5bec6d85741ad19db80 Mon Sep 17 00:00:00 2001 From: Vincent Riquer Date: Sat, 28 Sep 2013 00:20:36 +0200 Subject: [PATCH 040/112] Write bitdepth to DB --- lib/tags/update | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/lib/tags/update b/lib/tags/update index 652bdd2..f63387c 100644 --- a/lib/tags/update +++ b/lib/tags/update @@ -14,6 +14,7 @@ updateTags() { tags.albumartist, tags.artist, tags.composer, + tags.depth, tags.disc, tags.genre, tags.performer, @@ -68,6 +69,8 @@ updateTags() { rest=${rest#*::AtOM:SQL:Sep::} oldcomposer=${rest%%::AtOM:SQL:Sep::*} rest=${rest#*::AtOM:SQL:Sep::} + olddepth=${rest%%::AtOM:SQL:Sep::*} + rest=${rest#*::AtOM:SQL:Sep::} olddisc=${rest%%::AtOM:SQL:Sep::*} rest=${rest#*::AtOM:SQL:Sep::} oldgenre=${rest%%::AtOM:SQL:Sep::*} @@ -99,6 +102,7 @@ updateTags() { [[ $oldalbumartist != "$albumartist" ]]&&uaa=1 [[ $oldartist != "$artist" ]]&& uar=1 [[ $oldcomposer != "$composer" ]]&& uco=1 + [[ $olddepth != "$bitdepth" ]]&& ude=1 [[ $olddisc != "$disc" ]]&& udi=1 [[ $oldgenre != "$genre" ]]&& uge=1 [[ $oldperformer != "$performer" ]]&& upe=1 @@ -113,6 +117,7 @@ updateTags() { ${uaa:+albumartist "${albumartist:+::AtOM:FT::}${albumartist:-NULL}"}\ ${uar:+artist "${artist:+::AtOM:FT::}${artist:-NULL}"}\ ${uco:+composer "${composer:+::AtOM:FT::}${composer:-NULL}"}\ + ${ude:+depth "${bitdepth:-NULL}"} \ ${udi:+disc "${disc:-NULL}"} \ ${uge:+genre "${genre:-NULL}"} \ ${upe:+performer "${performer:+::AtOM:FT::}${performer:-NULL}"}\ @@ -136,12 +141,14 @@ updateTags() { composer \ performer \ rate \ + bitdepth \ bitrate \ channels \ ual \ uaa \ uar \ uco \ + ude \ udi \ uge \ upe \ From 783b1946c85d740c3b48eb6583bfb2473bef6940 Mon Sep 17 00:00:00 2001 From: Vincent Riquer Date: Sat, 28 Sep 2013 13:38:16 +0200 Subject: [PATCH 041/112] Document -q, -B --- README | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/README b/README index 96ac15e..23c6430 100644 --- a/README +++ b/README @@ -77,6 +77,17 @@ If, for whatever reason, you need to force the regeneration of a destination, after changing the quality settings for example, run $ atom -F +Running as a cronjob: +--------------------- +If you want to run AtOM as a cronjob, atom -q will give you a cleaner output, +more suitable for mail or logfile output. You may also want to limit the size of +each batch with -B . AtOM will not create or update more than + destination files. + +For example: +#m h dom mon dow command +0 5 * * * atom -B 1000 -q + ================= Technical details ----------------- From a3e26350aab7ed55de8b6c1be9074e2f2eb4d694 Mon Sep 17 00:00:00 2001 From: Vincent Riquer Date: Sun, 6 Oct 2013 02:44:25 +0200 Subject: [PATCH 042/112] -q: fix display --- lib/files/getFiles | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/files/getFiles b/lib/files/getFiles index cd79f5f..dfb8731 100644 --- a/lib/files/getFiles +++ b/lib/files/getFiles @@ -59,7 +59,7 @@ getFiles() { find "$sourcepath" $prunes -type f -printf "%T@ %s %P\n" ) echo 'COMMIT;' >&3 - (( cron )) || echo $'\r' + (( cron )) || echo -n $'\r' echo "${count:-0} files found, ${new:=0} new or changed." unset count } From abb87fd3a1ae8fd4a13b583f45733019057b3c6d Mon Sep 17 00:00:00 2001 From: Vincent Riquer Date: Sun, 6 Oct 2013 23:23:34 +0200 Subject: [PATCH 043/112] -B also limit number of files scanned for tags --- lib/tags/update | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/lib/tags/update b/lib/tags/update index f63387c..515f551 100644 --- a/lib/tags/update +++ b/lib/tags/update @@ -40,7 +40,10 @@ updateTags() { CAST(source_files.last_change AS TEXT) OR ('"$tagreaderclause"') ) - AND mime_type_actions.action = 1; + AND mime_type_actions.action = 1' >&3 +(( maxbatch )) && echo "LIMIT $maxbatch" >&3 +echo ' +; SELECT "AtOM:NoMoreFiles";' >&3 read -u4 line From e7a2c00f2bd4c18d43f94a7c25bfbe988e83fe53 Mon Sep 17 00:00:00 2001 From: Vincent Riquer Date: Sun, 6 Oct 2013 23:26:29 +0200 Subject: [PATCH 044/112] -q: fix output for file copy --- lib/copy/action | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/copy/action b/lib/copy/action index 95bbb4a..a4589dd 100644 --- a/lib/copy/action +++ b/lib/copy/action @@ -44,7 +44,7 @@ copyFiles_action() { destfileid=${rest%%::AtOM:SQL:Sep::*} rest=${rest#*::AtOM:SQL:Sep::} (( count++ )) - printf '\b\b\b\b%3i%%' $(( (count * 100) / ${#copyfiles[@]} )) + (( cron )) || printf '\b\b\b\b%3i%%' $(( (count * 100) / ${#copyfiles[@]} )) if [ -n "${renamepath["$destination"]}" ] then destdir="$(guessPath)" || continue From 2dffcdb8b86cf28dcda7dfa6722f2bb4d68a8b26 Mon Sep 17 00:00:00 2001 From: Vincent Riquer Date: Mon, 7 Oct 2013 01:37:41 +0200 Subject: [PATCH 045/112] Vorbis metadata: * soxi is faster than ogginfo * soxi provides bit-depth (as Precision) --- atom | 9 ++++++++- lib/tags/gettags | 4 ++-- lib/tags/{ogg => soxi} | 22 +++++++++++----------- 3 files changed, 21 insertions(+), 14 deletions(-) rename lib/tags/{ogg => soxi} (56%) diff --git a/atom b/atom index 8fcf787..8e2db2c 100755 --- a/atom +++ b/atom @@ -201,11 +201,18 @@ if ! which ogginfo >/dev/null then echo "[WARNING] Tool ogginfo (from vorbis-tools) is not" \ "installed or not in PATH - Vorbis metadata disabled WebM metadata disabled" >&2 disableogginfo=1 (( sanitywarn++ )) fi +if ! which soxi >/dev/null +then + echo "[WARNING] Tool soxi (from sox) is not" \ + "installed or not in PATH + Vorbis metadata disabled" >&2 + disablesoxi=1 + (( sanitywarn++ )) +fi if (( oggencneeded )) && ! which oggenc >/dev/null then echo "[WARNING] Tool oggenc (from vorbis-tools) is not" \ diff --git a/lib/tags/gettags b/lib/tags/gettags index dcaa0e9..2556140 100644 --- a/lib/tags/gettags +++ b/lib/tags/gettags @@ -12,8 +12,8 @@ getTags() { (( disableopusinfo )) && unset type ;; application/ogg*) - type=Ogg - (( disableogginfo )) && unset type + type=soxi + (( disablesoxi )) && unset type ;; audio/x-flac) type=FLAC diff --git a/lib/tags/ogg b/lib/tags/soxi similarity index 56% rename from lib/tags/ogg rename to lib/tags/soxi index 658c3ab..606bfc0 100644 --- a/lib/tags/ogg +++ b/lib/tags/soxi @@ -1,11 +1,10 @@ #!/bin/bash -getInfosOgg_version='Ogg-2' -tagreaders+=( "$getInfosOgg_version" ) -getInfos::Ogg() { - tagreader="$getInfosOgg_version" +getInfosSoxi_version='soxi-1' +tagreaders+=( "$getInfosSoxi_version" ) +getInfos::soxi() { + tagreader="$getInfosSoxi_version" infos=$( - ogginfo "$sourcepath/$filename" \ - | sed 's/\t//' + soxi "$sourcepath/$filename" ) albumartist=$(gettag albumartist) album=$(gettag album) @@ -22,9 +21,10 @@ getInfos::Ogg() { tracknum="$tracknum/$tracktotal" fi year=$(gettag date) - infos="${infos//: /=}" - rate=$(gettag rate|head -n1) - channels=$(gettag channels|head -n1) - bitrate=$(gettag 'average bitrate') - bitrate=${bitrate%%,*} + infos="${infos//*( ): /=}" + rate=$(gettag 'sample rate') + channels=$(gettag channels) + bitrate=$(gettag 'bit rate') + bitrate=${bitrate%%k} + bitdepth=$(gettag precision) } From 86493da2e6fd3199ad6ffbabd83ae7b1e00da5c1 Mon Sep 17 00:00:00 2001 From: Vincent Riquer Date: Mon, 7 Oct 2013 02:09:02 +0200 Subject: [PATCH 046/112] Soxi: remove garbage from bitrate & bitdepth --- lib/tags/soxi | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/tags/soxi b/lib/tags/soxi index 606bfc0..ccd60fd 100644 --- a/lib/tags/soxi +++ b/lib/tags/soxi @@ -25,6 +25,8 @@ getInfos::soxi() { rate=$(gettag 'sample rate') channels=$(gettag channels) bitrate=$(gettag 'bit rate') - bitrate=${bitrate%%k} + bitrate=${bitrate%k} + bitrate=${bitrate%%.*} bitdepth=$(gettag precision) + bitdepth=${bitdepth%-bit} } From 472f966698cd3d5718410c973ceda80ad29fa905 Mon Sep 17 00:00:00 2001 From: Vincent Riquer Date: Mon, 7 Oct 2013 03:13:18 +0200 Subject: [PATCH 047/112] updateMimes(): fix $IFS * updateMimes() did not restore IFS in some cases, causing all kinds of mayhem --- lib/destinations/updateMimes | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/destinations/updateMimes b/lib/destinations/updateMimes index 189572b..6835e01 100644 --- a/lib/destinations/updateMimes +++ b/lib/destinations/updateMimes @@ -29,4 +29,5 @@ updateMimes() { ) done done + IFS="$oldIFS" } From 34b31e597498287819b994ea788595afa0ce2aca Mon Sep 17 00:00:00 2001 From: Vincent Riquer Date: Tue, 8 Oct 2013 12:40:38 +0200 Subject: [PATCH 048/112] max batch was inefficient: order destination files by source files while creating tasks --- atom | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/atom b/atom index 8e2db2c..d141199 100755 --- a/atom +++ b/atom @@ -479,7 +479,8 @@ echo ' WHERE CAST(destination_files.last_change AS TEXT) <> CAST(source_files.last_change AS TEXT) AND mime_type_actions.destination_id = destinations.id - AND mime_type_actions.action = 1' >&3 + AND mime_type_actions.action = 1 + ORDER BY source_files.id' >&3 (( maxbatch )) && echo "LIMIT $maxbatch" >&3 echo '; SELECT "AtOM:NoMoreFiles";' >&3 From 142c5438e63dba5b48da34a864adc68677e1796e Mon Sep 17 00:00:00 2001 From: Vincent Riquer Date: Tue, 8 Oct 2013 13:40:03 +0200 Subject: [PATCH 049/112] SQL Schema versioning, VACUUM on exit --- lib/database/checkVersion | 26 ++++++++++++++++++++++++++ lib/database/close | 1 + lib/database/open | 1 + share/schema.sql | 9 ++++++--- 4 files changed, 34 insertions(+), 3 deletions(-) create mode 100644 lib/database/checkVersion diff --git a/lib/database/checkVersion b/lib/database/checkVersion new file mode 100644 index 0000000..eb80d20 --- /dev/null +++ b/lib/database/checkVersion @@ -0,0 +1,26 @@ +#!/bin/bash +currentdbversion=1 +checkDatabaseVersion() { + local dbversion + if dbversion=$(Select atom version <<<"1 = 1") + then + if (( dbversion == currentdbversion )) + then + return 0 + elif (( dbversion < currentversion )) + then + until (( dbversion == currentversion )) + do + upgradedatabase_${dbversion}_$((dbversion+1)) + dbversion=$(Select atom version <<<"1 = 1") + done + else + echo "Database schema version $dbversion is higher than + that of this version of AtOM + ($currentdbversion). Bailing out." >&2 + exit 1 + fi + else + Insert atom 1 <<<"version $currentdbversion" + fi +} diff --git a/lib/database/close b/lib/database/close index 2d6e1a2..19a412b 100644 --- a/lib/database/close +++ b/lib/database/close @@ -1,5 +1,6 @@ #!/bin/bash closeDatabase() { + echo 'vacuum;' >&3 echo .quit >&3 (( debug )) && echo -n "Waiting for SQLite to terminate... " wait diff --git a/lib/database/open b/lib/database/open index ff829fe..c20caca 100644 --- a/lib/database/open +++ b/lib/database/open @@ -15,4 +15,5 @@ openDatabase() { cat $schema >&3 echo '.separator ::AtOM:SQL:Sep::' >&3 echo 'PRAGMA foreign_keys = ON;' >&3 + checkDatabaseVersion } diff --git a/share/schema.sql b/share/schema.sql index 9de9a43..d8f6888 100644 --- a/share/schema.sql +++ b/share/schema.sql @@ -1,4 +1,7 @@ BEGIN TRANSACTION; +CREATE TABLE IF NOT EXISTS atom ( + version INTEGER +); CREATE TABLE IF NOT EXISTS source_files ( id INTEGER PRIMARY KEY, filename TEXT UNIQUE NOT NULL, @@ -111,8 +114,7 @@ CREATE TRIGGER IF NOT EXISTS create_tags AFTER INSERT ON source_files BEGIN INSERT INTO tags (source_file,last_change) VALUES (new.id,0); END; -DROP TRIGGER IF EXISTS force_destination_update_on_tag_update; -CREATE TRIGGER force_destination_update_on_tag_update +CREATE TRIGGER IF NOT EXISTS force_destination_update_on_tag_update AFTER UPDATE OF genre, albumartist, @@ -126,7 +128,8 @@ CREATE TRIGGER force_destination_update_on_tag_update performer, rate, channels, - bitrate + bitrate, + bitdepth ON tags BEGIN UPDATE destination_files SET last_change=0 From 1b801cbe12b555bf6ff3dc019c5013b40f325b85 Mon Sep 17 00:00:00 2001 From: Vincent Riquer Date: Tue, 8 Oct 2013 16:55:01 +0200 Subject: [PATCH 050/112] fat32compat & ascii as separate columns in DB --- lib/database/checkVersion | 11 +++++----- lib/database/upgradedatabase_1_2 | 37 ++++++++++++++++++++++++++++++++ lib/tools/ascii | 14 ++++++++++++ share/schema.sql | 2 ++ 4 files changed, 59 insertions(+), 5 deletions(-) create mode 100644 lib/database/upgradedatabase_1_2 create mode 100644 lib/tools/ascii diff --git a/lib/database/checkVersion b/lib/database/checkVersion index eb80d20..8fe090b 100644 --- a/lib/database/checkVersion +++ b/lib/database/checkVersion @@ -1,18 +1,18 @@ #!/bin/bash -currentdbversion=1 +currentdbversion=2 checkDatabaseVersion() { local dbversion - if dbversion=$(Select atom version <<<"1 = 1") + if dbversion=$(Select atom version <<<"\"1\" = 1") then if (( dbversion == currentdbversion )) then return 0 - elif (( dbversion < currentversion )) + elif (( dbversion < currentdbversion )) then - until (( dbversion == currentversion )) + until (( dbversion == currentdbversion )) do upgradedatabase_${dbversion}_$((dbversion+1)) - dbversion=$(Select atom version <<<"1 = 1") + dbversion=$(Select atom version <<<"\"1\" = 1") done else echo "Database schema version $dbversion is higher than @@ -23,4 +23,5 @@ checkDatabaseVersion() { else Insert atom 1 <<<"version $currentdbversion" fi + exit } diff --git a/lib/database/upgradedatabase_1_2 b/lib/database/upgradedatabase_1_2 new file mode 100644 index 0000000..888cd64 --- /dev/null +++ b/lib/database/upgradedatabase_1_2 @@ -0,0 +1,37 @@ +#!/bin/bash + +upgradedatabase_1_2() { + echo "Upgrading database to version 2... (backup is $database.bak_v1)" + cp "$database" "$database.bak_v1" + echo 'ALTER TABLE destination_files ADD COLUMN fat32compat INTEGER;' >&3 + echo 'ALTER TABLE destination_files ADD COLUMN ascii INTEGER;' >&3 + echo ' +SELECT id, + rename_pattern +FROM destination_files; + +SELECT "AtOM::NoMoreData";' >&3 + + read -u4 data + while [[ $data != AtOM::NoMoreData ]] + do + datas+=( "$data" ) + read -u4 data + done + echo 'BEGIN TRANSACTION;' >&3 + for data in "${datas[@]}" + do + id="${data%%::AtOM:SQL:Sep::*}" + rename_pattern="${data#*::AtOM:SQL:Sep::}" + IFS=':' + read pattern fat32 <<<"$rename_pattern" + IFS="$oldIFS" + Update destination_files \ + rename_pattern "$pattern" \ + fat32compat "$fat32" \ + ascii 0 \ + <<<"id = $id" + done + echo 'COMMIT;' >&3 + Update atom version 2 <<<"1 = 1" +} diff --git a/lib/tools/ascii b/lib/tools/ascii new file mode 100644 index 0000000..74b7b65 --- /dev/null +++ b/lib/tools/ascii @@ -0,0 +1,14 @@ +#!/bin/bash + +ascii() { + coproc toascii { + perl -e ' + use utf8; + use Text::Unidecode; + use open qw(:std :utf8); + $| = 1; + while (<>) { + print(unidecode($_)); + }' + } +} diff --git a/share/schema.sql b/share/schema.sql index d8f6888..2e4f3c6 100644 --- a/share/schema.sql +++ b/share/schema.sql @@ -22,6 +22,8 @@ CREATE TABLE IF NOT EXISTS destination_files ( filename TEXT, old_filename TEXT, rename_pattern TEXT, + fat32compat INTEGER, + ascii INTEGER, last_change FLOAT NOT NULL DEFAULT 0, source_file_id INTEGER, destination_id INTEGER, From 94c149e1845597d10f20194b6f144087b9dee46b Mon Sep 17 00:00:00 2001 From: Vincent Riquer Date: Wed, 9 Oct 2013 00:50:49 +0200 Subject: [PATCH 051/112] DB upgrade: prevent variable leakage --- lib/database/upgradedatabase_1_2 | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/lib/database/upgradedatabase_1_2 b/lib/database/upgradedatabase_1_2 index 888cd64..b1d3ed1 100644 --- a/lib/database/upgradedatabase_1_2 +++ b/lib/database/upgradedatabase_1_2 @@ -1,6 +1,12 @@ #!/bin/bash upgradedatabase_1_2() { + local data \ + datas \ + id \ + rename_pattern \ + pattern \ + fat32 echo "Upgrading database to version 2... (backup is $database.bak_v1)" cp "$database" "$database.bak_v1" echo 'ALTER TABLE destination_files ADD COLUMN fat32compat INTEGER;' >&3 From e16b8ff4a9eed381e8363f748576fbfa9f191c2d Mon Sep 17 00:00:00 2001 From: Vincent Riquer Date: Wed, 9 Oct 2013 00:52:36 +0200 Subject: [PATCH 052/112] config parser, setup for ascii-only --- atom | 1 + lib/config/getDestination | 16 ++++++++++++++++ lib/config/print | 1 + lib/config/write | 11 +++++++++++ lib/database/checkVersion | 1 - lib/setup/destination | 30 ++++++++++++++++++++++++++++++ 6 files changed, 59 insertions(+), 1 deletion(-) diff --git a/atom b/atom index d141199..1984bb2 100755 --- a/atom +++ b/atom @@ -13,6 +13,7 @@ EFMTINVPARM=49 # config structures declare -A \ + destinationascii \ destinationchannels \ destinationfat32compat \ destinationcopymime \ diff --git a/lib/config/getDestination b/lib/config/getDestination index 6620dff..1e239c4 100644 --- a/lib/config/getDestination +++ b/lib/config/getDestination @@ -154,6 +154,22 @@ getConfigDestination() { ;; esac ;; + 'ascii-only') + case $value in + 'true'|'on'|'yes') + destinationascii["$destination"]=1 + textunidecodeneeded=1 + ;; + 'false'|'off'|'no') + destinationascii["$destination"]=0 + ;; + *) + echo "ascii-only takes values:" \ + "'yes' ,'true' ,'on', 'no', 'false',"\ + "'off'" + ;; + esac + ;; 'skip_mime-type') destinationskipmime[$destination]="${destinationskipmime[$destination]:+${destinationskipmime[$destination]}|}$value" ;; diff --git a/lib/config/print b/lib/config/print index bd296e6..06d65b3 100644 --- a/lib/config/print +++ b/lib/config/print @@ -42,6 +42,7 @@ printConfig() { |Frequency|${destinationfrequency["$destination"]} |Higher than|${destinationmaxbps["$destination"]} |Fat32 Compat.|${destinationfat32compat["$destination"]} + |ASCII Compat.|${destinationascii["$destination"]} |Path Change|${destinationrenamepath["$destination"]} |File Rename|${destinationrename["$destination"]} EOF diff --git a/lib/config/write b/lib/config/write index 6564832..0891648 100644 --- a/lib/config/write +++ b/lib/config/write @@ -168,6 +168,17 @@ bitrate ${destinationquality["$destination"]} fi cat <<-EOCfg +# * ascii-only /: Rename files for compatibility with ASCII-only +# systems. + EOCfg + if (( ${destinationascii["$destination"]} )) + then + echo $'ascii-only\t\tyes' + else + echo $'ascii-only\t\tno' + fi + cat <<-EOCfg + # * skip_mime-type : Files with mime-type will not # be included in that destination. For more than one mime-type, use multiple # times, as needed. The '*' character is a wildcard. diff --git a/lib/database/checkVersion b/lib/database/checkVersion index 8fe090b..3491709 100644 --- a/lib/database/checkVersion +++ b/lib/database/checkVersion @@ -23,5 +23,4 @@ checkDatabaseVersion() { else Insert atom 1 <<<"version $currentdbversion" fi - exit } diff --git a/lib/setup/destination b/lib/setup/destination index 40f4ed1..e9402a1 100644 --- a/lib/setup/destination +++ b/lib/setup/destination @@ -295,6 +295,36 @@ setupDestination() { comeagain cat <<-EODesc + ASCII-only filenames (boolean): + Rename files for compatibility with ASCII-only systems (most car + radios). + EODesc + case ${destinationascii["$destination"]} in + 0) initialvalue=n ;; + 1) initialvalue=y ;; + *) unset initialvalue ;; + esac + comeagain() { + read \ + -e \ + ${initialvalue+-i $initialvalue}\ + -p'ASCII-only filenames (y/N): '\ + value + case $value in + [yY]) + destinationascii["$destination"]=1 + ;; + ''|[nN]) + destinationascii["$destination"]=0 + ;; + *) + comeagain + ;; + esac + } + comeagain + cat <<-EODesc + Skip mime-type (mime-type, string): Files with mime-type will not be included in that destination. The '*' character is a wildcard. From 4ef24fc3335e1f90369500d8432eaeeb368e3e3d Mon Sep 17 00:00:00 2001 From: Vincent Riquer Date: Wed, 9 Oct 2013 00:56:11 +0200 Subject: [PATCH 053/112] Sanity check (perl module Text::Unidecode) --- atom | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/atom b/atom index 1984bb2..d2dd2ec 100755 --- a/atom +++ b/atom @@ -295,6 +295,14 @@ then disablevideo=1 (( sanitywarn++ )) fi +if (( textunidecodeneeded )) && ! perl -MText::Unidecode -e 'exit;' 2>/dev/null +then + echo "[WARNING] Perl module Text::Unidecode is not available + Renaming to ASCII-only disabled" >&2 + unset destinationascii + destinationascii=0 + (( sanitywarn++ )) +fi if (( sanityfail )) then echo " From eba2adf2c272eaef41fd636534d3a74071f37600 Mon Sep 17 00:00:00 2001 From: Vincent Riquer Date: Wed, 9 Oct 2013 01:30:47 +0200 Subject: [PATCH 054/112] Implement new DB schema --- atom | 12 ++++++++++-- lib/copy/action | 4 +++- lib/copy/matching | 7 +++++-- lib/encode/mp3 | 4 +++- lib/encode/opus | 4 +++- lib/encode/vorbis | 4 +++- lib/workers/cleaner | 10 ++++++++++ 7 files changed, 37 insertions(+), 8 deletions(-) diff --git a/atom b/atom index d2dd2ec..bcacbf4 100755 --- a/atom +++ b/atom @@ -388,6 +388,8 @@ echo ' id INTEGER PRIMARY KEY, key TEXT UNIQUE, rename_pattern TEXT, + fat32compat INTEGER, + ascii INTEGER, source_file INTEGER, fileid INTEGER, filename TEXT, @@ -793,7 +795,9 @@ do WHERE destinations.name="'"$destination"'" AND (destination_files.rename_pattern != -"'"${destinationrenamepath[$destination]}/${destinationrename[$destination]}:${destinationfat32compat["$destination"]}"'" +"'"${destinationrenamepath[$destination]}/${destinationrename[$destination]}"'" + OR fat32compat != ${destinationfat32compat["$destination"]} + OR ascii != ${destinationascii["$destination"]} OR destination_files.rename_pattern is NULL) AND destination_files.last_change > 0 ; @@ -859,7 +863,11 @@ do echo "UPDATE destination_files" \ "SET filename=\"${destfilename//\"/\"\"}\"," \ " rename_pattern=" \ -"\"${destinationrenamepath[$destination]}/${destinationrename[$destination]}:${destinationfat32compat["$destination"]}\"" \ +"\"${destinationrenamepath[$destination]}/${destinationrename[$destination]}" \ + " fat32compat=" +"${destinationfat32compat["$destination"]}\"" \ + " ascii=" \ +"${destinationascii["$destination"]}" \ "WHERE id=$destfileid;" \ >&3 if (( commit )) diff --git a/lib/copy/action b/lib/copy/action index a4589dd..401f080 100644 --- a/lib/copy/action +++ b/lib/copy/action @@ -74,7 +74,9 @@ copyFiles_action() { then Update destination_files \ filename "$destdir/${sourcefilename##*/}"\ - rename_pattern "${destinationrenamepath[$destination]}/${destinationrename[$destination]}:${destinationfat32compat["$destination"]}"\ + rename_pattern "${destinationrenamepath[$destination]}/${destinationrename[$destination]}"\ + fat32compat ${destinationfat32compat["$destination"]}\ + ascii ${destinationascii["$destination"]}\ last_change $lastchange \ <<-EOWhere id = $destfileid diff --git a/lib/copy/matching b/lib/copy/matching index 64a3744..6c52329 100644 --- a/lib/copy/matching +++ b/lib/copy/matching @@ -22,8 +22,11 @@ copyFiles_matching() { " FROM destination_files" \ " WHERE id=$destfileid" \ " )," \ - " rename_pattern=" \ -"\"${destinationrenamepath[$destination]}/${destinationrename[$destination]}:${destinationfat32compat["$destination"]}\""\ + " rename_pattern" \ +"\"${destinationrenamepath[$destination]}/${destinationrename[$destination]}\""\ + " fat32compat" \ + " ${destinationfat32compat["$destination"]}"\ + " ascii ${destinationascii["$destination"]}"\ "WHERE id=$destfileid;" \ >&3 (( ++copies )) diff --git a/lib/encode/mp3 b/lib/encode/mp3 index 7474ab2..4963b5e 100644 --- a/lib/encode/mp3 +++ b/lib/encode/mp3 @@ -65,7 +65,9 @@ encodeFile::mp3() { cleanup $tempdir/$tmpfile.wav source_file $fileid status 0 - rename_pattern ${destinationrenamepath[$destination]}/${destinationrename[$destination]}:${destinationfat32compat["$destination"]} + rename_pattern ${destinationrenamepath[$destination]}/${destinationrename[$destination]} + fat32compat ${destinationfat32compat["$destination"]} + ascii ${destinationascii["$destination"]} EOInsert ) progressSpin diff --git a/lib/encode/opus b/lib/encode/opus index 8e9bbcd..3aa993d 100644 --- a/lib/encode/opus +++ b/lib/encode/opus @@ -37,7 +37,9 @@ encodeFile::opus() { cleanup $tempdir/$tmpfile.wav source_file $fileid status 0 - rename_pattern ${destinationrenamepath[$destination]}/${destinationrename[$destination]}:${destinationfat32compat["$destination"]} + rename_pattern ${destinationrenamepath[$destination]}/${destinationrename[$destination]} + fat32compat ${destinationfat32compat["$destination"]} + ascii ${destinationascii["$destination"]} EOInsert ) progressSpin diff --git a/lib/encode/vorbis b/lib/encode/vorbis index 7c32538..7eee65c 100644 --- a/lib/encode/vorbis +++ b/lib/encode/vorbis @@ -33,7 +33,9 @@ encodeFile::vorbis() { cleanup $tempdir/$tmpfile.wav source_file $fileid status 0 - rename_pattern ${destinationrenamepath[$destination]}/${destinationrename[$destination]}:${destinationfat32compat["$destination"]} + rename_pattern ${destinationrenamepath[$destination]}/${destinationrename[$destination]} + fat32compat ${destinationfat32compat["$destination"]} + ascii ${destinationascii["$destination"]} EOInsert ) progressSpin diff --git a/lib/workers/cleaner b/lib/workers/cleaner index 56071ea..177def4 100644 --- a/lib/workers/cleaner +++ b/lib/workers/cleaner @@ -47,6 +47,16 @@ cleaner() { " SELECT rename_pattern" \ " FROM tasks" \ " WHERE id=$taskid" \ + " )," \ + " fat32compat=(" \ + " SELECT fat32compat" \ + " FROM tasks" \ + " WHERE id=$taskid" \ + " )," \ + " ascii=(" \ + " SELECT ascii" \ + " FROM tasks" \ + " WHERE id=$taskid" \ " )" \ "WHERE id=$destfileid;" \ >&3 From f9143525d028958f677bada7f0b6084dbae9dd55 Mon Sep 17 00:00:00 2001 From: Vincent Riquer Date: Wed, 9 Oct 2013 17:20:36 +0200 Subject: [PATCH 055/112] Fix rename_pattern, permit optional fields with [] --- lib/files/getDestDir | 72 +++++++++++++++++++++++------------------- lib/files/getDestFile | 73 ++++++++++++++++++++++++------------------- 2 files changed, 80 insertions(+), 65 deletions(-) diff --git a/lib/files/getDestDir b/lib/files/getDestDir index ff4f26b..f28fb49 100644 --- a/lib/files/getDestDir +++ b/lib/files/getDestDir @@ -1,52 +1,60 @@ #!/bin/bash getDestDir() { destdir="${destinationpath[$destination]}/" - if [ -n "${destinationrenamepath[$destination]}" ] \ + if [ -n "${destinationrenamepath[$destination]}" ] \ && ( ( - ! [[ ${destinationrenamepath[$destination]} =~ %\{album\} ]] \ - || [ -n "$album" ] - ) && ( - ! [[ ${destinationrenamepath[$destination]} =~ %\{albumartist\} ]] \ - || [ -n "$albumartist" ] - ) && ( - ! [[ ${destinationrenamepath[$destination]} =~ %\{artist\} ]] \ - || [ -n "$artist" ] - ) && ( - ! [[ ${destinationrenamepath[$destination]} =~ %\{genre\} ]] \ - || [ -n "$genre" ] - ) && ( - ! [[ ${destinationrenamepath[$destination]} =~ %\{title\} ]] \ - || [ -n "$title" ] - ) && ( - ! [[ ${destinationrenamepath[$destination]} =~ %\{track\} ]] \ - || [ -n "$track" ] - ) && ( - ! [[ ${destinationrenamepath[$destination]} =~ %\{year\} ]] \ - || [ -n "$year" ] - ) && ( - ! [[ ${destinationrenamepath[$destination]} =~ %\{disc\} ]] \ - || [ -n "$disc" ] + [[ ${destinationrenamepath[$destination]} == \ + *?([^[])%\{album\}?([^\]])* ]] \ + && [ -n "$album" ] + ) || ( + [[ ${destinationrenamepath[$destination]} == \ + *?([^[])%\{albumartist\}?([^\]])* ]] \ + && [ -n "$albumartist" ] + ) || ( + [[ ${destinationrenamepath[$destination]} == \ + *?([^[])%\{artist\}?([^\]])* ]] \ + && [ -n "$artist" ] + ) || ( + [[ ${destinationrenamepath[$destination]} == \ + *?([^[])%\{genre\}?([^\]])* ]] \ + && [ -n "$genre" ] + ) || ( + [[ ${destinationrenamepath[$destination]} == \ + *?([^[])%\{title\}?([^\]])* ]] \ + && [ -n "$title" ] + ) || ( + [[ ${destinationrenamepath[$destination]} == \ + *?([^[])%\{track\}?([^\]])* ]] \ + && [ -n "$track" ] + ) || ( + [[ ${destinationrenamepath[$destination]} == \ + *?([^[])%\{year\}?([^\]])* ]] \ + && [ -n "$year" ] + ) || ( + [[ ${destinationrenamepath[$destination]} == \ + *?([^[])%\{disc\}?([^\]])* ]] \ + && [ -n "$disc" ] ) ) then replace=$(sanitizeFile "$album" dir) - destdir+="${destinationrenamepath[$destination]//%\{album\}/$replace}" + destdir+="${destinationrenamepath[$destination]//?(\[)%\{album\}?(\])/$replace}" replace=$(sanitizeFile "$albumartist" dir) - destdir="${destdir//%\{albumartist\}/$replace}" + destdir="${destdir//?(\[)%\{albumartist\}?(\])/$replace}" replace=$(sanitizeFile "$artist" dir) - destdir="${destdir//%\{artist\}/$replace}" + destdir="${destdir//?(\[)%\{artist\}?(\])/$replace}" replace=$(sanitizeFile "$genre" dir) - destdir="${destdir//%\{genre\}/$replace}" + destdir="${destdir//?(\[)%\{genre\}?(\])/$replace}" replace=$(sanitizeFile "$title" dir) - destdir="${destdir//%\{title\}/$replace}" + destdir="${destdir//?(\[)%\{title\}?(\])/$replace}" tracknumber="${track%/*}" replace=$(sanitizeFile "$tracknumber" dir) - destdir="${destdir//%\{track\}/$replace}" + destdir="${destdir//?(\[)%\{track\}?(\])/$replace}" replace=$(sanitizeFile "$year" dir) - destdir="${destdir//%\{year\}/$replace}" + destdir="${destdir//?(\[)%\{year\}?(\])/$replace}" replace=$(sanitizeFile "$disc" dir) - destdir="${destdir//%\{disc\}/$replace}" + destdir="${destdir//?(\[)%\{disc\}?(\])/$replace}" else destdir+=$(sanitizeFile "${filename%%/*}" dir) part=${filename#*/} diff --git a/lib/files/getDestFile b/lib/files/getDestFile index 1ec9087..41c2003 100644 --- a/lib/files/getDestFile +++ b/lib/files/getDestFile @@ -1,43 +1,50 @@ #!/bin/bash getDestFile() { - if [ -n "${destinationrename[$destination]}" ] \ + if [ -n "${destinationrename[$destination]}" ] \ && ( ( - ! [[ ${destinationrename[$destination]} =~ %\{album\} ]] \ - || [ -n "$album" ] - ) && ( - ! [[ ${destinationrename[$destination]} =~ %\{albumartist\} ]] \ - || [ -n "$albumartist" ] - ) && ( - ! [[ ${destinationrename[$destination]} =~ %\{artist\} ]] \ - || [ -n "$artist" ] - ) && ( - ! [[ ${destinationrename[$destination]} =~ %\{genre\} ]] \ - || [ -n "$genre" ] - ) && ( - ! [[ ${destinationrename[$destination]} =~ %\{title\} ]] \ - || [ -n "$title" ] - ) && ( - ! [[ ${destinationrename[$destination]} =~ %\{track\} ]] \ - || [ -n "$track" ] - ) && ( - ! [[ ${destinationrename[$destination]} =~ %\{year\} ]] \ - || [ -n "$year" ] - ) && ( - ! [[ ${destinationrename[$destination]} =~ %\{disc\} ]] \ - || [ -n "$disc" ] + [[ ${destinationrename[$destination]} == \ + *?([^[])%\{album\}?([^\]])* ]] \ + && [ -n "$album" ] + ) || ( + [[ ${destinationrename[$destination]} == \ + *?([^[])%\{albumartist\}?([^\]])* ]] \ + && [ -n "$albumartist" ] + ) || ( + [[ ${destinationrename[$destination]} == \ + *?([^[])%\{artist\}?([^\]])* ]] \ + && [ -n "$artist" ] + ) || ( + [[ ${destinationrename[$destination]} == \ + *?([^[])%\{genre\}?([^\]])* ]] \ + && [ -n "$genre" ] + ) || ( + [[ ${destinationrename[$destination]} == \ + *?([^[])%\{title\}?([^\]])* ]] \ + && [ -n "$title" ] + ) || ( + [[ ${destinationrename[$destination]} == \ + *?([^[])%\{track\}?([^\]])* ]] \ + && [ -n "$track" ] + ) || ( + [[ ${destinationrename[$destination]} == \ + *?([^[])%\{year\}?([^\]])* ]] \ + && [ -n "$year" ] + ) || ( + [[ ${destinationrename[$destination]} == \ + *?([^[])%\{disc\}?([^\]])* ]] \ + && [ -n "$disc" ] ) ) - then - destfile="${destinationrename[$destination]//%\{album\}/$album}" - destfile="${destfile//%\{albumartist\}/$albumartist}" - destfile="${destfile//%\{artist\}/$artist}" - destfile="${destfile//%\{genre\}/$genre}" - destfile="${destfile//%\{title\}/$title}" + destfile="${destinationrename[$destination]//?(\[)%\{album\}?(\])/$album}" + destfile="${destfile//?(\[)%\{albumartist\}?(\])/$albumartist}" + destfile="${destfile//?(\[)%\{artist\}?(\])/$artist}" + destfile="${destfile//?(\[)%\{genre\}?(\])/$genre}" + destfile="${destfile//?(\[)%\{title\}?(\])/$title}" tracknumber="${track%/*}" - destfile="${destfile//%\{track\}/$tracknumber}" - destfile="${destfile//%\{year\}/$year}" - destfile="${destfile//%\{disc\}/$disc}" + destfile="${destfile//?(\[)%\{track\}?(\])/$tracknumber}" + destfile="${destfile//?(\[)%\{year\}?(\])/$year}" + destfile="${destfile//?(\[)%\{disc\}?(\])/$disc}" else destfile="${filename##*/}" destfile="${destfile%.*}" From 6f2a29f0d520e6eaa3af77bdf707b74c0ee0b862 Mon Sep 17 00:00:00 2001 From: Vincent Riquer Date: Wed, 9 Oct 2013 17:24:08 +0200 Subject: [PATCH 056/112] Implement ascii-only --- atom | 24 ++++++++++++++++++------ lib/copy/matching | 40 ++++++++++++++++++++-------------------- lib/files/getDestDir | 5 +++++ lib/files/getDestFile | 6 ++++++ 4 files changed, 49 insertions(+), 26 deletions(-) diff --git a/atom b/atom index bcacbf4..03b417d 100755 --- a/atom +++ b/atom @@ -301,6 +301,7 @@ then Renaming to ASCII-only disabled" >&2 unset destinationascii destinationascii=0 + textunidecodeneeded=0 (( sanitywarn++ )) fi if (( sanityfail )) @@ -503,6 +504,8 @@ do done (( cron )) || echo -n 'Creating tasks... ' +(( textunidecodeneeded )) && ascii + echo 'BEGIN TRANSACTION;' >&3 for line in "${decodefiles[@]}" do @@ -591,6 +594,9 @@ echo 'COMMIT;' >&3 (( cron )) || echo -n $'\r' echo "Created ${count:-0} tasks for $filecount files ${togo:+($togo left) }(${copies:-0} immediate copies)" +# remove perl unicode to ascii coprocess +(( textunidecodeneeded )) && eval exec "${toascii[1]}>&-" + concurrency=$(( maxload / 2 )) (( concurrency )) || concurrency=1 active=0 @@ -764,7 +770,6 @@ then exit fi -#set -x for destination in "${!destinationpath[@]}" do echo ' @@ -792,14 +797,18 @@ do INNER JOIN source_files ON destination_files.source_file_id =source_files.id + INNER JOIN mime_actions + ON source_files.mime_type + =mime_actions.mime_type WHERE destinations.name="'"$destination"'" AND (destination_files.rename_pattern != "'"${destinationrenamepath[$destination]}/${destinationrename[$destination]}"'" - OR fat32compat != ${destinationfat32compat["$destination"]} - OR ascii != ${destinationascii["$destination"]} + OR fat32compat != '${destinationfat32compat["$destination"]}' + OR ascii != '${destinationascii["$destination"]}' OR destination_files.rename_pattern is NULL) AND destination_files.last_change > 0 + AND mime_actions.action=1 ; SELECT "AtOM:NoMoreFiles"; @@ -819,6 +828,7 @@ do 'vorbis') extension=ogg ;; esac (( cron )) || echo -n "$destination: rename pattern changed, renaming files... " + (( textunidecodeneeded )) && ascii echo 'BEGIN TRANSACTION;' >&3 for line in "${renamefiles[@]}" do @@ -863,9 +873,9 @@ do echo "UPDATE destination_files" \ "SET filename=\"${destfilename//\"/\"\"}\"," \ " rename_pattern=" \ -"\"${destinationrenamepath[$destination]}/${destinationrename[$destination]}" \ - " fat32compat=" -"${destinationfat32compat["$destination"]}\"" \ +"\"${destinationrenamepath[$destination]}/${destinationrename[$destination]}\","\ + " fat32compat=" \ +"${destinationfat32compat["$destination"]}," \ " ascii=" \ "${destinationascii["$destination"]}" \ "WHERE id=$destfileid;" \ @@ -877,6 +887,8 @@ do fi fi done + # remove perl unicode to ascii coprocess + (( textunidecodeneeded )) && eval exec "${toascii[1]}>&-" echo 'COMMIT;' >&3 (( cron )) || echo -n $'\r' echo -n "$destination: Renamed ${changedcount:-0} files" diff --git a/lib/copy/matching b/lib/copy/matching index 6c52329..b0f6516 100644 --- a/lib/copy/matching +++ b/lib/copy/matching @@ -8,26 +8,26 @@ copyFiles_matching() { || cp -a \ "$sourcepath/$filename" \ "$destdir/$destfile.$extension" - echo \ - "UPDATE destination_files" \ - "SET filename=" \ - "\"${destdir//\"/\"\"}/${destfile//\"/\"\"}.$extension\"," \ - " last_change=(" \ - " SELECT last_change" \ - " FROM source_files" \ - " WHERE id=$fileid" \ - " )," \ - " old_filename=(" \ - " SELECT filename" \ - " FROM destination_files" \ - " WHERE id=$destfileid" \ - " )," \ - " rename_pattern" \ -"\"${destinationrenamepath[$destination]}/${destinationrename[$destination]}\""\ - " fat32compat" \ - " ${destinationfat32compat["$destination"]}"\ - " ascii ${destinationascii["$destination"]}"\ - "WHERE id=$destfileid;" \ + echo \ + "UPDATE destination_files" \ + "SET filename=" \ + "\"${destdir//\"/\"\"}/${destfile//\"/\"\"}.$extension\"," \ + " last_change=(" \ + " SELECT last_change" \ + " FROM source_files" \ + " WHERE id=$fileid" \ + " )," \ + " old_filename=(" \ + " SELECT filename" \ + " FROM destination_files" \ + " WHERE id=$destfileid" \ + " )," \ + " rename_pattern=" \ +"\"${destinationrenamepath[$destination]}/${destinationrename[$destination]}\","\ + " fat32compat=" \ + "${destinationfat32compat["$destination"]}," \ + " ascii=${destinationascii["$destination"]}" \ + "WHERE id=$destfileid;" \ >&3 (( ++copies )) } diff --git a/lib/files/getDestDir b/lib/files/getDestDir index f28fb49..8ac3714 100644 --- a/lib/files/getDestDir +++ b/lib/files/getDestDir @@ -64,6 +64,11 @@ getDestDir() { part=${part#*/} done fi + if (( ${destinationascii["$destination"]} )) + then + echo "$destdir" >&${toascii[1]} + read -r -u${toascii[0]} destdir + fi if ! [ -d "$destdir" ] then mkdir -p "$destdir" diff --git a/lib/files/getDestFile b/lib/files/getDestFile index 41c2003..aae78b8 100644 --- a/lib/files/getDestFile +++ b/lib/files/getDestFile @@ -36,6 +36,7 @@ getDestFile() { && [ -n "$disc" ] ) ) + then destfile="${destinationrename[$destination]//?(\[)%\{album\}?(\])/$album}" destfile="${destfile//?(\[)%\{albumartist\}?(\])/$albumartist}" destfile="${destfile//?(\[)%\{artist\}?(\])/$artist}" @@ -49,5 +50,10 @@ getDestFile() { destfile="${filename##*/}" destfile="${destfile%.*}" fi + if (( ${destinationascii["$destination"]} )) + then + echo "$destfile" >&${toascii[1]} + read -r -u${toascii[0]} destfile + fi destfile=$(sanitizeFile "$destfile") } From bd0b5b2d267a2232c5fffde9d3f9f2d9524d81e8 Mon Sep 17 00:00:00 2001 From: Vincent Riquer Date: Wed, 9 Oct 2013 18:21:44 +0200 Subject: [PATCH 057/112] Document ascii-only --- doc/config | 5 +++++ lib/setup/destination | 4 ++++ 2 files changed, 9 insertions(+) diff --git a/doc/config b/doc/config index fbe078e..9068e64 100644 --- a/doc/config +++ b/doc/config @@ -70,8 +70,13 @@ Sections: %{track}, %{year}. Untagged files or files in unrecognized formats will not be changed. + Surrounding a field with [] makes it optional, meaning renaming will still + happen if the corresponding tag is not defined. * fat32compat /: Rename files for compatibility with FAT32 filesystems. + * ascii-only /: Rename files for compatibility with ASCII-only + systems. Uses Perl with Text::Unidecode to replace cyrillic or kanji + with an ASCII representation. * skip_mime-type : Files with mime-type will not be included in that destination. For more than one mime-type, use multiple times, as needed. The '*' character is a wildcard. diff --git a/lib/setup/destination b/lib/setup/destination index e9402a1..7293fbf 100644 --- a/lib/setup/destination +++ b/lib/setup/destination @@ -246,6 +246,8 @@ setupDestination() { %{track}, %{year}. Untagged files or files in unrecognized formats will not be changed. + Surrounding a field with [] makes it optional, meaning renaming + will still happen if the corresponding tag is not defined. Leave blank if you don't want file renaming. EODesc @@ -298,6 +300,8 @@ setupDestination() { ASCII-only filenames (boolean): Rename files for compatibility with ASCII-only systems (most car radios). + Uses Perl with Text::Unidecode to replace cyrillic or kanji + with an ASCII representation. EODesc case ${destinationascii["$destination"]} in 0) initialvalue=n ;; From 09f7753e9fbbb34bc5049ff27eff101ec24749be Mon Sep 17 00:00:00 2001 From: Vincent Riquer Date: Thu, 17 Oct 2013 20:25:47 +0200 Subject: [PATCH 058/112] Also order tag reader to ensure format properties are known before transcoding --- lib/tags/update | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/tags/update b/lib/tags/update index 515f551..dad68c4 100644 --- a/lib/tags/update +++ b/lib/tags/update @@ -40,7 +40,8 @@ updateTags() { CAST(source_files.last_change AS TEXT) OR ('"$tagreaderclause"') ) - AND mime_type_actions.action = 1' >&3 + AND mime_type_actions.action = 1 + ORDER BY source_files.id' >&3 (( maxbatch )) && echo "LIMIT $maxbatch" >&3 echo ' ; From 9e0666ccbc8b1276b3ed31f582d95a8a42564d9c Mon Sep 17 00:00:00 2001 From: Vincent Riquer Date: Fri, 25 Oct 2013 22:17:01 +0200 Subject: [PATCH 059/112] Fix path change --- lib/files/getDestDir | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/files/getDestDir b/lib/files/getDestDir index 8ac3714..82d5667 100644 --- a/lib/files/getDestDir +++ b/lib/files/getDestDir @@ -73,4 +73,5 @@ getDestDir() { then mkdir -p "$destdir" fi + destdir="${destdir//+(\/)/\/}" } From c7331f074510a4c925181290121602dec7fb9763 Mon Sep 17 00:00:00 2001 From: Vincent Riquer Date: Fri, 8 Nov 2013 22:33:35 +0100 Subject: [PATCH 060/112] show failed tasks --- atom | 57 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) diff --git a/atom b/atom index 03b417d..e72a157 100755 --- a/atom +++ b/atom @@ -763,6 +763,63 @@ echo -n "Ran ${ran:=0} tasks, $failed of which failed, in $days" \ "days, $hours hours, $minutes minutes and $seconds seconds." (( cron )) || echo -en "\033[K" echo +if (( failed )) +then + echo $'\nFailed tasks:\n' + echo ' + SELECT source_files.filename, + tasks.cmd_arg0, + tasks.cmd_arg1, + tasks.cmd_arg2, + tasks.cmd_arg3, + tasks.cmd_arg4, + tasks.cmd_arg5, + tasks.cmd_arg6, + tasks.cmd_arg7, + tasks.cmd_arg8, + tasks.cmd_arg9, + tasks.cmd_arg10, + tasks.cmd_arg11, + tasks.cmd_arg12, + tasks.cmd_arg13, + tasks.cmd_arg14, + tasks.cmd_arg15, + tasks.cmd_arg16, + tasks.cmd_arg17, + tasks.cmd_arg18, + tasks.cmd_arg19, + tasks.cmd_arg20, + tasks.cmd_arg21, + tasks.cmd_arg22, + tasks.cmd_arg23, + tasks.cmd_arg24, + tasks.cmd_arg25, + tasks.cmd_arg26, + tasks.cmd_arg27, + tasks.cmd_arg28, + tasks.cmd_arg29 + FROM tasks + INNER JOIN source_files + ON tasks.source_file=source_files.id + WHERE tasks.status = 2 + AND requires is NULL; + + SELECT "AtOM:NoMoreFiles";' >&3 + read -u4 line + while ! [[ $line = AtOM:NoMoreFiles ]] + do + failedtasks+=("$line") + read -u4 line + done + for line in "${failedtasks[@]}" + do + echo "${line%%::AtOM:SQL:Sep::*}" + line="${line#*::AtOM:SQL:Sep::}" + line="${line//::AtOM:SQL:Sep::/ }" + echo $'\t'"${line/+( )$/}" + echo + done +fi if [ -n "$quit" ] then From f34025a9891d0a10429b28fea865d29d6377e9b9 Mon Sep 17 00:00:00 2001 From: Vincent Riquer Date: Wed, 13 Nov 2013 17:24:04 +0100 Subject: [PATCH 061/112] Add known bug: early temporary file deletion --- BUGS | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/BUGS b/BUGS index d6e5d6f..879080f 100644 --- a/BUGS +++ b/BUGS @@ -22,3 +22,9 @@ Symptoms: Please let me know if this hits you, or if you have any idea as to why this is happening and how to avoid it. + +3. Premature temp file removal +------------------------------ +Sometimes, a temporary file gets deleted while one or more task(s) still needs +it. Ensuing failure is correctly handled and the failed files will be generated +during the next batch. From ee5714f97a1de143c5f592b16e3ae25bb951b58d Mon Sep 17 00:00:00 2001 From: Vincent Riquer Date: Wed, 13 Nov 2013 23:05:13 +0100 Subject: [PATCH 062/112] FLAC: fix handling of tracknumber / tracktotal --- lib/tags/flac | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/tags/flac b/lib/tags/flac index 32d3f4a..32a37be 100644 --- a/lib/tags/flac +++ b/lib/tags/flac @@ -1,5 +1,5 @@ #!/bin/bash -getInfosFLAC_version='FLAC-2' +getInfosFLAC_version='FLAC-3' tagreaders+=( "$getInfosFLAC_version" ) getInfos::FLAC() { tagreader="$getInfosFLAC_version" @@ -26,7 +26,8 @@ getInfos::FLAC() { genre=$(gettag genre) performer=$(gettag performer) title=$(gettag title) - tracknum="$(gettag tracknumber)/$(gettag tracktotal)" + tracknum=$(gettag tracknumber) + tracktotal=$(gettag tracktotal) year=$(gettag date) if [ -n "$tracknum" -a -n "$tracktotal" ] then From 3d715ef18e426e63a77e35a8e53854f5edc27c17 Mon Sep 17 00:00:00 2001 From: Vincent Riquer Date: Wed, 20 Nov 2013 13:02:38 +0100 Subject: [PATCH 063/112] fix file copy: don't copy files if we can't guess the destination path --- lib/copy/action | 17 ++++++++++++++--- lib/copy/guessPath | 28 ++++++++++++++++++++++++++++ 2 files changed, 42 insertions(+), 3 deletions(-) diff --git a/lib/copy/action b/lib/copy/action index 401f080..63e7d75 100644 --- a/lib/copy/action +++ b/lib/copy/action @@ -45,9 +45,19 @@ copyFiles_action() { rest=${rest#*::AtOM:SQL:Sep::} (( count++ )) (( cron )) || printf '\b\b\b\b%3i%%' $(( (count * 100) / ${#copyfiles[@]} )) - if [ -n "${renamepath["$destination"]}" ] + if [ -n "${destinationrenamepath["$destination"]}" ] then - destdir="$(guessPath)" || continue + destdir="$(guessPath)" + guessstatus=$? + case $guessstatus in + 1) + continue + ;; + 2) + (( postponed++ )) + continue + ;; + esac else destdir="${destinationpath["$destination"]}/" if [[ $sourcefilename =~ / ]] @@ -88,7 +98,8 @@ copyFiles_action() { if (( count )) then (( cron )) || echo -n $'\r' - echo -n "Copied ${done:-0} of $count files." + echo -n "Copied ${done:-0} of $count" \ + "files${postponed+ ($postponed postponed)}." (( cron )) || echo -en "\033[K" echo else diff --git a/lib/copy/guessPath b/lib/copy/guessPath index ccb0e4b..75e1914 100644 --- a/lib/copy/guessPath +++ b/lib/copy/guessPath @@ -1,6 +1,31 @@ #!/bin/bash guessPath() { + echo 'SELECT IFNULL( ( + SELECT destination_files.last_change + FROM destination_files + INNER JOIN source_files + ON destination_files.source_file_id=source_files.id + INNER JOIN mime_type_actions + ON + mime_type_actions.id=source_files.mime_type + INNER JOIN destinations + ON destinations.id=destination_files.destination_id + WHERE destinations.id = '$destinationid' + AND source_files.filename LIKE + "'"${sourcedir//\"/\"\"}"'/%" + AND NOT source_files.filename LIKE + "'"${sourcedir//\"/\"\"}"'/%/%" + AND mime_type_actions.action = 1 + ORDER BY destination_files.last_change DESC + LIMIT 1 + ),"0.0"); + '>&3 + read -u4 timestamp + if (( ${timestamp/./} == 0 )) + then + return 2 + fi echo 'SELECT IFNULL( ( SELECT destination_files.filename FROM destination_files @@ -14,7 +39,10 @@ guessPath() { WHERE destinations.id = '$destinationid' AND source_files.filename LIKE "'"${sourcedir//\"/\"\"}"'/%" + AND NOT source_files.filename LIKE + "'"${sourcedir//\"/\"\"}"'/%/%" AND mime_type_actions.action = 1 + ORDER BY destination_files.last_change DESC LIMIT 1 ),"AtOM:NotFound"); '>&3 From 64906e2ffb8cf5cca9cd08a4c38fdc49731e059d Mon Sep 17 00:00:00 2001 From: Vincent Riquer Date: Wed, 20 Nov 2013 22:00:43 +0100 Subject: [PATCH 064/112] ascii-only: safely handle non-unicode characters --- lib/tools/ascii | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/tools/ascii b/lib/tools/ascii index 74b7b65..a950e22 100644 --- a/lib/tools/ascii +++ b/lib/tools/ascii @@ -4,11 +4,12 @@ ascii() { coproc toascii { perl -e ' use utf8; + use Encode; use Text::Unidecode; use open qw(:std :utf8); $| = 1; while (<>) { - print(unidecode($_)); + print(unidecode(encode("utf-8", $_))); }' } } From c6068006d4ab0ebb21a17a576c24f44ca45f7242 Mon Sep 17 00:00:00 2001 From: Vincent Riquer Date: Thu, 21 Nov 2013 10:41:50 +0100 Subject: [PATCH 065/112] Add bug: ascii-only + immediate copies fail --- BUGS | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/BUGS b/BUGS index 879080f..de3f383 100644 --- a/BUGS +++ b/BUGS @@ -28,3 +28,13 @@ happening and how to avoid it. Sometimes, a temporary file gets deleted while one or more task(s) still needs it. Ensuing failure is correctly handled and the failed files will be generated during the next batch. + +4. ascii-only fails for immediate copies +---------------------------------------- +Example output: +cp: cannot create regular file `\\/mnt\\/Musique-mp3\\/Punk\\/Tagada +Jones\\/2003-L\'Envers du dA(c)cor/11--Tagada Jones-Star System.mp3': No such +file or directory + +Moreover, failure is not detected, and files are marked as done. + From 2e407068bf1933e18e4391a60f522da108234687 Mon Sep 17 00:00:00 2001 From: Vincent Riquer Date: Thu, 21 Nov 2013 18:47:36 +0100 Subject: [PATCH 066/112] Refer to the InDefero forge Add informations on video tagging tools --- BUGS | 40 +++++++++------------------------------- 1 file changed, 9 insertions(+), 31 deletions(-) diff --git a/BUGS b/BUGS index de3f383..5c3935f 100644 --- a/BUGS +++ b/BUGS @@ -1,4 +1,5 @@ Known bugs: +(Real bugs have been moved to http://indefero.riquer.fr/p/AtOM/issues/) 1. Video formats suck --------------------- @@ -7,34 +8,11 @@ writer, director, main actors, original author, etc. But most of them only provide ONE tag: title... Planned workaround: tag guessing (see TODO). -2. Random hang --------------- -Something I encountered and don't understand: sometimes, seemingly randomly, -the call to wait, to retrieve the exit status of a worker, does not return, -though the worker *has* exited. This happens rarely, but it will probably hit -you if you have more than 20.000 tasks to run. When this happens, just hit -Control-C and re-run AtOM to resume transcoding. - -Symptoms: -- Progress does not update anymore -- a pstree only show one child process: sqlite3 -- strace()ing shows the script process stuck on waitpid() - -Please let me know if this hits you, or if you have any idea as to why this is -happening and how to avoid it. - -3. Premature temp file removal ------------------------------- -Sometimes, a temporary file gets deleted while one or more task(s) still needs -it. Ensuing failure is correctly handled and the failed files will be generated -during the next batch. - -4. ascii-only fails for immediate copies ----------------------------------------- -Example output: -cp: cannot create regular file `\\/mnt\\/Musique-mp3\\/Punk\\/Tagada -Jones\\/2003-L\'Envers du dA(c)cor/11--Tagada Jones-Star System.mp3': No such -file or directory - -Moreover, failure is not detected, and files are marked as done. - +Tools you can use to add metadata to video files: +(use ALBUM_ARTIST, ALBUM, ARTIST, COMPOSER, GENRE, PERFORMER, TITLE, TRACK, +DATE) + * AVI: ffmpeg + * FLV: ffmpeg, or flvmeta >= 1.1 + * MKV/WebM: ffmpeg (only allows TITLE) + * MPEG: ffmpeg + * ASF/WMA/WMV: ffmpeg, exfalso From eba9bfffd61076e9b4e023d3c22509616af7b841 Mon Sep 17 00:00:00 2001 From: Vincent Riquer Date: Mon, 25 Nov 2013 12:54:26 +0100 Subject: [PATCH 067/112] Fix "/" handling in getDestDir (fixes ticket 3) --- lib/files/getDestDir | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/files/getDestDir b/lib/files/getDestDir index 82d5667..6ec50cc 100644 --- a/lib/files/getDestDir +++ b/lib/files/getDestDir @@ -73,5 +73,5 @@ getDestDir() { then mkdir -p "$destdir" fi - destdir="${destdir//+(\/)/\/}" + destdir="${destdir//+(\/)//}" } From 459d8c06ce137dc85f7b523f0152828f01a8a135 Mon Sep 17 00:00:00 2001 From: Vincent Riquer Date: Wed, 27 Nov 2013 17:26:15 +0100 Subject: [PATCH 068/112] Fix (again!) handling of non-UTF8 characters --- lib/tools/ascii | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/tools/ascii b/lib/tools/ascii index a950e22..5f474af 100644 --- a/lib/tools/ascii +++ b/lib/tools/ascii @@ -6,10 +6,10 @@ ascii() { use utf8; use Encode; use Text::Unidecode; - use open qw(:std :utf8); + binmode STDIN, ":encoding(UTF-8)"; $| = 1; while (<>) { - print(unidecode(encode("utf-8", $_))); + print(unidecode($_)); }' } } From 421c690e5df4f4ec72f5a04307fb9e6deb556540 Mon Sep 17 00:00:00 2001 From: Vincent Riquer Date: Tue, 10 Dec 2013 13:37:24 +0100 Subject: [PATCH 069/112] Albumartist, composer, performer for MP3 --- lib/encode/mp3 | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/encode/mp3 b/lib/encode/mp3 index 4963b5e..68a93ab 100644 --- a/lib/encode/mp3 +++ b/lib/encode/mp3 @@ -8,6 +8,10 @@ encodeFile::mp3() { [ -n "$title" ] && lameopts+=(--tt "$title") [ -n "$track" ] && lameopts+=(--tn "$track") [ -n "$year" ] && lameopts+=(--ty "$year") + [ -n "$albumartist" ] && lameopts+=(--tv TPE2="$albumartist") + [ -n "$composer" ] && lameopts+=(--tv TCOM="$composer") + [ -n "$performer" ] && lameopts+=(--tv TOPE="$performer") + [ -n "$disc" ] && lameopts+=(--tv TPOS="$disc") if (( ${destinationnoresample[$destination]:-0} == 1 )) then # If 'rate' is not one of these value, it cannot be encoded to From 377f332acd660ccb3606cacc44efac3fa1ae4a3b Mon Sep 17 00:00:00 2001 From: Vincent Riquer Date: Tue, 26 Aug 2014 11:13:35 +0200 Subject: [PATCH 070/112] include ffmpeg tagreader version in tagreaders depending on it --- lib/tags/ffmpeg | 2 +- lib/tags/getInfos::WebM | 2 +- lib/tags/getInfos::ffmpeg_other | 2 +- lib/tags/getInfos::ffmpeg_video | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/tags/ffmpeg b/lib/tags/ffmpeg index 6b836a3..9009295 100644 --- a/lib/tags/ffmpeg +++ b/lib/tags/ffmpeg @@ -1,5 +1,5 @@ #!/bin/bash -getInfosffmpeg_version='ffmpeg-4' +getInfosffmpeg_version='ffmpeg-5' tagreaders+=( "$getInfosffmpeg_version" ) getInfos::ffmpeg() { tagreader="$getInfosffmpeg_version" diff --git a/lib/tags/getInfos::WebM b/lib/tags/getInfos::WebM index 03a0f75..a40a6b7 100644 --- a/lib/tags/getInfos::WebM +++ b/lib/tags/getInfos::WebM @@ -1,5 +1,5 @@ #!/bin/bash -getInfoswebm_version='webm-2' +getInfoswebm_version='webm-3:'"$getInfosffmpeg_version" tagreaders+=( "$getInfoswebm_version" ) getInfos::WebM() { getInfos::ffmpeg_video diff --git a/lib/tags/getInfos::ffmpeg_other b/lib/tags/getInfos::ffmpeg_other index 1ac3528..273627d 100644 --- a/lib/tags/getInfos::ffmpeg_other +++ b/lib/tags/getInfos::ffmpeg_other @@ -1,5 +1,5 @@ #!/bin/bash -getInfosffmpeg_other_version='ffmpeg_other-2' +getInfosffmpeg_other_version='ffmpeg_other-3:'"$getInfosffmpeg_version" tagreaders+=( "$getInfosffmpeg_other_version" ) getInfos::ffmpeg_other() { getInfos::ffmpeg diff --git a/lib/tags/getInfos::ffmpeg_video b/lib/tags/getInfos::ffmpeg_video index 9004d7f..cb04459 100644 --- a/lib/tags/getInfos::ffmpeg_video +++ b/lib/tags/getInfos::ffmpeg_video @@ -1,5 +1,5 @@ #!/bin/bash -getInfosffmpeg_video_version='ffmpeg_video-2' +getInfosffmpeg_video_version='ffmpeg_video-3:'"$getInfosffmpeg_version" tagreaders+=( "$getInfosffmpeg_video_version" ) getInfos::ffmpeg_video() { getInfos::ffmpeg From a43398f4ac2e0c5f5d3a840c19a6ddf50e21fe87 Mon Sep 17 00:00:00 2001 From: Vincent Riquer Date: Tue, 26 Aug 2014 11:14:13 +0200 Subject: [PATCH 071/112] ffmpeg: add "disc" tag --- lib/tags/ffmpeg | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/tags/ffmpeg b/lib/tags/ffmpeg index 9009295..da91fbc 100644 --- a/lib/tags/ffmpeg +++ b/lib/tags/ffmpeg @@ -28,6 +28,7 @@ getInfos::ffmpeg() { album=$(gettag album) artist=$(gettag artist) composer=$(gettag composer) + disc=$(gettag disc) genre=$(gettag genre) performer=$(gettag TOPE) title=$(gettag title) From 3fd7a11bed3f809cb27d9cc82ab2cf6774dde123 Mon Sep 17 00:00:00 2001 From: Vincent Riquer Date: Tue, 10 Mar 2015 01:26:08 +0100 Subject: [PATCH 072/112] setup: fix higher-than in rare cases, no pattern would be recognized as a valid bitrate for "higher-than" --- lib/setup/destination | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/setup/destination b/lib/setup/destination index 7293fbf..c67c63d 100644 --- a/lib/setup/destination +++ b/lib/setup/destination @@ -451,7 +451,6 @@ setupDestination() { echo "Invalid frequency value: $value" >&2 comeagain fi - unset expr } comeagain if [ -n "${destinationfrequency["$destination"]}" ] \ @@ -492,4 +491,5 @@ setupDestination() { fi destinationmaxbps[$destination]="$value" unset regen + unset expr } From 6510eb17fce404d2db10d47777b1957579d23079 Mon Sep 17 00:00:00 2001 From: Vincent Riquer Date: Tue, 10 Mar 2015 10:24:33 +0100 Subject: [PATCH 073/112] ionice may not be available --- lib/config/getGeneral | 83 ++++++++++++++++++++++--------------------- 1 file changed, 43 insertions(+), 40 deletions(-) diff --git a/lib/config/getGeneral b/lib/config/getGeneral index ec183ab..1e7797d 100644 --- a/lib/config/getGeneral +++ b/lib/config/getGeneral @@ -24,47 +24,50 @@ getConfigGeneral() { unset expr ;; 'ionice') - read class niceness <<<"$value" - case $class in - 1) - # real-time class, only root can do that - if (( UID )) - then - echo "IO class 'realtime' is"\ - "not available to unprivileged"\ - "users" >&2 + if which ionice >/dev/null + then + read class niceness <<<"$value" + case $class in + 1) + # real-time class, only root can do that + if (( UID )) + then + echo "IO class 'realtime' is"\ + "not available to unprivileged"\ + "users" >&2 + exit $EIONICE + fi + if [ -n "$niceness" ] \ + && (( niceness >= 0 && niceness <= 7 )) + then + ionice="ionice -c1 -n$niceness " + else + echo "Invalid IO priority"\ + "'$niceness'" >&2 + exit $EIONICE + fi + ;; + 2) + if [ -n "$niceness" ] \ + && (( niceness >= 0 && niceness <= 7 )) + then + ionice="ionice -c2 -n$niceness " + else + echo "Invalid IO priority"\ + "'$niceness'" >&2 + exit $EIONICE + fi + ;; + 3) + ionice="ionice -c3 " + ;; + *) + echo "Invalid ionice parameters $value"\ + >&2 exit $EIONICE - fi - if [ -n "$niceness" ] \ - && (( niceness >= 0 && niceness <= 7 )) - then - ionice="ionice -c1 -n$niceness " - else - echo "Invalid IO priority"\ - "'$niceness'" >&2 - exit $EIONICE - fi - ;; - 2) - if [ -n "$niceness" ] \ - && (( niceness >= 0 && niceness <= 7 )) - then - ionice="ionice -c2 -n$niceness " - else - echo "Invalid IO priority"\ - "'$niceness'" >&2 - exit $EIONICE - fi - ;; - 3) - ionice="ionice -c3 " - ;; - *) - echo "Invalid ionice parameters $value"\ - >&2 - exit $EIONICE - ;; - esac + ;; + esac + fi ;; 'temporary-directory') tempdir="$value" From fdae75956c057acee97d368211aeb8f31b5f0e69 Mon Sep 17 00:00:00 2001 From: Vincent Riquer Date: Thu, 12 Mar 2015 19:28:16 +0100 Subject: [PATCH 074/112] update README --- README | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README b/README index 23c6430..74f1826 100644 --- a/README +++ b/README @@ -1,8 +1,8 @@ AtOM: Anything to Ogg and Mp3 -URL: http://gitorious.org/atom +URL: https://forge.riquer.fr/p/AtOM/ Author: Vincent Riquer -Copyright/left: 2012-2013 Vincent Riquer - GPLv3 (see doc/GPL-3) +Copyright/left: 2012-2013,2015 Vincent Riquer - GPLv3 (see doc/GPL-3) except: transogg: WTFPL 2.0 ============ From 14b7a54a93ed361149094b05e20bc50c6eea3e01 Mon Sep 17 00:00:00 2001 From: Vincent Riquer Date: Thu, 12 Mar 2015 20:08:20 +0100 Subject: [PATCH 075/112] Portability: shebang --- atom | 2 +- toys/checkextensions | 2 +- toys/cleandestinations | 2 +- toys/createindex | 2 +- toys/lowquality | 2 +- toys/missingtags | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/atom b/atom index e72a157..b18c18e 100755 --- a/atom +++ b/atom @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash ## Define exit codes # General config errors [10-19] diff --git a/toys/checkextensions b/toys/checkextensions index 3a427ff..08da0b0 100755 --- a/toys/checkextensions +++ b/toys/checkextensions @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # config structures declare -A \ diff --git a/toys/cleandestinations b/toys/cleandestinations index b80e767..b5a35e8 100755 --- a/toys/cleandestinations +++ b/toys/cleandestinations @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # config structures declare -A \ diff --git a/toys/createindex b/toys/createindex index 6963104..9f2a0a7 100755 --- a/toys/createindex +++ b/toys/createindex @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash #!/bin/bash diff --git a/toys/lowquality b/toys/lowquality index 0e3395b..1adfdab 100755 --- a/toys/lowquality +++ b/toys/lowquality @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # config structures declare -A \ diff --git a/toys/missingtags b/toys/missingtags index 72f723b..417444a 100755 --- a/toys/missingtags +++ b/toys/missingtags @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # config structures declare -A \ From b7ddf653314ae98cdaa66af850033adbef9c02ad Mon Sep 17 00:00:00 2001 From: Vincent Riquer Date: Thu, 12 Mar 2015 20:15:25 +0100 Subject: [PATCH 076/112] Portability: column -n is Debian-specific --- lib/config/print | 44 ++++++++++++++++++++++---------------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/lib/config/print b/lib/config/print index 06d65b3..5dbf3a9 100644 --- a/lib/config/print +++ b/lib/config/print @@ -4,19 +4,19 @@ printConfig() { echo "General|Config file|$cffile" [ -n "$ionice" ] && echo "|IO Nice|$ionice" cat <<-EOF - |Load|$maxload - |Load Interval|$loadinterval - |Temp Dir|$tempdir - |Database|$database - |Debug|$debug + |Load|$maxload + |Load Interval|$loadinterval + |Temp Dir|$tempdir + |Database|$database + |Debug|$debug Source|Path|$sourcepath EOF for prune_expression in "${skippeddirectories[@]}" do (( printed )) \ - && echo -n '||' \ - || echo -n '|Skipped directories|' + && echo -n ' | |' \ + || echo -n ' |Skipped directories|' echo "$prune_expression" printed=1 done @@ -26,32 +26,32 @@ printConfig() { cat <<-EOF $destination|Path|${destinationpath["$destination"]} - |Format|${destinationformat["$destination"]} - |Quality|${destinationquality["$destination"]} + |Format|${destinationformat["$destination"]} + |Quality|${destinationquality["$destination"]} EOF if [[ ${destinationformat["$destination"]} == opus ]] then - echo "|Expected loss|${destinationloss["$destination"]}" + echo " |Expected loss|${destinationloss["$destination"]}" elif [[ ${destinationformat["$destination"]} == mp3 ]] then - echo "|Prevent resampling|${destinationnoresample["$destination"]}" + echo " |Prevent resampling|${destinationnoresample["$destination"]}" fi cat <<-EOF - |Normalize|${destinationnormalize["$destination"]} - |Channels|${destinationchannels["$destination"]} - |Frequency|${destinationfrequency["$destination"]} - |Higher than|${destinationmaxbps["$destination"]} - |Fat32 Compat.|${destinationfat32compat["$destination"]} - |ASCII Compat.|${destinationascii["$destination"]} - |Path Change|${destinationrenamepath["$destination"]} - |File Rename|${destinationrename["$destination"]} + |Normalize|${destinationnormalize["$destination"]} + |Channels|${destinationchannels["$destination"]} + |Frequency|${destinationfrequency["$destination"]} + |Higher than|${destinationmaxbps["$destination"]} + |Fat32 Compat.|${destinationfat32compat["$destination"]} + |ASCII Compat.|${destinationascii["$destination"]} + |Path Change|${destinationrenamepath["$destination"]} + |File Rename|${destinationrename["$destination"]} EOF [ -n "${destinationskipmime["$destination"]}" ] \ - && echo "|Skipped mime-types|${destinationskipmime["$destination"]//\|/ + && echo " |Skipped mime-types|${destinationskipmime["$destination"]//\|/ ||}" [ -n "${destinationmskipime["$destination"]}" ] \ - && echo "|Copied mime-types|${destinationcopymime["$destination"]//\|/ + && echo " |Copied mime-types|${destinationcopymime["$destination"]//\|/ ||}" done - }|column -t -s'|' -n + }|column -t -s'|' } From 2f93e25d2422435ffeaf45e508cc522f364de89f Mon Sep 17 00:00:00 2001 From: Vincent Riquer Date: Sat, 16 Apr 2016 01:04:15 +0200 Subject: [PATCH 077/112] "30 columns ought to be enough for anyone" (enlarge your table) The task temporary table has now 60 columns. --- atom | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/atom b/atom index b18c18e..45c4f7e 100755 --- a/atom +++ b/atom @@ -424,6 +424,36 @@ echo ' cmd_arg27 TEXT, cmd_arg28 TEXT, cmd_arg29 TEXT, + cmd_arg30 TEXT, + cmd_arg31 TEXT, + cmd_arg32 TEXT, + cmd_arg33 TEXT, + cmd_arg34 TEXT, + cmd_arg35 TEXT, + cmd_arg36 TEXT, + cmd_arg37 TEXT, + cmd_arg38 TEXT, + cmd_arg39 TEXT, + cmd_arg40 TEXT, + cmd_arg41 TEXT, + cmd_arg42 TEXT, + cmd_arg43 TEXT, + cmd_arg44 TEXT, + cmd_arg45 TEXT, + cmd_arg46 TEXT, + cmd_arg47 TEXT, + cmd_arg48 TEXT, + cmd_arg49 TEXT, + cmd_arg50 TEXT, + cmd_arg51 TEXT, + cmd_arg52 TEXT, + cmd_arg53 TEXT, + cmd_arg54 TEXT, + cmd_arg55 TEXT, + cmd_arg56 TEXT, + cmd_arg57 TEXT, + cmd_arg58 TEXT, + cmd_arg59 TEXT, requires INTEGER, required INTEGER, status INTEGER NOT NULL, From 008db8c93936ecb744f1bdbc0c85ed5c081948c5 Mon Sep 17 00:00:00 2001 From: Vincent Riquer Date: Sat, 16 Apr 2016 13:55:50 +0200 Subject: [PATCH 078/112] Work around pipe size limit removing obsolete files --- atom | 35 +++++++++++++++++++++++++---------- 1 file changed, 25 insertions(+), 10 deletions(-) diff --git a/atom b/atom index 45c4f7e..f8b1d8e 100755 --- a/atom +++ b/atom @@ -339,21 +339,36 @@ updateMimes removeObsoleteFiles echo ' - SELECT id, - filename + SELECT COUNT(id) FROM destination_files - WHERE source_file_id is NULL; + WHERE source_file_id is NULL;' >&3 - SELECT "AtOM:NoMoreFiles"; -' >&3 +read -u4 removecount +until (( ${#removefile[@]} == removecount )) +do + echo ' + SELECT id, + filename + FROM destination_files + WHERE source_file_id is NULL + LIMIT 500 OFFSET '${#removefile[@]}'; + + SELECT "AtOM:NoMoreFiles"; + ' >&3 + + read -u4 line + until [[ $line == AtOM:NoMoreFiles ]] + do + removefile[${line%::AtOM:SQL:Sep::*}]="${line%::AtOM:SQL:Sep::*}" + read -u4 line + done +done deleted=0 removed=0 -read -u4 line -until [[ $line == AtOM:NoMoreFiles ]] +for id in ${!removefile[@]} do - id=${line%::AtOM:SQL:Sep::*} - filename=${line#*::AtOM:SQL:Sep::} + filename=${removefile[id]} if [ -n "$filename" ] then if rm -f "$filename" @@ -365,9 +380,9 @@ do Delete destination_files <<<"id = $id" (( ++removed )) fi - read -u4 line done echo "Suppressed $deleted files, $removed removed from database" +unset removecount deleted removed removefile updateTags From 6cba3cf80fbdbd5afac1150ec56b70144327b465 Mon Sep 17 00:00:00 2001 From: Vincent Riquer Date: Mon, 18 Apr 2016 14:35:14 +0200 Subject: [PATCH 079/112] tasks: use the new columns! --- atom | 30 ++++++++++++++++ lib/workers/master | 90 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 120 insertions(+) diff --git a/atom b/atom index 45c4f7e..4a13055 100755 --- a/atom +++ b/atom @@ -828,6 +828,36 @@ then tasks.cmd_arg27, tasks.cmd_arg28, tasks.cmd_arg29 + tasks.cmd_arg30, + tasks.cmd_arg31, + tasks.cmd_arg32, + tasks.cmd_arg33, + tasks.cmd_arg34, + tasks.cmd_arg35, + tasks.cmd_arg36, + tasks.cmd_arg37, + tasks.cmd_arg38, + tasks.cmd_arg39, + tasks.cmd_arg40, + tasks.cmd_arg41, + tasks.cmd_arg42, + tasks.cmd_arg43, + tasks.cmd_arg44, + tasks.cmd_arg45, + tasks.cmd_arg46, + tasks.cmd_arg47, + tasks.cmd_arg48, + tasks.cmd_arg49, + tasks.cmd_arg50, + tasks.cmd_arg51, + tasks.cmd_arg52, + tasks.cmd_arg53, + tasks.cmd_arg54, + tasks.cmd_arg55, + tasks.cmd_arg56, + tasks.cmd_arg57, + tasks.cmd_arg58, + tasks.cmd_arg59 FROM tasks INNER JOIN source_files ON tasks.source_file=source_files.id diff --git a/lib/workers/master b/lib/workers/master index a6d0c99..b51b064 100644 --- a/lib/workers/master +++ b/lib/workers/master @@ -48,6 +48,36 @@ master() { cmd_arg27, cmd_arg28, cmd_arg29, + cmd_arg30, + cmd_arg31, + cmd_arg32, + cmd_arg33, + cmd_arg34, + cmd_arg35, + cmd_arg36, + cmd_arg37, + cmd_arg38, + cmd_arg39, + cmd_arg40, + cmd_arg41, + cmd_arg42, + cmd_arg43, + cmd_arg44, + cmd_arg45, + cmd_arg46, + cmd_arg47, + cmd_arg48, + cmd_arg49, + cmd_arg50, + cmd_arg51, + cmd_arg52, + cmd_arg53, + cmd_arg54, + cmd_arg55, + cmd_arg56, + cmd_arg57, + cmd_arg58, + cmd_arg59, cleanup, fileid, filename @@ -135,6 +165,66 @@ master() { rest=${rest#*::AtOM:SQL:Sep::} cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}") rest=${rest#*::AtOM:SQL:Sep::} + cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}") + rest=${rest#*::AtOM:SQL:Sep::} + cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}") + rest=${rest#*::AtOM:SQL:Sep::} + cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}") + rest=${rest#*::AtOM:SQL:Sep::} + cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}") + rest=${rest#*::AtOM:SQL:Sep::} + cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}") + rest=${rest#*::AtOM:SQL:Sep::} + cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}") + rest=${rest#*::AtOM:SQL:Sep::} + cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}") + rest=${rest#*::AtOM:SQL:Sep::} + cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}") + rest=${rest#*::AtOM:SQL:Sep::} + cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}") + rest=${rest#*::AtOM:SQL:Sep::} + cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}") + rest=${rest#*::AtOM:SQL:Sep::} + cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}") + rest=${rest#*::AtOM:SQL:Sep::} + cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}") + rest=${rest#*::AtOM:SQL:Sep::} + cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}") + rest=${rest#*::AtOM:SQL:Sep::} + cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}") + rest=${rest#*::AtOM:SQL:Sep::} + cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}") + rest=${rest#*::AtOM:SQL:Sep::} + cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}") + rest=${rest#*::AtOM:SQL:Sep::} + cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}") + rest=${rest#*::AtOM:SQL:Sep::} + cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}") + rest=${rest#*::AtOM:SQL:Sep::} + cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}") + rest=${rest#*::AtOM:SQL:Sep::} + cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}") + rest=${rest#*::AtOM:SQL:Sep::} + cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}") + rest=${rest#*::AtOM:SQL:Sep::} + cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}") + rest=${rest#*::AtOM:SQL:Sep::} + cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}") + rest=${rest#*::AtOM:SQL:Sep::} + cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}") + rest=${rest#*::AtOM:SQL:Sep::} + cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}") + rest=${rest#*::AtOM:SQL:Sep::} + cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}") + rest=${rest#*::AtOM:SQL:Sep::} + cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}") + rest=${rest#*::AtOM:SQL:Sep::} + cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}") + rest=${rest#*::AtOM:SQL:Sep::} + cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}") + rest=${rest#*::AtOM:SQL:Sep::} + cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}") + rest=${rest#*::AtOM:SQL:Sep::} cleanup=${rest%%::AtOM:SQL:Sep::*} rest=${rest#*::AtOM:SQL:Sep::} destfileid=${rest%%::AtOM:SQL:Sep::*} From f5a7b658807ab4e56ae536b417f3a04918e0c129 Mon Sep 17 00:00:00 2001 From: Vincent Riquer Date: Mon, 25 Apr 2016 00:50:48 +0200 Subject: [PATCH 080/112] fix failed tasks recollection query --- atom | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/atom b/atom index b3c7a9e..1227d39 100755 --- a/atom +++ b/atom @@ -842,7 +842,7 @@ then tasks.cmd_arg26, tasks.cmd_arg27, tasks.cmd_arg28, - tasks.cmd_arg29 + tasks.cmd_arg29, tasks.cmd_arg30, tasks.cmd_arg31, tasks.cmd_arg32, From db1ed90eaa8f6b33290c73929ff830fc284f476c Mon Sep 17 00:00:00 2001 From: Vincent Riquer Date: Tue, 26 Apr 2016 16:47:23 +0200 Subject: [PATCH 081/112] progress during cleanup --- atom | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/atom b/atom index 1227d39..171b6a5 100755 --- a/atom +++ b/atom @@ -380,8 +380,12 @@ do Delete destination_files <<<"id = $id" (( ++removed )) fi + (( cron )) || echo -en "\rClean obsolete data: $(((deleted+removed)*100/removecount))%" done -echo "Suppressed $deleted files, $removed removed from database" +(( cron )) || echo -n $'\r' +echo -n "Suppressed $deleted files, $removed removed from database" +(( cron )) || echo -ne "\033[K" +echo unset removecount deleted removed removefile updateTags From fe61ee8c53089bec33014747e9fbb3f891a9c9bd Mon Sep 17 00:00:00 2001 From: Vincent Riquer Date: Tue, 26 Apr 2016 16:47:58 +0200 Subject: [PATCH 082/112] Optimize DB writes during cleanup --- atom | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/atom b/atom index 171b6a5..34acc56 100755 --- a/atom +++ b/atom @@ -366,6 +366,7 @@ done deleted=0 removed=0 +echo 'BEGIN TRANSACTION;' >&3 for id in ${!removefile[@]} do filename=${removefile[id]} @@ -380,8 +381,13 @@ do Delete destination_files <<<"id = $id" (( ++removed )) fi + if (( (deleted + removed) % 1000 == 0 )) + then + echo 'COMMIT;BEGIN TRANSACTION;' >&3 + fi (( cron )) || echo -en "\rClean obsolete data: $(((deleted+removed)*100/removecount))%" done +echo 'COMMIT;' >&3 (( cron )) || echo -n $'\r' echo -n "Suppressed $deleted files, $removed removed from database" (( cron )) || echo -ne "\033[K" From 5977629cab6f99cdefa1a4ee3587fc0abe3f4972 Mon Sep 17 00:00:00 2001 From: Vincent Riquer Date: Thu, 1 Sep 2016 18:43:18 +0200 Subject: [PATCH 083/112] Use TRIGGER to fail tasks depending upon a failed task --- atom | 17 ++++++++++++++--- lib/database/open | 1 + lib/workers/cleaner | 1 - 3 files changed, 15 insertions(+), 4 deletions(-) diff --git a/atom b/atom index 34acc56..7977ce2 100755 --- a/atom +++ b/atom @@ -412,6 +412,9 @@ done echo ' CREATE TEMPORARY TABLE tasks( id INTEGER PRIMARY KEY, + requires INTEGER, + required INTEGER, + status INTEGER NOT NULL, key TEXT UNIQUE, rename_pattern TEXT, fat32compat INTEGER, @@ -479,15 +482,23 @@ echo ' cmd_arg57 TEXT, cmd_arg58 TEXT, cmd_arg59 TEXT, - requires INTEGER, - required INTEGER, - status INTEGER NOT NULL, cleanup TEXT, FOREIGN KEY(requires) REFERENCES tasks(id) ON DELETE SET NULL ); CREATE INDEX tasks_by_key ON tasks ( key ); CREATE INDEX tasks_by_sourcefile ON tasks ( source_file ); + + CREATE TEMPORARY TRIGGER fail_depends + AFTER UPDATE OF + status + ON + tasks + WHEN + NEW.status=2 + BEGIN + UPDATE tasks SET status=2 WHERE requires=NEW.id; + END; ' >&3 echo ' diff --git a/lib/database/open b/lib/database/open index c20caca..ce86d95 100644 --- a/lib/database/open +++ b/lib/database/open @@ -15,5 +15,6 @@ openDatabase() { cat $schema >&3 echo '.separator ::AtOM:SQL:Sep::' >&3 echo 'PRAGMA foreign_keys = ON;' >&3 + echo 'PRAGMA recursive_triggers = ON;' >&3 checkDatabaseVersion } diff --git a/lib/workers/cleaner b/lib/workers/cleaner index 177def4..b72a53e 100644 --- a/lib/workers/cleaner +++ b/lib/workers/cleaner @@ -12,7 +12,6 @@ cleaner() { (( failed+=faildepends )) (( ran+=faildepends )) Update tasks status 2 <<<"id = $taskid" - Update tasks status 2 <<<"requires = $taskid" echo "SELECT COUNT(*) FROM tasks WHERE ( status = 0 OR status = 1 ) From b4afedb70fa339b8e12b65461e77729af71d8c28 Mon Sep 17 00:00:00 2001 From: Vincent Riquer Date: Thu, 1 Sep 2016 18:44:02 +0200 Subject: [PATCH 084/112] Always try ffmpeg on unknown filetypes --- lib/decode/file | 19 ++++++++++++++++++- lib/encode/mp3 | 1 + lib/encode/opus | 1 + lib/encode/vorbis | 1 + lib/video/extractaudio | 4 ++-- lib/workers/master | 24 ++++++++++++++++++++++++ 6 files changed, 47 insertions(+), 3 deletions(-) diff --git a/lib/decode/file b/lib/decode/file index 1571222..5883f83 100644 --- a/lib/decode/file +++ b/lib/decode/file @@ -1,4 +1,5 @@ #!/bin/bash +declare soxtaskid decodeFile() { case "$mimetype" in 'video/'*) @@ -77,7 +78,23 @@ decodeFile() { fi ;; *) - decodeSox + if (( disablevideo )) + then + decodeSox + else + extractAudio + if (( ${destinationnormalize["$destination"]}))\ + || ( + [ -n "${destinationfrequency["$destination"]}" ]\ + && (( ${rate:-0} != ${destinationfrequency["$destination"]}))\ + ) || ( + [ -n "${destinationchannels["$destination"]}" ]\ + && (( ${channels:-0} != ${destinationchannels["$destination"]} )) + ) + then + sox_needed=1 + fi + fi ;; esac ;; diff --git a/lib/encode/mp3 b/lib/encode/mp3 index 68a93ab..d0d6800 100644 --- a/lib/encode/mp3 +++ b/lib/encode/mp3 @@ -75,4 +75,5 @@ encodeFile::mp3() { EOInsert ) progressSpin + soxtaskid='' } diff --git a/lib/encode/opus b/lib/encode/opus index 3aa993d..bfa48ca 100644 --- a/lib/encode/opus +++ b/lib/encode/opus @@ -43,4 +43,5 @@ encodeFile::opus() { EOInsert ) progressSpin + soxtaskid='' } diff --git a/lib/encode/vorbis b/lib/encode/vorbis index 7eee65c..a96523f 100644 --- a/lib/encode/vorbis +++ b/lib/encode/vorbis @@ -39,4 +39,5 @@ encodeFile::vorbis() { EOInsert ) progressSpin + soxtaskid='' } diff --git a/lib/video/extractaudio b/lib/video/extractaudio index d8d3344..afa1392 100644 --- a/lib/video/extractaudio +++ b/lib/video/extractaudio @@ -1,6 +1,6 @@ #!/bin/bash extractAudio() { - tmpfile="${fileid}ffmpeg" + tmpfile="${fileid}ffmpeg.wav" commandline=(${ionice}ffmpeg -v 0 -vn -y) - commandline+=(-i "$sourcepath/$filename" "$tempdir/$tmpfile.wav") + commandline+=(-i "$sourcepath/$filename" "$tempdir/$tmpfile") } diff --git a/lib/workers/master b/lib/workers/master index b51b064..96e3937 100644 --- a/lib/workers/master +++ b/lib/workers/master @@ -93,6 +93,30 @@ master() { then sleep 0.1 continue + elif (( active == 0 && ready == 0 )) + then + dumpfile=tasks-$(date +%Y%m%d%H%M%S).csv + cat <<-EOF + + + $remaining TASKS LEFT, NONE READY! + + Something went wrong, dumping tasks table to $dumpfile + EOF + cat >&3 <<-EOSQL + .mode csv + .headers on + .output $dumpfile + SELECT * from tasks; + .mode list + .headers off + .output stdout + EOSQL + closeDatabase + echo "Waiting for children to come back home..." + wait + echo $'\nGood luck!' + exit 1 elif (( ready == 0 )) then sleep 0.1 From dc29803438be2c336435f14466d6cf35e21d0b13 Mon Sep 17 00:00:00 2001 From: Vincent Riquer Date: Fri, 9 Jun 2017 10:11:37 +0200 Subject: [PATCH 085/112] Remove mysql pipes early --- lib/database/close | 1 - lib/database/open | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/database/close b/lib/database/close index 19a412b..9fb38ca 100644 --- a/lib/database/close +++ b/lib/database/close @@ -7,5 +7,4 @@ closeDatabase() { (( debug )) && echo OK exec 3>&- exec 4<&- - rm "$tempdir"/sqlite.{in,out} } diff --git a/lib/database/open b/lib/database/open index ce86d95..e04131f 100644 --- a/lib/database/open +++ b/lib/database/open @@ -7,6 +7,7 @@ openDatabase() { > "$tempdir/sqlite.out" & exec 3> "$tempdir"/sqlite.in exec 4< "$tempdir"/sqlite.out + rm "$tempdir"/sqlite.in "$tempdir"/sqlite.out if (( debug > 2 )) then exec 5>&3 From 6c438c4c904dc30526bbe99efa506286bb1a4e10 Mon Sep 17 00:00:00 2001 From: Vincent Riquer Date: Fri, 9 Jun 2017 10:12:32 +0200 Subject: [PATCH 086/112] Fix double extension of temp files --- lib/decode/file | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/decode/file b/lib/decode/file index 5883f83..c931fa0 100644 --- a/lib/decode/file +++ b/lib/decode/file @@ -123,7 +123,7 @@ decodeFile() { if (( sox_needed )) then cleanup="$tempdir/$tmpfile" - decodeSox "$tempdir/$tmpfile.wav" + decodeSox "$tempdir/$tmpfile" if ! soxtaskid=$( Select tasks id <<<"key = $tmpfile" ) From d16b31f479f2d1fd7234229fc64b92ad9da5ece3 Mon Sep 17 00:00:00 2001 From: Vincent Riquer Date: Sun, 18 Jun 2017 18:21:18 +0200 Subject: [PATCH 087/112] add "copy" destination type --- lib/config/getDestination | 6 + lib/config/print | 8 +- lib/config/write | 4 +- lib/decode/file | 193 ++++++++++++----------- lib/files/getDestDir | 32 +++- lib/files/sanitizeFile | 2 +- lib/setup/destination | 320 ++++++++++++++++++++------------------ lib/setup/destinations | 4 +- lib/video/extractaudio | 2 +- lib/workers/master | 2 +- 10 files changed, 308 insertions(+), 265 deletions(-) diff --git a/lib/config/getDestination b/lib/config/getDestination index 1e239c4..2926648 100644 --- a/lib/config/getDestination +++ b/lib/config/getDestination @@ -9,6 +9,9 @@ getConfigDestination() { 'mp3') destinationformat["$destination"]=mp3 lameneeded=1 + # MP3 can't handfle more than 2 channels + [[ -z ${destinationchannels["$destination"]} ]] \ + && destinationchannels["$destination"]=2 ;; 'opus') destinationformat["$destination"]=opus @@ -18,6 +21,9 @@ getConfigDestination() { destinationformat["$destination"]=vorbis oggencneeded=1 ;; + 'copy') + destinationformat["$destination"]=copy + ;; *) echo "Unsupported destination format: $value" >&2 exit $EFORMAT diff --git a/lib/config/print b/lib/config/print index 5dbf3a9..567c547 100644 --- a/lib/config/print +++ b/lib/config/print @@ -24,7 +24,7 @@ printConfig() { for destination in ${!destinationpath[@]} do cat <<-EOF - + $destination|Path|${destinationpath["$destination"]} |Format|${destinationformat["$destination"]} |Quality|${destinationquality["$destination"]} @@ -48,10 +48,10 @@ printConfig() { EOF [ -n "${destinationskipmime["$destination"]}" ] \ && echo " |Skipped mime-types|${destinationskipmime["$destination"]//\|/ -||}" - [ -n "${destinationmskipime["$destination"]}" ] \ +| | |}" + [ -n "${destinationcopymime["$destination"]}" ] \ && echo " |Copied mime-types|${destinationcopymime["$destination"]//\|/ -||}" +| | |}" done }|column -t -s'|' } diff --git a/lib/config/write b/lib/config/write index 0891648..d8b6b36 100644 --- a/lib/config/write +++ b/lib/config/write @@ -63,8 +63,8 @@ path $sourcepath # * path: Where files will be written path ${destinationpath["$destination"]} -# * format: ogg, opus or mp3. Other formats may appear in the future - feel -# free to implement your preferred format. +# * format: copy, ogg, opus or mp3. Other formats may appear in the future - +# feel free to implement your preferred format. format ${destinationformat["$destination"]} EOCfg diff --git a/lib/decode/file b/lib/decode/file index c931fa0..b0f350a 100644 --- a/lib/decode/file +++ b/lib/decode/file @@ -1,39 +1,14 @@ #!/bin/bash declare soxtaskid decodeFile() { - case "$mimetype" in - 'video/'*) - (( disablevideo )) && continue - extractAudio - if (( ${destinationnormalize["$destination"]}))\ - || ( - [ -n "${destinationfrequency["$destination"]}" ]\ - && (( ${rate:-0} != ${destinationfrequency["$destination"]}))\ - ) || ( - [ -n "${destinationchannels["$destination"]}" ]\ - && (( ${channels:-0} != ${destinationchannels["$destination"]} )) - ) - then - sox_needed=1 - fi - ;; - 'audio/mpeg') - if [[ ${destinationformat[$destination]} = mp3 ]] \ - && checkCopy - then - copied=1 - else - decodeSox - fi - ;; - 'application/ogg opus') - if [[ ${destinationformat[$destination]} = opus ]] \ - && checkCopy - then - copied=1 - else - (( disableopusdec )) && continue - decodeOpusdec + if [[ ${destinationformat["$destination"]} == copy ]] + then + copied=1 + else + case "$mimetype" in + 'video/'*) + (( disablevideo )) && continue + extractAudio if (( ${destinationnormalize["$destination"]}))\ || ( [ -n "${destinationfrequency["$destination"]}" ]\ @@ -45,26 +20,24 @@ decodeFile() { then sox_needed=1 fi - fi - ;; - 'application/ogg'*) - if [[ ${destinationformat[$destination]} = vorbis ]] \ - && checkCopy - then - copied=1 - else - decodeSox - fi - ;; - 'audio/x-flac') - decodeSox - ;; - *) - extendedtype=$(file -b "$sourcepath/$filename") - case "$extendedtype" in - *'Musepack '*) - (( disablempcdec )) && continue - decodeMpcdec + ;; + 'audio/mpeg') + if [[ ${destinationformat[$destination]} = mp3 ]] \ + && checkCopy + then + copied=1 + else + decodeSox + fi + ;; + 'application/ogg opus') + if [[ ${destinationformat[$destination]} = opus ]] \ + && checkCopy + then + copied=1 + else + (( disableopusdec )) && continue + decodeOpusdec if (( ${destinationnormalize["$destination"]}))\ || ( [ -n "${destinationfrequency["$destination"]}" ]\ @@ -76,13 +49,26 @@ decodeFile() { then sox_needed=1 fi - ;; - *) - if (( disablevideo )) - then - decodeSox - else - extractAudio + fi + ;; + 'application/ogg'*) + if [[ ${destinationformat[$destination]} = vorbis ]] \ + && checkCopy + then + copied=1 + else + decodeSox + fi + ;; + 'audio/x-flac') + decodeSox + ;; + *) + extendedtype=$(file -b "$sourcepath/$filename") + case "$extendedtype" in + *'Musepack '*) + (( disablempcdec )) && continue + decodeMpcdec if (( ${destinationnormalize["$destination"]}))\ || ( [ -n "${destinationfrequency["$destination"]}" ]\ @@ -94,41 +80,36 @@ decodeFile() { then sox_needed=1 fi - fi - ;; - esac - ;; - esac - if ! (( copied )) - then - if ! decodetaskid=$( - Select tasks id <<<"key = $tmpfile" - ) + ;; + *) + if (( disablevideo )) + then + decodeSox + else + extractAudio + if (( ${destinationnormalize["$destination"]}))\ + || ( + [ -n "${destinationfrequency["$destination"]}" ]\ + && (( ${rate:-0} != ${destinationfrequency["$destination"]}))\ + ) || ( + [ -n "${destinationchannels["$destination"]}" ]\ + && (( ${channels:-0} != ${destinationchannels["$destination"]} )) + ) + then + sox_needed=1 + fi + fi + ;; + esac + ;; + esac + if ! (( copied )) then - decodetaskid=$( - Insert tasks <<-EOInsert - key $tmpfile - source_file $fileid - $( - for key in ${!commandline[@]} - do - echo "cmd_arg$key ${commandline[key]}" - done - ) - status 0 - EOInsert - ) - progressSpin - fi - if (( sox_needed )) - then - cleanup="$tempdir/$tmpfile" - decodeSox "$tempdir/$tmpfile" - if ! soxtaskid=$( + if ! decodetaskid=$( Select tasks id <<<"key = $tmpfile" ) then - soxtaskid=$( + decodetaskid=$( Insert tasks <<-EOInsert key $tmpfile source_file $fileid @@ -138,14 +119,38 @@ decodeFile() { echo "cmd_arg$key ${commandline[key]}" done ) - requires $decodetaskid - required $decodetaskid status 0 - cleanup $cleanup EOInsert ) progressSpin fi + if (( sox_needed )) + then + cleanup="$tempdir/$tmpfile" + decodeSox "$tempdir/$tmpfile" + if ! soxtaskid=$( + Select tasks id <<<"key = $tmpfile" + ) + then + soxtaskid=$( + Insert tasks <<-EOInsert + key $tmpfile + source_file $fileid + $( + for key in ${!commandline[@]} + do + echo "cmd_arg$key ${commandline[key]}" + done + ) + requires $decodetaskid + required $decodetaskid + status 0 + cleanup $cleanup + EOInsert + ) + progressSpin + fi + fi fi fi } diff --git a/lib/files/getDestDir b/lib/files/getDestDir index 6ec50cc..436d1bd 100644 --- a/lib/files/getDestDir +++ b/lib/files/getDestDir @@ -38,6 +38,25 @@ getDestDir() { ) ) then + if (( ${destinationascii["$destination"]} )) + then + echo "$album" >&${toascii[1]} + read -r -u${toascii[0]} album + echo "$albumartist" >&${toascii[1]} + read -r -u${toascii[0]} albumartist + echo "$artist" >&${toascii[1]} + read -r -u${toascii[0]} artist + echo "$genre" >&${toascii[1]} + read -r -u${toascii[0]} genre + echo "$title" >&${toascii[1]} + read -r -u${toascii[0]} title + echo "$tracknumber" >&${toascii[1]} + read -r -u${toascii[0]} tracknumber + echo "$year" >&${toascii[1]} + read -r -u${toascii[0]} year + echo "$disc" >&${toascii[1]} + read -r -u${toascii[0]} disc + fi replace=$(sanitizeFile "$album" dir) destdir+="${destinationrenamepath[$destination]//?(\[)%\{album\}?(\])/$replace}" replace=$(sanitizeFile "$albumartist" dir) @@ -60,15 +79,16 @@ getDestDir() { part=${filename#*/} while [[ $part =~ / ]] do - destdir+="/$(sanitizeFile "${part%%/*}" dir)" + thispart="${part%%/*}" + if (( ${destinationascii["$destination"]} )) + then + echo "$thispart" >&${toascii[1]} + read -r -u${toascii[0]} thispart + fi + destdir+="/$(sanitizeFile "$thispart" dir)" part=${part#*/} done fi - if (( ${destinationascii["$destination"]} )) - then - echo "$destdir" >&${toascii[1]} - read -r -u${toascii[0]} destdir - fi if ! [ -d "$destdir" ] then mkdir -p "$destdir" diff --git a/lib/files/sanitizeFile b/lib/files/sanitizeFile index 07e8e50..164ca9b 100644 --- a/lib/files/sanitizeFile +++ b/lib/files/sanitizeFile @@ -13,7 +13,7 @@ sanitizeFile() { string=${string//>/ } string=${string//:/ } string=${string//\*/ } - string=${string//|/ } + string=${string//\|/ } string=${string//\"/ } # Filenames can't begin or end with ' ' diff --git a/lib/setup/destination b/lib/setup/destination index c67c63d..c6ea323 100644 --- a/lib/setup/destination +++ b/lib/setup/destination @@ -4,7 +4,7 @@ setupDestination() { cat <<-EODesc Format: - vorbis, opus or mp3. Other formats may appear in the future. + copy, vorbis, opus or mp3. Other formats may appear in the future. EODesc comeagain() { read \ @@ -25,6 +25,9 @@ setupDestination() { destinationformat["$destination"]=vorbis oggencneeded=1 ;; + 'copy') + destinationformat["$destination"]=copy + ;; *) echo "Unsupported destination format: $value" >&2 comeagain @@ -44,6 +47,9 @@ setupDestination() { ${destinationpath["$destination"]+-i"${destinationpath["$destination"]}"}\ destinationpath["$destination"] case ${destinationformat["$destination"]} in + copy) + : + ;; vorbis) cat <<-EODesc @@ -199,39 +205,42 @@ setupDestination() { Now you will have the opportunity to configure "advanced" parameters for $destination. You may leave any of these fields blank. EODesc - cat <<-EODesc + if [[ ${destinationformat["$destination"]} != copy ]] + then + cat <<-EODesc - Normalize (boolean): - Normalize output files. - EODesc - case ${destinationnormalize["$destination"]} in - 0) initialvalue=n ;; - 1) initialvalue=y ;; - *) unset initialvalue ;; - esac - comeagain() { - read \ - -e \ - ${initialvalue+-i $initialvalue}\ - -p'Normalize (y/N): ' \ - value - case $value in - [yY]) - [[ $initialvalue == n ]] \ - && setupRegen normalize - destinationnormalize["$destination"]=1 - ;; - ''|[nN]) - [[ $initialvalue == y ]] \ - && setupRegen normalize - destinationnormalize["$destination"]=0 - ;; - *) - comeagain - ;; + Normalize (boolean): + Normalize output files. + EODesc + case ${destinationnormalize["$destination"]} in + 0) initialvalue=n ;; + 1) initialvalue=y ;; + *) unset initialvalue ;; esac - } - comeagain + comeagain() { + read \ + -e \ + ${initialvalue+-i $initialvalue}\ + -p'Normalize (y/N): ' \ + value + case $value in + [yY]) + [[ $initialvalue == n ]] \ + && setupRegen normalize + destinationnormalize["$destination"]=1 + ;; + ''|[nN]) + [[ $initialvalue == y ]] \ + && setupRegen normalize + destinationnormalize["$destination"]=0 + ;; + *) + comeagain + ;; + esac + } + comeagain + fi cat <<-EODesc Rename (string): @@ -362,134 +371,137 @@ setupDestination() { fi done unset skippedmimes - cat <<-EODesc - - Copy mime-type (mime-type, string): - Files with mime-type will be copied as-is to the - destination. E.g. image/* will copy covers and other images to the - destination. The '*' character is a wildcard. - - This prompt will loop until an empty string is encountered. - EODesc - while [[ ${destinationcopymime["$destination"]} =~ \| ]] - do - copiedmimes+=("${destinationcopymime["$destination"]%%|*}") - destinationcopymime["$destination"]="${destinationcopymime["$destination"]#*|}" - done - [ -n "${destinationcopymime["$destination"]}" ] \ - && copiedmimes+=("${destinationcopymime["$destination"]}") - count=${#copiedmimes[@]} - unset destinationcopymime["$destination"] - for (( i=0 ; 1 ; i++ )) - do - read \ - -e \ - ${copiedmimes[i]+-i"${copiedmimes[i]}"} \ - -p 'Copy mime-type: ' \ - value - if [ -n "$value" ] - then - destinationcopymime[$destination]="${destinationcopymime[$destination]:+${destinationcopymime[$destination]}|}$value" - elif (( i < count )) - then - continue - else - break - fi - done - unset copiedmimes - cat <<-EODesc - - Channels (integer): - Produced files should have this many channels, no more, no less. - EODesc - expr='^[0-9]*$' - comeagain() { - read \ - -e \ - ${destinationchannels["$destination"]+-i${destinationchannels["$destination"]}}\ - -p'Channel count: ' \ - value - if ! [[ $value =~ $expr ]] - then - echo "Invalid channel count: $value" >&2 - comeagain - fi - } - comeagain - if [ -n "${destinationchannels["$destination"]}" ] \ - && (( value != ${destinationchannels["$destination"]} )) - then - setupRegen channels - fi - destinationchannels["$destination"]=$value - cat <<-EODesc - - Sampling rate (Hertz, integer): - Files will be resampled as needed to the specified sampling-rate. - Shoutcast/Icecast streams require a constant sampling-rate. - Telephony systems often require a sampling-rate of 8000Hz. - EODesc - if [[ ${destinationformat["$destination"]} == opus ]] + if [[ ${destinationformat["$destination"]} != copy ]] then cat <<-EODesc - Please note that Opus supports only the following sample-rates: - 8000, 12000, 16000, 24000, and 48000 Hz. So don't set - resampling on an Opus destination to any other value or files - will be resampled twice. + Copy mime-type (mime-type, string): + Files with mime-type will be copied as-is to the + destination. E.g. image/* will copy covers and other images to the + destination. The '*' character is a wildcard. + + This prompt will loop until an empty string is encountered. EODesc - fi - comeagain() { - read \ - -e \ - ${destinationfrequency["$destination"]+-i${destinationfrequency["$destination"]}}\ - -p'Sampling-rate: ' \ - value - if ! [[ $value =~ $expr ]] - then - echo "Invalid frequency value: $value" >&2 - comeagain - fi - } - comeagain - if [ -n "${destinationfrequency["$destination"]}" ] \ - && (( value != ${destinationfrequency["$destination"]} )) - then - setupRegen frequency - fi - destinationfrequency["$destination"]=$value - cat <<-EODesc + while [[ ${destinationcopymime["$destination"]} =~ \| ]] + do + copiedmimes+=("${destinationcopymime["$destination"]%%|*}") + destinationcopymime["$destination"]="${destinationcopymime["$destination"]#*|}" + done + [ -n "${destinationcopymime["$destination"]}" ] \ + && copiedmimes+=("${destinationcopymime["$destination"]}") + count=${#copiedmimes[@]} + unset destinationcopymime["$destination"] + for (( i=0 ; 1 ; i++ )) + do + read \ + -e \ + ${copiedmimes[i]+-i"${copiedmimes[i]}"} \ + -p 'Copy mime-type: ' \ + value + if [ -n "$value" ] + then + destinationcopymime[$destination]="${destinationcopymime[$destination]:+${destinationcopymime[$destination]}|}$value" + elif (( i < count )) + then + continue + else + break + fi + done + unset copiedmimes + cat <<-EODesc - Higher-Than (bitrate, integer): - Only reencode files with bitrates higher then kbps. This - only applies if sample-rate, channel count and of course format are - equal. If unset, only files with bitrates equal to that of the - target will be copied (actually, hardlinking will be attempted - first). - - As Ogg Vorbis target quality is not defined by its bitrate, Ogg - Vorbis files will always be reencoded if unset. - EODesc - comeagain() { - read \ - -e \ - ${destinationmaxbps["$destination"]+-i${destinationmaxbps["$destination"]}}\ - -p'Higher-Than: ' \ - value - if ! [[ $value =~ $expr ]] + Channels (integer): + Produced files should have this many channels, no more, no less. + EODesc + expr='^[0-9]*$' + comeagain() { + read \ + -e \ + ${destinationchannels["$destination"]+-i${destinationchannels["$destination"]}}\ + -p'Channel count: ' \ + value + if ! [[ $value =~ $expr ]] + then + echo "Invalid channel count: $value" >&2 + comeagain + fi + } + comeagain + if [ -n "${destinationchannels["$destination"]}" ] \ + && (( value != ${destinationchannels["$destination"]} )) then - echo "Invalid higher-than bitrate value: $value" >&2 - comeagain + setupRegen channels fi - } - comeagain - if [ -n "${destinationmaxbps["$destination"]}" ] \ - && (( value != ${destinationmaxbps["$destination"]} )) - then - setupRegen maxbps + destinationchannels["$destination"]=$value + cat <<-EODesc + + Sampling rate (Hertz, integer): + Files will be resampled as needed to the specified sampling-rate. + Shoutcast/Icecast streams require a constant sampling-rate. + Telephony systems often require a sampling-rate of 8000Hz. + EODesc + if [[ ${destinationformat["$destination"]} == opus ]] + then + cat <<-EODesc + + Please note that Opus supports only the following sample-rates: + 8000, 12000, 16000, 24000, and 48000 Hz. So don't set + resampling on an Opus destination to any other value or files + will be resampled twice. + EODesc + fi + comeagain() { + read \ + -e \ + ${destinationfrequency["$destination"]+-i${destinationfrequency["$destination"]}}\ + -p'Sampling-rate: ' \ + value + if ! [[ $value =~ $expr ]] + then + echo "Invalid frequency value: $value" >&2 + comeagain + fi + } + comeagain + if [ -n "${destinationfrequency["$destination"]}" ] \ + && (( value != ${destinationfrequency["$destination"]} )) + then + setupRegen frequency + fi + destinationfrequency["$destination"]=$value + cat <<-EODesc + + Higher-Than (bitrate, integer): + Only reencode files with bitrates higher then kbps. This + only applies if sample-rate, channel count and of course format are + equal. If unset, only files with bitrates equal to that of the + target will be copied (actually, hardlinking will be attempted + first). + + As Ogg Vorbis target quality is not defined by its bitrate, Ogg + Vorbis files will always be reencoded if unset. + EODesc + comeagain() { + read \ + -e \ + ${destinationmaxbps["$destination"]+-i${destinationmaxbps["$destination"]}}\ + -p'Higher-Than: ' \ + value + if ! [[ $value =~ $expr ]] + then + echo "Invalid higher-than bitrate value: $value" >&2 + comeagain + fi + } + comeagain + if [ -n "${destinationmaxbps["$destination"]}" ] \ + && (( value != ${destinationmaxbps["$destination"]} )) + then + setupRegen maxbps + fi + destinationmaxbps[$destination]="$value" + unset regen + unset expr fi - destinationmaxbps[$destination]="$value" - unset regen - unset expr } diff --git a/lib/setup/destinations b/lib/setup/destinations index 305852b..4e1c2b3 100644 --- a/lib/setup/destinations +++ b/lib/setup/destinations @@ -65,7 +65,7 @@ setupDestinations() { expr='^[A-z0-9]*$' comeagain() { read -p'Name: ' value - [ -z "$value" ] && break + [ -z "$value" ] && return 1 if ! [[ $value =~ $expr ]] then echo "Invalid name $value. Please use" \ @@ -73,7 +73,7 @@ setupDestinations() { comeagain fi } - comeagain + comeagain || break destination="$value" setupDestination done diff --git a/lib/video/extractaudio b/lib/video/extractaudio index afa1392..785f1ce 100644 --- a/lib/video/extractaudio +++ b/lib/video/extractaudio @@ -2,5 +2,5 @@ extractAudio() { tmpfile="${fileid}ffmpeg.wav" commandline=(${ionice}ffmpeg -v 0 -vn -y) - commandline+=(-i "$sourcepath/$filename" "$tempdir/$tmpfile") + commandline+=(-i "$sourcepath/$filename" -map a:0 "$tempdir/$tmpfile") } diff --git a/lib/workers/master b/lib/workers/master index 96e3937..e24956f 100644 --- a/lib/workers/master +++ b/lib/workers/master @@ -92,7 +92,7 @@ master() { if (( remaining == 0 )) then sleep 0.1 - continue + return 0 elif (( active == 0 && ready == 0 )) then dumpfile=tasks-$(date +%Y%m%d%H%M%S).csv From 890dfc5e9cdada339db6cf63724669e79fb38bb4 Mon Sep 17 00:00:00 2001 From: Vincent Riquer Date: Wed, 12 Feb 2020 23:14:47 +0100 Subject: [PATCH 088/112] Remove interactivity (improve performances) --- atom | 32 +------------------------------- 1 file changed, 1 insertion(+), 31 deletions(-) diff --git a/atom b/atom index 7977ce2..9e2531a 100755 --- a/atom +++ b/atom @@ -673,36 +673,6 @@ remaining=$taskcount failed=0 while (( (remaining || ${#workers[@]}) && ! quit )) do - if read -n 1 -t 0.1 userinput - then - case $userinput in - '+') - ((maxload++)) - ;; - '-') - ((--maxload)) || ((maxload=1)) - ;; - [qQ]) - quit=1 - ;; - [pP]) - if (( pause )) - then - pause=0 - (( - pausedtime += - $(date +%s) - pausestart - )) - concurrencychange=$(date +%s) - else - pause=1 - pausestart=$(date +%s) - (( concurrency = maxload / 2 )) - (( concurrency )) || concurrency=1 - fi - ;; - esac - fi read humanload garbage < /proc/loadavg load=${humanload%.*} if [ -z "$quit" ] \ @@ -980,7 +950,7 @@ do 'opus') extension=opus ;; 'vorbis') extension=ogg ;; esac - (( cron )) || echo -n "$destination: rename pattern changed, renaming files... " + (( cron )) || echo -en "$destination: rename pattern changed, renaming files...\033[K" (( textunidecodeneeded )) && ascii echo 'BEGIN TRANSACTION;' >&3 for line in "${renamefiles[@]}" From 5c38663f50f71a1ffcfa0d44237e74bc188f536b Mon Sep 17 00:00:00 2001 From: Vincent Riquer Date: Wed, 12 Feb 2020 23:15:38 +0100 Subject: [PATCH 089/112] Fix ffmpeg temp file naming --- lib/video/extractaudio | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/video/extractaudio b/lib/video/extractaudio index 785f1ce..b77a048 100644 --- a/lib/video/extractaudio +++ b/lib/video/extractaudio @@ -1,6 +1,6 @@ #!/bin/bash extractAudio() { - tmpfile="${fileid}ffmpeg.wav" + tmpfile="${fileid}ffmpeg" commandline=(${ionice}ffmpeg -v 0 -vn -y) - commandline+=(-i "$sourcepath/$filename" -map a:0 "$tempdir/$tmpfile") + commandline+=(-i "$sourcepath/$filename" -map a:0 "$tempdir/$tmpfile".wav) } From 55b3c1ee81677c3e2a3916b74c304204c36cbea9 Mon Sep 17 00:00:00 2001 From: Vincent Riquer Date: Wed, 12 Feb 2020 23:17:31 +0100 Subject: [PATCH 090/112] Fix getDestDir when no renaming is done --- lib/files/getDestDir | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/files/getDestDir b/lib/files/getDestDir index 436d1bd..e92a899 100644 --- a/lib/files/getDestDir +++ b/lib/files/getDestDir @@ -1,6 +1,5 @@ #!/bin/bash getDestDir() { - destdir="${destinationpath[$destination]}/" if [ -n "${destinationrenamepath[$destination]}" ] \ && ( ( @@ -38,6 +37,7 @@ getDestDir() { ) ) then + destdir="${destinationpath[$destination]}/" if (( ${destinationascii["$destination"]} )) then echo "$album" >&${toascii[1]} @@ -75,6 +75,7 @@ getDestDir() { replace=$(sanitizeFile "$disc" dir) destdir="${destdir//?(\[)%\{disc\}?(\])/$replace}" else + destdir="${destinationpath[$destination]}/" destdir+=$(sanitizeFile "${filename%%/*}" dir) part=${filename#*/} while [[ $part =~ / ]] From 001c927375380e9c71b7b93774ee9bfbfd5ff753 Mon Sep 17 00:00:00 2001 From: Vincent Riquer Date: Wed, 12 Feb 2020 23:18:16 +0100 Subject: [PATCH 091/112] Fix opus encoding arguments (--music option removed) --- lib/encode/opus | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/encode/opus b/lib/encode/opus index bfa48ca..2c25f9a 100644 --- a/lib/encode/opus +++ b/lib/encode/opus @@ -1,6 +1,6 @@ #!/bin/bash encodeFile::opus() { - opusencopts=(${ionice}opusenc --music --quiet) + opusencopts=(${ionice}opusenc --quiet) opusencopts+=(--bitrate ${destinationquality[$destination]}) [ -n "${destinationloss["$destination"]}" ] \ && opusencopts+=(--expect-loss "${destinationloss["$destination"]}") @@ -15,7 +15,7 @@ encodeFile::opus() { [ -n "$track" ] && opusencopts+=(--comment "TRACKNUMBER=${track%/*}") [ -n "${track#*/}" ] && opusencopts+=(--comment "TRACKTOTAL=${track#*/}") [ -n "$year" ] && opusencopts+=(--comment "DATE=$year") - opusencopts+=("$tempdir/$tmpfile.wav" "$destdir/$destfile.opus") + opusencopts+=("$tempdir/$tmpfile".wav "$destdir/$destfile.opus") encodetaskid=$( Insert tasks <<-EOInsert key ${fileid}opusenc$destination From d4ddb7ac72163d32e7e3ef2c4938467229cf40d8 Mon Sep 17 00:00:00 2001 From: Vincent Riquer Date: Wed, 12 Feb 2020 23:23:13 +0100 Subject: [PATCH 092/112] Ignore .vscode --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 21963fd..530d6c9 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ *.ex *.EX trace.log +.vscode/ \ No newline at end of file From dcf158a5a21a95fd719d8f5e53c35c6fd9da2e6d Mon Sep 17 00:00:00 2001 From: Vincent Riquer Date: Thu, 13 Feb 2020 00:22:29 +0100 Subject: [PATCH 093/112] Show item count with spinner (nope! reverted!) --- lib/tools/progressSpin | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/tools/progressSpin b/lib/tools/progressSpin index 3cd54b8..b30a18c 100644 --- a/lib/tools/progressSpin +++ b/lib/tools/progressSpin @@ -5,10 +5,10 @@ progressSpin() { (( ++count )) else case $(( ++count % 40 )) in - 0) echo -ne '\b|' ;; - 10) echo -ne '\b/' ;; - 20) echo -en '\b-' ;; - 30) echo -ne '\b\\' ;; + 0) echo -ne $count '\b|' ;; + 10) echo -ne $count '\b/' ;; + 20) echo -en $count '\b-' ;; + 30) echo -ne $count '\b\\' ;; *) ;; esac fi From ed0191e4500eda42810fdfe2c40b70fcf98ded7d Mon Sep 17 00:00:00 2001 From: Vincent Riquer Date: Thu, 13 Feb 2020 00:32:48 +0100 Subject: [PATCH 094/112] Allow disabling destinations --- atom | 17 +++++++++++++++-- lib/config/getDestination | 3 +++ lib/config/print | 1 + lib/config/write | 3 +++ lib/database/checkVersion | 2 +- lib/database/upgradedatabase_2_3 | 14 ++++++++++++++ lib/destinations/create | 2 +- lib/tools/progressSpin | 8 ++++---- 8 files changed, 42 insertions(+), 8 deletions(-) create mode 100644 lib/database/upgradedatabase_2_3 diff --git a/atom b/atom index 9e2531a..05f719e 100755 --- a/atom +++ b/atom @@ -13,6 +13,7 @@ EFMTINVPARM=49 # config structures declare -A \ + destinationenabled \ destinationascii \ destinationchannels \ destinationfat32compat \ @@ -330,6 +331,16 @@ fi openDatabase +for destination in "${!destinationpath[@]}" +do + if (( ${destinationenabled["$destination"]} )) + then + Update destinations enabled 1 <<<"name = $destination" + else + Update destinations enabled 0 <<<"name = $destination" + fi +done + createDestinations getFiles @@ -513,7 +524,8 @@ echo ' ON mime_type_actions.id = source_files.mime_type INNER JOIN tags ON source_files.id = tags.source_file - WHERE CAST(destination_files.last_change AS TEXT) + WHERE destinations.enabled = 1 + AND CAST(destination_files.last_change AS TEXT) <> CAST(source_files.last_change AS TEXT) AND mime_type_actions.destination_id = destinations.id AND mime_type_actions.action = 1;' >&3 @@ -554,7 +566,8 @@ echo ' ON mime_type_actions.id = source_files.mime_type INNER JOIN tags ON source_files.id = tags.source_file - WHERE CAST(destination_files.last_change AS TEXT) + WHERE destinations.enabled = 1 + AND CAST(destination_files.last_change AS TEXT) <> CAST(source_files.last_change AS TEXT) AND mime_type_actions.destination_id = destinations.id AND mime_type_actions.action = 1 diff --git a/lib/config/getDestination b/lib/config/getDestination index 2926648..20773db 100644 --- a/lib/config/getDestination +++ b/lib/config/getDestination @@ -1,6 +1,9 @@ #!/bin/bash getConfigDestination() { case "$key" in + 'enabled') + destinationenabled["$destination"]="$value" + ;; 'path') destinationpath["$destination"]="$value" ;; diff --git a/lib/config/print b/lib/config/print index 567c547..ddbc65e 100644 --- a/lib/config/print +++ b/lib/config/print @@ -26,6 +26,7 @@ printConfig() { cat <<-EOF $destination|Path|${destinationpath["$destination"]} + |Enabled|${destinationenabled["$destination"]} |Format|${destinationformat["$destination"]} |Quality|${destinationquality["$destination"]} EOF diff --git a/lib/config/write b/lib/config/write index d8b6b36..40da076 100644 --- a/lib/config/write +++ b/lib/config/write @@ -60,6 +60,9 @@ path $sourcepath # Common parameters: # Mandatory parameters: +# * enabled: Whether or not to treat this destination (1=tue/0=false) +enabled 1 + # * path: Where files will be written path ${destinationpath["$destination"]} diff --git a/lib/database/checkVersion b/lib/database/checkVersion index 3491709..dcef357 100644 --- a/lib/database/checkVersion +++ b/lib/database/checkVersion @@ -1,5 +1,5 @@ #!/bin/bash -currentdbversion=2 +currentdbversion=3 checkDatabaseVersion() { local dbversion if dbversion=$(Select atom version <<<"\"1\" = 1") diff --git a/lib/database/upgradedatabase_2_3 b/lib/database/upgradedatabase_2_3 new file mode 100644 index 0000000..998556f --- /dev/null +++ b/lib/database/upgradedatabase_2_3 @@ -0,0 +1,14 @@ +#!/bin/bash + +upgradedatabase_2_3() { + local data \ + datas \ + id \ + destination + echo "Upgrading database to version 3... (backup is $database.bak_v2)" + cp "$database" "$database.bak_v2" + echo 'ALTER TABLE destinations ADD COLUMN enabled INTEGER DEFAULT 1;' >&3 + Update destinations enabled 1 <<< "1 = 1" + + Update atom version 3 <<<"1 = 1" +} \ No newline at end of file diff --git a/lib/destinations/create b/lib/destinations/create index fc6e3bc..e30b835 100644 --- a/lib/destinations/create +++ b/lib/destinations/create @@ -11,7 +11,7 @@ createDestinations() { fi fi destinationid["$destination"]=$( - InsertIfUnset destinations <<<"name $destination" + InsertIfUnset destinations <<<"name $destination ${destinationenabled[\"$destination\"]}" ) done } diff --git a/lib/tools/progressSpin b/lib/tools/progressSpin index b30a18c..40dbb52 100644 --- a/lib/tools/progressSpin +++ b/lib/tools/progressSpin @@ -5,10 +5,10 @@ progressSpin() { (( ++count )) else case $(( ++count % 40 )) in - 0) echo -ne $count '\b|' ;; - 10) echo -ne $count '\b/' ;; - 20) echo -en $count '\b-' ;; - 30) echo -ne $count '\b\\' ;; + 0) echo -n $'\b|' ;; + 10) echo -n $'\b/' ;; + 20) echo -n $'\b-' ;; + 30) echo -n $'\b\\' ;; *) ;; esac fi From b38138667ab307442e1597da1858fa0193a870ee Mon Sep 17 00:00:00 2001 From: Vincent Riquer Date: Thu, 7 May 2020 20:40:52 +0200 Subject: [PATCH 095/112] Add copy_extension --- atom | 8 ++++++++ lib/config/getDestination | 3 +++ lib/config/print | 3 +++ lib/config/write | 11 +++++++++++ 4 files changed, 25 insertions(+) diff --git a/atom b/atom index 05f719e..5a5e8e2 100755 --- a/atom +++ b/atom @@ -634,6 +634,14 @@ do decodeFile getDestDir getDestFile + for copy_ext in "${destinationcopyext[@]}" + do + if [[ $filename =~ '.*\.'$copy_ext'$' ]] + then + copied=1 + break + fi + done if (( copied )) then copyFiles_matching diff --git a/lib/config/getDestination b/lib/config/getDestination index 20773db..5284c75 100644 --- a/lib/config/getDestination +++ b/lib/config/getDestination @@ -185,6 +185,9 @@ getConfigDestination() { 'copy_mime-type') destinationcopymime[$destination]="${destinationcopymime[$destination]:+${destinationcopymime[$destination]}|}$value" ;; + 'copy_extension') + destinationcopyext[$destination]="${destinationcopyext[$destination]:+${destinationcopyext[$destination]}|}$value" + ;; 'higher-than') expr='^[0-9]*$' if ! [[ $value =~ $expr ]] diff --git a/lib/config/print b/lib/config/print index ddbc65e..dea0d16 100644 --- a/lib/config/print +++ b/lib/config/print @@ -52,6 +52,9 @@ printConfig() { | | |}" [ -n "${destinationcopymime["$destination"]}" ] \ && echo " |Copied mime-types|${destinationcopymime["$destination"]//\|/ +| | |}" + [ -n "${destinationcopyext["$destination"]}" ] \ + && echo " |Copied extensions|${destinationcopyext["$destination"]//\|/ | | |}" done }|column -t -s'|' diff --git a/lib/config/write b/lib/config/write index 40da076..1b85c7b 100644 --- a/lib/config/write +++ b/lib/config/write @@ -207,6 +207,17 @@ bitrate ${destinationquality["$destination"]} done cat <<-EOCfg +# * copy_extension : Same as skip_extension, except that files +# matching will be copied as-is to the destination. + EOCfg + destinationcopyext["$destination"]="${destinationcopyext["$destination"]}|" + while [[ ${destinationcopyext["$destination"]} =~ \| ]] + do + echo $'copy_extension\t\t'"${destinationcopyext["$destination"]%%|*}" + destinationcopyext["$destination"]="${destinationcopyext["$destination"]#*|}" + done + cat <<-EOCfg + # * channels : Files with more than channels will be # downmixed. Useful if you create files for telephony music-on-hold. EOCfg From 82f1648a4d7aa0b690939ad674aa4134214274a7 Mon Sep 17 00:00:00 2001 From: Vincent Riquer Date: Mon, 20 Jan 2025 19:20:30 +0100 Subject: [PATCH 096/112] support for new audio/ogg & audio/flac mimetypes --- lib/files/getFiles | 2 +- lib/tags/gettags | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/lib/files/getFiles b/lib/files/getFiles index dfb8731..06cfb65 100644 --- a/lib/files/getFiles +++ b/lib/files/getFiles @@ -17,7 +17,7 @@ getFiles() { EOWhere then mimetype=$(file -b --mime-type "$sourcepath/$filename") - if [[ $mimetype == application/ogg ]] + if [[ $mimetype == application/ogg ]] || [[ $mimetype == audio/ogg ]] then case "$(head -n5 "$sourcepath/$filename")" in *'vorbis'*) diff --git a/lib/tags/gettags b/lib/tags/gettags index 2556140..8afae86 100644 --- a/lib/tags/gettags +++ b/lib/tags/gettags @@ -15,10 +15,18 @@ getTags() { type=soxi (( disablesoxi )) && unset type ;; + audio/ogg) + type=soxi + (( disablesoxi )) && unset type + ;; audio/x-flac) type=FLAC (( disableflac )) && unset type ;; + audio/flac) + type=FLAC + (( disableflac )) && unset type + ;; video/webm) type=WebM (( disablemkvextract || disableogginfo )) && unset type From 04bda0178cf5dbbe0806129b06a409ceb4df495b Mon Sep 17 00:00:00 2001 From: Vincent Riquer Date: Mon, 20 Jan 2025 19:43:26 +0100 Subject: [PATCH 097/112] Support for nex flac and ogg mime-type: decode --- lib/decode/file | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/lib/decode/file b/lib/decode/file index b0f350a..5797b9d 100644 --- a/lib/decode/file +++ b/lib/decode/file @@ -51,6 +51,27 @@ decodeFile() { fi fi ;; + 'audio/ogg opus') + if [[ ${destinationformat[$destination]} = opus ]] \ + && checkCopy + then + copied=1 + else + (( disableopusdec )) && continue + decodeOpusdec + if (( ${destinationnormalize["$destination"]}))\ + || ( + [ -n "${destinationfrequency["$destination"]}" ]\ + && (( ${rate:-0} != ${destinationfrequency["$destination"]}))\ + ) || ( + [ -n "${destinationchannels["$destination"]}" ]\ + && (( ${channels:-0} != ${destinationchannels["$destination"]} )) + ) + then + sox_needed=1 + fi + fi + ;; 'application/ogg'*) if [[ ${destinationformat[$destination]} = vorbis ]] \ && checkCopy @@ -60,9 +81,21 @@ decodeFile() { decodeSox fi ;; + 'audio/ogg'*) + if [[ ${destinationformat[$destination]} = vorbis ]] \ + && checkCopy + then + copied=1 + else + decodeSox + fi + ;; 'audio/x-flac') decodeSox ;; + 'audio/flac') + decodeSox + ;; *) extendedtype=$(file -b "$sourcepath/$filename") case "$extendedtype" in From 47250615235ac11f956fd06b3d938bba91f48b37 Mon Sep 17 00:00:00 2001 From: Vincent Riquer Date: Mon, 20 Jan 2025 20:21:39 +0100 Subject: [PATCH 098/112] Performance --- doc/performances.txt | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 doc/performances.txt diff --git a/doc/performances.txt b/doc/performances.txt new file mode 100644 index 0000000..45ca95d --- /dev/null +++ b/doc/performances.txt @@ -0,0 +1,29 @@ +# TEST DATA + +155GB, 14430 files in 2716 directories and subdirectories: +$ du -sh /var/lib/mpd/music +155G /var/lib/mpd/music +$ find /var/lib/mpd/music -type d|wc -l +2716 +$ find /var/lib/mpd/music -type f|wc -l +14430 + +# SCANNING + +Initial scan takes 5 minutes, probably because of mime-type detection: +$ time ./atom +[...] +14430 files found, 14430 new or changed. + +real 5m4.144s +user 0m19.821s +sys 0m17.393s + +A second scan takes less than 14 seconds: +$ time ./atom +[...] +14430 files found, 0 new or changed. + +real 0m13.770s +user 0m11.169s +sys 0m1.844s From b4909b46d6b20c8b65d0e7bf8cf135555599b8bf Mon Sep 17 00:00:00 2001 From: Vincent Riquer Date: Mon, 20 Jan 2025 21:33:47 +0100 Subject: [PATCH 099/112] Update copyright --- README | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README b/README index 74f1826..d94b0aa 100644 --- a/README +++ b/README @@ -2,7 +2,7 @@ AtOM: Anything to Ogg and Mp3 URL: https://forge.riquer.fr/p/AtOM/ Author: Vincent Riquer -Copyright/left: 2012-2013,2015 Vincent Riquer - GPLv3 (see doc/GPL-3) +Copyright/left: 2012-2013,2015,2025 Vincent Riquer - GPLv3 (see doc/GPL-3) except: transogg: WTFPL 2.0 ============ From d9031f784f47abedf91cc9909f8c9c98d69e7a05 Mon Sep 17 00:00:00 2001 From: Vincent Riquer Date: Mon, 20 Jan 2025 22:58:32 +0100 Subject: [PATCH 100/112] fix vorbis/opus differentiation --- lib/files/getFiles | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/files/getFiles b/lib/files/getFiles index 06cfb65..51feaf3 100644 --- a/lib/files/getFiles +++ b/lib/files/getFiles @@ -19,7 +19,7 @@ getFiles() { mimetype=$(file -b --mime-type "$sourcepath/$filename") if [[ $mimetype == application/ogg ]] || [[ $mimetype == audio/ogg ]] then - case "$(head -n5 "$sourcepath/$filename")" in + case "$(head -n5 "$sourcepath/$filename" | tr -d '\0')" in *'vorbis'*) mimetype+=' vorbis' ;; From 5ec035874ecac6e6de6ab40a1d729672b823b0b6 Mon Sep 17 00:00:00 2001 From: Vincent Riquer Date: Mon, 20 Jan 2025 23:04:40 +0100 Subject: [PATCH 101/112] destination enable: update schema --- share/schema.sql | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/share/schema.sql b/share/schema.sql index 2e4f3c6..62fa9d1 100644 --- a/share/schema.sql +++ b/share/schema.sql @@ -15,7 +15,8 @@ CREATE TABLE IF NOT EXISTS source_files ( ); CREATE TABLE IF NOT EXISTS destinations ( id INTEGER PRIMARY KEY, - name TEXT UNIQUE NOT NULL + name TEXT UNIQUE NOT NULL, + enabled INTEGER DEFAULT 1 ); CREATE TABLE IF NOT EXISTS destination_files ( id INTEGER PRIMARY KEY, From 3f26d983c8138a0d9ee288ddf71857416dd45151 Mon Sep 17 00:00:00 2001 From: Vincent Riquer Date: Tue, 21 Jan 2025 00:22:28 +0100 Subject: [PATCH 102/112] Force temp table to memory --- lib/database/open | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/database/open b/lib/database/open index e04131f..3abf236 100644 --- a/lib/database/open +++ b/lib/database/open @@ -17,5 +17,6 @@ openDatabase() { echo '.separator ::AtOM:SQL:Sep::' >&3 echo 'PRAGMA foreign_keys = ON;' >&3 echo 'PRAGMA recursive_triggers = ON;' >&3 + echo 'PRAGMA temp_store = 2;' >&3 checkDatabaseVersion } From 4e59449d40629a1698744dcb31167fc90f331444 Mon Sep 17 00:00:00 2001 From: Vincent Riquer Date: Tue, 21 Jan 2025 00:31:06 +0100 Subject: [PATCH 103/112] Exclusively lock database --- lib/database/open | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/database/open b/lib/database/open index 3abf236..9d9fdc3 100644 --- a/lib/database/open +++ b/lib/database/open @@ -18,5 +18,8 @@ openDatabase() { echo 'PRAGMA foreign_keys = ON;' >&3 echo 'PRAGMA recursive_triggers = ON;' >&3 echo 'PRAGMA temp_store = 2;' >&3 + echo 'PRAGMA locking_mode = EXCLUSIVE;' >&3 + read -u4 + unset REPLY checkDatabaseVersion } From 220e756866ac535820ebeefc26b3493155d33667 Mon Sep 17 00:00:00 2001 From: Vincent Riquer Date: Tue, 21 Jan 2025 01:52:23 +0100 Subject: [PATCH 104/112] Treat destination in the order they are declared --- atom | 2 +- lib/config/print | 2 +- lib/config/write | 2 +- lib/destinations/create | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/atom b/atom index 5a5e8e2..671ca4e 100755 --- a/atom +++ b/atom @@ -331,7 +331,7 @@ fi openDatabase -for destination in "${!destinationpath[@]}" +for destination in "${destinations[@]}" do if (( ${destinationenabled["$destination"]} )) then diff --git a/lib/config/print b/lib/config/print index dea0d16..db25e0e 100644 --- a/lib/config/print +++ b/lib/config/print @@ -21,7 +21,7 @@ printConfig() { printed=1 done unset printed - for destination in ${!destinationpath[@]} + for destination in ${destinations[@]} do cat <<-EOF diff --git a/lib/config/write b/lib/config/write index 1b85c7b..7da7b3a 100644 --- a/lib/config/write +++ b/lib/config/write @@ -52,7 +52,7 @@ path $sourcepath EOCfg - for destination in "${!destinationpath[@]}" + for destination in "${destinations[@]}" do cat <<-EOCfg [$destination] diff --git a/lib/destinations/create b/lib/destinations/create index e30b835..5c9910b 100644 --- a/lib/destinations/create +++ b/lib/destinations/create @@ -1,6 +1,6 @@ #!/bin/bash createDestinations() { - for destination in ${!destinationpath[@]} + for destination in ${destinations[@]} do if ! [ -d "${destinationpath["$destination"]}" ] then From a83957fe09f32e89af7c48eaaaf44d4d77bf6513 Mon Sep 17 00:00:00 2001 From: Vincent Riquer Date: Wed, 22 Jan 2025 22:45:28 +0100 Subject: [PATCH 105/112] README: convert to markdown --- README => README.md | 154 ++++++++++++++++++-------------------------- 1 file changed, 63 insertions(+), 91 deletions(-) rename README => README.md (69%) diff --git a/README b/README.md similarity index 69% rename from README rename to README.md index d94b0aa..54dc5cd 100644 --- a/README +++ b/README.md @@ -1,71 +1,57 @@ -AtOM: Anything to Ogg and Mp3 +# AtOM: Anything to Ogg and Mp3 URL: https://forge.riquer.fr/p/AtOM/ Author: Vincent Riquer Copyright/left: 2012-2013,2015,2025 Vincent Riquer - GPLv3 (see doc/GPL-3) except: transogg: WTFPL 2.0 -============ -Dependencies ------------- -Required: -* bash (>= 4.0) - http://www.gnu.org/software/bash/bash.html -* SoX - http://sox.sourceforge.net/ -* SQLite - http://www.sqlite.org/ +## Dependencies +### Required: +* [bash (>= 4.0)](http://www.gnu.org/software/bash/bash.html) +* [SoX](http://sox.sourceforge.net/) +* [SQLite](http://www.sqlite.org/) -Optional: -* vorbis-tools - http://www.vorbis.com/ - * ogginfo (Ogg Vorbis metadata) - * oggenc (Ogg Vorbis encoding) -* opus-tools - http://opus-codec.org/ - * opusinfo (Opus metadata) - * opusenc (Opus encoding) - * opusdec (Opus decoding) -* LAME MP3 Encoder - http://lame.sourceforge.net/ - * lame (MP3 encoding) -* FLAC - http://flac.sourceforge.net/ - * metaflac (FLAC metadata) -* Musepack - http://www.musepack.net/ - * mpcdec (Musepack decoding) -* FFmpeg - http://ffmpeg.org/ - * ffprobe (ID3v2, Musepack, Windows Media and video metadata) - * ffmpeg (Windows Media and video decoding) +### Optional: +* [vorbis-tools](http://www.vorbis.com/) + * `ogginfo` (Ogg Vorbis metadata) + * `oggenc` (Ogg Vorbis encoding) +* [opus-tools](http://opus-codec.org/) + * `opusinfo` (Opus metadata) + * `opusenc` (Opus encoding) + * `opusdec` (Opus decoding) +* [LAME MP3 Encoder](http://lame.sourceforge.net/) + * `lame` (MP3 encoding) +* [FLAC](http://flac.sourceforge.net/) + * `metaflac` (FLAC metadata) +* [Musepack](http://www.musepack.net/) + * `mpcdec` (Musepack decoding) +* [FFmpeg](http://ffmpeg.org/) + * `ffprobe` (ID3v2, Musepack, Windows Media and video metadata) + * `ffmpeg` (Windows Media and video decoding) -================== -Using the software ------------------- +## Using the software -Configuration: --------------- +### Configuration: On first run, AtOM will ask a set of questions to help you create a configuration file. -You can run atom -S at any time to re-run the setup. It will be prefilled with +You can run `atom -S` at any time to re-run the setup. It will be prefilled with your current configuration. If, however, you still want to make changes manually, please read doc/config. There are a lot of comments in the generated config file too. -Preparing data: ---------------- +### Preparing data: Nothing specific needs to be done. You can edit ypur tags, rename files, move them around how you see fit. However, make sure you setup your tag editor to *do* update the files' timestamps: though it was initially plan to make this optional, using checksums or tags, it was abandoned due to the huge amount of IO required. -Running: --------- +### Running: Make sure your configuration is correct by running +``` $ atom -C +``` This will produce a human-readable dump of your current configuration. If all settings are correct, simply run atom with no argument. Go get a beer. Meet some friends. Go to bed. Depending on the size of your collection, the @@ -75,24 +61,27 @@ much faster this time, as only changed data will be treated. If, for whatever reason, you need to force the regeneration of a destination, after changing the quality settings for example, run +``` $ atom -F +``` -Running as a cronjob: ---------------------- -If you want to run AtOM as a cronjob, atom -q will give you a cleaner output, +### Running as a cronjob: +If you want to run AtOM as a cronjob, `atom -q` will give you a cleaner output, more suitable for mail or logfile output. You may also want to limit the size of -each batch with -B . AtOM will not create or update more than - destination files. +each batch with `-B `. AtOM will not create or update more than +`` destination files. For example: +``` #m h dom mon dow command 0 5 * * * atom -B 1000 -q +``` -================= -Technical details ------------------ -I. Source scan --------------- +### Full options list +Please refer to `atom -h`, as this will always be the most up to date. + +## Technical details +### I. Source scan After reading its configuration file, AtOM uses find to get a list of all files in the source directory. Each file is checked against the database. If it's already there, and its last @@ -101,25 +90,22 @@ If its mtime has changed, mime-type scan is attempted. It is updated in the database, along with last_seen. If the file is new, its mime-type is scanned, and it is added to the database. -II. Obsolete files ------------------- +### II. Obsolete files Using the last_seen field, AtOM removes from destinations each files which are not present anymore in the source directory. AtOM never touches files not present in its database (unless there is a filename conflict, in which case your file *WILL* be overwritten). If you wish to clear unknown files from your destinations, have a look at toys/cleandestinations. -III. Reading metadata ---------------------- +### III. Reading metadata AtOM then tries to read metadata from each new or changed file. It also re-reads metadata from files scanned with an older version of AtOM, if the parser for that format has changed. The actual data read depends on the format, but at the very least, AtOM should identify the sampling rate, bitrate and number of -channels. Unknown file types are scanned with ffprobe, so you may still have +channels. Unknown file types are scanned with `ffprobe`, so you may still have some luck, depending on your FFmpeg setup. -IV. Task creation ------------------ +### IV. Task creation For every destination files having their last change field different from their corresponding source file entry, we create one or more tasks, to generate or update (overwrite) the destination file. AtOM tries to generate as few tasks as @@ -138,27 +124,20 @@ The steps required for each file depend on the format and destination parameters decoded/resampled file) to 3 (if format can't be decoded using sox and resampling/normalization is required) tasks. -V.1 Running tasks ------------------ -While running tasks, AtOM responds to the following keypresses: -+/- Increase/decrease max-load -q Quit (will skip all following steps) +### V. Running tasks +#### V.1 Running tasks Progress display: -L:/ -W:/ -T:/ (F:) -% (A:s/task) -ETA: +``` +L:/ W:/ T:/ (F:) % (A:s/task) ETA: +``` -V.2 Renaming files ------------------- +#### V.2 Renaming files If rename pattern (or FAT32 compatibility) for one or more destinations has changed, files already transcoded will be renamed. Otherwise, this step is skipped. -VI. Copies ----------- +### VI. Copies During that stage, files which mime-types matched copy_mime-type directives are copied (symlinked where possible) to the destination. When a rename pattern is defined and impacts path, files that are not in the @@ -166,38 +145,31 @@ same directories as files which have been successfully transcoded are ignored, as AtOM cannot guess what the destination path should be. Otherwise, files are copied with their name and path unchanged. -VII. Obsolete files 2 -------------------- +### VII. Obsolete files 2 Whenever a file is transcoded, if it was already present in the database but its name changed, following a rename pattern change, the old file is removed during that stage. -==== -Toys ----- +## Toys AtOM requires a database to function. Now that we have a database containing various information about our media files, why not use it? AtOM comes with a small set of tools in the toys/ directory. These are -documented in toys/README. +documented in `toys/README`. -======================== -Shameless Self Promotion ------------------------- +# Shameless Self Promotion I am the author of free (Creative Commons CC-By-SA) music which you can stream -for free, or buy to get high quality and bonuses from Bandcamp -(http://djblackred.bandcamp.com). If you like electronic music taking its -inspiration from Trance, Drum & Bass, Ambient and (rarely) Free Jazz, please +for free, or buy to get high quality and bonuses from +[Bandcamp](http://djblackred.bandcamp.com). If you like electronic music taking +its inspiration from Trance, Drum & Bass, Ambient and (rarely) Free Jazz, please check it out! Downloads are available in FLAC, Ogg, MP3, and more, and includes the "source code" (sequencer files and the likes) for most tracks. -I am receiving 85% of the money you'll spend, so you won't be feeding some +I am receiving 80% of the money you'll spend, so you won't be feeding some greedy BigCorp producer or distributor. And if you don't like it, you can still spread the word to friends who may like. You can see this as a way to thank me for this piece of code. -===== -Legal ------ +# Legal Some of the format and/or tool names cited above are trademarks belonging to their rightful owners. AtOM and its authors are not linked in any way to those companies or individuals. Said companies do not endorse nor support From 5def910f36677caf6921d5e966246150eade11da Mon Sep 17 00:00:00 2001 From: Vincent Riquer Date: Wed, 22 Jan 2025 23:35:06 +0100 Subject: [PATCH 106/112] Change URL to framagit --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 54dc5cd..052e2d2 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # AtOM: Anything to Ogg and Mp3 -URL: https://forge.riquer.fr/p/AtOM/ +URL: https://framagit.org/ScriptFanix/AtOM/ Author: Vincent Riquer Copyright/left: 2012-2013,2015,2025 Vincent Riquer - GPLv3 (see doc/GPL-3) except: transogg: WTFPL 2.0 From f09a4a30d9f6e25c6ed5b06c3ae7035c7893d351 Mon Sep 17 00:00:00 2001 From: Vincent Riquer Date: Wed, 22 Jan 2025 23:37:42 +0100 Subject: [PATCH 107/112] README: markdown fix --- README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 052e2d2..611f29b 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,9 @@ # AtOM: Anything to Ogg and Mp3 -URL: https://framagit.org/ScriptFanix/AtOM/ -Author: Vincent Riquer -Copyright/left: 2012-2013,2015,2025 Vincent Riquer - GPLv3 (see doc/GPL-3) - except: transogg: WTFPL 2.0 +* URL: https://framagit.org/ScriptFanix/AtOM/ +* Author: Vincent Riquer +* Copyright/left: 2012-2013,2015,2025 Vincent Riquer - GPLv3 (see doc/GPL-3) + - except: transogg: WTFPL 2.0 ## Dependencies ### Required: From f18eb00caaff5f5d2a7ee0991b4c7a8778b68c09 Mon Sep 17 00:00:00 2001 From: Vincent Riquer Date: Wed, 22 Jan 2025 22:42:07 +0000 Subject: [PATCH 108/112] Add license file --- LICENSE | 674 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 674 insertions(+) create mode 100644 LICENSE diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..f238ad7 --- /dev/null +++ b/LICENSE @@ -0,0 +1,674 @@ + GNU GENERAL PUBLIC LICENSE + Version 3, 29 June 2007 + + Copyright (C) 2007 Free Software Foundation, Inc. + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The GNU General Public License is a free, copyleft license for +software and other kinds of works. + + The licenses for most software and other practical works are designed +to take away your freedom to share and change the works. By contrast, +the GNU General Public License is intended to guarantee your freedom to +share and change all versions of a program--to make sure it remains free +software for all its users. We, the Free Software Foundation, use the +GNU General Public License for most of our software; it applies also to +any other work released this way by its authors. You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +them if you wish), that you receive source code or can get it if you +want it, that you can change the software or use pieces of it in new +free programs, and that you know you can do these things. + + To protect your rights, we need to prevent others from denying you +these rights or asking you to surrender the rights. Therefore, you have +certain responsibilities if you distribute copies of the software, or if +you modify it: responsibilities to respect the freedom of others. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must pass on to the recipients the same +freedoms that you received. You must make sure that they, too, receive +or can get the source code. And you must show them these terms so they +know their rights. + + Developers that use the GNU GPL protect your rights with two steps: +(1) assert copyright on the software, and (2) offer you this License +giving you legal permission to copy, distribute and/or modify it. + + For the developers' and authors' protection, the GPL clearly explains +that there is no warranty for this free software. For both users' and +authors' sake, the GPL requires that modified versions be marked as +changed, so that their problems will not be attributed erroneously to +authors of previous versions. + + Some devices are designed to deny users access to install or run +modified versions of the software inside them, although the manufacturer +can do so. This is fundamentally incompatible with the aim of +protecting users' freedom to change the software. The systematic +pattern of such abuse occurs in the area of products for individuals to +use, which is precisely where it is most unacceptable. Therefore, we +have designed this version of the GPL to prohibit the practice for those +products. If such problems arise substantially in other domains, we +stand ready to extend this provision to those domains in future versions +of the GPL, as needed to protect the freedom of users. + + Finally, every program is threatened constantly by software patents. +States should not allow patents to restrict development and use of +software on general-purpose computers, but in those that do, we wish to +avoid the special danger that patents applied to a free program could +make it effectively proprietary. To prevent this, the GPL assures that +patents cannot be used to render the program non-free. + + The precise terms and conditions for copying, distribution and +modification follow. + + TERMS AND CONDITIONS + + 0. Definitions. + + "This License" refers to version 3 of the GNU General Public License. + + "Copyright" also means copyright-like laws that apply to other kinds of +works, such as semiconductor masks. + + "The Program" refers to any copyrightable work licensed under this +License. Each licensee is addressed as "you". "Licensees" and +"recipients" may be individuals or organizations. + + To "modify" a work means to copy from or adapt all or part of the work +in a fashion requiring copyright permission, other than the making of an +exact copy. The resulting work is called a "modified version" of the +earlier work or a work "based on" the earlier work. + + A "covered work" means either the unmodified Program or a work based +on the Program. + + To "propagate" a work means to do anything with it that, without +permission, would make you directly or secondarily liable for +infringement under applicable copyright law, except executing it on a +computer or modifying a private copy. Propagation includes copying, +distribution (with or without modification), making available to the +public, and in some countries other activities as well. + + To "convey" a work means any kind of propagation that enables other +parties to make or receive copies. Mere interaction with a user through +a computer network, with no transfer of a copy, is not conveying. + + An interactive user interface displays "Appropriate Legal Notices" +to the extent that it includes a convenient and prominently visible +feature that (1) displays an appropriate copyright notice, and (2) +tells the user that there is no warranty for the work (except to the +extent that warranties are provided), that licensees may convey the +work under this License, and how to view a copy of this License. If +the interface presents a list of user commands or options, such as a +menu, a prominent item in the list meets this criterion. + + 1. Source Code. + + The "source code" for a work means the preferred form of the work +for making modifications to it. "Object code" means any non-source +form of a work. + + A "Standard Interface" means an interface that either is an official +standard defined by a recognized standards body, or, in the case of +interfaces specified for a particular programming language, one that +is widely used among developers working in that language. + + The "System Libraries" of an executable work include anything, other +than the work as a whole, that (a) is included in the normal form of +packaging a Major Component, but which is not part of that Major +Component, and (b) serves only to enable use of the work with that +Major Component, or to implement a Standard Interface for which an +implementation is available to the public in source code form. A +"Major Component", in this context, means a major essential component +(kernel, window system, and so on) of the specific operating system +(if any) on which the executable work runs, or a compiler used to +produce the work, or an object code interpreter used to run it. + + The "Corresponding Source" for a work in object code form means all +the source code needed to generate, install, and (for an executable +work) run the object code and to modify the work, including scripts to +control those activities. However, it does not include the work's +System Libraries, or general-purpose tools or generally available free +programs which are used unmodified in performing those activities but +which are not part of the work. For example, Corresponding Source +includes interface definition files associated with source files for +the work, and the source code for shared libraries and dynamically +linked subprograms that the work is specifically designed to require, +such as by intimate data communication or control flow between those +subprograms and other parts of the work. + + The Corresponding Source need not include anything that users +can regenerate automatically from other parts of the Corresponding +Source. + + The Corresponding Source for a work in source code form is that +same work. + + 2. Basic Permissions. + + All rights granted under this License are granted for the term of +copyright on the Program, and are irrevocable provided the stated +conditions are met. This License explicitly affirms your unlimited +permission to run the unmodified Program. The output from running a +covered work is covered by this License only if the output, given its +content, constitutes a covered work. This License acknowledges your +rights of fair use or other equivalent, as provided by copyright law. + + You may make, run and propagate covered works that you do not +convey, without conditions so long as your license otherwise remains +in force. You may convey covered works to others for the sole purpose +of having them make modifications exclusively for you, or provide you +with facilities for running those works, provided that you comply with +the terms of this License in conveying all material for which you do +not control copyright. Those thus making or running the covered works +for you must do so exclusively on your behalf, under your direction +and control, on terms that prohibit them from making any copies of +your copyrighted material outside their relationship with you. + + Conveying under any other circumstances is permitted solely under +the conditions stated below. Sublicensing is not allowed; section 10 +makes it unnecessary. + + 3. Protecting Users' Legal Rights From Anti-Circumvention Law. + + No covered work shall be deemed part of an effective technological +measure under any applicable law fulfilling obligations under article +11 of the WIPO copyright treaty adopted on 20 December 1996, or +similar laws prohibiting or restricting circumvention of such +measures. + + When you convey a covered work, you waive any legal power to forbid +circumvention of technological measures to the extent such circumvention +is effected by exercising rights under this License with respect to +the covered work, and you disclaim any intention to limit operation or +modification of the work as a means of enforcing, against the work's +users, your or third parties' legal rights to forbid circumvention of +technological measures. + + 4. Conveying Verbatim Copies. + + You may convey verbatim copies of the Program's source code as you +receive it, in any medium, provided that you conspicuously and +appropriately publish on each copy an appropriate copyright notice; +keep intact all notices stating that this License and any +non-permissive terms added in accord with section 7 apply to the code; +keep intact all notices of the absence of any warranty; and give all +recipients a copy of this License along with the Program. + + You may charge any price or no price for each copy that you convey, +and you may offer support or warranty protection for a fee. + + 5. Conveying Modified Source Versions. + + You may convey a work based on the Program, or the modifications to +produce it from the Program, in the form of source code under the +terms of section 4, provided that you also meet all of these conditions: + + a) The work must carry prominent notices stating that you modified + it, and giving a relevant date. + + b) The work must carry prominent notices stating that it is + released under this License and any conditions added under section + 7. This requirement modifies the requirement in section 4 to + "keep intact all notices". + + c) You must license the entire work, as a whole, under this + License to anyone who comes into possession of a copy. This + License will therefore apply, along with any applicable section 7 + additional terms, to the whole of the work, and all its parts, + regardless of how they are packaged. This License gives no + permission to license the work in any other way, but it does not + invalidate such permission if you have separately received it. + + d) If the work has interactive user interfaces, each must display + Appropriate Legal Notices; however, if the Program has interactive + interfaces that do not display Appropriate Legal Notices, your + work need not make them do so. + + A compilation of a covered work with other separate and independent +works, which are not by their nature extensions of the covered work, +and which are not combined with it such as to form a larger program, +in or on a volume of a storage or distribution medium, is called an +"aggregate" if the compilation and its resulting copyright are not +used to limit the access or legal rights of the compilation's users +beyond what the individual works permit. Inclusion of a covered work +in an aggregate does not cause this License to apply to the other +parts of the aggregate. + + 6. Conveying Non-Source Forms. + + You may convey a covered work in object code form under the terms +of sections 4 and 5, provided that you also convey the +machine-readable Corresponding Source under the terms of this License, +in one of these ways: + + a) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by the + Corresponding Source fixed on a durable physical medium + customarily used for software interchange. + + b) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by a + written offer, valid for at least three years and valid for as + long as you offer spare parts or customer support for that product + model, to give anyone who possesses the object code either (1) a + copy of the Corresponding Source for all the software in the + product that is covered by this License, on a durable physical + medium customarily used for software interchange, for a price no + more than your reasonable cost of physically performing this + conveying of source, or (2) access to copy the + Corresponding Source from a network server at no charge. + + c) Convey individual copies of the object code with a copy of the + written offer to provide the Corresponding Source. This + alternative is allowed only occasionally and noncommercially, and + only if you received the object code with such an offer, in accord + with subsection 6b. + + d) Convey the object code by offering access from a designated + place (gratis or for a charge), and offer equivalent access to the + Corresponding Source in the same way through the same place at no + further charge. You need not require recipients to copy the + Corresponding Source along with the object code. If the place to + copy the object code is a network server, the Corresponding Source + may be on a different server (operated by you or a third party) + that supports equivalent copying facilities, provided you maintain + clear directions next to the object code saying where to find the + Corresponding Source. Regardless of what server hosts the + Corresponding Source, you remain obligated to ensure that it is + available for as long as needed to satisfy these requirements. + + e) Convey the object code using peer-to-peer transmission, provided + you inform other peers where the object code and Corresponding + Source of the work are being offered to the general public at no + charge under subsection 6d. + + A separable portion of the object code, whose source code is excluded +from the Corresponding Source as a System Library, need not be +included in conveying the object code work. + + A "User Product" is either (1) a "consumer product", which means any +tangible personal property which is normally used for personal, family, +or household purposes, or (2) anything designed or sold for incorporation +into a dwelling. In determining whether a product is a consumer product, +doubtful cases shall be resolved in favor of coverage. For a particular +product received by a particular user, "normally used" refers to a +typical or common use of that class of product, regardless of the status +of the particular user or of the way in which the particular user +actually uses, or expects or is expected to use, the product. A product +is a consumer product regardless of whether the product has substantial +commercial, industrial or non-consumer uses, unless such uses represent +the only significant mode of use of the product. + + "Installation Information" for a User Product means any methods, +procedures, authorization keys, or other information required to install +and execute modified versions of a covered work in that User Product from +a modified version of its Corresponding Source. The information must +suffice to ensure that the continued functioning of the modified object +code is in no case prevented or interfered with solely because +modification has been made. + + If you convey an object code work under this section in, or with, or +specifically for use in, a User Product, and the conveying occurs as +part of a transaction in which the right of possession and use of the +User Product is transferred to the recipient in perpetuity or for a +fixed term (regardless of how the transaction is characterized), the +Corresponding Source conveyed under this section must be accompanied +by the Installation Information. But this requirement does not apply +if neither you nor any third party retains the ability to install +modified object code on the User Product (for example, the work has +been installed in ROM). + + The requirement to provide Installation Information does not include a +requirement to continue to provide support service, warranty, or updates +for a work that has been modified or installed by the recipient, or for +the User Product in which it has been modified or installed. Access to a +network may be denied when the modification itself materially and +adversely affects the operation of the network or violates the rules and +protocols for communication across the network. + + Corresponding Source conveyed, and Installation Information provided, +in accord with this section must be in a format that is publicly +documented (and with an implementation available to the public in +source code form), and must require no special password or key for +unpacking, reading or copying. + + 7. Additional Terms. + + "Additional permissions" are terms that supplement the terms of this +License by making exceptions from one or more of its conditions. +Additional permissions that are applicable to the entire Program shall +be treated as though they were included in this License, to the extent +that they are valid under applicable law. If additional permissions +apply only to part of the Program, that part may be used separately +under those permissions, but the entire Program remains governed by +this License without regard to the additional permissions. + + When you convey a copy of a covered work, you may at your option +remove any additional permissions from that copy, or from any part of +it. (Additional permissions may be written to require their own +removal in certain cases when you modify the work.) You may place +additional permissions on material, added by you to a covered work, +for which you have or can give appropriate copyright permission. + + Notwithstanding any other provision of this License, for material you +add to a covered work, you may (if authorized by the copyright holders of +that material) supplement the terms of this License with terms: + + a) Disclaiming warranty or limiting liability differently from the + terms of sections 15 and 16 of this License; or + + b) Requiring preservation of specified reasonable legal notices or + author attributions in that material or in the Appropriate Legal + Notices displayed by works containing it; or + + c) Prohibiting misrepresentation of the origin of that material, or + requiring that modified versions of such material be marked in + reasonable ways as different from the original version; or + + d) Limiting the use for publicity purposes of names of licensors or + authors of the material; or + + e) Declining to grant rights under trademark law for use of some + trade names, trademarks, or service marks; or + + f) Requiring indemnification of licensors and authors of that + material by anyone who conveys the material (or modified versions of + it) with contractual assumptions of liability to the recipient, for + any liability that these contractual assumptions directly impose on + those licensors and authors. + + All other non-permissive additional terms are considered "further +restrictions" within the meaning of section 10. If the Program as you +received it, or any part of it, contains a notice stating that it is +governed by this License along with a term that is a further +restriction, you may remove that term. If a license document contains +a further restriction but permits relicensing or conveying under this +License, you may add to a covered work material governed by the terms +of that license document, provided that the further restriction does +not survive such relicensing or conveying. + + If you add terms to a covered work in accord with this section, you +must place, in the relevant source files, a statement of the +additional terms that apply to those files, or a notice indicating +where to find the applicable terms. + + Additional terms, permissive or non-permissive, may be stated in the +form of a separately written license, or stated as exceptions; +the above requirements apply either way. + + 8. Termination. + + You may not propagate or modify a covered work except as expressly +provided under this License. Any attempt otherwise to propagate or +modify it is void, and will automatically terminate your rights under +this License (including any patent licenses granted under the third +paragraph of section 11). + + However, if you cease all violation of this License, then your +license from a particular copyright holder is reinstated (a) +provisionally, unless and until the copyright holder explicitly and +finally terminates your license, and (b) permanently, if the copyright +holder fails to notify you of the violation by some reasonable means +prior to 60 days after the cessation. + + Moreover, your license from a particular copyright holder is +reinstated permanently if the copyright holder notifies you of the +violation by some reasonable means, this is the first time you have +received notice of violation of this License (for any work) from that +copyright holder, and you cure the violation prior to 30 days after +your receipt of the notice. + + Termination of your rights under this section does not terminate the +licenses of parties who have received copies or rights from you under +this License. If your rights have been terminated and not permanently +reinstated, you do not qualify to receive new licenses for the same +material under section 10. + + 9. Acceptance Not Required for Having Copies. + + You are not required to accept this License in order to receive or +run a copy of the Program. Ancillary propagation of a covered work +occurring solely as a consequence of using peer-to-peer transmission +to receive a copy likewise does not require acceptance. However, +nothing other than this License grants you permission to propagate or +modify any covered work. These actions infringe copyright if you do +not accept this License. Therefore, by modifying or propagating a +covered work, you indicate your acceptance of this License to do so. + + 10. Automatic Licensing of Downstream Recipients. + + Each time you convey a covered work, the recipient automatically +receives a license from the original licensors, to run, modify and +propagate that work, subject to this License. You are not responsible +for enforcing compliance by third parties with this License. + + An "entity transaction" is a transaction transferring control of an +organization, or substantially all assets of one, or subdividing an +organization, or merging organizations. If propagation of a covered +work results from an entity transaction, each party to that +transaction who receives a copy of the work also receives whatever +licenses to the work the party's predecessor in interest had or could +give under the previous paragraph, plus a right to possession of the +Corresponding Source of the work from the predecessor in interest, if +the predecessor has it or can get it with reasonable efforts. + + You may not impose any further restrictions on the exercise of the +rights granted or affirmed under this License. For example, you may +not impose a license fee, royalty, or other charge for exercise of +rights granted under this License, and you may not initiate litigation +(including a cross-claim or counterclaim in a lawsuit) alleging that +any patent claim is infringed by making, using, selling, offering for +sale, or importing the Program or any portion of it. + + 11. Patents. + + A "contributor" is a copyright holder who authorizes use under this +License of the Program or a work on which the Program is based. The +work thus licensed is called the contributor's "contributor version". + + A contributor's "essential patent claims" are all patent claims +owned or controlled by the contributor, whether already acquired or +hereafter acquired, that would be infringed by some manner, permitted +by this License, of making, using, or selling its contributor version, +but do not include claims that would be infringed only as a +consequence of further modification of the contributor version. For +purposes of this definition, "control" includes the right to grant +patent sublicenses in a manner consistent with the requirements of +this License. + + Each contributor grants you a non-exclusive, worldwide, royalty-free +patent license under the contributor's essential patent claims, to +make, use, sell, offer for sale, import and otherwise run, modify and +propagate the contents of its contributor version. + + In the following three paragraphs, a "patent license" is any express +agreement or commitment, however denominated, not to enforce a patent +(such as an express permission to practice a patent or covenant not to +sue for patent infringement). To "grant" such a patent license to a +party means to make such an agreement or commitment not to enforce a +patent against the party. + + If you convey a covered work, knowingly relying on a patent license, +and the Corresponding Source of the work is not available for anyone +to copy, free of charge and under the terms of this License, through a +publicly available network server or other readily accessible means, +then you must either (1) cause the Corresponding Source to be so +available, or (2) arrange to deprive yourself of the benefit of the +patent license for this particular work, or (3) arrange, in a manner +consistent with the requirements of this License, to extend the patent +license to downstream recipients. "Knowingly relying" means you have +actual knowledge that, but for the patent license, your conveying the +covered work in a country, or your recipient's use of the covered work +in a country, would infringe one or more identifiable patents in that +country that you have reason to believe are valid. + + If, pursuant to or in connection with a single transaction or +arrangement, you convey, or propagate by procuring conveyance of, a +covered work, and grant a patent license to some of the parties +receiving the covered work authorizing them to use, propagate, modify +or convey a specific copy of the covered work, then the patent license +you grant is automatically extended to all recipients of the covered +work and works based on it. + + A patent license is "discriminatory" if it does not include within +the scope of its coverage, prohibits the exercise of, or is +conditioned on the non-exercise of one or more of the rights that are +specifically granted under this License. You may not convey a covered +work if you are a party to an arrangement with a third party that is +in the business of distributing software, under which you make payment +to the third party based on the extent of your activity of conveying +the work, and under which the third party grants, to any of the +parties who would receive the covered work from you, a discriminatory +patent license (a) in connection with copies of the covered work +conveyed by you (or copies made from those copies), or (b) primarily +for and in connection with specific products or compilations that +contain the covered work, unless you entered into that arrangement, +or that patent license was granted, prior to 28 March 2007. + + Nothing in this License shall be construed as excluding or limiting +any implied license or other defenses to infringement that may +otherwise be available to you under applicable patent law. + + 12. No Surrender of Others' Freedom. + + If conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot convey a +covered work so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you may +not convey it at all. For example, if you agree to terms that obligate you +to collect a royalty for further conveying from those to whom you convey +the Program, the only way you could satisfy both those terms and this +License would be to refrain entirely from conveying the Program. + + 13. Use with the GNU Affero General Public License. + + Notwithstanding any other provision of this License, you have +permission to link or combine any covered work with a work licensed +under version 3 of the GNU Affero General Public License into a single +combined work, and to convey the resulting work. The terms of this +License will continue to apply to the part which is the covered work, +but the special requirements of the GNU Affero General Public License, +section 13, concerning interaction through a network will apply to the +combination as such. + + 14. Revised Versions of this License. + + The Free Software Foundation may publish revised and/or new versions of +the GNU General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + + Each version is given a distinguishing version number. If the +Program specifies that a certain numbered version of the GNU General +Public License "or any later version" applies to it, you have the +option of following the terms and conditions either of that numbered +version or of any later version published by the Free Software +Foundation. If the Program does not specify a version number of the +GNU General Public License, you may choose any version ever published +by the Free Software Foundation. + + If the Program specifies that a proxy can decide which future +versions of the GNU General Public License can be used, that proxy's +public statement of acceptance of a version permanently authorizes you +to choose that version for the Program. + + Later license versions may give you additional or different +permissions. However, no additional obligations are imposed on any +author or copyright holder as a result of your choosing to follow a +later version. + + 15. Disclaimer of Warranty. + + THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY +APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT +HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY +OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM +IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF +ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. Limitation of Liability. + + IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS +THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY +GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE +USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF +DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD +PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), +EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF +SUCH DAMAGES. + + 17. Interpretation of Sections 15 and 16. + + If the disclaimer of warranty and limitation of liability provided +above cannot be given local legal effect according to their terms, +reviewing courts shall apply local law that most closely approximates +an absolute waiver of all civil liability in connection with the +Program, unless a warranty or assumption of liability accompanies a +copy of the Program in return for a fee. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +state the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) 2025 Vincent Riquer + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + +Also add information on how to contact you by electronic and paper mail. + + If the program does terminal interaction, make it output a short +notice like this when it starts in an interactive mode: + + Copyright (C) 2025 Vincent Riquer + This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, your program's commands +might be different; for a GUI interface, you would use an "about box". + + You should also get your employer (if you work as a programmer) or school, +if any, to sign a "copyright disclaimer" for the program, if necessary. +For more information on this, and how to apply and follow the GNU GPL, see +. + + The GNU General Public License does not permit incorporating your program +into proprietary programs. If your program is a subroutine library, you +may consider it more useful to permit linking proprietary applications with +the library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. But first, please read +. From 40f638c83395c25fbf881fb5894ebd89839e1887 Mon Sep 17 00:00:00 2001 From: Vincent Riquer Date: Wed, 22 Jan 2025 22:44:43 +0000 Subject: [PATCH 109/112] Edit BUGS: framagit URL --- BUGS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/BUGS b/BUGS index 5c3935f..d79a49f 100644 --- a/BUGS +++ b/BUGS @@ -1,5 +1,5 @@ Known bugs: -(Real bugs have been moved to http://indefero.riquer.fr/p/AtOM/issues/) +(Real bugs have been moved to https://framagit.org/ScriptFanix/AtOM/-/issues) 1. Video formats suck --------------------- From 8b69f8e1ee16756f55583986b60d36676ab5857a Mon Sep 17 00:00:00 2001 From: Vincent Riquer Date: Wed, 22 Jan 2025 22:48:31 +0000 Subject: [PATCH 110/112] Configure SAST in `.gitlab-ci.yml`, creating this file if it does not already exist --- .gitlab-ci.yml | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 .gitlab-ci.yml diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 0000000..195d65e --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,13 @@ +# You can override the included template(s) by including variable overrides +# SAST customization: https://docs.gitlab.com/ee/user/application_security/sast/#customizing-the-sast-settings +# Secret Detection customization: https://docs.gitlab.com/ee/user/application_security/secret_detection/pipeline/#customization +# Dependency Scanning customization: https://docs.gitlab.com/ee/user/application_security/dependency_scanning/#customizing-the-dependency-scanning-settings +# Container Scanning customization: https://docs.gitlab.com/ee/user/application_security/container_scanning/#customizing-the-container-scanning-settings +# Note that environment variables can be set in several places +# See https://docs.gitlab.com/ee/ci/variables/#cicd-variable-precedence +stages: +- test +sast: + stage: test +include: +- template: Security/SAST.gitlab-ci.yml From d45f73ec243853886c7effb8b97ee744a37ccb4a Mon Sep 17 00:00:00 2001 From: Vincent Riquer Date: Wed, 22 Jan 2025 22:54:14 +0000 Subject: [PATCH 111/112] Configure Secret Detection in `.gitlab-ci.yml`, creating this file if it does not already exist --- .gitlab-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 195d65e..e35fec0 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -11,3 +11,4 @@ sast: stage: test include: - template: Security/SAST.gitlab-ci.yml +- template: Security/Secret-Detection.gitlab-ci.yml