From d16b31f479f2d1fd7234229fc64b92ad9da5ece3 Mon Sep 17 00:00:00 2001 From: Vincent Riquer Date: Sun, 18 Jun 2017 18:21:18 +0200 Subject: [PATCH] add "copy" destination type --- lib/config/getDestination | 6 + lib/config/print | 8 +- lib/config/write | 4 +- lib/decode/file | 193 ++++++++++++----------- lib/files/getDestDir | 32 +++- lib/files/sanitizeFile | 2 +- lib/setup/destination | 320 ++++++++++++++++++++------------------ lib/setup/destinations | 4 +- lib/video/extractaudio | 2 +- lib/workers/master | 2 +- 10 files changed, 308 insertions(+), 265 deletions(-) diff --git a/lib/config/getDestination b/lib/config/getDestination index 1e239c4..2926648 100644 --- a/lib/config/getDestination +++ b/lib/config/getDestination @@ -9,6 +9,9 @@ getConfigDestination() { 'mp3') destinationformat["$destination"]=mp3 lameneeded=1 + # MP3 can't handfle more than 2 channels + [[ -z ${destinationchannels["$destination"]} ]] \ + && destinationchannels["$destination"]=2 ;; 'opus') destinationformat["$destination"]=opus @@ -18,6 +21,9 @@ getConfigDestination() { destinationformat["$destination"]=vorbis oggencneeded=1 ;; + 'copy') + destinationformat["$destination"]=copy + ;; *) echo "Unsupported destination format: $value" >&2 exit $EFORMAT diff --git a/lib/config/print b/lib/config/print index 5dbf3a9..567c547 100644 --- a/lib/config/print +++ b/lib/config/print @@ -24,7 +24,7 @@ printConfig() { for destination in ${!destinationpath[@]} do cat <<-EOF - + $destination|Path|${destinationpath["$destination"]} |Format|${destinationformat["$destination"]} |Quality|${destinationquality["$destination"]} @@ -48,10 +48,10 @@ printConfig() { EOF [ -n "${destinationskipmime["$destination"]}" ] \ && echo " |Skipped mime-types|${destinationskipmime["$destination"]//\|/ -||}" - [ -n "${destinationmskipime["$destination"]}" ] \ +| | |}" + [ -n "${destinationcopymime["$destination"]}" ] \ && echo " |Copied mime-types|${destinationcopymime["$destination"]//\|/ -||}" +| | |}" done }|column -t -s'|' } diff --git a/lib/config/write b/lib/config/write index 0891648..d8b6b36 100644 --- a/lib/config/write +++ b/lib/config/write @@ -63,8 +63,8 @@ path $sourcepath # * path: Where files will be written path ${destinationpath["$destination"]} -# * format: ogg, opus or mp3. Other formats may appear in the future - feel -# free to implement your preferred format. +# * format: copy, ogg, opus or mp3. Other formats may appear in the future - +# feel free to implement your preferred format. format ${destinationformat["$destination"]} EOCfg diff --git a/lib/decode/file b/lib/decode/file index c931fa0..b0f350a 100644 --- a/lib/decode/file +++ b/lib/decode/file @@ -1,39 +1,14 @@ #!/bin/bash declare soxtaskid decodeFile() { - case "$mimetype" in - 'video/'*) - (( disablevideo )) && continue - extractAudio - if (( ${destinationnormalize["$destination"]}))\ - || ( - [ -n "${destinationfrequency["$destination"]}" ]\ - && (( ${rate:-0} != ${destinationfrequency["$destination"]}))\ - ) || ( - [ -n "${destinationchannels["$destination"]}" ]\ - && (( ${channels:-0} != ${destinationchannels["$destination"]} )) - ) - then - sox_needed=1 - fi - ;; - 'audio/mpeg') - if [[ ${destinationformat[$destination]} = mp3 ]] \ - && checkCopy - then - copied=1 - else - decodeSox - fi - ;; - 'application/ogg opus') - if [[ ${destinationformat[$destination]} = opus ]] \ - && checkCopy - then - copied=1 - else - (( disableopusdec )) && continue - decodeOpusdec + if [[ ${destinationformat["$destination"]} == copy ]] + then + copied=1 + else + case "$mimetype" in + 'video/'*) + (( disablevideo )) && continue + extractAudio if (( ${destinationnormalize["$destination"]}))\ || ( [ -n "${destinationfrequency["$destination"]}" ]\ @@ -45,26 +20,24 @@ decodeFile() { then sox_needed=1 fi - fi - ;; - 'application/ogg'*) - if [[ ${destinationformat[$destination]} = vorbis ]] \ - && checkCopy - then - copied=1 - else - decodeSox - fi - ;; - 'audio/x-flac') - decodeSox - ;; - *) - extendedtype=$(file -b "$sourcepath/$filename") - case "$extendedtype" in - *'Musepack '*) - (( disablempcdec )) && continue - decodeMpcdec + ;; + 'audio/mpeg') + if [[ ${destinationformat[$destination]} = mp3 ]] \ + && checkCopy + then + copied=1 + else + decodeSox + fi + ;; + 'application/ogg opus') + if [[ ${destinationformat[$destination]} = opus ]] \ + && checkCopy + then + copied=1 + else + (( disableopusdec )) && continue + decodeOpusdec if (( ${destinationnormalize["$destination"]}))\ || ( [ -n "${destinationfrequency["$destination"]}" ]\ @@ -76,13 +49,26 @@ decodeFile() { then sox_needed=1 fi - ;; - *) - if (( disablevideo )) - then - decodeSox - else - extractAudio + fi + ;; + 'application/ogg'*) + if [[ ${destinationformat[$destination]} = vorbis ]] \ + && checkCopy + then + copied=1 + else + decodeSox + fi + ;; + 'audio/x-flac') + decodeSox + ;; + *) + extendedtype=$(file -b "$sourcepath/$filename") + case "$extendedtype" in + *'Musepack '*) + (( disablempcdec )) && continue + decodeMpcdec if (( ${destinationnormalize["$destination"]}))\ || ( [ -n "${destinationfrequency["$destination"]}" ]\ @@ -94,41 +80,36 @@ decodeFile() { then sox_needed=1 fi - fi - ;; - esac - ;; - esac - if ! (( copied )) - then - if ! decodetaskid=$( - Select tasks id <<<"key = $tmpfile" - ) + ;; + *) + if (( disablevideo )) + then + decodeSox + else + extractAudio + if (( ${destinationnormalize["$destination"]}))\ + || ( + [ -n "${destinationfrequency["$destination"]}" ]\ + && (( ${rate:-0} != ${destinationfrequency["$destination"]}))\ + ) || ( + [ -n "${destinationchannels["$destination"]}" ]\ + && (( ${channels:-0} != ${destinationchannels["$destination"]} )) + ) + then + sox_needed=1 + fi + fi + ;; + esac + ;; + esac + if ! (( copied )) then - decodetaskid=$( - Insert tasks <<-EOInsert - key $tmpfile - source_file $fileid - $( - for key in ${!commandline[@]} - do - echo "cmd_arg$key ${commandline[key]}" - done - ) - status 0 - EOInsert - ) - progressSpin - fi - if (( sox_needed )) - then - cleanup="$tempdir/$tmpfile" - decodeSox "$tempdir/$tmpfile" - if ! soxtaskid=$( + if ! decodetaskid=$( Select tasks id <<<"key = $tmpfile" ) then - soxtaskid=$( + decodetaskid=$( Insert tasks <<-EOInsert key $tmpfile source_file $fileid @@ -138,14 +119,38 @@ decodeFile() { echo "cmd_arg$key ${commandline[key]}" done ) - requires $decodetaskid - required $decodetaskid status 0 - cleanup $cleanup EOInsert ) progressSpin fi + if (( sox_needed )) + then + cleanup="$tempdir/$tmpfile" + decodeSox "$tempdir/$tmpfile" + if ! soxtaskid=$( + Select tasks id <<<"key = $tmpfile" + ) + then + soxtaskid=$( + Insert tasks <<-EOInsert + key $tmpfile + source_file $fileid + $( + for key in ${!commandline[@]} + do + echo "cmd_arg$key ${commandline[key]}" + done + ) + requires $decodetaskid + required $decodetaskid + status 0 + cleanup $cleanup + EOInsert + ) + progressSpin + fi + fi fi fi } diff --git a/lib/files/getDestDir b/lib/files/getDestDir index 6ec50cc..436d1bd 100644 --- a/lib/files/getDestDir +++ b/lib/files/getDestDir @@ -38,6 +38,25 @@ getDestDir() { ) ) then + if (( ${destinationascii["$destination"]} )) + then + echo "$album" >&${toascii[1]} + read -r -u${toascii[0]} album + echo "$albumartist" >&${toascii[1]} + read -r -u${toascii[0]} albumartist + echo "$artist" >&${toascii[1]} + read -r -u${toascii[0]} artist + echo "$genre" >&${toascii[1]} + read -r -u${toascii[0]} genre + echo "$title" >&${toascii[1]} + read -r -u${toascii[0]} title + echo "$tracknumber" >&${toascii[1]} + read -r -u${toascii[0]} tracknumber + echo "$year" >&${toascii[1]} + read -r -u${toascii[0]} year + echo "$disc" >&${toascii[1]} + read -r -u${toascii[0]} disc + fi replace=$(sanitizeFile "$album" dir) destdir+="${destinationrenamepath[$destination]//?(\[)%\{album\}?(\])/$replace}" replace=$(sanitizeFile "$albumartist" dir) @@ -60,15 +79,16 @@ getDestDir() { part=${filename#*/} while [[ $part =~ / ]] do - destdir+="/$(sanitizeFile "${part%%/*}" dir)" + thispart="${part%%/*}" + if (( ${destinationascii["$destination"]} )) + then + echo "$thispart" >&${toascii[1]} + read -r -u${toascii[0]} thispart + fi + destdir+="/$(sanitizeFile "$thispart" dir)" part=${part#*/} done fi - if (( ${destinationascii["$destination"]} )) - then - echo "$destdir" >&${toascii[1]} - read -r -u${toascii[0]} destdir - fi if ! [ -d "$destdir" ] then mkdir -p "$destdir" diff --git a/lib/files/sanitizeFile b/lib/files/sanitizeFile index 07e8e50..164ca9b 100644 --- a/lib/files/sanitizeFile +++ b/lib/files/sanitizeFile @@ -13,7 +13,7 @@ sanitizeFile() { string=${string//>/ } string=${string//:/ } string=${string//\*/ } - string=${string//|/ } + string=${string//\|/ } string=${string//\"/ } # Filenames can't begin or end with ' ' diff --git a/lib/setup/destination b/lib/setup/destination index c67c63d..c6ea323 100644 --- a/lib/setup/destination +++ b/lib/setup/destination @@ -4,7 +4,7 @@ setupDestination() { cat <<-EODesc Format: - vorbis, opus or mp3. Other formats may appear in the future. + copy, vorbis, opus or mp3. Other formats may appear in the future. EODesc comeagain() { read \ @@ -25,6 +25,9 @@ setupDestination() { destinationformat["$destination"]=vorbis oggencneeded=1 ;; + 'copy') + destinationformat["$destination"]=copy + ;; *) echo "Unsupported destination format: $value" >&2 comeagain @@ -44,6 +47,9 @@ setupDestination() { ${destinationpath["$destination"]+-i"${destinationpath["$destination"]}"}\ destinationpath["$destination"] case ${destinationformat["$destination"]} in + copy) + : + ;; vorbis) cat <<-EODesc @@ -199,39 +205,42 @@ setupDestination() { Now you will have the opportunity to configure "advanced" parameters for $destination. You may leave any of these fields blank. EODesc - cat <<-EODesc + if [[ ${destinationformat["$destination"]} != copy ]] + then + cat <<-EODesc - Normalize (boolean): - Normalize output files. - EODesc - case ${destinationnormalize["$destination"]} in - 0) initialvalue=n ;; - 1) initialvalue=y ;; - *) unset initialvalue ;; - esac - comeagain() { - read \ - -e \ - ${initialvalue+-i $initialvalue}\ - -p'Normalize (y/N): ' \ - value - case $value in - [yY]) - [[ $initialvalue == n ]] \ - && setupRegen normalize - destinationnormalize["$destination"]=1 - ;; - ''|[nN]) - [[ $initialvalue == y ]] \ - && setupRegen normalize - destinationnormalize["$destination"]=0 - ;; - *) - comeagain - ;; + Normalize (boolean): + Normalize output files. + EODesc + case ${destinationnormalize["$destination"]} in + 0) initialvalue=n ;; + 1) initialvalue=y ;; + *) unset initialvalue ;; esac - } - comeagain + comeagain() { + read \ + -e \ + ${initialvalue+-i $initialvalue}\ + -p'Normalize (y/N): ' \ + value + case $value in + [yY]) + [[ $initialvalue == n ]] \ + && setupRegen normalize + destinationnormalize["$destination"]=1 + ;; + ''|[nN]) + [[ $initialvalue == y ]] \ + && setupRegen normalize + destinationnormalize["$destination"]=0 + ;; + *) + comeagain + ;; + esac + } + comeagain + fi cat <<-EODesc Rename (string): @@ -362,134 +371,137 @@ setupDestination() { fi done unset skippedmimes - cat <<-EODesc - - Copy mime-type (mime-type, string): - Files with mime-type will be copied as-is to the - destination. E.g. image/* will copy covers and other images to the - destination. The '*' character is a wildcard. - - This prompt will loop until an empty string is encountered. - EODesc - while [[ ${destinationcopymime["$destination"]} =~ \| ]] - do - copiedmimes+=("${destinationcopymime["$destination"]%%|*}") - destinationcopymime["$destination"]="${destinationcopymime["$destination"]#*|}" - done - [ -n "${destinationcopymime["$destination"]}" ] \ - && copiedmimes+=("${destinationcopymime["$destination"]}") - count=${#copiedmimes[@]} - unset destinationcopymime["$destination"] - for (( i=0 ; 1 ; i++ )) - do - read \ - -e \ - ${copiedmimes[i]+-i"${copiedmimes[i]}"} \ - -p 'Copy mime-type: ' \ - value - if [ -n "$value" ] - then - destinationcopymime[$destination]="${destinationcopymime[$destination]:+${destinationcopymime[$destination]}|}$value" - elif (( i < count )) - then - continue - else - break - fi - done - unset copiedmimes - cat <<-EODesc - - Channels (integer): - Produced files should have this many channels, no more, no less. - EODesc - expr='^[0-9]*$' - comeagain() { - read \ - -e \ - ${destinationchannels["$destination"]+-i${destinationchannels["$destination"]}}\ - -p'Channel count: ' \ - value - if ! [[ $value =~ $expr ]] - then - echo "Invalid channel count: $value" >&2 - comeagain - fi - } - comeagain - if [ -n "${destinationchannels["$destination"]}" ] \ - && (( value != ${destinationchannels["$destination"]} )) - then - setupRegen channels - fi - destinationchannels["$destination"]=$value - cat <<-EODesc - - Sampling rate (Hertz, integer): - Files will be resampled as needed to the specified sampling-rate. - Shoutcast/Icecast streams require a constant sampling-rate. - Telephony systems often require a sampling-rate of 8000Hz. - EODesc - if [[ ${destinationformat["$destination"]} == opus ]] + if [[ ${destinationformat["$destination"]} != copy ]] then cat <<-EODesc - Please note that Opus supports only the following sample-rates: - 8000, 12000, 16000, 24000, and 48000 Hz. So don't set - resampling on an Opus destination to any other value or files - will be resampled twice. + Copy mime-type (mime-type, string): + Files with mime-type will be copied as-is to the + destination. E.g. image/* will copy covers and other images to the + destination. The '*' character is a wildcard. + + This prompt will loop until an empty string is encountered. EODesc - fi - comeagain() { - read \ - -e \ - ${destinationfrequency["$destination"]+-i${destinationfrequency["$destination"]}}\ - -p'Sampling-rate: ' \ - value - if ! [[ $value =~ $expr ]] - then - echo "Invalid frequency value: $value" >&2 - comeagain - fi - } - comeagain - if [ -n "${destinationfrequency["$destination"]}" ] \ - && (( value != ${destinationfrequency["$destination"]} )) - then - setupRegen frequency - fi - destinationfrequency["$destination"]=$value - cat <<-EODesc + while [[ ${destinationcopymime["$destination"]} =~ \| ]] + do + copiedmimes+=("${destinationcopymime["$destination"]%%|*}") + destinationcopymime["$destination"]="${destinationcopymime["$destination"]#*|}" + done + [ -n "${destinationcopymime["$destination"]}" ] \ + && copiedmimes+=("${destinationcopymime["$destination"]}") + count=${#copiedmimes[@]} + unset destinationcopymime["$destination"] + for (( i=0 ; 1 ; i++ )) + do + read \ + -e \ + ${copiedmimes[i]+-i"${copiedmimes[i]}"} \ + -p 'Copy mime-type: ' \ + value + if [ -n "$value" ] + then + destinationcopymime[$destination]="${destinationcopymime[$destination]:+${destinationcopymime[$destination]}|}$value" + elif (( i < count )) + then + continue + else + break + fi + done + unset copiedmimes + cat <<-EODesc - Higher-Than (bitrate, integer): - Only reencode files with bitrates higher then kbps. This - only applies if sample-rate, channel count and of course format are - equal. If unset, only files with bitrates equal to that of the - target will be copied (actually, hardlinking will be attempted - first). - - As Ogg Vorbis target quality is not defined by its bitrate, Ogg - Vorbis files will always be reencoded if unset. - EODesc - comeagain() { - read \ - -e \ - ${destinationmaxbps["$destination"]+-i${destinationmaxbps["$destination"]}}\ - -p'Higher-Than: ' \ - value - if ! [[ $value =~ $expr ]] + Channels (integer): + Produced files should have this many channels, no more, no less. + EODesc + expr='^[0-9]*$' + comeagain() { + read \ + -e \ + ${destinationchannels["$destination"]+-i${destinationchannels["$destination"]}}\ + -p'Channel count: ' \ + value + if ! [[ $value =~ $expr ]] + then + echo "Invalid channel count: $value" >&2 + comeagain + fi + } + comeagain + if [ -n "${destinationchannels["$destination"]}" ] \ + && (( value != ${destinationchannels["$destination"]} )) then - echo "Invalid higher-than bitrate value: $value" >&2 - comeagain + setupRegen channels fi - } - comeagain - if [ -n "${destinationmaxbps["$destination"]}" ] \ - && (( value != ${destinationmaxbps["$destination"]} )) - then - setupRegen maxbps + destinationchannels["$destination"]=$value + cat <<-EODesc + + Sampling rate (Hertz, integer): + Files will be resampled as needed to the specified sampling-rate. + Shoutcast/Icecast streams require a constant sampling-rate. + Telephony systems often require a sampling-rate of 8000Hz. + EODesc + if [[ ${destinationformat["$destination"]} == opus ]] + then + cat <<-EODesc + + Please note that Opus supports only the following sample-rates: + 8000, 12000, 16000, 24000, and 48000 Hz. So don't set + resampling on an Opus destination to any other value or files + will be resampled twice. + EODesc + fi + comeagain() { + read \ + -e \ + ${destinationfrequency["$destination"]+-i${destinationfrequency["$destination"]}}\ + -p'Sampling-rate: ' \ + value + if ! [[ $value =~ $expr ]] + then + echo "Invalid frequency value: $value" >&2 + comeagain + fi + } + comeagain + if [ -n "${destinationfrequency["$destination"]}" ] \ + && (( value != ${destinationfrequency["$destination"]} )) + then + setupRegen frequency + fi + destinationfrequency["$destination"]=$value + cat <<-EODesc + + Higher-Than (bitrate, integer): + Only reencode files with bitrates higher then kbps. This + only applies if sample-rate, channel count and of course format are + equal. If unset, only files with bitrates equal to that of the + target will be copied (actually, hardlinking will be attempted + first). + + As Ogg Vorbis target quality is not defined by its bitrate, Ogg + Vorbis files will always be reencoded if unset. + EODesc + comeagain() { + read \ + -e \ + ${destinationmaxbps["$destination"]+-i${destinationmaxbps["$destination"]}}\ + -p'Higher-Than: ' \ + value + if ! [[ $value =~ $expr ]] + then + echo "Invalid higher-than bitrate value: $value" >&2 + comeagain + fi + } + comeagain + if [ -n "${destinationmaxbps["$destination"]}" ] \ + && (( value != ${destinationmaxbps["$destination"]} )) + then + setupRegen maxbps + fi + destinationmaxbps[$destination]="$value" + unset regen + unset expr fi - destinationmaxbps[$destination]="$value" - unset regen - unset expr } diff --git a/lib/setup/destinations b/lib/setup/destinations index 305852b..4e1c2b3 100644 --- a/lib/setup/destinations +++ b/lib/setup/destinations @@ -65,7 +65,7 @@ setupDestinations() { expr='^[A-z0-9]*$' comeagain() { read -p'Name: ' value - [ -z "$value" ] && break + [ -z "$value" ] && return 1 if ! [[ $value =~ $expr ]] then echo "Invalid name $value. Please use" \ @@ -73,7 +73,7 @@ setupDestinations() { comeagain fi } - comeagain + comeagain || break destination="$value" setupDestination done diff --git a/lib/video/extractaudio b/lib/video/extractaudio index afa1392..785f1ce 100644 --- a/lib/video/extractaudio +++ b/lib/video/extractaudio @@ -2,5 +2,5 @@ extractAudio() { tmpfile="${fileid}ffmpeg.wav" commandline=(${ionice}ffmpeg -v 0 -vn -y) - commandline+=(-i "$sourcepath/$filename" "$tempdir/$tmpfile") + commandline+=(-i "$sourcepath/$filename" -map a:0 "$tempdir/$tmpfile") } diff --git a/lib/workers/master b/lib/workers/master index 96e3937..e24956f 100644 --- a/lib/workers/master +++ b/lib/workers/master @@ -92,7 +92,7 @@ master() { if (( remaining == 0 )) then sleep 0.1 - continue + return 0 elif (( active == 0 && ready == 0 )) then dumpfile=tasks-$(date +%Y%m%d%H%M%S).csv