ScriptFanix/AtOM
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

New 'ascii-only' parameter

Browse Source 
* ascii:
  Implement ascii-only
  Fix rename_pattern, permit optional fields with []
  Implement new DB schema
  Sanity check (perl module Text::Unidecode)
  config parser, setup for ascii-only
  DB upgrade: prevent variable leakage
  fat32compat & ascii as separate columns in DB
This commit is contained in:
Vincent Riquer 2013-10-09 17:24:36 +02:00
commit 0c91e93580
17 changed files with 286 additions and 93 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

35
atom
View File

@ -13,6 +13,7 @@ EFMTINVPARM=49
# config structures
declare -A \
destinationascii \
destinationchannels \
destinationfat32compat \
destinationcopymime \
@ -294,6 +295,15 @@ then
disablevideo=1
(( sanitywarn++ ))
fi
if (( textunidecodeneeded )) && ! perl -MText::Unidecode -e 'exit;' 2>/dev/null
then
echo "[WARNING] Perl module Text::Unidecode is not available
Renaming to ASCII-only disabled" >&2
unset destinationascii
destinationascii=0
textunidecodeneeded=0
(( sanitywarn++ ))
fi
if (( sanityfail ))
then
echo "
@ -379,6 +389,8 @@ echo '
id INTEGER PRIMARY KEY,
key TEXT UNIQUE,
rename_pattern TEXT,
fat32compat INTEGER,
ascii INTEGER,
source_file INTEGER,
fileid INTEGER,
filename TEXT,
@ -492,6 +504,8 @@ do
done
(( cron )) || echo -n 'Creating tasks... '
(( textunidecodeneeded )) && ascii
echo 'BEGIN TRANSACTION;' >&3
for line in "${decodefiles[@]}"
do
@ -580,6 +594,9 @@ echo 'COMMIT;' >&3
(( cron )) || echo -n $'\r'
echo "Created ${count:-0} tasks for $filecount files ${togo:+($togo left) }(${copies:-0} immediate copies)"
# remove perl unicode to ascii coprocess
(( textunidecodeneeded )) && eval exec "${toascii[1]}>&-"
concurrency=$(( maxload / 2 ))
(( concurrency )) || concurrency=1
active=0
@ -753,7 +770,6 @@ then
exit
fi
#set -x
for destination in "${!destinationpath[@]}"
do
echo '
@ -781,12 +797,18 @@ do
INNER JOIN source_files
ON destination_files.source_file_id
=source_files.id
INNER JOIN mime_actions
ON source_files.mime_type
=mime_actions.mime_type
WHERE destinations.name="'"$destination"'"
AND (destination_files.rename_pattern
!=
"'"${destinationrenamepath[$destination]}/${destinationrename[$destination]}:${destinationfat32compat["$destination"]}"'"
"'"${destinationrenamepath[$destination]}/${destinationrename[$destination]}"'"
OR fat32compat != '${destinationfat32compat["$destination"]}'
OR ascii != '${destinationascii["$destination"]}'
OR destination_files.rename_pattern is NULL)
AND destination_files.last_change > 0
AND mime_actions.action=1
;
SELECT "AtOM:NoMoreFiles";
@ -806,6 +828,7 @@ do
'vorbis') extension=ogg ;;
esac
(( cron )) || echo -n "$destination: rename pattern changed, renaming files... "
(( textunidecodeneeded )) && ascii
echo 'BEGIN TRANSACTION;' >&3
for line in "${renamefiles[@]}"
do
@ -850,7 +873,11 @@ do
echo "UPDATE destination_files" \
"SET filename=\"${destfilename//\"/\"\"}\"," \
" rename_pattern=" \
"\"${destinationrenamepath[$destination]}/${destinationrename[$destination]}:${destinationfat32compat["$destination"]}\"" \
"\"${destinationrenamepath[$destination]}/${destinationrename[$destination]}\","\
" fat32compat=" \
"${destinationfat32compat["$destination"]}," \
" ascii=" \
"${destinationascii["$destination"]}" \
"WHERE id=$destfileid;" \
>&3
if (( commit ))
@ -860,6 +887,8 @@ do
fi
fi
done
# remove perl unicode to ascii coprocess
(( textunidecodeneeded )) && eval exec "${toascii[1]}>&-"
echo 'COMMIT;' >&3
(( cron )) || echo -n $'\r'
echo -n "$destination: Renamed ${changedcount:-0} files"

16
lib/config/getDestination
View File

@ -154,6 +154,22 @@ getConfigDestination() {
;;
esac
;;
'ascii-only')
case $value in
'true'|'on'|'yes')
destinationascii["$destination"]=1
textunidecodeneeded=1
;;
'false'|'off'|'no')
destinationascii["$destination"]=0
;;
*)
echo "ascii-only takes values:" \
"'yes' ,'true' ,'on', 'no', 'false',"\
"'off'"
;;
esac
;;
'skip_mime-type')
destinationskipmime[$destination]="${destinationskipmime[$destination]:+${destinationskipmime[$destination]}|}$value"
;;

1
lib/config/print
View File

@ -42,6 +42,7 @@ printConfig() {
|Frequency|${destinationfrequency["$destination"]}
|Higher than|${destinationmaxbps["$destination"]}
|Fat32 Compat.|${destinationfat32compat["$destination"]}
|ASCII Compat.|${destinationascii["$destination"]}
|Path Change|${destinationrenamepath["$destination"]}
|File Rename|${destinationrename["$destination"]}
EOF

11
lib/config/write
View File

@ -168,6 +168,17 @@ bitrate ${destinationquality["$destination"]}
fi
cat <<-EOCfg
# * ascii-only <yes>/<no>: Rename files for compatibility with ASCII-only
# systems.
EOCfg
if (( ${destinationascii["$destination"]} ))
then
echo $'ascii-only\t\tyes'
else
echo $'ascii-only\t\tno'
fi
cat <<-EOCfg
# * skip_mime-type <mime-type>: Files with mime-type <mime-type> will not
# be included in that destination. For more than one mime-type, use multiple
# times, as needed. The '*' character is a wildcard.

4
lib/copy/action
View File

@ -74,7 +74,9 @@ copyFiles_action() {
then
Update destination_files \
filename "$destdir/${sourcefilename##*/}"\
rename_pattern "${destinationrenamepath[$destination]}/${destinationrename[$destination]}:${destinationfat32compat["$destination"]}"\
rename_pattern "${destinationrenamepath[$destination]}/${destinationrename[$destination]}"\
fat32compat ${destinationfat32compat["$destination"]}\
ascii ${destinationascii["$destination"]}\
last_change $lastchange \
<<-EOWhere
id = $destfileid

5
lib/copy/matching
View File

@ -23,7 +23,10 @@ copyFiles_matching() {
" WHERE id=$destfileid" \
" )," \
" rename_pattern=" \
"\"${destinationrenamepath[$destination]}/${destinationrename[$destination]}:${destinationfat32compat["$destination"]}\""\
"\"${destinationrenamepath[$destination]}/${destinationrename[$destination]}\","\
" fat32compat=" \
"${destinationfat32compat["$destination"]}," \
" ascii=${destinationascii["$destination"]}" \
"WHERE id=$destfileid;" \
>&3
(( ++copies ))

10
lib/database/checkVersion
View File

@ -1,18 +1,18 @@
#!/bin/bash
currentdbversion=1
currentdbversion=2
checkDatabaseVersion() {
local dbversion
if dbversion=$(Select atom version <<<"1 = 1")
if dbversion=$(Select atom version <<<"\"1\" = 1")
then
if (( dbversion == currentdbversion ))
then
return 0
elif (( dbversion < currentversion ))
elif (( dbversion < currentdbversion ))
then
until (( dbversion == currentversion ))
until (( dbversion == currentdbversion ))
do
upgradedatabase_${dbversion}_$((dbversion+1))
dbversion=$(Select atom version <<<"1 = 1")
dbversion=$(Select atom version <<<"\"1\" = 1")
done
else
echo "Database schema version $dbversion is higher than

43
lib/database/upgradedatabase_1_2 Normal file
View File

@ -0,0 +1,43 @@
#!/bin/bash
upgradedatabase_1_2() {
local data \
datas \
id \
rename_pattern \
pattern \
fat32
echo "Upgrading database to version 2... (backup is $database.bak_v1)"
cp "$database" "$database.bak_v1"
echo 'ALTER TABLE destination_files ADD COLUMN fat32compat INTEGER;' >&3
echo 'ALTER TABLE destination_files ADD COLUMN ascii INTEGER;' >&3
echo '
SELECT id,
rename_pattern
FROM destination_files;
SELECT "AtOM::NoMoreData";' >&3
read -u4 data
while [[ $data != AtOM::NoMoreData ]]
do
datas+=( "$data" )
read -u4 data
done
echo 'BEGIN TRANSACTION;' >&3
for data in "${datas[@]}"
do
id="${data%%::AtOM:SQL:Sep::*}"
rename_pattern="${data#*::AtOM:SQL:Sep::}"
IFS=':'
read pattern fat32 <<<"$rename_pattern"
IFS="$oldIFS"
Update destination_files \
rename_pattern "$pattern" \
fat32compat "$fat32" \
ascii 0 \
<<<"id = $id"
done
echo 'COMMIT;' >&3
Update atom version 2 <<<"1 = 1"
}

4
lib/encode/mp3
View File

@ -65,7 +65,9 @@ encodeFile::mp3() {
cleanup $tempdir/$tmpfile.wav
source_file $fileid
status 0
rename_pattern ${destinationrenamepath[$destination]}/${destinationrename[$destination]}:${destinationfat32compat["$destination"]}
rename_pattern ${destinationrenamepath[$destination]}/${destinationrename[$destination]}
fat32compat ${destinationfat32compat["$destination"]}
ascii ${destinationascii["$destination"]}
EOInsert
)
progressSpin

4
lib/encode/opus
View File

@ -37,7 +37,9 @@ encodeFile::opus() {
cleanup $tempdir/$tmpfile.wav
source_file $fileid
status 0
rename_pattern ${destinationrenamepath[$destination]}/${destinationrename[$destination]}:${destinationfat32compat["$destination"]}
rename_pattern ${destinationrenamepath[$destination]}/${destinationrename[$destination]}
fat32compat ${destinationfat32compat["$destination"]}
ascii ${destinationascii["$destination"]}
EOInsert
)
progressSpin

4
lib/encode/vorbis
View File

@ -33,7 +33,9 @@ encodeFile::vorbis() {
cleanup $tempdir/$tmpfile.wav
source_file $fileid
status 0
rename_pattern ${destinationrenamepath[$destination]}/${destinationrename[$destination]}:${destinationfat32compat["$destination"]}
rename_pattern ${destinationrenamepath[$destination]}/${destinationrename[$destination]}
fat32compat ${destinationfat32compat["$destination"]}
ascii ${destinationascii["$destination"]}
EOInsert
)
progressSpin

75
lib/files/getDestDir
View File

@ -4,49 +4,57 @@ getDestDir() {
if [ -n "${destinationrenamepath[$destination]}" ] \
&& (
(
! [[ ${destinationrenamepath[$destination]} =~ %\{album\} ]] \
|| [ -n "$album" ]
) && (
! [[ ${destinationrenamepath[$destination]} =~ %\{albumartist\} ]] \
|| [ -n "$albumartist" ]
) && (
! [[ ${destinationrenamepath[$destination]} =~ %\{artist\} ]] \
|| [ -n "$artist" ]
) && (
! [[ ${destinationrenamepath[$destination]} =~ %\{genre\} ]] \
|| [ -n "$genre" ]
) && (
! [[ ${destinationrenamepath[$destination]} =~ %\{title\} ]] \
|| [ -n "$title" ]
) && (
! [[ ${destinationrenamepath[$destination]} =~ %\{track\} ]] \
|| [ -n "$track" ]
) && (
! [[ ${destinationrenamepath[$destination]} =~ %\{year\} ]] \
|| [ -n "$year" ]
) && (
! [[ ${destinationrenamepath[$destination]} =~ %\{disc\} ]] \
|| [ -n "$disc" ]
[[ ${destinationrenamepath[$destination]} == \
*?([^[])%\{album\}?([^\]])* ]] \
&& [ -n "$album" ]
) || (
[[ ${destinationrenamepath[$destination]} == \
*?([^[])%\{albumartist\}?([^\]])* ]] \
&& [ -n "$albumartist" ]
) || (
[[ ${destinationrenamepath[$destination]} == \
*?([^[])%\{artist\}?([^\]])* ]] \
&& [ -n "$artist" ]
) || (
[[ ${destinationrenamepath[$destination]} == \
*?([^[])%\{genre\}?([^\]])* ]] \
&& [ -n "$genre" ]
) || (
[[ ${destinationrenamepath[$destination]} == \
*?([^[])%\{title\}?([^\]])* ]] \
&& [ -n "$title" ]
) || (
[[ ${destinationrenamepath[$destination]} == \
*?([^[])%\{track\}?([^\]])* ]] \
&& [ -n "$track" ]
) || (
[[ ${destinationrenamepath[$destination]} == \
*?([^[])%\{year\}?([^\]])* ]] \
&& [ -n "$year" ]
) || (
[[ ${destinationrenamepath[$destination]} == \
*?([^[])%\{disc\}?([^\]])* ]] \
&& [ -n "$disc" ]
)
)
then
replace=$(sanitizeFile "$album" dir)
destdir+="${destinationrenamepath[$destination]//%\{album\}/$replace}"
destdir+="${destinationrenamepath[$destination]//?(\[)%\{album\}?(\])/$replace}"
replace=$(sanitizeFile "$albumartist" dir)
destdir="${destdir//%\{albumartist\}/$replace}"
destdir="${destdir//?(\[)%\{albumartist\}?(\])/$replace}"
replace=$(sanitizeFile "$artist" dir)
destdir="${destdir//%\{artist\}/$replace}"
destdir="${destdir//?(\[)%\{artist\}?(\])/$replace}"
replace=$(sanitizeFile "$genre" dir)
destdir="${destdir//%\{genre\}/$replace}"
destdir="${destdir//?(\[)%\{genre\}?(\])/$replace}"
replace=$(sanitizeFile "$title" dir)
destdir="${destdir//%\{title\}/$replace}"
destdir="${destdir//?(\[)%\{title\}?(\])/$replace}"
tracknumber="${track%/*}"
replace=$(sanitizeFile "$tracknumber" dir)
destdir="${destdir//%\{track\}/$replace}"
destdir="${destdir//?(\[)%\{track\}?(\])/$replace}"
replace=$(sanitizeFile "$year" dir)
destdir="${destdir//%\{year\}/$replace}"
destdir="${destdir//?(\[)%\{year\}?(\])/$replace}"
replace=$(sanitizeFile "$disc" dir)
destdir="${destdir//%\{disc\}/$replace}"
destdir="${destdir//?(\[)%\{disc\}?(\])/$replace}"
else
destdir+=$(sanitizeFile "${filename%%/*}" dir)
part=${filename#*/}
@ -56,6 +64,11 @@ getDestDir() {
part=${part#*/}
done
fi
if (( ${destinationascii["$destination"]} ))
then
echo "$destdir" >&${toascii[1]}
read -r -u${toascii[0]} destdir
fi
if ! [ -d "$destdir" ]
then
mkdir -p "$destdir"

75
lib/files/getDestFile
View File

@ -3,44 +3,57 @@ getDestFile() {
if [ -n "${destinationrename[$destination]}" ] \
&& (
(
! [[ ${destinationrename[$destination]} =~ %\{album\} ]] \
|| [ -n "$album" ]
) && (
! [[ ${destinationrename[$destination]} =~ %\{albumartist\} ]] \
|| [ -n "$albumartist" ]
) && (
! [[ ${destinationrename[$destination]} =~ %\{artist\} ]] \
|| [ -n "$artist" ]
) && (
! [[ ${destinationrename[$destination]} =~ %\{genre\} ]] \
|| [ -n "$genre" ]
) && (
! [[ ${destinationrename[$destination]} =~ %\{title\} ]] \
|| [ -n "$title" ]
) && (
! [[ ${destinationrename[$destination]} =~ %\{track\} ]] \
|| [ -n "$track" ]
) && (
! [[ ${destinationrename[$destination]} =~ %\{year\} ]] \
|| [ -n "$year" ]
) && (
! [[ ${destinationrename[$destination]} =~ %\{disc\} ]] \
|| [ -n "$disc" ]
[[ ${destinationrename[$destination]} == \
*?([^[])%\{album\}?([^\]])* ]] \
&& [ -n "$album" ]
) || (
[[ ${destinationrename[$destination]} == \
*?([^[])%\{albumartist\}?([^\]])* ]] \
&& [ -n "$albumartist" ]
) || (
[[ ${destinationrename[$destination]} == \
*?([^[])%\{artist\}?([^\]])* ]] \
&& [ -n "$artist" ]
) || (
[[ ${destinationrename[$destination]} == \
*?([^[])%\{genre\}?([^\]])* ]] \
&& [ -n "$genre" ]
) || (
[[ ${destinationrename[$destination]} == \
*?([^[])%\{title\}?([^\]])* ]] \
&& [ -n "$title" ]
) || (
[[ ${destinationrename[$destination]} == \
*?([^[])%\{track\}?([^\]])* ]] \
&& [ -n "$track" ]
) || (
[[ ${destinationrename[$destination]} == \
*?([^[])%\{year\}?([^\]])* ]] \
&& [ -n "$year" ]
) || (
[[ ${destinationrename[$destination]} == \
*?([^[])%\{disc\}?([^\]])* ]] \
&& [ -n "$disc" ]
)
)
then
destfile="${destinationrename[$destination]//%\{album\}/$album}"
destfile="${destfile//%\{albumartist\}/$albumartist}"
destfile="${destfile//%\{artist\}/$artist}"
destfile="${destfile//%\{genre\}/$genre}"
destfile="${destfile//%\{title\}/$title}"
destfile="${destinationrename[$destination]//?(\[)%\{album\}?(\])/$album}"
destfile="${destfile//?(\[)%\{albumartist\}?(\])/$albumartist}"
destfile="${destfile//?(\[)%\{artist\}?(\])/$artist}"
destfile="${destfile//?(\[)%\{genre\}?(\])/$genre}"
destfile="${destfile//?(\[)%\{title\}?(\])/$title}"
tracknumber="${track%/*}"
destfile="${destfile//%\{track\}/$tracknumber}"
destfile="${destfile//%\{year\}/$year}"
destfile="${destfile//%\{disc\}/$disc}"
destfile="${destfile//?(\[)%\{track\}?(\])/$tracknumber}"
destfile="${destfile//?(\[)%\{year\}?(\])/$year}"
destfile="${destfile//?(\[)%\{disc\}?(\])/$disc}"
else
destfile="${filename##*/}"
destfile="${destfile%.*}"
fi
if (( ${destinationascii["$destination"]} ))
then
echo "$destfile" >&${toascii[1]}
read -r -u${toascii[0]} destfile
fi
destfile=$(sanitizeFile "$destfile")
}

30
lib/setup/destination
View File

@ -295,6 +295,36 @@ setupDestination() {
comeagain
cat <<-EODesc
ASCII-only filenames (boolean):
Rename files for compatibility with ASCII-only systems (most car
radios).
EODesc
case ${destinationascii["$destination"]} in
0) initialvalue=n ;;
1) initialvalue=y ;;
*) unset initialvalue ;;
esac
comeagain() {
read \
-e \
${initialvalue+-i $initialvalue}\
-p'ASCII-only filenames (y/N): '\
value
case $value in
[yY])
destinationascii["$destination"]=1
;;
''|[nN])
destinationascii["$destination"]=0
;;
*)
comeagain
;;
esac
}
comeagain
cat <<-EODesc
Skip mime-type (mime-type, string):
Files with mime-type <mime-type> will not be included in that
destination. The '*' character is a wildcard.

14
lib/tools/ascii Normal file
View File

@ -0,0 +1,14 @@
#!/bin/bash
ascii() {
coproc toascii {
perl -e '
use utf8;
use Text::Unidecode;
use open qw(:std :utf8);
$| = 1;
while (<>) {
print(unidecode($_));
}'
}
}

10
lib/workers/cleaner
View File

@ -47,6 +47,16 @@ cleaner() {
" SELECT rename_pattern" \
" FROM tasks" \
" WHERE id=$taskid" \
" )," \
" fat32compat=(" \
" SELECT fat32compat" \
" FROM tasks" \
" WHERE id=$taskid" \
" )," \
" ascii=(" \
" SELECT ascii" \
" FROM tasks" \
" WHERE id=$taskid" \
" )" \
"WHERE id=$destfileid;" \
>&3

2
share/schema.sql
View File

@ -22,6 +22,8 @@ CREATE TABLE IF NOT EXISTS destination_files (
filename TEXT,
old_filename TEXT,
rename_pattern TEXT,
fat32compat INTEGER,
ascii INTEGER,
last_change FLOAT NOT NULL DEFAULT 0,
source_file_id INTEGER,
destination_id INTEGER,