ScriptFanix/AtOM
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Compare commits

base: ScriptFanix:c282e9c361ed3e149660cdd27bedb978ed312872
Branches Tags
ScriptFanix:master
ScriptFanix:25-filenames-can-have-linefeeds-in-them
ScriptFanix:28-store-dest-filename-relative
ScriptFanix:dev
ScriptFanix:27-script-is-uselessly-verbose
ScriptFanix:29-many-error-codes-are-not-defined
ScriptFanix:26-ffmpeg-tags-are-always-null
ScriptFanix:18-tag-guessing
ScriptFanix:17-read-and-copy-replaygain
ScriptFanix:16-unable-to-saturate-cpus-on-sufficiently-powerfulm-hartdware
ScriptFanix:15-temp-directory-not-created-for-toys
ScriptFanix:10-bug-releasecountry-change-should-update-dest-files
ScriptFanix:8-feat-support-for-releasecountry-tag
ScriptFanix:7-provide-a-proper-way-to-install
ScriptFanix:xdg-compliance
ScriptFanix:4-with-rename-pattern-destination-files-can-have-in-db-if-tags-missing
ScriptFanix:6-readme-md-doesn-t-explain-what-this-is-doing
ScriptFanix:5-looks-like-too-many-tasks-fail
ScriptFanix:get-rid-of-transogg
ScriptFanix:v1.0.4
ScriptFanix:v1.0.3
ScriptFanix:v1.0.2
ScriptFanix:v1.0.1
ScriptFanix:v1.0.0
..
compare: ScriptFanix:434152daba95cb8d724bda8cdc45ba9c7ca74c6a
Branches Tags
ScriptFanix:25-filenames-can-have-linefeeds-in-them
ScriptFanix:28-store-dest-filename-relative
ScriptFanix:dev
ScriptFanix:27-script-is-uselessly-verbose
ScriptFanix:29-many-error-codes-are-not-defined
ScriptFanix:master
ScriptFanix:26-ffmpeg-tags-are-always-null
ScriptFanix:18-tag-guessing
ScriptFanix:17-read-and-copy-replaygain
ScriptFanix:16-unable-to-saturate-cpus-on-sufficiently-powerfulm-hartdware
ScriptFanix:15-temp-directory-not-created-for-toys
ScriptFanix:10-bug-releasecountry-change-should-update-dest-files
ScriptFanix:8-feat-support-for-releasecountry-tag
ScriptFanix:7-provide-a-proper-way-to-install
ScriptFanix:xdg-compliance
ScriptFanix:4-with-rename-pattern-destination-files-can-have-in-db-if-tags-missing
ScriptFanix:6-readme-md-doesn-t-explain-what-this-is-doing
ScriptFanix:5-looks-like-too-many-tasks-fail
ScriptFanix:get-rid-of-transogg
ScriptFanix:v1.0.4
ScriptFanix:v1.0.3
ScriptFanix:v1.0.2
ScriptFanix:v1.0.1
ScriptFanix:v1.0.0

No commits in common. "c282e9c361ed3e149660cdd27bedb978ed312872" and "434152daba95cb8d724bda8cdc45ba9c7ca74c6a" have entirely different histories.
c282e9c361 ... 434152daba

11 changed files with 192 additions and 319 deletions
Show Stats Expand all files Collapse all files

5
atom
View File

@ -429,7 +429,7 @@ echo '
CREATE TEMPORARY TABLE tasks( CREATE TEMPORARY TABLE tasks(
id INTEGER PRIMARY KEY, id INTEGER PRIMARY KEY,
requires INTEGER, requires INTEGER,
required_by INTEGER DEFAULT 0, required INTEGER,
status INTEGER NOT NULL, status INTEGER NOT NULL,
key TEXT UNIQUE, key TEXT UNIQUE,
rename_pattern TEXT, rename_pattern TEXT,
@ -906,7 +906,8 @@ then
FROM tasks FROM tasks
INNER JOIN source_files INNER JOIN source_files
ON tasks.source_file=source_files.id ON tasks.source_file=source_files.id
WHERE tasks.status = 2; WHERE tasks.status = 2
AND requires is NULL;
SELECT "AtOM:NoMoreFiles";' >&3 SELECT "AtOM:NoMoreFiles";' >&3
read -u4 line read -u4 line

2
lib/database/open
View File

@ -11,7 +11,7 @@ openDatabase() {
if (( debug > 2 )) if (( debug > 2 ))
then then
exec 5>&3 exec 5>&3
exec 3> >(tee -a "$tempdir/debug.log" >&5) exec 3> >(tee -a $tempdir/debug.log >&5)
fi fi
cat $schema >&3 cat $schema >&3
echo '.separator ::AtOM:SQL:Sep::' >&3 echo '.separator ::AtOM:SQL:Sep::' >&3

11
lib/decode/file
View File

@ -153,24 +153,18 @@ decodeFile() {
done done
) )
status 0 status 0
cleanup $tmpfile.wav
EOInsert EOInsert
) )
progressSpin progressSpin
fi fi
if (( sox_needed )) if (( sox_needed ))
then then
cleanup="$tempdir/$tmpfile"
decodeSox "$tempdir/$tmpfile.wav" decodeSox "$tempdir/$tmpfile.wav"
if ! soxtaskid=$( if ! soxtaskid=$(
Select tasks id <<<"key = $tmpfile" Select tasks id <<<"key = $tmpfile"
) )
then then
parent_required=$(
Select tasks required_by \
<<<"id = $decodetaskid"
)
Update tasks required_by $((++parent_required)) \
<<<"id = $decodetaskid"
soxtaskid=$( soxtaskid=$(
Insert tasks <<-EOInsert Insert tasks <<-EOInsert
key $tmpfile key $tmpfile
@ -182,8 +176,9 @@ decodeFile() {
done done
) )
requires $decodetaskid requires $decodetaskid
required $decodetaskid
status 0 status 0
cleanup $tmpfile.wav cleanup $cleanup
EOInsert EOInsert
) )
progressSpin progressSpin

8
lib/encode/mp3
View File

@ -52,6 +52,7 @@ encodeFile::mp3() {
Insert tasks <<-EOInsert Insert tasks <<-EOInsert
key ${fileid}lame$destination key ${fileid}lame$destination
requires ${soxtaskid:-$decodetaskid} requires ${soxtaskid:-$decodetaskid}
required ${soxtaskid:-$decodetaskid}
fileid $destfileid fileid $destfileid
filename $destdir/$destfile.mp3 filename $destdir/$destfile.mp3
$( $(
@ -65,6 +66,7 @@ encodeFile::mp3() {
echo "cmd_arg$key $cleanedopts" echo "cmd_arg$key $cleanedopts"
done done
) )
cleanup $tempdir/$tmpfile.wav
source_file $fileid source_file $fileid
status 0 status 0
rename_pattern ${destinationrenamepath[$destination]}/${destinationrename[$destination]} rename_pattern ${destinationrenamepath[$destination]}/${destinationrename[$destination]}
@ -72,12 +74,6 @@ encodeFile::mp3() {
ascii ${destinationascii["$destination"]} ascii ${destinationascii["$destination"]}
EOInsert EOInsert
) )
parent_required=$(
Select tasks required_by \
<<<"id = ${soxtaskid:-$decodetaskid}"
)
Update tasks required_by $((++parent_required)) \
<<<"id = ${soxtaskid:-$decodetaskid}"
progressSpin progressSpin
soxtaskid='' soxtaskid=''
} }

8
lib/encode/opus
View File

@ -20,6 +20,7 @@ encodeFile::opus() {
Insert tasks <<-EOInsert Insert tasks <<-EOInsert
key ${fileid}opusenc$destination key ${fileid}opusenc$destination
requires ${soxtaskid:-$decodetaskid} requires ${soxtaskid:-$decodetaskid}
required ${soxtaskid:-$decodetaskid}
fileid $destfileid fileid $destfileid
filename $destdir/$destfile.opus filename $destdir/$destfile.opus
$( $(
@ -33,6 +34,7 @@ encodeFile::opus() {
echo "cmd_arg$key $cleanedopts" echo "cmd_arg$key $cleanedopts"
done done
) )
cleanup $tempdir/$tmpfile.wav
source_file $fileid source_file $fileid
status 0 status 0
rename_pattern ${destinationrenamepath[$destination]}/${destinationrename[$destination]} rename_pattern ${destinationrenamepath[$destination]}/${destinationrename[$destination]}
@ -40,12 +42,6 @@ encodeFile::opus() {
ascii ${destinationascii["$destination"]} ascii ${destinationascii["$destination"]}
EOInsert EOInsert
) )
parent_required=$(
Select tasks required_by \
<<<"id = ${soxtaskid:-$decodetaskid}"
)
Update tasks required_by $((++parent_required)) \
<<<"id = ${soxtaskid:-$decodetaskid}"
progressSpin progressSpin
soxtaskid='' soxtaskid=''
} }

8
lib/encode/vorbis
View File

@ -16,6 +16,7 @@ encodeFile::vorbis() {
Insert tasks <<-EOInsert Insert tasks <<-EOInsert
key ${fileid}oggenc$destination key ${fileid}oggenc$destination
requires ${soxtaskid:-$decodetaskid} requires ${soxtaskid:-$decodetaskid}
required ${soxtaskid:-$decodetaskid}
fileid $destfileid fileid $destfileid
filename $destdir/$destfile.ogg filename $destdir/$destfile.ogg
$( $(
@ -29,6 +30,7 @@ encodeFile::vorbis() {
echo "cmd_arg$key ${oggencopts[key]}" echo "cmd_arg$key ${oggencopts[key]}"
done done
) )
cleanup $tempdir/$tmpfile.wav
source_file $fileid source_file $fileid
status 0 status 0
rename_pattern ${destinationrenamepath[$destination]}/${destinationrename[$destination]} rename_pattern ${destinationrenamepath[$destination]}/${destinationrename[$destination]}
@ -36,12 +38,6 @@ encodeFile::vorbis() {
ascii ${destinationascii["$destination"]} ascii ${destinationascii["$destination"]}
EOInsert EOInsert
) )
parent_required=$(
Select tasks required_by \
<<<"id = ${soxtaskid:-$decodetaskid}"
)
Update tasks required_by $((++parent_required)) \
<<<"id = ${soxtaskid:-$decodetaskid}"
progressSpin progressSpin
soxtaskid='' soxtaskid=''
} }

3
lib/tasks/gettaskinfos
View File

@ -4,6 +4,7 @@ gettaskinfos() {
SELECT SELECT
id, id,
source_file, source_file,
required,
cleanup, cleanup,
fileid, fileid,
filename filename
@ -15,6 +16,8 @@ gettaskinfos() {
rest="${line#*::AtOM:SQL:Sep::}::AtOM:SQL:Sep::" rest="${line#*::AtOM:SQL:Sep::}::AtOM:SQL:Sep::"
sourcefileid=${rest%%::AtOM:SQL:Sep::*} sourcefileid=${rest%%::AtOM:SQL:Sep::*}
rest=${rest#*::AtOM:SQL:Sep::} rest=${rest#*::AtOM:SQL:Sep::}
required=${rest%%::AtOM:SQL:Sep::*}
rest=${rest#*::AtOM:SQL:Sep::}
cleanup=${rest%%::AtOM:SQL:Sep::*} cleanup=${rest%%::AtOM:SQL:Sep::*}
rest=${rest#*::AtOM:SQL:Sep::} rest=${rest#*::AtOM:SQL:Sep::}
destfileid=${rest%%::AtOM:SQL:Sep::*} destfileid=${rest%%::AtOM:SQL:Sep::*}

17
lib/workers/check
View File

@ -1,9 +1,5 @@
#!/bin/bash #!/bin/bash
checkworkers() { checkworkers() {
local \
taskid \
parent_required \
parent_task
for key in ${!workers[@]} for key in ${!workers[@]}
do do
if ! kill -0 ${workers[key]} 2>/dev/null if ! kill -0 ${workers[key]} 2>/dev/null
@ -18,19 +14,6 @@ checkworkers() {
failedtasks+=($taskid) failedtasks+=($taskid)
(( ++failed )) (( ++failed ))
fi fi
parent_task=$(
Select tasks requires \
<<<"id = $taskid"
)
if (( parent_task ))
then
parent_required=$(
Select tasks required_by \
<<<"id = $parent_task"
)
Update tasks required_by $((--parent_required)) \
<<<"id = $parent_task"
fi
unset workertasks[key] unset workertasks[key]
fi fi
done done

40
lib/workers/cleaner
View File

@ -1,23 +1,26 @@
#!/bin/bash #!/bin/bash
cleaner() { cleaner() {
local \
key \
faildepends \
taskid \
count
for key in ${!failedtasks[@]} for key in ${!failedtasks[@]}
do do
taskid=${failedtasks[key]} taskid=${failedtasks[key]}
gettaskinfos $taskid gettaskinfos $taskid
faildepends=$( faildepends=$(
Select tasks required_by <<-EOWhere Select tasks 'COUNT(*)' <<-EOWhere
id = $taskid requires = $taskid
EOWhere EOWhere
) )
(( failed+=faildepends )) (( failed+=faildepends ))
(( ran+=faildepends )) (( ran+=faildepends ))
Update tasks status 2 <<<"id = $taskid" Update tasks status 2 <<<"id = $taskid"
rm -f "$cleanup" echo "SELECT COUNT(*)
FROM tasks
WHERE ( status = 0 OR status = 1 )
AND required = $taskid;">&3
read -u4 count
if (( count == 0 ))
then
rm -f "$cleanup"
fi
unset failedtasks[key] unset failedtasks[key]
done done
for key in ${!finishedtasks[@]} for key in ${!finishedtasks[@]}
@ -44,28 +47,29 @@ cleaner() {
" FROM tasks" \ " FROM tasks" \
" WHERE id=$taskid" \ " WHERE id=$taskid" \
" )," \ " )," \
" fat32compat=(" \ " fat32compat=(" \
" SELECT fat32compat" \ " SELECT fat32compat" \
" FROM tasks" \ " FROM tasks" \
" WHERE id=$taskid" \ " WHERE id=$taskid" \
" )," \ " )," \
" ascii=(" \ " ascii=(" \
" SELECT ascii" \ " SELECT ascii" \
" FROM tasks" \ " FROM tasks" \
" WHERE id=$taskid" \ " WHERE id=$taskid" \
" )" \ " )" \
"WHERE id=$destfileid;" \ "WHERE id=$destfileid;" \
>&3 >&3
fi fi
count=$(Select tasks required_by <<<"id = $taskid") echo "SELECT COUNT(*)
FROM tasks
WHERE ( status = 0 OR status = 1 )
AND required = $taskid;">&3
read -u4 count
if (( count == 0 )) if (( count == 0 ))
then then
[[ -n "$cleanup" ]] && rm -f "$tempdir/$cleanup" rm -f "$cleanup"
Delete tasks <<<"id = $taskid"
unset finishedtasks[key]
else
Update tasks status 4 \
<<<"id = $taskid"
fi fi
Delete tasks <<<"id = $taskid"
unset finishedtasks[key]
done done
} }

148
lib/workers/create
View File

@ -1,145 +1,5 @@
#!/usr/bin/env bash #!/bin/bash
createworker() { createworker() {
(( ++active )) worker $1 &
read -u4 line workers[$1]=$!
taskid=${line%%::AtOM:SQL:Sep::*} }
rest="${line#*::AtOM:SQL:Sep::}::AtOM:SQL:Sep::"
sourcefileid=${rest%%::AtOM:SQL:Sep::*}
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cleanup=${rest%%::AtOM:SQL:Sep::*}
rest=${rest#*::AtOM:SQL:Sep::}
destfileid=${rest%%::AtOM:SQL:Sep::*}
rest=${rest#*::AtOM:SQL:Sep::}
destfilename=${rest%%::AtOM:SQL:Sep::*}
rest=${rest#*::AtOM:SQL:Sep::}
for key in ${!cmd_arg[@]}
do
[ -z "${cmd_arg[key]}" ] && unset cmd_arg[key]
done
workerid=$(getworkerid)
workertasks[workerid]=$taskid
Update tasks status 1 <<<"id = $taskid"
worker $workerid &
workers[$workerid]=$!
}

261
lib/workers/master
View File

@ -8,112 +8,7 @@ master() {
SELECT COUNT(*) SELECT COUNT(*)
FROM tasks FROM tasks
WHERE status = 0; WHERE status = 0;
'>&3
read -u4 remaining
if (( remaining == 0 ))
then
sleep 0.1
return 0
fi
echo '
SELECT COUNT(*)
FROM tasks
WHERE status = 0
AND requires IN (
SELECT id
FROM tasks
WHERE status = 4
);
SELECT
id,
source_file,
cmd_arg0,
cmd_arg1,
cmd_arg2,
cmd_arg3,
cmd_arg4,
cmd_arg5,
cmd_arg6,
cmd_arg7,
cmd_arg8,
cmd_arg9,
cmd_arg10,
cmd_arg11,
cmd_arg12,
cmd_arg13,
cmd_arg14,
cmd_arg15,
cmd_arg16,
cmd_arg17,
cmd_arg18,
cmd_arg19,
cmd_arg20,
cmd_arg21,
cmd_arg22,
cmd_arg23,
cmd_arg24,
cmd_arg25,
cmd_arg26,
cmd_arg27,
cmd_arg28,
cmd_arg29,
cmd_arg30,
cmd_arg31,
cmd_arg32,
cmd_arg33,
cmd_arg34,
cmd_arg35,
cmd_arg36,
cmd_arg37,
cmd_arg38,
cmd_arg39,
cmd_arg40,
cmd_arg41,
cmd_arg42,
cmd_arg43,
cmd_arg44,
cmd_arg45,
cmd_arg46,
cmd_arg47,
cmd_arg48,
cmd_arg49,
cmd_arg50,
cmd_arg51,
cmd_arg52,
cmd_arg53,
cmd_arg54,
cmd_arg55,
cmd_arg56,
cmd_arg57,
cmd_arg58,
cmd_arg59,
cleanup,
fileid,
filename
FROM tasks
WHERE status = 0
AND requires IN (
SELECT id
FROM tasks
WHERE status = 4
ORDER BY source_file
/* LIMIT 1 */
)
ORDER BY source_file
LIMIT 1;
'>&3
read -u4 ready
if (( ready > 0 ))
then
createworker
return 0
fi
echo '
SELECT COUNT(*) SELECT COUNT(*)
FROM tasks FROM tasks
WHERE status = 0 WHERE status = 0
@ -122,6 +17,7 @@ master() {
SELECT SELECT
id, id,
source_file, source_file,
required,
cmd_arg0, cmd_arg0,
cmd_arg1, cmd_arg1,
cmd_arg2, cmd_arg2,
@ -191,12 +87,15 @@ master() {
ORDER BY source_file ORDER BY source_file
LIMIT 1; LIMIT 1;
' >&3 ' >&3
read -u4 remaining
read -u4 ready read -u4 ready
if (( remaining == 0 ))
if (( active == 0 && ready == 0 ))
then then
dumpfile="$tempdir/tasks-$(date -Iseconds).csv" sleep 0.1
return 0
elif (( active == 0 && ready == 0 ))
then
dumpfile=tasks-$(date +%Y%m%d%H%M%S).csv
cat <<-EOF cat <<-EOF
@ -212,7 +111,6 @@ master() {
.mode list .mode list
.headers off .headers off
.output stdout .output stdout
COMMIT;
EOSQL EOSQL
closeDatabase closeDatabase
echo "Waiting for children to come back home..." echo "Waiting for children to come back home..."
@ -223,7 +121,148 @@ master() {
then then
sleep 0.1 sleep 0.1
else else
createworker (( ++active ))
read -u4 line
taskid=${line%%::AtOM:SQL:Sep::*}
rest="${line#*::AtOM:SQL:Sep::}::AtOM:SQL:Sep::"
sourcefileid=${rest%%::AtOM:SQL:Sep::*}
rest=${rest#*::AtOM:SQL:Sep::}
required=${rest%%::AtOM:SQL:Sep::*}
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cleanup=${rest%%::AtOM:SQL:Sep::*}
rest=${rest#*::AtOM:SQL:Sep::}
destfileid=${rest%%::AtOM:SQL:Sep::*}
rest=${rest#*::AtOM:SQL:Sep::}
destfilename=${rest%%::AtOM:SQL:Sep::*}
rest=${rest#*::AtOM:SQL:Sep::}
for key in ${!cmd_arg[@]}
do
[ -z "${cmd_arg[key]}" ] && unset cmd_arg[key]
done
workerid=$(getworkerid)
workertasks[workerid]=$taskid
Update tasks status 1 <<<"id = $taskid"
createworker $workerid
fi fi
fi fi
} }