ScriptFanix/AtOM
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Compare commits

merge into: ScriptFanix:master
Branches Tags
ScriptFanix:master
ScriptFanix:25-filenames-can-have-linefeeds-in-them
ScriptFanix:28-store-dest-filename-relative
ScriptFanix:dev
ScriptFanix:27-script-is-uselessly-verbose
ScriptFanix:29-many-error-codes-are-not-defined
ScriptFanix:26-ffmpeg-tags-are-always-null
ScriptFanix:18-tag-guessing
ScriptFanix:17-read-and-copy-replaygain
ScriptFanix:16-unable-to-saturate-cpus-on-sufficiently-powerfulm-hartdware
ScriptFanix:15-temp-directory-not-created-for-toys
ScriptFanix:10-bug-releasecountry-change-should-update-dest-files
ScriptFanix:8-feat-support-for-releasecountry-tag
ScriptFanix:7-provide-a-proper-way-to-install
ScriptFanix:xdg-compliance
ScriptFanix:4-with-rename-pattern-destination-files-can-have-in-db-if-tags-missing
ScriptFanix:6-readme-md-doesn-t-explain-what-this-is-doing
ScriptFanix:5-looks-like-too-many-tasks-fail
ScriptFanix:get-rid-of-transogg
ScriptFanix:v1.0.4
ScriptFanix:v1.0.3
ScriptFanix:v1.0.2
ScriptFanix:v1.0.1
ScriptFanix:v1.0.0
...
pull from: ScriptFanix:5-looks-like-too-many-tasks-fail
Branches Tags
ScriptFanix:25-filenames-can-have-linefeeds-in-them
ScriptFanix:28-store-dest-filename-relative
ScriptFanix:dev
ScriptFanix:27-script-is-uselessly-verbose
ScriptFanix:29-many-error-codes-are-not-defined
ScriptFanix:master
ScriptFanix:26-ffmpeg-tags-are-always-null
ScriptFanix:18-tag-guessing
ScriptFanix:17-read-and-copy-replaygain
ScriptFanix:16-unable-to-saturate-cpus-on-sufficiently-powerfulm-hartdware
ScriptFanix:15-temp-directory-not-created-for-toys
ScriptFanix:10-bug-releasecountry-change-should-update-dest-files
ScriptFanix:8-feat-support-for-releasecountry-tag
ScriptFanix:7-provide-a-proper-way-to-install
ScriptFanix:xdg-compliance
ScriptFanix:4-with-rename-pattern-destination-files-can-have-in-db-if-tags-missing
ScriptFanix:6-readme-md-doesn-t-explain-what-this-is-doing
ScriptFanix:5-looks-like-too-many-tasks-fail
ScriptFanix:get-rid-of-transogg
ScriptFanix:v1.0.4
ScriptFanix:v1.0.3
ScriptFanix:v1.0.2
ScriptFanix:v1.0.1
ScriptFanix:v1.0.0

8 Commits
master ... 5-looks-li

Author SHA1 Message Date
Vincent Riquer
485f1c7f50 fix: On inconsistency, COMMIT before closing database 2025-01-28 23:29:19 +01:00
Vincent Riquer
60d259b200 fix: file cleanup 2025-01-28 23:28:11 +01:00
Vincent Riquer
9b8113fdff fix: use required_by, check child task status 2025-01-28 23:26:50 +01:00
Vincent Riquer
a3e80e9416 cleanup: move task creation to a function 2025-01-28 23:23:40 +01:00
Vincent Riquer
ba459feb90 fix: change tasks table schema 
replace required with required_by
change meaning of field
 2025-01-28 22:54:09 +01:00
Vincent Riquer
5f9bd027f3 fix: quote variable in database debug output 2025-01-28 15:49:38 +01:00
Vincent Riquer
7963754379 remove unnecessary createworker function 2025-01-28 02:52:44 +01:00
Vincent Riquer
9aedb06c03 workers/master: dump DB to $tmpdir on failure 2025-01-28 00:24:54 +01:00
11 changed files with 318 additions and 190 deletions
Show Stats Expand all files Collapse all files

2
atom
View File

@ -429,7 +429,7 @@ echo '
CREATE TEMPORARY TABLE tasks( CREATE TEMPORARY TABLE tasks(
id INTEGER PRIMARY KEY, id INTEGER PRIMARY KEY,
requires INTEGER, requires INTEGER,
required INTEGER, required_by INTEGER DEFAULT 0,
status INTEGER NOT NULL, status INTEGER NOT NULL,
key TEXT UNIQUE, key TEXT UNIQUE,
rename_pattern TEXT, rename_pattern TEXT,

2
lib/database/open
View File

@ -11,7 +11,7 @@ openDatabase() {
if (( debug > 2 )) if (( debug > 2 ))
then then
exec 5>&3 exec 5>&3
exec 3> >(tee -a $tempdir/debug.log >&5) exec 3> >(tee -a "$tempdir/debug.log" >&5)
fi fi
cat $schema >&3 cat $schema >&3
echo '.separator ::AtOM:SQL:Sep::' >&3 echo '.separator ::AtOM:SQL:Sep::' >&3

11
lib/decode/file
View File

@ -153,18 +153,24 @@ decodeFile() {
done done
) )
status 0 status 0
cleanup $tmpfile.wav
EOInsert EOInsert
) )
progressSpin progressSpin
fi fi
if (( sox_needed )) if (( sox_needed ))
then then
cleanup="$tempdir/$tmpfile"
decodeSox "$tempdir/$tmpfile.wav" decodeSox "$tempdir/$tmpfile.wav"
if ! soxtaskid=$( if ! soxtaskid=$(
Select tasks id <<<"key = $tmpfile" Select tasks id <<<"key = $tmpfile"
) )
then then
parent_required=$(
Select tasks required_by \
<<<"id = $decodetaskid"
)
Update tasks required_by $((++parent_required)) \
<<<"id = $decodetaskid"
soxtaskid=$( soxtaskid=$(
Insert tasks <<-EOInsert Insert tasks <<-EOInsert
key $tmpfile key $tmpfile
@ -176,9 +182,8 @@ decodeFile() {
done done
) )
requires $decodetaskid requires $decodetaskid
required $decodetaskid
status 0 status 0
cleanup $cleanup cleanup $tmpfile.wav
EOInsert EOInsert
) )
progressSpin progressSpin

8
lib/encode/mp3
View File

@ -52,7 +52,6 @@ encodeFile::mp3() {
Insert tasks <<-EOInsert Insert tasks <<-EOInsert
key ${fileid}lame$destination key ${fileid}lame$destination
requires ${soxtaskid:-$decodetaskid} requires ${soxtaskid:-$decodetaskid}
required ${soxtaskid:-$decodetaskid}
fileid $destfileid fileid $destfileid
filename $destdir/$destfile.mp3 filename $destdir/$destfile.mp3
$( $(
@ -66,7 +65,6 @@ encodeFile::mp3() {
echo "cmd_arg$key $cleanedopts" echo "cmd_arg$key $cleanedopts"
done done
) )
cleanup $tempdir/$tmpfile.wav
source_file $fileid source_file $fileid
status 0 status 0
rename_pattern ${destinationrenamepath[$destination]}/${destinationrename[$destination]} rename_pattern ${destinationrenamepath[$destination]}/${destinationrename[$destination]}
@ -74,6 +72,12 @@ encodeFile::mp3() {
ascii ${destinationascii["$destination"]} ascii ${destinationascii["$destination"]}
EOInsert EOInsert
) )
parent_required=$(
Select tasks required_by \
<<<"id = ${soxtaskid:-$decodetaskid}"
)
Update tasks required_by $((++parent_required)) \
<<<"id = ${soxtaskid:-$decodetaskid}"
progressSpin progressSpin
soxtaskid='' soxtaskid=''
} }

8
lib/encode/opus
View File

@ -20,7 +20,6 @@ encodeFile::opus() {
Insert tasks <<-EOInsert Insert tasks <<-EOInsert
key ${fileid}opusenc$destination key ${fileid}opusenc$destination
requires ${soxtaskid:-$decodetaskid} requires ${soxtaskid:-$decodetaskid}
required ${soxtaskid:-$decodetaskid}
fileid $destfileid fileid $destfileid
filename $destdir/$destfile.opus filename $destdir/$destfile.opus
$( $(
@ -34,7 +33,6 @@ encodeFile::opus() {
echo "cmd_arg$key $cleanedopts" echo "cmd_arg$key $cleanedopts"
done done
) )
cleanup $tempdir/$tmpfile.wav
source_file $fileid source_file $fileid
status 0 status 0
rename_pattern ${destinationrenamepath[$destination]}/${destinationrename[$destination]} rename_pattern ${destinationrenamepath[$destination]}/${destinationrename[$destination]}
@ -42,6 +40,12 @@ encodeFile::opus() {
ascii ${destinationascii["$destination"]} ascii ${destinationascii["$destination"]}
EOInsert EOInsert
) )
parent_required=$(
Select tasks required_by \
<<<"id = ${soxtaskid:-$decodetaskid}"
)
Update tasks required_by $((++parent_required)) \
<<<"id = ${soxtaskid:-$decodetaskid}"
progressSpin progressSpin
soxtaskid='' soxtaskid=''
} }

8
lib/encode/vorbis
View File

@ -16,7 +16,6 @@ encodeFile::vorbis() {
Insert tasks <<-EOInsert Insert tasks <<-EOInsert
key ${fileid}oggenc$destination key ${fileid}oggenc$destination
requires ${soxtaskid:-$decodetaskid} requires ${soxtaskid:-$decodetaskid}
required ${soxtaskid:-$decodetaskid}
fileid $destfileid fileid $destfileid
filename $destdir/$destfile.ogg filename $destdir/$destfile.ogg
$( $(
@ -30,7 +29,6 @@ encodeFile::vorbis() {
echo "cmd_arg$key ${oggencopts[key]}" echo "cmd_arg$key ${oggencopts[key]}"
done done
) )
cleanup $tempdir/$tmpfile.wav
source_file $fileid source_file $fileid
status 0 status 0
rename_pattern ${destinationrenamepath[$destination]}/${destinationrename[$destination]} rename_pattern ${destinationrenamepath[$destination]}/${destinationrename[$destination]}
@ -38,6 +36,12 @@ encodeFile::vorbis() {
ascii ${destinationascii["$destination"]} ascii ${destinationascii["$destination"]}
EOInsert EOInsert
) )
parent_required=$(
Select tasks required_by \
<<<"id = ${soxtaskid:-$decodetaskid}"
)
Update tasks required_by $((++parent_required)) \
<<<"id = ${soxtaskid:-$decodetaskid}"
progressSpin progressSpin
soxtaskid='' soxtaskid=''
} }

3
lib/tasks/gettaskinfos
View File

@ -4,7 +4,6 @@ gettaskinfos() {
SELECT SELECT
id, id,
source_file, source_file,
required,
cleanup, cleanup,
fileid, fileid,
filename filename
@ -16,8 +15,6 @@ gettaskinfos() {
rest="${line#*::AtOM:SQL:Sep::}::AtOM:SQL:Sep::" rest="${line#*::AtOM:SQL:Sep::}::AtOM:SQL:Sep::"
sourcefileid=${rest%%::AtOM:SQL:Sep::*} sourcefileid=${rest%%::AtOM:SQL:Sep::*}
rest=${rest#*::AtOM:SQL:Sep::} rest=${rest#*::AtOM:SQL:Sep::}
required=${rest%%::AtOM:SQL:Sep::*}
rest=${rest#*::AtOM:SQL:Sep::}
cleanup=${rest%%::AtOM:SQL:Sep::*} cleanup=${rest%%::AtOM:SQL:Sep::*}
rest=${rest#*::AtOM:SQL:Sep::} rest=${rest#*::AtOM:SQL:Sep::}
destfileid=${rest%%::AtOM:SQL:Sep::*} destfileid=${rest%%::AtOM:SQL:Sep::*}

17
lib/workers/check
View File

@ -1,5 +1,9 @@
#!/bin/bash #!/bin/bash
checkworkers() { checkworkers() {
local \
taskid \
parent_required \
parent_task
for key in ${!workers[@]} for key in ${!workers[@]}
do do
if ! kill -0 ${workers[key]} 2>/dev/null if ! kill -0 ${workers[key]} 2>/dev/null
@ -14,6 +18,19 @@ checkworkers() {
failedtasks+=($taskid) failedtasks+=($taskid)
(( ++failed )) (( ++failed ))
fi fi
parent_task=$(
Select tasks requires \
<<<"id = $taskid"
)
if (( parent_task ))
then
parent_required=$(
Select tasks required_by \
<<<"id = $parent_task"
)
Update tasks required_by $((--parent_required)) \
<<<"id = $parent_task"
fi
unset workertasks[key] unset workertasks[key]
fi fi
done done

30
lib/workers/cleaner
View File

@ -1,26 +1,23 @@
#!/bin/bash #!/bin/bash
cleaner() { cleaner() {
local \
key \
faildepends \
taskid \
count
for key in ${!failedtasks[@]} for key in ${!failedtasks[@]}
do do
taskid=${failedtasks[key]} taskid=${failedtasks[key]}
gettaskinfos $taskid gettaskinfos $taskid
faildepends=$( faildepends=$(
Select tasks 'COUNT(*)' <<-EOWhere Select tasks required_by <<-EOWhere
requires = $taskid id = $taskid
EOWhere EOWhere
) )
(( failed+=faildepends )) (( failed+=faildepends ))
(( ran+=faildepends )) (( ran+=faildepends ))
Update tasks status 2 <<<"id = $taskid" Update tasks status 2 <<<"id = $taskid"
echo "SELECT COUNT(*)
FROM tasks
WHERE ( status = 0 OR status = 1 )
AND required = $taskid;">&3
read -u4 count
if (( count == 0 ))
then
rm -f "$cleanup" rm -f "$cleanup"
fi
unset failedtasks[key] unset failedtasks[key]
done done
for key in ${!finishedtasks[@]} for key in ${!finishedtasks[@]}
@ -60,16 +57,15 @@ cleaner() {
"WHERE id=$destfileid;" \ "WHERE id=$destfileid;" \
>&3 >&3
fi fi
echo "SELECT COUNT(*) count=$(Select tasks required_by <<<"id = $taskid")
FROM tasks
WHERE ( status = 0 OR status = 1 )
AND required = $taskid;">&3
read -u4 count
if (( count == 0 )) if (( count == 0 ))
then then
rm -f "$cleanup" [[ -n "$cleanup" ]] && rm -f "$tempdir/$cleanup"
fi
Delete tasks <<<"id = $taskid" Delete tasks <<<"id = $taskid"
unset finishedtasks[key] unset finishedtasks[key]
else
Update tasks status 4 \
<<<"id = $taskid"
fi
done done
} }

146
lib/workers/create
View File

@ -1,5 +1,145 @@
#!/bin/bash #!/usr/bin/env bash
createworker() { createworker() {
worker $1 & (( ++active ))
workers[$1]=$! read -u4 line
taskid=${line%%::AtOM:SQL:Sep::*}
rest="${line#*::AtOM:SQL:Sep::}::AtOM:SQL:Sep::"
sourcefileid=${rest%%::AtOM:SQL:Sep::*}
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cleanup=${rest%%::AtOM:SQL:Sep::*}
rest=${rest#*::AtOM:SQL:Sep::}
destfileid=${rest%%::AtOM:SQL:Sep::*}
rest=${rest#*::AtOM:SQL:Sep::}
destfilename=${rest%%::AtOM:SQL:Sep::*}
rest=${rest#*::AtOM:SQL:Sep::}
for key in ${!cmd_arg[@]}
do
[ -z "${cmd_arg[key]}" ] && unset cmd_arg[key]
done
workerid=$(getworkerid)
workertasks[workerid]=$taskid
Update tasks status 1 <<<"id = $taskid"
worker $workerid &
workers[$workerid]=$!
} }

261
lib/workers/master
View File

@ -8,7 +8,112 @@ master() {
SELECT COUNT(*) SELECT COUNT(*)
FROM tasks FROM tasks
WHERE status = 0; WHERE status = 0;
'>&3
read -u4 remaining
if (( remaining == 0 ))
then
sleep 0.1
return 0
fi
echo '
SELECT COUNT(*)
FROM tasks
WHERE status = 0
AND requires IN (
SELECT id
FROM tasks
WHERE status = 4
);
SELECT
id,
source_file,
cmd_arg0,
cmd_arg1,
cmd_arg2,
cmd_arg3,
cmd_arg4,
cmd_arg5,
cmd_arg6,
cmd_arg7,
cmd_arg8,
cmd_arg9,
cmd_arg10,
cmd_arg11,
cmd_arg12,
cmd_arg13,
cmd_arg14,
cmd_arg15,
cmd_arg16,
cmd_arg17,
cmd_arg18,
cmd_arg19,
cmd_arg20,
cmd_arg21,
cmd_arg22,
cmd_arg23,
cmd_arg24,
cmd_arg25,
cmd_arg26,
cmd_arg27,
cmd_arg28,
cmd_arg29,
cmd_arg30,
cmd_arg31,
cmd_arg32,
cmd_arg33,
cmd_arg34,
cmd_arg35,
cmd_arg36,
cmd_arg37,
cmd_arg38,
cmd_arg39,
cmd_arg40,
cmd_arg41,
cmd_arg42,
cmd_arg43,
cmd_arg44,
cmd_arg45,
cmd_arg46,
cmd_arg47,
cmd_arg48,
cmd_arg49,
cmd_arg50,
cmd_arg51,
cmd_arg52,
cmd_arg53,
cmd_arg54,
cmd_arg55,
cmd_arg56,
cmd_arg57,
cmd_arg58,
cmd_arg59,
cleanup,
fileid,
filename
FROM tasks
WHERE status = 0
AND requires IN (
SELECT id
FROM tasks
WHERE status = 4
ORDER BY source_file
/* LIMIT 1 */
)
ORDER BY source_file
LIMIT 1;
'>&3
read -u4 ready
if (( ready > 0 ))
then
createworker
return 0
fi
echo '
SELECT COUNT(*) SELECT COUNT(*)
FROM tasks FROM tasks
WHERE status = 0 WHERE status = 0
@ -17,7 +122,6 @@ master() {
SELECT SELECT
id, id,
source_file, source_file,
required,
cmd_arg0, cmd_arg0,
cmd_arg1, cmd_arg1,
cmd_arg2, cmd_arg2,
@ -87,15 +191,12 @@ master() {
ORDER BY source_file ORDER BY source_file
LIMIT 1; LIMIT 1;
' >&3 ' >&3
read -u4 remaining
read -u4 ready read -u4 ready
if (( remaining == 0 ))
if (( active == 0 && ready == 0 ))
then then
sleep 0.1 dumpfile="$tempdir/tasks-$(date -Iseconds).csv"
return 0
elif (( active == 0 && ready == 0 ))
then
dumpfile=tasks-$(date +%Y%m%d%H%M%S).csv
cat <<-EOF cat <<-EOF
@ -111,6 +212,7 @@ master() {
.mode list .mode list
.headers off .headers off
.output stdout .output stdout
COMMIT;
EOSQL EOSQL
closeDatabase closeDatabase
echo "Waiting for children to come back home..." echo "Waiting for children to come back home..."
@ -121,148 +223,7 @@ master() {
then then
sleep 0.1 sleep 0.1
else else
(( ++active )) createworker
read -u4 line
taskid=${line%%::AtOM:SQL:Sep::*}
rest="${line#*::AtOM:SQL:Sep::}::AtOM:SQL:Sep::"
sourcefileid=${rest%%::AtOM:SQL:Sep::*}
rest=${rest#*::AtOM:SQL:Sep::}
required=${rest%%::AtOM:SQL:Sep::*}
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cmd_arg+=("${rest%%::AtOM:SQL:Sep::*}")
rest=${rest#*::AtOM:SQL:Sep::}
cleanup=${rest%%::AtOM:SQL:Sep::*}
rest=${rest#*::AtOM:SQL:Sep::}
destfileid=${rest%%::AtOM:SQL:Sep::*}
rest=${rest#*::AtOM:SQL:Sep::}
destfilename=${rest%%::AtOM:SQL:Sep::*}
rest=${rest#*::AtOM:SQL:Sep::}
for key in ${!cmd_arg[@]}
do
[ -z "${cmd_arg[key]}" ] && unset cmd_arg[key]
done
workerid=$(getworkerid)
workertasks[workerid]=$taskid
Update tasks status 1 <<<"id = $taskid"
createworker $workerid
fi fi
fi fi
} }