ScriptFanix/AtOM
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

get rid of pipes

Browse Source
This commit is contained in:
Vincent Riquer 2013-04-03 16:15:56 +02:00
parent ec9201caa9
commit 7141787728
1 changed files with 256 additions and 362 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

402
atom
View File

@ -1269,27 +1269,87 @@ encodeFile::vorbis() {
} }
worker() { worker() {
debug=2
trap "kill -USR1 $masterpid" EXIT trap "kill -USR1 $masterpid" EXIT
trap - USR1 ALRM PIPE trap - USR1 ALRM PIPE
exec 2>>"$tempdir/errors.log" exec 2>>"$tempdir/worker$1.log"
while : (( debug >= 2 )) && echo "${cmd_arg[@]}" >&2
do "${cmd_arg[@]}" >/dev/null
echo work status=$?
read -t 10 line kill -USR1 $masterpid
until [ -n "$line" ] && [[ $line != AtOM:ComFail ]] exit $status
do }
echo work
read -t 10 line master() {
done if (( active >= concurrency)) || [ -n "$quit" ]
if [[ $line == AtOM:Die ]]
then then
trap EXIT sleep 0.1
break else
elif [[ $line == AtOM:Sleep ]] echo '
SELECT COUNT(*)
FROM tasks
WHERE status = 0;
SELECT COUNT(*)
FROM tasks
WHERE status = 0
AND requires is NULL;
SELECT
id,
source_file,
required,
cmd_arg0,
cmd_arg1,
cmd_arg2,
cmd_arg3,
cmd_arg4,
cmd_arg5,
cmd_arg6,
cmd_arg7,
cmd_arg8,
cmd_arg9,
cmd_arg10,
cmd_arg11,
cmd_arg12,
cmd_arg13,
cmd_arg14,
cmd_arg15,
cmd_arg16,
cmd_arg17,
cmd_arg18,
cmd_arg19,
cmd_arg20,
cmd_arg21,
cmd_arg22,
cmd_arg23,
cmd_arg24,
cmd_arg25,
cmd_arg26,
cmd_arg27,
cmd_arg28,
cmd_arg29,
cleanup,
fileid,
filename
FROM tasks
WHERE status = 0
AND requires is NULL
ORDER BY source_file
LIMIT 1;
' >&3
read -u4 remaining
read -u4 ready
if (( remaining == 0 ))
then then
sleep 1 sleep 0.1
continue continue
fi elif (( ready == 0 ))
then
sleep 0.1
else
(( ++active ))
read -u4 line
taskid=${line%%|*} taskid=${line%%|*}
rest="${line#*|}|" rest="${line#*|}|"
sourcefileid=${rest%%|*} sourcefileid=${rest%%|*}
@ -1366,197 +1426,86 @@ worker() {
do do
[ -z "${cmd_arg[key]}" ] && unset cmd_arg[key] [ -z "${cmd_arg[key]}" ] && unset cmd_arg[key]
done done
(( debug >= 2 )) && echo "${cmd_arg[@]}" >&2 workerid=$(getworkerid)
if "${cmd_arg[@]}" >/dev/null workertasks[workerid]=$taskid
then Update tasks status 1 <<<"id = $taskid"
echo "finished $taskid|$sourcefileid|$destfileid|$destfilename" export cmd_arg
read -t 10 line createworker $workerid
until [[ $line == AtOM:OK ]]
do
echo "finished $taskid|$sourcefileid|$destfileid|$destfilename"
read -t 10 line
done
else
echo "failed $taskid"
read -t 10 line
until [[ $line == AtOM:OK ]]
do
echo "failed $taskid"
read -t 10 line
done
[ -n "$filename" ] \
&& eval rm -f $filename
fi
unset cmd_arg unset cmd_arg
if [ -n "$cleanup" -a -n "$required" ]
then
echo "cleanup $required"
read -t 10 answer
until [[ $answer != AtOM:ComFail ]]
do
echo "cleanup $required"
read -t 10 answer
done
if (( answer == 1 ))
then
eval rm -f $cleanup
fi fi
fi fi
done
return
} }
master() { getworkerid() {
for workerid in ${!workers[@]} local i
for (( i=0 ; i >= 0 ; i++ ))
do do
if read -t0.001 -u$((200+workerid)) workercommand workerquery if [ -z "${workers[i]}" ]
then then
break echo $i
return
fi fi
done done
(( ${#workers[@]} == 0 )) && break # If we reach this, we have reached the signed long limit
if [ -n "$workercommand" ] # (2^63 - 1 = 9223372036854775807 - Got a supercomputer?)
(( concurrency-- ))
}
createworker() {
worker &
workers[$1]=$!
}
destroyworker() {
dyingworker=${workers[$1]}
unset workers[$1]
wait $dyingworker
}
checkworkers() {
for key in ${!workers[@]}
do
if ! kill -0 ${workers[key]} 2>/dev/null
then then
case $workercommand in if read -u4 -t 0.01 alienquery
'work')
if [ -n "$quit" ]
then
destroyworker $workerid
elif [ -n "${workertasks[workerid]}" ]
then then
alienresults=(${alienquery%%|*})
rest="${alienquery#*|}|"
while [ -n "$rest" ]
do
alienresults+=("${rest%%|*}")
rest=${rest#*|}
done
fi
echo ' echo '
SELECT SELECT
id, id,
source_file, source_file,
required, required,
cmd_arg0,
cmd_arg1,
cmd_arg2,
cmd_arg3,
cmd_arg4,
cmd_arg5,
cmd_arg6,
cmd_arg7,
cmd_arg8,
cmd_arg9,
cmd_arg10,
cmd_arg11,
cmd_arg12,
cmd_arg13,
cmd_arg14,
cmd_arg15,
cmd_arg16,
cmd_arg17,
cmd_arg18,
cmd_arg19,
cmd_arg20,
cmd_arg21,
cmd_arg22,
cmd_arg23,
cmd_arg24,
cmd_arg25,
cmd_arg26,
cmd_arg27,
cmd_arg28,
cmd_arg29,
cleanup, cleanup,
fileid, fileid,
filename filename
FROM tasks FROM tasks
WHERE WHERE
id='${workertasks[workerid]}'; id='${workertasks[key]}';
' >&3 ' >&3
read -u4 line read -u4 line
eval echo '"$line" >&'$((100+workerid))
elif (( active < concurrency ))
then
echo '
SELECT COUNT(*)
FROM tasks
WHERE status = 0;
SELECT COUNT(*)
FROM tasks
WHERE status = 0
AND requires is NULL;
SELECT
id,
source_file,
required,
cmd_arg0,
cmd_arg1,
cmd_arg2,
cmd_arg3,
cmd_arg4,
cmd_arg5,
cmd_arg6,
cmd_arg7,
cmd_arg8,
cmd_arg9,
cmd_arg10,
cmd_arg11,
cmd_arg12,
cmd_arg13,
cmd_arg14,
cmd_arg15,
cmd_arg16,
cmd_arg17,
cmd_arg18,
cmd_arg19,
cmd_arg20,
cmd_arg21,
cmd_arg22,
cmd_arg23,
cmd_arg24,
cmd_arg25,
cmd_arg26,
cmd_arg27,
cmd_arg28,
cmd_arg29,
cleanup,
fileid,
filename
FROM tasks
WHERE status = 0
AND requires is NULL
ORDER BY source_file
LIMIT 1;
' >&3
read -u4 remaining
read -u4 ready
if (( remaining == 0 ))
then
destroyworker $workerid
continue
elif (( ready == 0 ))
then
line=AtOM:Sleep
else
(( ++active ))
read -u4 line
taskid=${line%%|*} taskid=${line%%|*}
workertasks[workerid]=$taskid rest="${line#*|}|"
Update tasks status 1 <<<"id = $taskid"
fi
eval echo '"$line" >&'$((100+workerid))
else
destroyworker $workerid
fi
;;
'finished')
eval 'echo AtOM:OK >&'$((workerid+100))
[ -z "${workertasks[workerid]}" ] && continue
(( ++ran ))
unset workertasks[workerid]
(( active-- )) || true
taskid=${workerquery%%|*}
rest="${workerquery#*|}|"
sourcefileid=${rest%%|*} sourcefileid=${rest%%|*}
rest=${rest#*|} rest=${rest#*|}
required=${rest%%|*}
rest=${rest#*|}
cleanup=${rest%%|*}
rest=${rest#*|}
destfileid=${rest%%|*} destfileid=${rest%%|*}
rest=${rest#*|} rest=${rest#*|}
destfilename=${rest%%|*} destfilename=${rest%%|*}
rest=${rest#*|}
if destroyworker $key
then
(( ++ran ))
(( active-- )) || true
Delete tasks <<<"id = $taskid" Delete tasks <<<"id = $taskid"
if [ -n "$destfilename" ] if [ -n "$destfilename" ]
then then
@ -1581,15 +1530,7 @@ master() {
"WHERE id=$destfileid;" \ "WHERE id=$destfileid;" \
>&3 >&3
fi fi
;; else
'failed')
eval 'echo AtOM:OK >&'$((workerid+100))
[ -z "${workertasks[workerid]}" ] && continue
unset workertasks[workerid]
(( --active )) || true
(( ++failed ))
(( ++ran ))
taskid=$workerquery
faildepends=$( faildepends=$(
Select tasks 'COUNT(*)' <<-EOWhere Select tasks 'COUNT(*)' <<-EOWhere
requires = $taskid requires = $taskid
@ -1598,79 +1539,33 @@ master() {
(( failed+=faildepends )) (( failed+=faildepends ))
Update tasks status 2 <<<"id = $taskid" Update tasks status 2 <<<"id = $taskid"
Update tasks status 2 <<<"requires = $taskid" Update tasks status 2 <<<"requires = $taskid"
;; fi
'cleanup')
required=$workerquery
echo "SELECT COUNT(*) echo "SELECT COUNT(*)
FROM tasks FROM tasks
WHERE ( status = 0 OR status = 1 ) WHERE ( status = 0 OR status = 1 )
AND required = $required;">&3 AND required = $taskid;">&3
read -u4 count read -u4 count
if (( count == 0 )) if (( count == 0 ))
then then
eval echo 1 '>&'$((100+workerid)) rm -f "$cleanup"
else
eval echo 0 '>&'$((100+workerid))
fi fi
;; if (( ${#alienresults[@]} ))
*)
eval 'echo "AtOM:ComFail" >&'$((100+workerid))
;;
esac
fi
}
getworkerid() {
local i
for (( i=0 ; i < 100 ; i++ ))
do
if [ -z "${workers[i]}" ]
then then
echo $i alienquery='SELECT '
break for key in ${!alienresults[@]}
do
(( key > 0 )) && alienquery+=,
expr='^[0-9]*$'
if [[ ${alienresults[key]} =~ $expr ]]
then
alienquery+=${alienresults[key]}
else
alienquery+="\"${alienresults[@]//\"/\"\"}\""
fi fi
done done
# If we reach this, we have reached the hardcoded 100 workers limit echo "$alienquery;" >&3
(( concurrency-- )) unset alienquery alienresults
}
createworker() {
mkfifo "$tempdir"/worker$1{in,out}
worker $1 <"$tempdir"/worker$1in >"$tempdir"/worker$1out &
workers[$1]=$!
eval exec $((100+$1))'>"$tempdir"/worker$1in'
eval exec $((200+$1))'<"$tempdir"/worker$1out'
}
destroyworker() {
dyingworker=${workers[$1]}
unset workers[$1]
[ -z "$2" ] && eval echo AtOM:Die '>&'$((100+$1))
wait $dyingworker
eval $((100+$1))'>&-'
eval $((200+$1))'<&-'
rm "$tempdir"/worker$1{in,out}
}
checkworkers() {
for key in ${!workers[@]}
do
if ! kill -0 ${workers[key]} 2>/dev/null
then
destroyworker $key nokill
if [ -n "${workertasks[key]}" ]
then
faildepends=$(
Select tasks 'COUNT(*)' <<-EOWhere
requires = ${workertasks[key]}
EOWhere
)
(( ++failed ))
(( failed+=faildepends ))
Update tasks status 2 <<<"id = ${workertasks[key]}"
Update tasks status 2 <<<"requires = ${workertasks[key]}"
fi fi
createworker $key
fi fi
done done
} }
@ -2119,21 +2014,21 @@ echo -e "\rCreated ${count:-0} tasks for $filecount files (${copies:-0} immediat
masterpid=$$ masterpid=$$
trap checkworkers USR1 ALRM PIPE trap checkworkers USR1 ALRM PIPE
rm -f "$tempdir"/worker*
concurrency=$(( maxload / 2 )) concurrency=$(( maxload / 2 ))
(( concurrency )) || concurrency=1 (( concurrency )) || concurrency=1
active=0 active=0
#set -x
for (( i=0 ; i < concurrency ; i++ )) for (( i=0 ; i < concurrency ; i++ ))
do do
createworker $(getworkerid) master
done done
concurrencychange=$(date +%s) concurrencychange=$(date +%s)
starttime=$concurrencychange starttime=$concurrencychange
taskcount=$count taskcount=$count
failed=0 failed=0
while : while (( ${#workers[@]} ))
do do
if read -n 1 -t 0.01 userinput if read -n 1 -t 1 userinput
then then
case $userinput in case $userinput in
'+') '+')
@ -2160,7 +2055,6 @@ do
then then
concurrencychange=$(date +%s) concurrencychange=$(date +%s)
(( ++concurrency )) (( ++concurrency ))
createworker $(getworkerid)
fi fi
fi fi
master master