From b8e8b8395ba963359ffdf2a14b205c1444ff0dca Mon Sep 17 00:00:00 2001 From: Vincent Riquer Date: Fri, 6 Feb 2015 09:40:30 +0100 Subject: [PATCH] Stable image import --- add.html | 1 + import | 1 + index | 12 ++++++++++-- ingrrd.css | 7 +++++++ lib/image | 29 ++++++++++++++++++++++++++--- lib/import_images | 17 ++++++++++------- lib/readlast | 5 +++-- lib/send_headers | 6 +++--- settings | 2 ++ 9 files changed, 63 insertions(+), 17 deletions(-) diff --git a/add.html b/add.html index d2385d1..55f5a24 100755 --- a/add.html +++ b/add.html @@ -22,6 +22,7 @@ for lib in lib/* do source $lib done +import_images & read_query_string send_headers <<-EOHead diff --git a/import b/import index ebbb93e..9d370f7 100755 --- a/import +++ b/import @@ -82,6 +82,7 @@ then rrdtool update "$rrdfile" -t $template "${data//:0/:U}" 2>&1 done <<<"${POST["data"]}" else + import_images & send_headers <<-EOHead content-type: text/xml EOHead diff --git a/index b/index index b40cdfd..dc8c823 100755 --- a/index +++ b/index @@ -22,16 +22,16 @@ for lib in lib/* do source $lib done -import_images & read_query_string (( start >= end)) && (( end = start + 604800 )) if [ -z "$graph" ] then + import_images & refresh=300 [[ -r .importing ]] && refresh=5 - send_headers <<-EOHead + send_headers $(( refresh - 1 )) <<-EOHead content-type: application/xhtml+xml refresh: $refresh EOHead @@ -112,6 +112,9 @@ then EOHTML show_current_level + [[ -r .importing ]] && cat <<-EOHTML +
Importing $(<.importing)
+ EOHTML cat <<-EOHTML
@@ -389,6 +392,11 @@ then + EOHTML + [[ -r .importing ]] && cat <<-EOHTML +
Importing $(<.importing)
+ EOHTML + cat <<-EOHTML
EOHTML diff --git a/ingrrd.css b/ingrrd.css index f847925..d85715b 100644 --- a/ingrrd.css +++ b/ingrrd.css @@ -59,6 +59,13 @@ td.submenu-inv { background: #444; color: white } +div.updating { + position: relative; + background: orange; + width: 100%; + text-align: center; + font-size: larger; +} div.main { position: absolute; diff --git a/lib/image b/lib/image index 671aee0..58e7e2e 100755 --- a/lib/image +++ b/lib/image @@ -21,14 +21,31 @@ image() { local timestamp=$2 declare -A values - tmpfile=$(mktemp --suffix=.png) - convert -contrast "$1" "$tmpfile" - readarray -t tactout < <(tesseract -l eng -psm 4 "$tmpfile" stdout) + tmpfile=$(mktemp --suffix=.tiff) + transform='-adaptive-resize 600%x400% -contrast -channel green -threshold 50%' + + if (( debug >= 3 )) + then + readarray -t tactout < <(convert $transform "$1" -format tiff - | tee "${1%.png}.tiff" | tesseract -l eng -psm 4 stdin stdout) + else + readarray -t tactout < <(convert $transform "$1" -format tiff - | tesseract -l eng -psm 4 stdin stdout) + fi + if ((debug_import >=2 )) + then + for index in ${!tactout[@]} + do + echo "${tactout[index]}" + done > "${1%.png}.txt" + fi rm "$tmpfile" for line in "${tactout[@]}" do case "$line" in + 'Error, please try again.') + # Ingress failed when generating image, ignore file. + return 3 + ;; #14,522,411 AP 144mm AP *' AP'*) read value junk <<<"$line" @@ -175,5 +192,11 @@ image() { template="${!values[@]}" updatevalues="${values[@]}" + if ((debug_import)) + then + echo $'\t'"$template" + echo $'\t'"$updatevalues" + fi + rrdtool update "$rrdfile" -t "${template// /:}" "$timestamp:${updatevalues// /:}" } diff --git a/lib/import_images b/lib/import_images index 685bfec..02264b7 100644 --- a/lib/import_images +++ b/lib/import_images @@ -1,30 +1,33 @@ #!/bin/bash import_images() { - local starttime=$(date +%s) imagetime timestamp elapsed + local starttime=$(date +%s) imagetime timestamp elapsed lasttime now exec >> images.log exec 2>> images.log declare -r oldIFS="$IFS" [[ -r .importing ]] && return 2 [[ -w "$rrdfile" ]] || rrdcreate readlast - for file in dav/* + for file in dav/profile_*.png do [[ -r "$file" ]] || continue + (( count++ )) IFS=_. read junk date time junk <<<"$file" IFS="$oldIFS" - if (( $(date '+%Y%m%d%H%M%S' -d @${last[time]}) < $date$time )) + if (( $(date '+%Y%m%d%H%M%S' -d "@${last[time]}") < $date$time )) then - [[ -r .importing ]] && return 2 - touch .importing + lasttime=${now:-$starttime} + echo "${file##*/}" > .importing imagetime="${date:0:4}/${date:4:2}/${date:6:2} ${time:0:2}:${time:2:2}:${time:4:2}" echo "$(date +"%Y/%m/%d:%H:%M:%S") - $file: $imagetime" timestamp=$(date +%s -d "$imagetime") image "$file" $timestamp + now=$(date +%s) + echo $'\tDuration: '$(( now - lasttime )) fi done elapsed=$(( $(date +%s) - starttime )) - echo "$(date +"%Y/%m/%d:%H:%M:%S"): Spent $elapsed seconds" - rm .importing + (( count )) && echo "$(date +"%Y/%m/%d:%H:%M:%S"): Spent $elapsed seconds" + rm -f .importing } diff --git a/lib/readlast b/lib/readlast index bd2a428..3d3944a 100644 --- a/lib/readlast +++ b/lib/readlast @@ -5,14 +5,15 @@ readlast() { index \ indices \ lastupdatelines \ + time \ values \ - last["time"]=0 readarray -t lastupdatelines < <( rrdtool lastupdate $rrdfile ) indices=( ${lastupdatelines[0]} ) - last["time"]=${lastupdatelines[2]%:*} + time=${lastupdatelines[2]%:*} + last["time"]=${time:-0} values=( ${lastupdatelines[2]#*:} ) for index in ${!indices[@]} do diff --git a/lib/send_headers b/lib/send_headers index d55c80a..f7a38ad 100644 --- a/lib/send_headers +++ b/lib/send_headers @@ -2,9 +2,9 @@ send_headers() { local timestamp do_exit header thisdate expiredate - read timestamp < <(stat -c %Y $0 settings "$rrdfile" lib/* |sort -r) + read timestamp < <(stat -c %Y $0 settings "$rrdfile" lib/* .importing 2>/dev/null|sort -r) thisdate=$(LC_ALL=C TZ=GMT date +'%a, %d %b %Y %X %Z' -d @$timestamp) - expiredate=$(LC_ALL=C TZ=GMT date +'%a, %d %b %Y %X %Z' -d '+1 minute') + expiredate=$(LC_ALL=C TZ=GMT date +'%a, %d %b %Y %X %Z' -d "+${1:-60} second") if [[ $thisdate == $HTTP_IF_MODIFIED_SINCE ]] then echo "Status: 304 Not Modified" @@ -12,7 +12,7 @@ send_headers() { fi echo "Last-Modified: $thisdate" echo "Expires: $expiredate" - echo "Cache-Control: must-revalidate, max-age=60" + echo "Cache-Control: must-revalidate, max-age=${1:-60}" while read header do echo "$header" diff --git a/settings b/settings index b2ee412..23fa2ee 100644 --- a/settings +++ b/settings @@ -8,6 +8,8 @@ # /cgi/$user/add.html IFS=/ read garbage garbage webuser garbage <<<"$REQUEST_URI" +debug_import=2 + rrdfile=ingress.rrd graph_width=800