Stable image import

add.html

@@ -22,6 +22,7 @@ for lib in lib/*
 do
 	source $lib
 done
+import_images &
 read_query_string
 
 send_headers <<-EOHead

import

@@ -82,6 +82,7 @@ then
 		rrdtool update "$rrdfile" -t $template "${data//:0/:U}" 2>&1
 	done <<<"${POST["data"]}"
 else
+	import_images &
 	send_headers <<-EOHead
 		content-type: text/xml
 	EOHead

index

@@ -22,16 +22,16 @@ for lib in lib/*
 do
 	source $lib
 done
-import_images &
 
 read_query_string
 (( start >= end)) && (( end = start + 604800 ))
 
 if [ -z "$graph" ]
 then
+	import_images &
 	refresh=300
 	[[ -r .importing ]] && refresh=5
-	send_headers <<-EOHead
+	send_headers $(( refresh - 1 )) <<-EOHead
 		content-type: application/xhtml+xml
 		refresh: $refresh
 	EOHead
@@ -112,6 +112,9 @@ then
 			</table>
 		EOHTML
 		show_current_level
+		[[ -r .importing ]] && cat <<-EOHTML
+			<div class="updating">Importing $(<.importing)</div>
+		EOHTML
 		cat <<-EOHTML
 			</div>
 			<div class="main">
@@ -389,6 +392,11 @@ then
 					</td>
 				</tr>
 			</table>
+		EOHTML
+		[[ -r .importing ]] && cat <<-EOHTML
+			<div class="updating">Importing $(<.importing)</div>
+		EOHTML
+		cat <<-EOHTML
 			</div>
 			<div class="maingraph">
 		EOHTML

ingrrd.css

@@ -59,6 +59,13 @@ td.submenu-inv {
 	background: #444;
 	color: white
 }
+div.updating {
+	position: relative;
+	background: orange;
+	width: 100%;
+	text-align: center;
+	font-size: larger;
+}
 
 div.main {
 	position: absolute;

lib/image

@@ -21,14 +21,31 @@ image() {
 	local timestamp=$2
 	declare -A values
 
-	tmpfile=$(mktemp --suffix=.png)
-	convert -contrast "$1" "$tmpfile"
-	readarray -t tactout < <(tesseract -l eng -psm 4 "$tmpfile" stdout)
+	tmpfile=$(mktemp --suffix=.tiff)
+	transform='-adaptive-resize 600%x400% -contrast -channel green -threshold 50%'
+
+	if (( debug >= 3 ))
+	then
+		readarray -t tactout < <(convert $transform "$1" -format tiff  - | tee "${1%.png}.tiff" | tesseract -l eng -psm 4 stdin stdout)
+	else
+		readarray -t tactout < <(convert $transform "$1" -format tiff  - | tesseract -l eng -psm 4 stdin stdout)
+	fi
+	if ((debug_import >=2 ))
+	then
+		for index in ${!tactout[@]}
+		do
+			echo "${tactout[index]}"
+		done > "${1%.png}.txt"
+	fi
 	rm "$tmpfile"
 
 	for line in "${tactout[@]}"
 	do
 		case "$line" in
+			'Error, please try again.')
+				# Ingress failed when generating image, ignore file.
+				return 3
+			;;
 			#14,522,411 AP 144mm AP
 			*' AP'*)
 				read value junk <<<"$line"
@@ -175,5 +192,11 @@ image() {
 	template="${!values[@]}"
 	updatevalues="${values[@]}"
 
+	if ((debug_import))
+	then
+		echo $'\t'"$template"
+		echo $'\t'"$updatevalues"
+	fi
+
 	rrdtool update "$rrdfile" -t "${template// /:}" "$timestamp:${updatevalues// /:}"
 }

lib/import_images

@@ -1,30 +1,33 @@
 #!/bin/bash
 
 import_images() {
-	local starttime=$(date +%s) imagetime timestamp elapsed
+	local starttime=$(date +%s) imagetime timestamp elapsed lasttime now
 	exec >> images.log
 	exec 2>> images.log
 	declare -r oldIFS="$IFS"
 	[[ -r .importing ]] && return 2
 	[[ -w "$rrdfile" ]] || rrdcreate
 	readlast
-	for file in dav/*
+	for file in dav/profile_*.png
 	do
 		[[ -r "$file" ]] || continue
+		(( count++ ))
 		IFS=_.
 		read junk date time junk <<<"$file"
 		IFS="$oldIFS"
-		if (( $(date '+%Y%m%d%H%M%S' -d @${last[time]}) <  $date$time ))
+		if (( $(date '+%Y%m%d%H%M%S' -d "@${last[time]}") <  $date$time ))
 		then
-			[[ -r .importing ]] && return 2
-			touch .importing
+			lasttime=${now:-$starttime}
+			echo "${file##*/}" > .importing
 			imagetime="${date:0:4}/${date:4:2}/${date:6:2} ${time:0:2}:${time:2:2}:${time:4:2}"
 			echo "$(date +"%Y/%m/%d:%H:%M:%S") - $file: $imagetime"
 			timestamp=$(date +%s -d "$imagetime")
 			image "$file" $timestamp
+			now=$(date +%s)
+			echo $'\tDuration: '$(( now - lasttime ))
 		fi
 	done
 	elapsed=$(( $(date +%s) - starttime ))
-	echo "$(date +"%Y/%m/%d:%H:%M:%S"): Spent $elapsed seconds"
-	rm .importing
+	(( count )) && echo "$(date +"%Y/%m/%d:%H:%M:%S"): Spent $elapsed seconds"
+	rm -f .importing
 }

lib/readlast

@@ -5,14 +5,15 @@ readlast() {
 		index		\
 		indices		\
 		lastupdatelines	\
+		time		\
 		values		\
 
-	last["time"]=0
 	readarray -t lastupdatelines < <(
 		rrdtool lastupdate $rrdfile
 	)
 	indices=( ${lastupdatelines[0]} )
-	last["time"]=${lastupdatelines[2]%:*}
+	time=${lastupdatelines[2]%:*}
+	last["time"]=${time:-0}
 	values=( ${lastupdatelines[2]#*:} )
 	for index in ${!indices[@]}
 	do

lib/send_headers

@@ -2,9 +2,9 @@
 
 send_headers() {
 	local timestamp do_exit header thisdate expiredate
-	read timestamp < <(stat -c %Y $0 settings "$rrdfile" lib/* |sort -r)
+	read timestamp < <(stat -c %Y $0 settings "$rrdfile" lib/* .importing 2>/dev/null|sort -r)
 	thisdate=$(LC_ALL=C TZ=GMT date +'%a, %d %b %Y %X %Z' -d @$timestamp)
-	expiredate=$(LC_ALL=C TZ=GMT date +'%a, %d %b %Y %X %Z' -d '+1 minute')
+	expiredate=$(LC_ALL=C TZ=GMT date +'%a, %d %b %Y %X %Z' -d "+${1:-60} second")
 	if [[ $thisdate == $HTTP_IF_MODIFIED_SINCE ]]
 	then
 		echo "Status: 304 Not Modified"
@@ -12,7 +12,7 @@ send_headers() {
 	fi
 	echo "Last-Modified: $thisdate"
 	echo "Expires: $expiredate"
-	echo "Cache-Control: must-revalidate, max-age=60"
+	echo "Cache-Control: must-revalidate, max-age=${1:-60}"
 	while read header
 	do
 		echo "$header"

settings

@@ -8,6 +8,8 @@
 # /cgi/$user/add.html
 IFS=/ read garbage garbage webuser garbage <<<"$REQUEST_URI"
 
+debug_import=2
+
 rrdfile=ingress.rrd
 
 graph_width=800