Merge remote-tracking branch 'origin/master' into prod

* origin/master:
  don't produce useless "Spent.." messages
  Fix RRD creation, quick & dirty
  fix unknown date handfling in "Overview"
  Stable image import
  prevision: only look 5y in the future (more precise)
  don't create lock for to early
  import from profile share

add.html

@@ -22,6 +22,7 @@ for lib in lib/*
 do
 	source $lib
 done
+import_images &
 read_query_string
 
 send_headers <<-EOHead

import

@@ -82,6 +82,7 @@ then
 		rrdtool update "$rrdfile" -t $template "${data//:0/:U}" 2>&1
 	done <<<"${POST["data"]}"
 else
+	import_images &
 	send_headers <<-EOHead
 		content-type: text/xml
 	EOHead

index

@@ -22,26 +22,18 @@ for lib in lib/*
 do
 	source $lib
 done
-if [ ! -w "$rrdfile" ]
-then
-	cat <<-EOHead
-		Content-type: text/plain
-		Last-Modified: $(LC_ALL=C TZ=GMT date +'%a, %d %b %Y %X %Z')
-		Expires: $(LC_ALL=C TZ=GMT date +'%a, %d %b %Y %X %Z')
-		refresh: 0,url=/cgi/$webuser/add.html
-
-	EOHead
-	exit
-fi
 
 read_query_string
 (( start >= end)) && (( end = start + 604800 ))
 
 if [ -z "$graph" ]
 then
-	send_headers <<-EOHead
+	import_images &
+	refresh=300
+	[[ -r .importing ]] && refresh=5
+	send_headers $(( refresh - 1 )) <<-EOHead
 		content-type: application/xhtml+xml
-		refresh: 300
+		refresh: $refresh
 	EOHead
 
 	readlast
@@ -120,6 +112,9 @@ then
 			</table>
 		EOHTML
 		show_current_level
+		[[ -r .importing ]] && cat <<-EOHTML
+			<div class="updating">Importing $(<.importing)</div>
+		EOHTML
 		cat <<-EOHTML
 			</div>
 			<div class="main">
@@ -302,9 +297,16 @@ then
 			then
 				(( maxtimestamp < prevtimestamp ))	\
 				&& maxtimestamp=$prevtimestamp
+				if (( maxtimestamp ))
+				then
 					cat <<-EOHTML
 						<td class="date">$(date +%Y-%m-%d -d @$maxtimestamp)</td>
 					EOHTML
+				else
+					cat <<-EOHTML
+						<td class="date">In more than 5 years</td>
+					EOHTML
+				fi
 			fi
 			echo '</tr>'
 			prevtimestamp=$maxtimestamp
@@ -397,6 +399,11 @@ then
 					</td>
 				</tr>
 			</table>
+		EOHTML
+		[[ -r .importing ]] && cat <<-EOHTML
+			<div class="updating">Importing $(<.importing)</div>
+		EOHTML
+		cat <<-EOHTML
 			</div>
 			<div class="maingraph">
 		EOHTML

ingrrd.css

@@ -59,6 +59,13 @@ td.submenu-inv {
 	background: #444;
 	color: white
 }
+div.updating {
+	position: relative;
+	background: orange;
+	width: 100%;
+	text-align: center;
+	font-size: larger;
+}
 
 div.main {
 	position: absolute;

lib/get_next_level

@@ -21,8 +21,8 @@ get_next_level() {
 			nextlevels[level]=$timestamp
 		done
 	} < <(
-		rrdtool graph /dev/null -s ${first["time"]%:} -e +15y	\
+		rrdtool graph /dev/null -s -$prevperiod -e +5y	\
-			DEF:data=$rrdfile:AP:AVERAGE:step=600		\
+			DEF:data=$rrdfile:AP:AVERAGE:step=3600		\
 			VDEF:intercept=data,LSLINT			\
 			VDEF:growth=data,LSLSLOPE			\
 			CDEF:trend=data,POP,COUNT,growth,*,intercept,+	\
@@ -96,7 +96,7 @@ get_next_level() {
 				esac
 			done
 		} < <(
-			rrdtool graph /dev/null -s ${first["time"]%:} -e +15y	\
+			rrdtool graph /dev/null -s -$prevperiod -e +5y	\
 				DEF:data=$rrdfile:$stat:AVERAGE:step=600	\
 				VDEF:intercept=data,LSLINT			\
 				VDEF:growth=data,LSLSLOPE			\

lib/image

@@ -0,0 +1,202 @@
+#!/bin/bash
+
+#    IngRRD (https://forge.riquer.fr/p/ingrrd/)
+#    Copyright (C) 2015  Vincent Riquer
+#
+#    This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU Affero General Public License as
+#    published by the Free Software Foundation, either version 3 of the
+#    License, or (at your option) any later version.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU Affero General Public License for more details.
+#
+#    You should have received a copy of the GNU Affero General Public License
+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+image() {
+	exec 2>&1
+	local timestamp=$2
+	declare -A values
+
+	tmpfile=$(mktemp --suffix=.tiff)
+	transform='-adaptive-resize 600%x400% -contrast -channel green -threshold 50%'
+
+	if (( debug >= 3 ))
+	then
+		readarray -t tactout < <(convert $transform "$1" -format tiff  - | tee "${1%.png}.tiff" | tesseract -l eng -psm 4 stdin stdout)
+	else
+		readarray -t tactout < <(convert $transform "$1" -format tiff  - | tesseract -l eng -psm 4 stdin stdout)
+	fi
+	if ((debug_import >=2 ))
+	then
+		for index in ${!tactout[@]}
+		do
+			echo "${tactout[index]}"
+		done > "${1%.png}.txt"
+	fi
+	rm "$tmpfile"
+
+	for line in "${tactout[@]}"
+	do
+		case "$line" in
+			'Error, please try again.')
+				# Ingress failed when generating image, ignore file.
+				return 3
+			;;
+			#14,522,411 AP 144mm AP
+			*' AP'*)
+				read value junk <<<"$line"
+				values[AP]=${value//,/}
+			;;
+			#Unique Portals Visited 718
+			'Unique Portals Visited'*)
+				value="${line#Unique Portals Visited }"
+				read values[Explorer] junk <<<"${value//,}"
+			;;
+			#Portals Discovered 9
+			'Portals Discovered'*)
+				value="${line#Portals Discovered }"
+				read values[Seer] junk <<<"${value//,/}"
+			;;
+			#XM Collected 130,908,527 XM
+			'XM Collected'*)
+				value="${line#XM Collected}"
+				read values[XM] junk <<<"${value//,/}"
+			;;
+			#Distance Walked 1,785 km
+			'Distance Walked'*)
+				value="${line#Distance Walked }"
+				read values[Walked] junk <<<"${value//,/}"
+			;;
+			#Resonators Deployed 24,451
+			'Resonators Deployed'*)
+				value="${line#Resonators Deployed }"
+				read values[Builder] junk <<<"${value//,/}"
+			;;
+			#Links Created 4,479
+			'Links Created'*)
+				value="${line#Links Created }"
+				read values[Connector] junk <<<"${value//,/}"
+			;;
+			#Control Fields Created 2,340
+			'Control Fields Created'*)
+				value="${line#Control Fields Created }"
+				read values[Mind_Controller] junk <<<"${value//,/}"
+			;;
+			#Mind Units Captured 155,815 MUs
+			'Mind Units Captured'*)
+				value="${line#Mind Units Captured }"
+				read values[MU] junk <<<"${value//,/}"
+			;;
+			#Longest Link Ever Created 3 km
+			'Longest Link Ever Created'*)
+				value="${line#Longest Link Ever Created }"
+				read values[Longest_Link] junk <<<"${value//,/}"
+			;;
+			#Largest Control Field 11,588 MUs
+			'Largest Control Field'*)
+				value="${line#Largest Control Field }"
+				read values[Largest_Field] junk <<<"${value//,/}"
+			;;
+			#XM Recharged 92,120,959 XM
+			'XM Recharged'*)
+				value="${line#XM Recharged }"
+				read values[Recharger] junk <<<"${value//,/}"
+			;;
+			#Portals Captured 2,022
+			'Portals Captured'*)
+				value="${line#Portals Captured }"
+				read values[Liberator] junk <<<"${value//,/}"
+			;;
+			#Unique Portals Captured 375
+			'Unique Portals Captured'*)
+				value="${line#Unique Portals Captured }"
+				read values[Pioneer] junk <<<"${value//,/}"
+			;;
+			#Mods Deployed 995
+			'Mods Deployed'*)
+				value="${line#Mods Deployed }"
+				read values[Engineer] junk <<<"${value//,/}"
+			;;
+			#Resonators Destroyed 15,855
+			'Resonators Destroyed'*)
+				value="${line#Resonators Destroyed }"
+				read values[Purifier] junk <<<"${value//,/}"
+			;;
+			#Portals Neutralized 2,145
+			'Portals Neutralized'*)
+				value="${line#Portals Neutralized }"
+				read values[Neutralized] junk <<<"${value//,/}"
+			;;
+			#Enemy Links Destroyed 2,725
+			'Enemy Links Destroyed'*)
+				value="${line#Enemy Links Destroyed }"
+				read values[Links_Destroyed] junk <<<"${value//,/}"
+			;;
+			#Enemy Control Fields Destroyed 1,418
+			'Enemy Control Fields Destroyed'*)
+				value="${line#Enemy Control Fields Destroyed }"
+				read values[Fields_Destroyed] junk <<<"${value//,/}"
+			;;
+			#Max Time Portal Held 105 days
+			'Max Time Portal Held'*)
+				value="${line#Max Time Portal Held }"
+				read values[Guardian] junk <<<"${value//,/}"
+			;;
+			#Max Time Link Maintained 75 days
+			'Max Time Link Maintained'*)
+				value="${line#Max Time Link Maintained }"
+				read values[Link_Maintained] junk <<<"${value//,/}"
+			;;
+			#Max Link Length x Days 89 km-days
+			'Max Link Length x Days'*)
+				value="${line#Max Link Length x Days }"
+				read values[Link_Length_x_Days] junk <<<"${value//,/}"
+			;;
+			#Max Time Field Held 55 days
+			'Max Time Field Held'*)
+				value="${line#Max Time Field Held }"
+				read values[Field_Held] junk <<<"${value//,/}"
+			;;
+			#Largest Field MUs x Days 58,959 MU-days
+			'Largest Field MUs x Days'*)
+				value="${line#Largest Field MUs x Days }"
+				read values[Field_x_Days] junk <<<"${value//,/}"
+			;;
+			#Unique Missions Completed 1
+			'Unique Missions Completed'*)
+				value="${line#Unique Missions Completed }"
+				read values[Mercenary] junk <<<"${value//,/}"
+			;;
+			#Hacks 25,741
+			'Hacks'*)
+				value="${line#Hacks }"
+				read values[Hacker] junk <<<"${value//,/}"
+			;;
+			#Glyph Hack Points 1,022
+			'Glyph Hack Points'*)
+				value="${line#Glyph Hack Points }"
+				read values[Translator] junk <<<"${value//,/}"
+			;;
+			#Agents Recruited 1
+			'Agents Recruited'*)
+				value="${line#Agents Recruited }"
+				read values[Recruiter] junk <<<"${value//,/}"
+			;;
+		esac
+	done
+
+	template="${!values[@]}"
+	updatevalues="${values[@]}"
+
+	if ((debug_import))
+	then
+		echo $'\t'"$template"
+		echo $'\t'"$updatevalues"
+	fi
+
+	rrdtool update "$rrdfile" -t "${template// /:}" "$timestamp:${updatevalues// /:}"
+}

lib/import_images

@@ -0,0 +1,33 @@
+#!/bin/bash
+
+import_images() {
+	local starttime=$(date +%s) imagetime timestamp elapsed lasttime now
+	exec >> images.log
+	exec 2>> images.log
+	declare -r oldIFS="$IFS"
+	[[ -r .importing ]] && return 2
+	[[ -w "$rrdfile" ]] || rrdcreate
+	readlast
+	for file in dav/profile_*.png
+	do
+		[[ -r "$file" ]] || continue
+		IFS=_.
+		read junk date time junk <<<"$file"
+		IFS="$oldIFS"
+		if (( $(date '+%Y%m%d%H%M%S' -d "@${last[time]}") <  $date$time ))
+		then
+			(( count++ ))
+			lasttime=${now:-$starttime}
+			echo "${file##*/}" > .importing
+			imagetime="${date:0:4}/${date:4:2}/${date:6:2} ${time:0:2}:${time:2:2}:${time:4:2}"
+			echo "$(date +"%Y/%m/%d:%H:%M:%S") - $file: $imagetime"
+			timestamp=$(date +%s -d "$imagetime")
+			image "$file" $timestamp
+			now=$(date +%s)
+			echo $'\tDuration: '$(( now - lasttime ))
+		fi
+	done
+	elapsed=$(( $(date +%s) - starttime ))
+	(( count )) && echo "$(date +"%Y/%m/%d:%H:%M:%S"): Imported $count file(s) in $elapsed seconds."
+	rm -f .importing
+}

lib/readlast

@@ -5,13 +5,15 @@ readlast() {
 		index		\
 		indices		\
 		lastupdatelines	\
+		time		\
 		values		\
 
 	readarray -t lastupdatelines < <(
 		rrdtool lastupdate $rrdfile
 	)
 	indices=( ${lastupdatelines[0]} )
-	last["time"]=${lastupdatelines[2]%:*}
+	time=${lastupdatelines[2]%:*}
+	last["time"]=${time:-0}
 	values=( ${lastupdatelines[2]#*:} )
 	for index in ${!indices[@]}
 	do

lib/rrdcreate

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+rrdcreate() {
+	rrdtool restore ingress.xml "$rrdfile"
+}

lib/send_headers

@@ -2,9 +2,9 @@
 
 send_headers() {
 	local timestamp do_exit header thisdate expiredate
-	read timestamp < <(stat -c %Y $0 settings "$rrdfile" lib/* |sort -r)
+	read timestamp < <(stat -c %Y $0 settings "$rrdfile" lib/* .importing 2>/dev/null|sort -r)
 	thisdate=$(LC_ALL=C TZ=GMT date +'%a, %d %b %Y %X %Z' -d @$timestamp)
-	expiredate=$(LC_ALL=C TZ=GMT date +'%a, %d %b %Y %X %Z' -d '+1 minute')
+	expiredate=$(LC_ALL=C TZ=GMT date +'%a, %d %b %Y %X %Z' -d "+${1:-60} second")
 	if [[ $thisdate == $HTTP_IF_MODIFIED_SINCE ]]
 	then
 		echo "Status: 304 Not Modified"
@@ -12,7 +12,7 @@ send_headers() {
 	fi
 	echo "Last-Modified: $thisdate"
 	echo "Expires: $expiredate"
-	echo "Cache-Control: must-revalidate, max-age=60"
+	echo "Cache-Control: must-revalidate, max-age=${1:-60}"
 	while read header
 	do
 		echo "$header"

settings

@@ -8,6 +8,8 @@ CACHE_DIR=/web/ingress-stats.riquer.fr/cache/
 # /cgi/$user/add.html
 IFS=/ read garbage garbage webuser garbage <<<"$REQUEST_URI"
 
+debug_import=2
+
 rrdfile=ingress.rrd
 
 graph_width=800
@@ -29,7 +31,7 @@ L14color=8B1479
 L15color=6B127D
 L16color=591494
 
-default_prevperiod=1m
+default_prevperiod=1month
 now=$(date +%s)
 
 (( default_start = now - 604800 ))