Package changes to support the aircraft DB.

debian/control

@@ -10,6 +10,7 @@ Vcs-Git: https://github.com/mutability/dump1090.git
 Package: dump1090-mutability
 Architecture: any
 Depends: ${shlibs:Depends}, ${misc:Depends}, adduser
+Recommends: ${python:Depends}, cron | cron-daemon
 Suggests: lighttpd
 Provides: fatsv-data-source
 Description: ADS-B Ground Station System for RTL-SDR

debian/cron-template

@@ -0,0 +1,12 @@
+## TEMPLATE FILE - This is used to create /etc/cron.d/dump1090-mutability ##
+## The first three lines will be discarded                                ##
+
+# Checks for updates to the VRS aircraft database once a day at around 5am;
+# when an update is available, downloads it and regenerates the JSON data
+# used by the webmap.
+
+# NB: the minute value below is randomly generated on install, to avoid 
+# all installs hitting the server at the same time.
+
+# m h dom mon dow user command
+@MIN@ 5 * * *  @USER@  test -x /usr/share/dump1090-mutability/update-aircraft-database.sh && /usr/share/dump1090-mutability/update-aircraft-database.sh --log-to-file

debian/dump1090-mutability.conffiles

@@ -5,3 +5,4 @@
 /usr/share/dump1090-mutability/html/style.css
 /usr/share/dump1090-mutability/html/markers.js
 /usr/share/dump1090-mutability/html/formatter.js
+/usr/share/dump1090-mutability/html/dbloader.js

debian/dump1090-mutability.config

@@ -184,13 +184,11 @@ is_port_number() {
 }
 
 db_input high $NAME/auto-start || true
+db_input_verify low $NAME/run-as-user is_non_root_user || true
+db_input_verify low $NAME/log-file is_not_empty || true
 
 db_go || true; db_get $NAME/auto-start; if [ "$RET" = "true" ]; then
  # all of these are only relevant if the init script is enabled
-
- db_input_verify low $NAME/run-as-user is_non_root_user || true
- db_input_verify low $NAME/log-file is_not_empty || true
-
  db_input medium $NAME/rtlsdr-device || true
 
  db_go || true; db_get $NAME/rtlsdr-device; if [ "x$RET" != "xnone" ]; then

debian/dump1090-mutability.init

@@ -130,6 +130,11 @@ do_start()
            fi
         fi
 
+        # create logfile with the appropriate permissions if not already there
+        # (the cronjob running as DUMP1090_USER wants to append to it)
+        touch $LOGFILE
+        chown "$DUMP1090_USER":root $LOGFILE
+
 	start-stop-daemon --start $NICELEVEL --quiet --pidfile $PIDFILE --user "$DUMP1090_USER" --chuid "$DUMP1090_USER" --make-pidfile --background --no-close --exec $DAEMON -- \
 		$ARGS >>$LOGFILE 2>&1 \
 		|| return 2

debian/dump1090-mutability.install

@@ -1,3 +1,6 @@
 public_html/* usr/share/dump1090-mutability/html
 debian/lighttpd/* etc/lighttpd/conf-available
 debian/config-template usr/share/dump1090-mutability
+debian/cron-template usr/share/dump1090-mutability
+tools/vrs-basicaircraft-to-json.py usr/share/dump1090-mutability
+tools/update-aircraft-database.sh usr/share/dump1090-mutability

debian/dump1090-mutability.postinst

@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/bin/bash
 # postinst script for dump1090
 #
 # see: dh_installdeb(1)
@@ -20,6 +20,8 @@ set -e
 NAME=dump1090-mutability
 CONFIGFILE=/etc/default/$NAME
 TEMPLATECONFIG=/usr/share/$NAME/config-template
+CRONFILE=/etc/cron.d/$NAME
+TEMPLATECRON=/usr/share/$NAME/cron-template
 SEDSCRIPT=$CONFIGFILE.sed.tmp
 
 subvar_raw() {
@@ -94,20 +96,35 @@ case "$1" in
         mv -f $CONFIGFILE.tmp $CONFIGFILE
         rm $SEDSCRIPT
 
-        db_get $NAME/auto-start
-        if [ "$RET" = "true" ]; then
-          db_get $NAME/run-as-user
-          if ! getent passwd "$RET" >/dev/null
-          then
-            adduser --system --home /usr/share/$NAME --no-create-home --quiet "$RET"
-          fi
+        db_get $NAME/run-as-user
+        RUNAS="$RET"
+        if ! getent passwd "$RUNAS" >/dev/null
+        then
+            adduser --system --home /usr/share/$NAME --no-create-home --quiet "$RUNAS"
         fi
 
-        # -10 changed the lighttpd config file, but didn't arrange to restart it.
-        # If we are upgrading from -10 or earlier, and lighttpd is in use,
-        # restart it.
+        # create log if missing; change ownership if needed so the cronjob works
+        db_get $NAME/log-file
+        touch $RET
+        chown $RUNAS $RET
+
+        # create cronjob
+        if ! test -e $CRONFILE; then
+            echo "Creating cronjob in $CRONFILE to periodically update the aircraft database.." >&2
+            MIN=$(($RANDOM % 60))
+            tail -n +4 $TEMPLATECRON | sed -e "s/@USER@/$RUNAS/g" -e "s/@MIN@/$MIN/g" >$CRONFILE
+        fi
+
+        # update the DB
+        echo "Updating aircraft database now.."
+        mkdir -m 0755 -p /var/cache/$NAME
+        chown $RUNAS /var/cache/$NAME
+        su $RUNAS -s /bin/bash -c /usr/share/$NAME/update-aircraft-database.sh || \
+          echo "Aircraft database update failed. It will be retried periodically from cron." >&2
+
+        # config file changed between 1.14 and 1.15
         if [ -e /etc/lighttpd/conf-enabled/89-dump1090.conf ]; then
-            if dpkg --compare-versions "$2" le "1.10.3010.14mu-10"; then
+            if dpkg --compare-versions "$2" le "1.14"; then
                 echo "Restarting lighttpd.." >&2
                 service lighttpd restart || echo "Warning: lighttpd failed to restart." >&2
             fi

debian/dump1090-mutability.postrm

@@ -22,6 +22,8 @@ set -e
 case "$1" in
     purge)
         rm -f /etc/default/dump1090-mutability
+        rm -f /etc/cron.d/dump1090-mutability
+        rm -rf /var/cache/dump1090-mutability
     ;;
 
     remove|upgrade|failed-upgrade|abort-install|abort-upgrade|disappear)

debian/lighttpd/89-dump1090.conf

@@ -9,10 +9,11 @@ url.redirect += (
 
 alias.url += (
   "/dump1090/data/" => "/run/dump1090-mutability/",
+  "/dump1090/db/" => "/var/cache/dump1090-mutability/db/",
   "/dump1090/" => "/usr/share/dump1090-mutability/html/"
 )
 
-# The stat cache must be disabled, as receiver.json changes
-# rapidly and lighttpd's stat cache often ends up with the
+# The stat cache must be disabled, as aircraft.json changes
+# frequently and lighttpd's stat cache often ends up with the
 # wrong content length.
 server.stat-cache-engine    = "disable"

tools/update-aircraft-database.sh

@@ -0,0 +1,109 @@
+#!/bin/bash
+
+# This script checks for a new version of BasicAircraftLookup from
+# the VRS website and, if one is available, downloads it and updates
+# the dump1090 webmap json files.
+
+set -e
+
+# defaults that can be overridden:
+VRS_URL=${VRS_URL:-http://www.virtualradarserver.co.uk/Files/BasicAircraftLookup.sqb.gz}
+CACHEDIR=${CACHEDIR:-/var/cache/dump1090-mutability}
+JSONDIR=${JSONDIR:-$CACHEDIR/db}
+SQBDIR=${SQBDIR:-$CACHEDIR/sqb}
+LOGFILE=${LOGFILE:-/var/log/dump1090-mutability.log}
+UPDATESCRIPT=${UPDATESCRIPT:-/usr/share/dump1090-mutability/vrs-basicaircraft-to-json.py}
+
+if [ -f /etc/default/dump1090-mutability ]
+then
+  . /etc/default/dump1090-mutability
+fi
+
+ETAGFILE=$SQBDIR/BasicAircraftLookup.sqb.etag
+SQBFILE=$SQBDIR/BasicAircraftLookup.sqb
+
+CHECKMODIFIED=true
+LOGTOFILE=false
+while [ "$#" -gt 0 ]
+do
+  case "$1" in
+    -f|--force) CHECKMODIFIED=false ;;
+    -l|--log-to-file) LOGTOFILE=true ;;
+    *) echo "unrecognized option: $1" >&2; exit 1 ;;
+  esac
+  shift
+done
+
+if $LOGTOFILE; then exec >>$LOGFILE 2>&1; fi
+
+log() {
+  date "+%c $*" >&2
+}
+
+mkdir -p $CACHEDIR $JSONDIR $SQBDIR
+rm -f $ETAGFILE.new $SQBFILE.new
+
+log "Checking VRS server for an updated database.."
+
+# get ETag
+curl --silent --fail --include --head $VRS_URL | grep ETag >$ETAGFILE.new
+
+# check for existing file
+RETRIEVE=true
+ARGS=""
+if $CHECKMODIFIED && [ -f $SQBFILE ]
+then
+  if [ -s $ETAGFILE -a -s $ETAGFILE.new ]
+  then
+    if cmp -s $ETAGFILE $ETAGFILE.new
+    then
+      log "Database not modified."
+      RETRIEVE=false
+    else
+      log "Database modified, will retrieve a new copy."
+    fi
+  else
+    # do an if-modified-since
+    log "Database possibly modified, will try to retrieve a new copy."
+    ARGS="-z $SQBFILE"
+  fi
+fi
+
+if $RETRIEVE
+then
+  log "Retrieving database.."
+  curl --silent --fail --remote-time --retry 2 $ARGS -o $SQBFILE.new $VRS_URL
+  mv $ETAGFILE.new $ETAGFILE
+  if [ -f $SQBFILE.new ]
+  then
+    log "Decompressing database.."
+    zcat $SQBFILE.new >$SQBFILE
+    touch -r $SQBFILE.new $SQBFILE
+    rm $SQBFILE.new
+  else
+    log "Database not modified."
+  fi
+fi
+
+UPDATE=true
+if $CHECKMODIFIED
+then
+  if test -f $JSONDIR/last_update; then
+    if ! test $SQBFILE -nt $JSONDIR/last_update; then UPDATE=false; fi
+  fi
+fi
+
+if $UPDATE
+then
+  log "Updating JSON files from database.."
+  mkdir -p $JSONDIR/new
+  $UPDATESCRIPT $SQBFILE $JSONDIR/new
+  rm -f $JSONDIR/*.json
+  mv $JSONDIR/new/*.json $JSONDIR/
+  touch -r $SQBFILE $JSONDIR/last_update
+  rmdir $JSONDIR/new
+  log "Done."
+else
+  log "No update to JSON files needed."
+fi
+

tools/vrs-basicaircraft-to-json.py

@@ -8,7 +8,7 @@
 import sqlite3, json
 from contextlib import closing
 
-def extract(dbfile, todir, blocklimit):
+def extract(dbfile, todir, blocklimit, debug):
     ac_count = 0
     block_count = 0
 
@@ -35,7 +35,7 @@ def extract(dbfile, todir, blocklimit):
 
         blockdata = blocks[bkey]
         if len(blockdata) > blocklimit:
-            print 'Splitting block', bkey, 'with', len(blockdata), 'entries..',
+            if debug: print 'Splitting block', bkey, 'with', len(blockdata), 'entries..',
 
             # split all children out
             children = {}
@@ -62,7 +62,7 @@ def extract(dbfile, todir, blocklimit):
                     retained += 1
                 del children[0]
 
-            print len(children), 'children created,', len(blockdata), 'entries retained in parent'
+            if debug: print len(children), 'children created,', len(blockdata), 'entries retained in parent'
             blockdata['children'] = sorted([x[0] for x in children])
             blocks[bkey] = blockdata
             for c_bkey, c_entries in children:
@@ -70,7 +70,7 @@ def extract(dbfile, todir, blocklimit):
                 queue.append(c_bkey)
 
         path = todir + '/' + bkey + '.json'
-        print 'Writing', len(blockdata), 'entries to', path
+        if debug: print 'Writing', len(blockdata), 'entries to', path
         block_count += 1
         with closing(open(path, 'w')) as f:
             json.dump(obj=blockdata, fp=f, check_circular=False, separators=(',',':'), sort_keys=True)
@@ -83,5 +83,5 @@ if __name__ == '__main__':
         print 'Syntax: %s <path to BasicAircraftLookup.sqb> <path to DB dir>' % sys.argv[0]
         sys.exit(1)
     else:
-        extract(sys.argv[1], sys.argv[2], 1000)
+        extract(sys.argv[1], sys.argv[2], 1000, False)
         sys.exit(0)