From 60eab2284f936aae26d1879f47a3ef1266289ced Mon Sep 17 00:00:00 2001 From: Oliver Jowett Date: Wed, 25 Feb 2015 13:16:54 +0000 Subject: [PATCH] Package changes to support the aircraft DB. --- debian/control | 1 + debian/cron-template | 12 +++ debian/dump1090-mutability.conffiles | 1 + debian/dump1090-mutability.config | 6 +- debian/dump1090-mutability.init | 5 ++ debian/dump1090-mutability.install | 3 + debian/dump1090-mutability.postinst | 41 +++++++--- debian/dump1090-mutability.postrm | 2 + debian/lighttpd/89-dump1090.conf | 5 +- tools/update-aircraft-database.sh | 109 +++++++++++++++++++++++++++ tools/vrs-basicaircraft-to-json.py | 10 +-- 11 files changed, 172 insertions(+), 23 deletions(-) create mode 100644 debian/cron-template create mode 100755 tools/update-aircraft-database.sh diff --git a/debian/control b/debian/control index d502518..8f4ac60 100644 --- a/debian/control +++ b/debian/control @@ -10,6 +10,7 @@ Vcs-Git: https://github.com/mutability/dump1090.git Package: dump1090-mutability Architecture: any Depends: ${shlibs:Depends}, ${misc:Depends}, adduser +Recommends: ${python:Depends}, cron | cron-daemon Suggests: lighttpd Provides: fatsv-data-source Description: ADS-B Ground Station System for RTL-SDR diff --git a/debian/cron-template b/debian/cron-template new file mode 100644 index 0000000..38b1210 --- /dev/null +++ b/debian/cron-template @@ -0,0 +1,12 @@ +## TEMPLATE FILE - This is used to create /etc/cron.d/dump1090-mutability ## +## The first three lines will be discarded ## + +# Checks for updates to the VRS aircraft database once a day at around 5am; +# when an update is available, downloads it and regenerates the JSON data +# used by the webmap. + +# NB: the minute value below is randomly generated on install, to avoid +# all installs hitting the server at the same time. + +# m h dom mon dow user command +@MIN@ 5 * * * @USER@ test -x /usr/share/dump1090-mutability/update-aircraft-database.sh && /usr/share/dump1090-mutability/update-aircraft-database.sh --log-to-file diff --git a/debian/dump1090-mutability.conffiles b/debian/dump1090-mutability.conffiles index 4a65fcc..1099d0c 100644 --- a/debian/dump1090-mutability.conffiles +++ b/debian/dump1090-mutability.conffiles @@ -5,3 +5,4 @@ /usr/share/dump1090-mutability/html/style.css /usr/share/dump1090-mutability/html/markers.js /usr/share/dump1090-mutability/html/formatter.js +/usr/share/dump1090-mutability/html/dbloader.js diff --git a/debian/dump1090-mutability.config b/debian/dump1090-mutability.config index b7bd76e..3e70c61 100644 --- a/debian/dump1090-mutability.config +++ b/debian/dump1090-mutability.config @@ -184,13 +184,11 @@ is_port_number() { } db_input high $NAME/auto-start || true +db_input_verify low $NAME/run-as-user is_non_root_user || true +db_input_verify low $NAME/log-file is_not_empty || true db_go || true; db_get $NAME/auto-start; if [ "$RET" = "true" ]; then # all of these are only relevant if the init script is enabled - - db_input_verify low $NAME/run-as-user is_non_root_user || true - db_input_verify low $NAME/log-file is_not_empty || true - db_input medium $NAME/rtlsdr-device || true db_go || true; db_get $NAME/rtlsdr-device; if [ "x$RET" != "xnone" ]; then diff --git a/debian/dump1090-mutability.init b/debian/dump1090-mutability.init index 3a81af3..39b2415 100644 --- a/debian/dump1090-mutability.init +++ b/debian/dump1090-mutability.init @@ -130,6 +130,11 @@ do_start() fi fi + # create logfile with the appropriate permissions if not already there + # (the cronjob running as DUMP1090_USER wants to append to it) + touch $LOGFILE + chown "$DUMP1090_USER":root $LOGFILE + start-stop-daemon --start $NICELEVEL --quiet --pidfile $PIDFILE --user "$DUMP1090_USER" --chuid "$DUMP1090_USER" --make-pidfile --background --no-close --exec $DAEMON -- \ $ARGS >>$LOGFILE 2>&1 \ || return 2 diff --git a/debian/dump1090-mutability.install b/debian/dump1090-mutability.install index bfc44bc..68b02bc 100644 --- a/debian/dump1090-mutability.install +++ b/debian/dump1090-mutability.install @@ -1,3 +1,6 @@ public_html/* usr/share/dump1090-mutability/html debian/lighttpd/* etc/lighttpd/conf-available debian/config-template usr/share/dump1090-mutability +debian/cron-template usr/share/dump1090-mutability +tools/vrs-basicaircraft-to-json.py usr/share/dump1090-mutability +tools/update-aircraft-database.sh usr/share/dump1090-mutability diff --git a/debian/dump1090-mutability.postinst b/debian/dump1090-mutability.postinst index 2b48170..ae7935e 100644 --- a/debian/dump1090-mutability.postinst +++ b/debian/dump1090-mutability.postinst @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/bash # postinst script for dump1090 # # see: dh_installdeb(1) @@ -20,6 +20,8 @@ set -e NAME=dump1090-mutability CONFIGFILE=/etc/default/$NAME TEMPLATECONFIG=/usr/share/$NAME/config-template +CRONFILE=/etc/cron.d/$NAME +TEMPLATECRON=/usr/share/$NAME/cron-template SEDSCRIPT=$CONFIGFILE.sed.tmp subvar_raw() { @@ -94,20 +96,35 @@ case "$1" in mv -f $CONFIGFILE.tmp $CONFIGFILE rm $SEDSCRIPT - db_get $NAME/auto-start - if [ "$RET" = "true" ]; then - db_get $NAME/run-as-user - if ! getent passwd "$RET" >/dev/null - then - adduser --system --home /usr/share/$NAME --no-create-home --quiet "$RET" - fi + db_get $NAME/run-as-user + RUNAS="$RET" + if ! getent passwd "$RUNAS" >/dev/null + then + adduser --system --home /usr/share/$NAME --no-create-home --quiet "$RUNAS" fi - # -10 changed the lighttpd config file, but didn't arrange to restart it. - # If we are upgrading from -10 or earlier, and lighttpd is in use, - # restart it. + # create log if missing; change ownership if needed so the cronjob works + db_get $NAME/log-file + touch $RET + chown $RUNAS $RET + + # create cronjob + if ! test -e $CRONFILE; then + echo "Creating cronjob in $CRONFILE to periodically update the aircraft database.." >&2 + MIN=$(($RANDOM % 60)) + tail -n +4 $TEMPLATECRON | sed -e "s/@USER@/$RUNAS/g" -e "s/@MIN@/$MIN/g" >$CRONFILE + fi + + # update the DB + echo "Updating aircraft database now.." + mkdir -m 0755 -p /var/cache/$NAME + chown $RUNAS /var/cache/$NAME + su $RUNAS -s /bin/bash -c /usr/share/$NAME/update-aircraft-database.sh || \ + echo "Aircraft database update failed. It will be retried periodically from cron." >&2 + + # config file changed between 1.14 and 1.15 if [ -e /etc/lighttpd/conf-enabled/89-dump1090.conf ]; then - if dpkg --compare-versions "$2" le "1.10.3010.14mu-10"; then + if dpkg --compare-versions "$2" le "1.14"; then echo "Restarting lighttpd.." >&2 service lighttpd restart || echo "Warning: lighttpd failed to restart." >&2 fi diff --git a/debian/dump1090-mutability.postrm b/debian/dump1090-mutability.postrm index 6bdd5ea..f5405fa 100644 --- a/debian/dump1090-mutability.postrm +++ b/debian/dump1090-mutability.postrm @@ -22,6 +22,8 @@ set -e case "$1" in purge) rm -f /etc/default/dump1090-mutability + rm -f /etc/cron.d/dump1090-mutability + rm -rf /var/cache/dump1090-mutability ;; remove|upgrade|failed-upgrade|abort-install|abort-upgrade|disappear) diff --git a/debian/lighttpd/89-dump1090.conf b/debian/lighttpd/89-dump1090.conf index 9856b9e..5eb563d 100644 --- a/debian/lighttpd/89-dump1090.conf +++ b/debian/lighttpd/89-dump1090.conf @@ -9,10 +9,11 @@ url.redirect += ( alias.url += ( "/dump1090/data/" => "/run/dump1090-mutability/", + "/dump1090/db/" => "/var/cache/dump1090-mutability/db/", "/dump1090/" => "/usr/share/dump1090-mutability/html/" ) -# The stat cache must be disabled, as receiver.json changes -# rapidly and lighttpd's stat cache often ends up with the +# The stat cache must be disabled, as aircraft.json changes +# frequently and lighttpd's stat cache often ends up with the # wrong content length. server.stat-cache-engine = "disable" diff --git a/tools/update-aircraft-database.sh b/tools/update-aircraft-database.sh new file mode 100755 index 0000000..af1c502 --- /dev/null +++ b/tools/update-aircraft-database.sh @@ -0,0 +1,109 @@ +#!/bin/bash + +# This script checks for a new version of BasicAircraftLookup from +# the VRS website and, if one is available, downloads it and updates +# the dump1090 webmap json files. + +set -e + +# defaults that can be overridden: +VRS_URL=${VRS_URL:-http://www.virtualradarserver.co.uk/Files/BasicAircraftLookup.sqb.gz} +CACHEDIR=${CACHEDIR:-/var/cache/dump1090-mutability} +JSONDIR=${JSONDIR:-$CACHEDIR/db} +SQBDIR=${SQBDIR:-$CACHEDIR/sqb} +LOGFILE=${LOGFILE:-/var/log/dump1090-mutability.log} +UPDATESCRIPT=${UPDATESCRIPT:-/usr/share/dump1090-mutability/vrs-basicaircraft-to-json.py} + +if [ -f /etc/default/dump1090-mutability ] +then + . /etc/default/dump1090-mutability +fi + +ETAGFILE=$SQBDIR/BasicAircraftLookup.sqb.etag +SQBFILE=$SQBDIR/BasicAircraftLookup.sqb + +CHECKMODIFIED=true +LOGTOFILE=false +while [ "$#" -gt 0 ] +do + case "$1" in + -f|--force) CHECKMODIFIED=false ;; + -l|--log-to-file) LOGTOFILE=true ;; + *) echo "unrecognized option: $1" >&2; exit 1 ;; + esac + shift +done + +if $LOGTOFILE; then exec >>$LOGFILE 2>&1; fi + +log() { + date "+%c $*" >&2 +} + +mkdir -p $CACHEDIR $JSONDIR $SQBDIR +rm -f $ETAGFILE.new $SQBFILE.new + +log "Checking VRS server for an updated database.." + +# get ETag +curl --silent --fail --include --head $VRS_URL | grep ETag >$ETAGFILE.new + +# check for existing file +RETRIEVE=true +ARGS="" +if $CHECKMODIFIED && [ -f $SQBFILE ] +then + if [ -s $ETAGFILE -a -s $ETAGFILE.new ] + then + if cmp -s $ETAGFILE $ETAGFILE.new + then + log "Database not modified." + RETRIEVE=false + else + log "Database modified, will retrieve a new copy." + fi + else + # do an if-modified-since + log "Database possibly modified, will try to retrieve a new copy." + ARGS="-z $SQBFILE" + fi +fi + +if $RETRIEVE +then + log "Retrieving database.." + curl --silent --fail --remote-time --retry 2 $ARGS -o $SQBFILE.new $VRS_URL + mv $ETAGFILE.new $ETAGFILE + if [ -f $SQBFILE.new ] + then + log "Decompressing database.." + zcat $SQBFILE.new >$SQBFILE + touch -r $SQBFILE.new $SQBFILE + rm $SQBFILE.new + else + log "Database not modified." + fi +fi + +UPDATE=true +if $CHECKMODIFIED +then + if test -f $JSONDIR/last_update; then + if ! test $SQBFILE -nt $JSONDIR/last_update; then UPDATE=false; fi + fi +fi + +if $UPDATE +then + log "Updating JSON files from database.." + mkdir -p $JSONDIR/new + $UPDATESCRIPT $SQBFILE $JSONDIR/new + rm -f $JSONDIR/*.json + mv $JSONDIR/new/*.json $JSONDIR/ + touch -r $SQBFILE $JSONDIR/last_update + rmdir $JSONDIR/new + log "Done." +else + log "No update to JSON files needed." +fi + diff --git a/tools/vrs-basicaircraft-to-json.py b/tools/vrs-basicaircraft-to-json.py index d6c0724..f449e35 100755 --- a/tools/vrs-basicaircraft-to-json.py +++ b/tools/vrs-basicaircraft-to-json.py @@ -8,7 +8,7 @@ import sqlite3, json from contextlib import closing -def extract(dbfile, todir, blocklimit): +def extract(dbfile, todir, blocklimit, debug): ac_count = 0 block_count = 0 @@ -35,7 +35,7 @@ def extract(dbfile, todir, blocklimit): blockdata = blocks[bkey] if len(blockdata) > blocklimit: - print 'Splitting block', bkey, 'with', len(blockdata), 'entries..', + if debug: print 'Splitting block', bkey, 'with', len(blockdata), 'entries..', # split all children out children = {} @@ -62,7 +62,7 @@ def extract(dbfile, todir, blocklimit): retained += 1 del children[0] - print len(children), 'children created,', len(blockdata), 'entries retained in parent' + if debug: print len(children), 'children created,', len(blockdata), 'entries retained in parent' blockdata['children'] = sorted([x[0] for x in children]) blocks[bkey] = blockdata for c_bkey, c_entries in children: @@ -70,7 +70,7 @@ def extract(dbfile, todir, blocklimit): queue.append(c_bkey) path = todir + '/' + bkey + '.json' - print 'Writing', len(blockdata), 'entries to', path + if debug: print 'Writing', len(blockdata), 'entries to', path block_count += 1 with closing(open(path, 'w')) as f: json.dump(obj=blockdata, fp=f, check_circular=False, separators=(',',':'), sort_keys=True) @@ -83,5 +83,5 @@ if __name__ == '__main__': print 'Syntax: %s ' % sys.argv[0] sys.exit(1) else: - extract(sys.argv[1], sys.argv[2], 1000) + extract(sys.argv[1], sys.argv[2], 1000, False) sys.exit(0)