Rearrange json DB stuff so it can work with CSV input.

tools/README.aircraft-db

@@ -0,0 +1,33 @@
+The dump1090 webmap uses a static database of json files to provide aircraft
+information.
+
+This directory has some tools to turn a CSV file with aircraft data into
+the json format that the dump1090 map expects.
+
+The default data comes from a database kindly provided by VRS (unfortunately
+no longer updated). This data is in vrs.csv. It was extracted by:
+
+  $ wget http://www.virtualradarserver.co.uk/Files/BasicAircraftLookup.sqb.gz
+  $ gunzip BasicAircraftLookup.sqb.gz
+  $ tools/vrs-to-csv.py BasicAircraftLookup.sqb >tools/vrs.csv
+
+You can modify vrs.csv (or build a new CSV entirely) and update the database
+via:
+
+  $ tools/csv-to-json.py tools/vrs.csv public_html/db
+
+The contents of public_html/db should be installed where the webmap can find
+them; the Debian packaging puts these in
+/usr/share/dump1090-mutability/html/db
+
+The CSV format is very simple. The first line must be a header line that names
+the columns. These columns are understood:
+
+ icao24: the 6-digit hex address of the aircraft
+ r: the registration / tail number of the aircraft
+ t: the ICAO aircraft type of the aircraft, e.g. B773
+
+Any other columns are put into the json DB under the name you give them, but
+the standard map code won't do anything special with them. You can pick these
+columns up in the PlaneObject constructor (see planeObject.js where it calls
+getAircraftData()) for later use.

tools/csv-to-json.py

@@ -5,10 +5,10 @@
 # into a bunch of json files suitable for use by the webmap
 #
 
-import sqlite3, json, sys
+import sqlite3, json, sys, csv
 from contextlib import closing
 
-def extract(dbfile, todir, blocklimit, debug):
+def extract(infile, todir, blocklimit, debug):
     ac_count = 0
     block_count = 0
 
@@ -16,18 +16,22 @@ def extract(dbfile, todir, blocklimit, debug):
     for i in xrange(16):
         blocks['%01X' % i] = {}
 
-    print >>sys.stderr, 'Reading', dbfile
+    reader = csv.DictReader(infile)
-    with closing(sqlite3.connect(dbfile)) as db:
+    if not 'icao24' in reader.fieldnames:
-        with closing(db.execute('SELECT a.Icao, a.Registration, m.Icao FROM Aircraft a, Model m WHERE a.ModelID = m.ModelID')) as c:
+        raise RuntimeError('CSV should have at least an "icao24" column')
-            for icao24, reg, icaotype in c:
+    for row in reader:
-                bkey = icao24[0:1].upper()
+        icao24 = row['icao24']
-                dkey = icao24[1:].upper()
-                blocks[bkey][dkey] = {}
-                if reg: blocks[bkey][dkey]['r'] = reg
-                if icaotype: blocks[bkey][dkey]['t'] = icaotype
-                ac_count += 1
-    print >>sys.stderr, 'Read', ac_count, 'aircraft'
 
+        bkey = icao24[0:1].upper()
+        dkey = icao24[1:].upper()
+        blocks[bkey][dkey] = {}
+
+        for k,v in row.items():
+            if k != 'icao24' and v != '':
+                blocks[bkey][dkey][k] = v
+        ac_count += 1
+
+    print >>sys.stderr, 'Read', ac_count, 'aircraft'
     print >>sys.stderr, 'Writing blocks:',
 
     queue = sorted(blocks.keys())
@@ -83,8 +87,14 @@ def extract(dbfile, todir, blocklimit, debug):
 
 if __name__ == '__main__':
     if len(sys.argv) < 3:
-        print >>sys.stderr, 'Syntax: %s <path to BasicAircraftLookup.sqb> <path to DB dir>' % sys.argv[0]
+        print >>sys.stderr, 'Reads a CSV file with aircraft information and produces a directory of JSON files'
+        print >>sys.stderr, 'Syntax: %s <path to CSV> <path to DB dir>' % sys.argv[0]
+        print >>sys.stderr, 'Use "-" as the CSV path to read from stdin'
         sys.exit(1)
     else:
-        extract(sys.argv[1], sys.argv[2], 1000, False)
+        if sys.argv[1] == '-':
+            extract(sys.stdin, sys.argv[2], 1000, False)
+        else:
+            with closing(open(sys.argv[1], 'r')) as infile:
+                extract(infile, sys.argv[2], 1000, False)
         sys.exit(0)

tools/vrs-to-csv.py

@@ -0,0 +1,31 @@
+#!/usr/bin/env python2
+
+#
+# Converts a Virtual Radar Server BasicAircraftLookup.sqb database
+# to a CSV file suitable for feeding to csv-to-json.py
+#
+
+import sqlite3, csv, sys
+from contextlib import closing
+
+def extract(dbfile):
+    writer = csv.DictWriter(sys.stdout,
+                            fieldnames=['icao24', 'r', 't'])
+    writer.writeheader()
+    with closing(sqlite3.connect(dbfile)) as db:
+        with closing(db.execute('SELECT a.Icao, a.Registration, m.Icao FROM Aircraft a, Model m WHERE a.ModelID = m.ModelID')) as c:
+            for icao24, reg, icaotype in c:
+                writer.writerow({
+                    'icao24': icao24,
+                    'r': reg,
+                    't': icaotype
+                })
+
+if __name__ == '__main__':
+    if len(sys.argv) < 2:
+        print >>sys.stderr, 'Reads a VRS sqlite database and writes a CSV to stdout'
+        print >>sys.stderr, 'Syntax: %s <path to BasicAircraftLookup.sqb>' % sys.argv[0]
+        sys.exit(1)
+    else:
+        extract(sys.argv[1])
+        sys.exit(0)