Rearrange json DB stuff so it can work with CSV input.

tools/csv-to-json.py

@@ -0,0 +1,100 @@
+#!/usr/bin/env python2
+
+#
+# Converts a Virtual Radar Server BasicAircraftLookup.sqb database
+# into a bunch of json files suitable for use by the webmap
+#
+
+import sqlite3, json, sys, csv
+from contextlib import closing
+
+def extract(infile, todir, blocklimit, debug):
+    ac_count = 0
+    block_count = 0
+
+    blocks = {}
+    for i in xrange(16):
+        blocks['%01X' % i] = {}
+
+    reader = csv.DictReader(infile)
+    if not 'icao24' in reader.fieldnames:
+        raise RuntimeError('CSV should have at least an "icao24" column')
+    for row in reader:
+        icao24 = row['icao24']
+
+        bkey = icao24[0:1].upper()
+        dkey = icao24[1:].upper()
+        blocks[bkey][dkey] = {}
+
+        for k,v in row.items():
+            if k != 'icao24' and v != '':
+                blocks[bkey][dkey][k] = v
+        ac_count += 1
+
+    print >>sys.stderr, 'Read', ac_count, 'aircraft'
+    print >>sys.stderr, 'Writing blocks:',
+
+    queue = sorted(blocks.keys())
+    while queue:
+        bkey = queue[0]
+        del queue[0]
+
+        blockdata = blocks[bkey]
+        if len(blockdata) > blocklimit:
+            if debug: print >>sys.stderr, 'Splitting block', bkey, 'with', len(blockdata), 'entries..',
+
+            # split all children out
+            children = {}
+            for dkey in blockdata.keys():
+                new_bkey = bkey + dkey[0]
+                new_dkey = dkey[1:]
+
+                if new_bkey not in children: children[new_bkey] = {}
+                children[new_bkey][new_dkey] = blockdata[dkey]
+
+            # look for small children we can retain in the parent, to
+            # reduce the total number of files needed. This reduces the
+            # number of blocks needed from 150 to 61
+            blockdata = {}
+            children = sorted(children.items(), key=lambda x: len(x[1]))
+            retained = 1
+
+            while len(children[0][1]) + retained < blocklimit:
+                # move this child back to the parent
+                c_bkey, c_entries = children[0]
+                for c_dkey, entry in c_entries.items():
+                    blockdata[c_bkey[-1] + c_dkey] = entry
+                    retained += 1
+                del children[0]
+
+            if debug: print >>sys.stderr, len(children), 'children created,', len(blockdata), 'entries retained in parent'
+            children = sorted(children, key=lambda x: x[0])
+            blockdata['children'] = [x[0] for x in children]
+            blocks[bkey] = blockdata
+            for c_bkey, c_entries in children:
+                blocks[c_bkey] = c_entries
+                queue.append(c_bkey)
+
+        path = todir + '/' + bkey + '.json'
+        if debug: print >>sys.stderr, 'Writing', len(blockdata), 'entries to', path
+        else: print >>sys.stderr, bkey,
+        block_count += 1
+        with closing(open(path, 'w')) as f:
+            json.dump(obj=blockdata, fp=f, check_circular=False, separators=(',',':'), sort_keys=True)
+
+    print >>sys.stderr, 'done.'
+    print >>sys.stderr, 'Wrote', block_count, 'blocks'
+
+if __name__ == '__main__':
+    if len(sys.argv) < 3:
+        print >>sys.stderr, 'Reads a CSV file with aircraft information and produces a directory of JSON files'
+        print >>sys.stderr, 'Syntax: %s <path to CSV> <path to DB dir>' % sys.argv[0]
+        print >>sys.stderr, 'Use "-" as the CSV path to read from stdin'
+        sys.exit(1)
+    else:
+        if sys.argv[1] == '-':
+            extract(sys.stdin, sys.argv[2], 1000, False)
+        else:
+            with closing(open(sys.argv[1], 'r')) as infile:
+                extract(infile, sys.argv[2], 1000, False)
+        sys.exit(0)