Rearrange json DB stuff so it can work with CSV input.

This commit is contained in:
Oliver Jowett 2016-08-27 16:12:52 +01:00
parent ca57fb27f0
commit 158a8298fc
4 changed files with 31197 additions and 15 deletions

33
tools/README.aircraft-db Normal file
View file

@ -0,0 +1,33 @@
The dump1090 webmap uses a static database of json files to provide aircraft
information.
This directory has some tools to turn a CSV file with aircraft data into
the json format that the dump1090 map expects.
The default data comes from a database kindly provided by VRS (unfortunately
no longer updated). This data is in vrs.csv. It was extracted by:
$ wget http://www.virtualradarserver.co.uk/Files/BasicAircraftLookup.sqb.gz
$ gunzip BasicAircraftLookup.sqb.gz
$ tools/vrs-to-csv.py BasicAircraftLookup.sqb >tools/vrs.csv
You can modify vrs.csv (or build a new CSV entirely) and update the database
via:
$ tools/csv-to-json.py tools/vrs.csv public_html/db
The contents of public_html/db should be installed where the webmap can find
them; the Debian packaging puts these in
/usr/share/dump1090-mutability/html/db
The CSV format is very simple. The first line must be a header line that names
the columns. These columns are understood:
icao24: the 6-digit hex address of the aircraft
r: the registration / tail number of the aircraft
t: the ICAO aircraft type of the aircraft, e.g. B773
Any other columns are put into the json DB under the name you give them, but
the standard map code won't do anything special with them. You can pick these
columns up in the PlaneObject constructor (see planeObject.js where it calls
getAircraftData()) for later use.

View file

@ -5,10 +5,10 @@
# into a bunch of json files suitable for use by the webmap # into a bunch of json files suitable for use by the webmap
# #
import sqlite3, json, sys import sqlite3, json, sys, csv
from contextlib import closing from contextlib import closing
def extract(dbfile, todir, blocklimit, debug): def extract(infile, todir, blocklimit, debug):
ac_count = 0 ac_count = 0
block_count = 0 block_count = 0
@ -16,18 +16,22 @@ def extract(dbfile, todir, blocklimit, debug):
for i in xrange(16): for i in xrange(16):
blocks['%01X' % i] = {} blocks['%01X' % i] = {}
print >>sys.stderr, 'Reading', dbfile reader = csv.DictReader(infile)
with closing(sqlite3.connect(dbfile)) as db: if not 'icao24' in reader.fieldnames:
with closing(db.execute('SELECT a.Icao, a.Registration, m.Icao FROM Aircraft a, Model m WHERE a.ModelID = m.ModelID')) as c: raise RuntimeError('CSV should have at least an "icao24" column')
for icao24, reg, icaotype in c: for row in reader:
bkey = icao24[0:1].upper() icao24 = row['icao24']
dkey = icao24[1:].upper()
blocks[bkey][dkey] = {}
if reg: blocks[bkey][dkey]['r'] = reg
if icaotype: blocks[bkey][dkey]['t'] = icaotype
ac_count += 1
print >>sys.stderr, 'Read', ac_count, 'aircraft'
bkey = icao24[0:1].upper()
dkey = icao24[1:].upper()
blocks[bkey][dkey] = {}
for k,v in row.items():
if k != 'icao24' and v != '':
blocks[bkey][dkey][k] = v
ac_count += 1
print >>sys.stderr, 'Read', ac_count, 'aircraft'
print >>sys.stderr, 'Writing blocks:', print >>sys.stderr, 'Writing blocks:',
queue = sorted(blocks.keys()) queue = sorted(blocks.keys())
@ -83,8 +87,14 @@ def extract(dbfile, todir, blocklimit, debug):
if __name__ == '__main__': if __name__ == '__main__':
if len(sys.argv) < 3: if len(sys.argv) < 3:
print >>sys.stderr, 'Syntax: %s <path to BasicAircraftLookup.sqb> <path to DB dir>' % sys.argv[0] print >>sys.stderr, 'Reads a CSV file with aircraft information and produces a directory of JSON files'
print >>sys.stderr, 'Syntax: %s <path to CSV> <path to DB dir>' % sys.argv[0]
print >>sys.stderr, 'Use "-" as the CSV path to read from stdin'
sys.exit(1) sys.exit(1)
else: else:
extract(sys.argv[1], sys.argv[2], 1000, False) if sys.argv[1] == '-':
extract(sys.stdin, sys.argv[2], 1000, False)
else:
with closing(open(sys.argv[1], 'r')) as infile:
extract(infile, sys.argv[2], 1000, False)
sys.exit(0) sys.exit(0)

31
tools/vrs-to-csv.py Executable file
View file

@ -0,0 +1,31 @@
#!/usr/bin/env python2
#
# Converts a Virtual Radar Server BasicAircraftLookup.sqb database
# to a CSV file suitable for feeding to csv-to-json.py
#
import sqlite3, csv, sys
from contextlib import closing
def extract(dbfile):
writer = csv.DictWriter(sys.stdout,
fieldnames=['icao24', 'r', 't'])
writer.writeheader()
with closing(sqlite3.connect(dbfile)) as db:
with closing(db.execute('SELECT a.Icao, a.Registration, m.Icao FROM Aircraft a, Model m WHERE a.ModelID = m.ModelID')) as c:
for icao24, reg, icaotype in c:
writer.writerow({
'icao24': icao24,
'r': reg,
't': icaotype
})
if __name__ == '__main__':
if len(sys.argv) < 2:
print >>sys.stderr, 'Reads a VRS sqlite database and writes a CSV to stdout'
print >>sys.stderr, 'Syntax: %s <path to BasicAircraftLookup.sqb>' % sys.argv[0]
sys.exit(1)
else:
extract(sys.argv[1])
sys.exit(0)

31108
tools/vrs.csv Normal file

File diff suppressed because it is too large Load diff