AFG/dump1090
3
0
Fork 0
Code Issues Pull requests Packages Projects Releases Wiki Activity

Support reading multiple CSVs in csv-to-json.py

Browse source
This commit is contained in:
Oliver Jowett 2016-09-10 17:26:55 +01:00
parent f45430e3b5
commit 2a3e4a0a0d
1 changed files with 34 additions and 20 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

54
tools/csv-to-json.py
View file

@ -8,13 +8,14 @@
import sqlite3, json, sys, csv import sqlite3, json, sys, csv
from contextlib import closing from contextlib import closing
def extract(infile, todir, blocklimit, debug): def readcsv(name, infile, blocks):
ac_count = 0 print >>sys.stderr, 'Reading from', name
block_count = 0
blocks = {} if len(blocks) == 0:
for i in xrange(16): for i in xrange(16):
blocks['%01X' % i] = {} blocks['%01X' % i] = {}
ac_count = 0
reader = csv.DictReader(infile) reader = csv.DictReader(infile)
if not 'icao24' in reader.fieldnames: if not 'icao24' in reader.fieldnames:
@ -22,16 +23,23 @@ def extract(infile, todir, blocklimit, debug):
for row in reader: for row in reader:
icao24 = row['icao24'] icao24 = row['icao24']
bkey = icao24[0:1].upper() entry = {}
dkey = icao24[1:].upper()
blocks[bkey][dkey] = {}
for k,v in row.items(): for k,v in row.items():
if k != 'icao24' and v != '': if k != 'icao24' and v != '':
blocks[bkey][dkey][k] = v entry[k] = v
ac_count += 1
if len(entry) > 0:
ac_count += 1
bkey = icao24[0:1].upper()
dkey = icao24[1:].upper()
blocks[bkey].setdefault(dkey, {}).update(entry)
print >>sys.stderr, 'Read', ac_count, 'aircraft from', name
def writedb(blocks, todir, blocklimit, debug):
block_count = 0
print >>sys.stderr, 'Read', ac_count, 'aircraft'
print >>sys.stderr, 'Writing blocks:', print >>sys.stderr, 'Writing blocks:',
queue = sorted(blocks.keys()) queue = sorted(blocks.keys())
@ -88,13 +96,19 @@ def extract(infile, todir, blocklimit, debug):
if __name__ == '__main__': if __name__ == '__main__':
if len(sys.argv) < 3: if len(sys.argv) < 3:
print >>sys.stderr, 'Reads a CSV file with aircraft information and produces a directory of JSON files' print >>sys.stderr, 'Reads a CSV file with aircraft information and produces a directory of JSON files'
print >>sys.stderr, 'Syntax: %s <path to CSV> <path to DB dir>' % sys.argv[0] print >>sys.stderr, 'Syntax: %s <path to CSV> [... additional CSV files ...] <path to DB dir>' % sys.argv[0]
print >>sys.stderr, 'Use "-" as the CSV path to read from stdin' print >>sys.stderr, 'Use "-" as the CSV path to read from stdin'
print >>sys.stderr, 'If multiple CSV files are specified and they provide conflicting data'
print >>sys.stderr, 'then the data from the last-listed CSV file is used'
sys.exit(1) sys.exit(1)
else:
if sys.argv[1] == '-': blocks = {}
extract(sys.stdin, sys.argv[2], 1000, False) for filename in sys.argv[1:-1]:
if filename == '-':
readcsv('stdin', sys.stdin, blocks)
else: else:
with closing(open(sys.argv[1], 'r')) as infile: with closing(open(filename, 'r')) as infile:
extract(infile, sys.argv[2], 1000, False) readcsv(filename, infile, blocks)
sys.exit(0)
writedb(blocks, sys.argv[-1], 1000, False)
sys.exit(0)