AFG/dump1090
3
0
Fork 0
Code Issues Pull requests Packages Projects Releases Wiki Activity

Have filter-regs emit a special value "-COMPUTED-" for filtered data values,

Browse source 
rather than blanking them out entirely.

This lets csv-to-json.py handle the case where an earlier input file has (e.g.)
a registration value which does not match the computed value, and then a later
input file has a value that does match. In this case we want to override the
older value with the newer value, then notice that the registration can be
omitted when writing the database. Previously in this case the older (incorrect)
value would be used.
This commit is contained in:
Oliver Jowett 2019-08-16 15:16:55 +08:00
parent 0d2bd6ae34
commit 35299c47ba
2 changed files with 24 additions and 7 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

11
tools/csv-to-json.py
View file

@ -37,6 +37,16 @@ def readcsv(name, infile, blocks):
print >>sys.stderr, 'Read', ac_count, 'aircraft from', name
def cleandb(blocks):
for blockdata in blocks.values():
for dkey in list(blockdata.keys()):
block = blockdata[dkey]
for key in list(block.keys()):
if block[key] == '-COMPUTED-':
del block[key]
if len(block) == 0:
del blockdata[dkey]
def writedb(blocks, todir, blocklimit, debug):
block_count = 0
@ -110,5 +120,6 @@ if __name__ == '__main__':
with closing(open(filename, 'r')) as infile:
readcsv(filename, infile, blocks)
cleandb(blocks)
writedb(blocks, sys.argv[-1], 2500, False)
sys.exit(0)

20
tools/filter-regs.js
View file

@ -3,10 +3,16 @@
// ICAO addresses (column 'icao24') and registrations
// (column 'r')
//
// It removes all registration entries that exactly match
// what dump1090 would have computed from the hexid anyway,
// reducing the size of the CSV in the cases where the
// two approaches match.
// It replaces all registration entries that exactly match
// what dump1090 would have computed from the hexid anyway
// with the special value "-COMPUTED-"; these values are
// removed from the final data when csv-to-json.py writes
// the json database, reducing the size of the database
// in the cases where the two approaches match.
//
// It does a similar transformation on 'desc' (aircraft type
// description) and 'wtc' (wake turbulence categogy) based
// on the aircraft type designator in 't', if provided.
//
// Any additional columns are passed through unchanged.
//
@ -28,7 +34,7 @@ var transformer = csv.transform(function (record, callback) {
if ('r' in record && record.r != '') {
var computed = reglookup(record.icao24);
if (computed === record.r) {
record.r = '';
record.r = '-COMPUTED-';
} else if (computed !== null) {
console.warn(record.icao24 + " computed registration " + computed + " but CSV data had " + record.r);
}
@ -38,7 +44,7 @@ var transformer = csv.transform(function (record, callback) {
if ('desc' in record && record.desc != '') {
var computed_desc = actypes[record.t].desc;
if (computed_desc === record.desc) {
record.desc = '';
record.desc = '-COMPUTED-';
} else if (computed_desc !== undefined) {
// too noisy, the icao descriptors are very coarse and reality often disagrees
//console.warn(record.icao24 + " (" + record.t + "): computed type description " + computed_desc + " but CSV data had " + record.desc);
@ -48,7 +54,7 @@ var transformer = csv.transform(function (record, callback) {
if ('wtc' in record && record.wtc != '') {
var computed_wtc = actypes[record.t].wtc;
if (computed_wtc === record.wtc) {
record.wtc = '';
record.wtc = '-COMPUTED-';
} else if (computed_desc !== undefined) {
//console.warn(record.icao24 + " (" + record.t + "): computed type WTC " + computed_wtc + " but CSV data had " + record.wtc);
}