35299c47ba
rather than blanking them out entirely. This lets csv-to-json.py handle the case where an earlier input file has (e.g.) a registration value which does not match the computed value, and then a later input file has a value that does match. In this case we want to override the older value with the newer value, then notice that the registration can be omitted when writing the database. Previously in this case the older (incorrect) value would be used.
69 lines
3.1 KiB
JavaScript
69 lines
3.1 KiB
JavaScript
//
|
|
// This script processes a CSV file that contains
|
|
// ICAO addresses (column 'icao24') and registrations
|
|
// (column 'r')
|
|
//
|
|
// It replaces all registration entries that exactly match
|
|
// what dump1090 would have computed from the hexid anyway
|
|
// with the special value "-COMPUTED-"; these values are
|
|
// removed from the final data when csv-to-json.py writes
|
|
// the json database, reducing the size of the database
|
|
// in the cases where the two approaches match.
|
|
//
|
|
// It does a similar transformation on 'desc' (aircraft type
|
|
// description) and 'wtc' (wake turbulence categogy) based
|
|
// on the aircraft type designator in 't', if provided.
|
|
//
|
|
// Any additional columns are passed through unchanged.
|
|
//
|
|
// To run it:
|
|
//
|
|
// sudo apt-get install nodejs
|
|
// sudo apt-get install npm
|
|
// npm install csv # must be done in the same dir as this script
|
|
// nodejs filter-regs.js <input.csv >output.csv
|
|
|
|
var reglookup = require('../public_html/registrations.js');
|
|
var actypes = require('../public_html/db/aircraft_types/icao_aircraft_types.json');
|
|
var csv = require('csv');
|
|
|
|
var parser = csv.parse({columns: true});
|
|
var writer = csv.stringify({header: true});
|
|
var transformer = csv.transform(function (record, callback) {
|
|
if ('icao24' in record) {
|
|
if ('r' in record && record.r != '') {
|
|
var computed = reglookup(record.icao24);
|
|
if (computed === record.r) {
|
|
record.r = '-COMPUTED-';
|
|
} else if (computed !== null) {
|
|
console.warn(record.icao24 + " computed registration " + computed + " but CSV data had " + record.r);
|
|
}
|
|
}
|
|
|
|
if ('t' in record && record.t in actypes) {
|
|
if ('desc' in record && record.desc != '') {
|
|
var computed_desc = actypes[record.t].desc;
|
|
if (computed_desc === record.desc) {
|
|
record.desc = '-COMPUTED-';
|
|
} else if (computed_desc !== undefined) {
|
|
// too noisy, the icao descriptors are very coarse and reality often disagrees
|
|
//console.warn(record.icao24 + " (" + record.t + "): computed type description " + computed_desc + " but CSV data had " + record.desc);
|
|
}
|
|
}
|
|
|
|
if ('wtc' in record && record.wtc != '') {
|
|
var computed_wtc = actypes[record.t].wtc;
|
|
if (computed_wtc === record.wtc) {
|
|
record.wtc = '-COMPUTED-';
|
|
} else if (computed_desc !== undefined) {
|
|
//console.warn(record.icao24 + " (" + record.t + "): computed type WTC " + computed_wtc + " but CSV data had " + record.wtc);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
callback(null, record);
|
|
});
|
|
|
|
process.stdin.pipe(parser).pipe(transformer).pipe(writer).pipe(process.stdout);
|