Add a script that filters redundant registrations from the json db

public_html/registrations.js

@@ -308,3 +308,8 @@ registration_from_hexid = (function () {
 
         return lookup;
 })();
+
+// make nodejs happy:
+if (module) {
+        module.exports = registration_from_hexid;
+}

tools/README.aircraft-db

@@ -11,10 +11,17 @@ no longer updated). This data is in vrs.csv. It was extracted by:
   $ gunzip BasicAircraftLookup.sqb.gz
   $ tools/vrs-to-csv.py BasicAircraftLookup.sqb >tools/vrs.csv
 
-You can modify vrs.csv (or build a new CSV entirely) and update the database
-via:
+You can modify vrs.csv (or build a new CSV entirely) and update the database.
 
-  $ tools/csv-to-json.py tools/vrs.csv public_html/db
+First, as an optional step, you can prune out all registrations which match
+what the in-browser hexid-to-registration logic would generate anyway. This
+requires nodejs, see the comments in filter-regs.js
+
+  $ nodejs tools/filter-regs.js <tools/vrs.csv >tools/vrs-pruned.csv
+
+Next, turn the pruned CSV into a set of json files:
+
+  $ tools/csv-to-json.py tools/vrs-pruned.csv public_html/db
 
 The contents of public_html/db should be installed where the webmap can find
 them; the Debian packaging puts these in

tools/filter-regs.js

@@ -0,0 +1,36 @@
+//
+// This script processes a CSV file that contains
+// ICAO addresses (column 'icao24') and registrations
+// (column 'r')
+//
+// It removes all registration entries that exactly match
+// what dump1090 would have computed from the hexid anyway,
+// reducing the size of the CSV in the cases where the
+// two approaches match.
+//
+// Any additional columns are passed through unchanged.
+//
+// To run it:
+//
+//   sudo apt-get install nodejs
+//   sudo apt-get install npm
+//   npm install csv     # must be done in the same dir as this script
+//   nodejs filter-regs.js <input.csv >output.csv
+
+var reglookup = require('../public_html/registrations.js');
+var csv = require('csv');
+
+var parser = csv.parse({columns: true});
+var writer = csv.stringify({header: true});
+var transformer = csv.transform(function (record, callback) {
+        if (('icao24' in record) && ('r' in record)) {
+                var computed = reglookup(record.icao24);
+                if (computed === record.r) {
+                        record.r = '';
+                }
+        }
+
+        callback(null, record);
+});
+
+process.stdin.pipe(parser).pipe(transformer).pipe(writer).pipe(process.stdout);