From d1a33f465543528c8bce70a4de37d53f09a99057 Mon Sep 17 00:00:00 2001
From: Oliver Jowett <oliver@mutability.co.uk>
Date: Sat, 10 Sep 2016 15:54:59 +0100
Subject: [PATCH] Add a script that filters redundant registrations from the
 json db

---
 public_html/registrations.js |  5 +++++
 tools/README.aircraft-db     | 13 ++++++++++---
 tools/filter-regs.js         | 36 ++++++++++++++++++++++++++++++++++++
 3 files changed, 51 insertions(+), 3 deletions(-)
 create mode 100644 tools/filter-regs.js

diff --git a/public_html/registrations.js b/public_html/registrations.js
index 299554e..72a4b48 100644
--- a/public_html/registrations.js
+++ b/public_html/registrations.js
@@ -308,3 +308,8 @@ registration_from_hexid = (function () {
 
         return lookup;
 })();
+
+// make nodejs happy:
+if (module) {
+        module.exports = registration_from_hexid;
+}
diff --git a/tools/README.aircraft-db b/tools/README.aircraft-db
index 10e6ca0..3b53c9a 100644
--- a/tools/README.aircraft-db
+++ b/tools/README.aircraft-db
@@ -11,10 +11,17 @@ no longer updated). This data is in vrs.csv. It was extracted by:
   $ gunzip BasicAircraftLookup.sqb.gz
   $ tools/vrs-to-csv.py BasicAircraftLookup.sqb >tools/vrs.csv
 
-You can modify vrs.csv (or build a new CSV entirely) and update the database
-via:
+You can modify vrs.csv (or build a new CSV entirely) and update the database.
 
-  $ tools/csv-to-json.py tools/vrs.csv public_html/db
+First, as an optional step, you can prune out all registrations which match
+what the in-browser hexid-to-registration logic would generate anyway. This
+requires nodejs, see the comments in filter-regs.js
+
+  $ nodejs tools/filter-regs.js <tools/vrs.csv >tools/vrs-pruned.csv
+
+Next, turn the pruned CSV into a set of json files:
+
+  $ tools/csv-to-json.py tools/vrs-pruned.csv public_html/db
 
 The contents of public_html/db should be installed where the webmap can find
 them; the Debian packaging puts these in
diff --git a/tools/filter-regs.js b/tools/filter-regs.js
new file mode 100644
index 0000000..74ef35d
--- /dev/null
+++ b/tools/filter-regs.js
@@ -0,0 +1,36 @@
+//
+// This script processes a CSV file that contains
+// ICAO addresses (column 'icao24') and registrations
+// (column 'r')
+//
+// It removes all registration entries that exactly match
+// what dump1090 would have computed from the hexid anyway,
+// reducing the size of the CSV in the cases where the
+// two approaches match.
+//
+// Any additional columns are passed through unchanged.
+//
+// To run it:
+//
+//   sudo apt-get install nodejs
+//   sudo apt-get install npm
+//   npm install csv     # must be done in the same dir as this script
+//   nodejs filter-regs.js <input.csv >output.csv
+
+var reglookup = require('../public_html/registrations.js');
+var csv = require('csv');
+
+var parser = csv.parse({columns: true});
+var writer = csv.stringify({header: true});
+var transformer = csv.transform(function (record, callback) {
+        if (('icao24' in record) && ('r' in record)) {
+                var computed = reglookup(record.icao24);
+                if (computed === record.r) {
+                        record.r = '';
+                }
+        }
+
+        callback(null, record);
+});
+
+process.stdin.pipe(parser).pipe(transformer).pipe(writer).pipe(process.stdout);