icao_filter.[ch] from the experimental branch.

Makefile

@@ -21,10 +21,10 @@ all: dump1090 view1090
 %.o: %.c dump1090.h
 	$(CC) $(CPPFLAGS) $(CFLAGS) $(EXTRACFLAGS) -c $< -o $@
 
-dump1090: dump1090.o anet.o interactive.o mode_ac.o mode_s.o net_io.o crc.o demod_2000.o demod_2400.o stats.o cpr.o
+dump1090: dump1090.o anet.o interactive.o mode_ac.o mode_s.o net_io.o crc.o demod_2000.o demod_2400.o stats.o cpr.o icao_filter.o
 	$(CC) -g -o $@ $^ $(LIBS) $(LIBS_RTL) $(LDFLAGS)
 
-view1090: view1090.o anet.o interactive.o mode_ac.o mode_s.o net_io.o crc.o stats.o cpr.o
+view1090: view1090.o anet.o interactive.o mode_ac.o mode_s.o net_io.o crc.o stats.o cpr.o icao_filter.o
 	$(CC) -g -o $@ $^ $(LIBS) $(LDFLAGS)
 
 clean:

dump1090.c

@@ -120,8 +120,7 @@ void modesInit(void) {
 
     // Allocate the various buffers used by Modes
     Modes.trailing_samples = (Modes.oversample ? (MODES_OS_PREAMBLE_SAMPLES + MODES_OS_LONG_MSG_SAMPLES) : (MODES_PREAMBLE_SAMPLES + MODES_LONG_MSG_SAMPLES)) + 16;
-    if ( ((Modes.icao_cache = (uint32_t *) malloc(sizeof(uint32_t) * MODES_ICAO_CACHE_LEN * 2)                  ) == NULL) ||
+    if ( ((Modes.pFileData  = (uint16_t *) malloc(MODES_ASYNC_BUF_SIZE)                                         ) == NULL) ||
-         ((Modes.pFileData  = (uint16_t *) malloc(MODES_ASYNC_BUF_SIZE)                                         ) == NULL) ||
          ((Modes.magnitude  = (uint16_t *) calloc(MODES_ASYNC_BUF_SAMPLES+Modes.trailing_samples, 2)            ) == NULL) ||
          ((Modes.maglut     = (uint16_t *) malloc(sizeof(uint16_t) * 256 * 256)                                 ) == NULL) ||
          ((Modes.log10lut   = (uint16_t *) malloc(sizeof(uint16_t) * 256 * 256)                                 ) == NULL) )
@@ -131,7 +130,6 @@ void modesInit(void) {
     }
 
     // Clear the buffers that have just been allocated, just in-case
-    memset(Modes.icao_cache, 0,   sizeof(uint32_t) * MODES_ICAO_CACHE_LEN * 2);
     memset(Modes.pFileData,127,   MODES_ASYNC_BUF_SIZE);
 
     // Validate the users Lat/Lon home location inputs
@@ -218,6 +216,7 @@ void modesInit(void) {
 
     // Prepare error correction tables
     modesChecksumInit(Modes.nfix_crc);
+    icaoFilterInit();
 }
 //
 // =============================== RTLSDR handling ==========================
@@ -607,6 +606,8 @@ void backgroundTasks(void) {
 
     time_t now = time(NULL);
 
+    icaoFilterExpire();
+
     if (Modes.net) {
 	modesNetPeriodicWork();
     }    

dump1090.h

@@ -153,8 +153,6 @@
 #define MODES_OUT_FLUSH_SIZE       (MODES_OUT_BUF_SIZE - 256)
 #define MODES_OUT_FLUSH_INTERVAL   (60)
 
-#define MODES_ICAO_CACHE_LEN 1024 // Power of two required
-#define MODES_ICAO_CACHE_TTL 60   // Time to live of cached addresses
 #define MODES_UNIT_FEET 0
 #define MODES_UNIT_METERS 1
 
@@ -230,6 +228,7 @@
 #include "demod_2400.h"
 #include "stats.h"
 #include "cpr.h"
+#include "icao_filter.h"
 
 
 //======================== structure declarations =========================
@@ -308,7 +307,6 @@ struct {                             // Internal state
     uint64_t        timestampBlk;    // Timestamp of the start of the current block
     struct timeb    stSystemTimeBlk; // System time when RTL passed us currently processing this block
     int             fd;              // --ifile option file descriptor
-    uint32_t       *icao_cache;      // Recently seen ICAO addresses cache
     uint16_t       *maglut;          // I/Q -> Magnitude lookup table
     uint16_t       *log10lut;        // Magnitude -> log10 lookup table
     int             exit;            // Exit from the main loop when true

icao_filter.c

@@ -0,0 +1,140 @@
+// Part of dump1090, a Mode S message decoder for RTLSDR devices.
+//
+// icao_filter.c: hashtable for ICAO addresses
+//
+// Copyright (c) 2014,2015 Oliver Jowett <oliver@mutability.co.uk>
+//
+// This file is free software: you may copy, redistribute and/or modify it  
+// under the terms of the GNU General Public License as published by the
+// Free Software Foundation, either version 2 of the License, or (at your  
+// option) any later version.  
+//
+// This file is distributed in the hope that it will be useful, but  
+// WITHOUT ANY WARRANTY; without even the implied warranty of  
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU  
+// General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License  
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+#include "dump1090.h"
+
+// hash table size, must be a power of two:
+#define ICAO_FILTER_SIZE 4096
+
+// Seconds between filter expiry flips:
+#define MODES_ICAO_FILTER_TTL 60
+
+// Open-addressed hash table with linear probing.
+// We store each address twice to handle Address/Parity and Data/Parity
+// which need to match on a partial address (top 16 bits only).
+
+// Maintain two tables and switch between them to age out entries.
+
+static uint32_t icao_filter_a[ICAO_FILTER_SIZE];
+static uint32_t icao_filter_b[ICAO_FILTER_SIZE];
+static uint32_t *icao_filter_active;
+
+static uint32_t icaoHash(uint32_t a)
+{
+    // Jenkins one-at-a-time hash, unrolled for 3 bytes
+    uint32_t hash = 0;
+
+    hash += a & 0xff;
+    hash += hash << 10;
+    hash ^= hash >> 6;
+
+    hash += (a >> 8) & 0xff;
+    hash += (hash << 10);
+    hash ^= (hash >> 6);
+
+    hash += (a >> 16) & 0xff;
+    hash += (hash << 10);
+    hash ^= (hash >> 6);
+             
+    hash += (hash << 3);
+    hash ^= (hash >> 11);
+    hash += (hash << 15);
+
+    return hash & (ICAO_FILTER_SIZE-1);
+}
+
+void icaoFilterInit()
+{
+    memset(icao_filter_a, 0, sizeof(icao_filter_a));
+    memset(icao_filter_b, 0, sizeof(icao_filter_b));
+    icao_filter_active = icao_filter_a;
+}
+
+void icaoFilterAdd(uint32_t addr)
+{
+    uint32_t h = icaoHash(addr);
+    while (icao_filter_active[h] && icao_filter_active[h] != addr)
+        h = (h+1) & (ICAO_FILTER_SIZE-1);
+    if (!icao_filter_active[h])
+        icao_filter_active[h] = addr;
+
+    // also add with a zeroed top byte, for handling DF20/21 with Data Parity
+    h = icaoHash(addr & 0x00ffff);
+    while (icao_filter_active[h] && (icao_filter_active[h] & 0x00ffff) != (addr & 0x00ffff))
+        h = (h+1) & (ICAO_FILTER_SIZE-1);
+    if (!icao_filter_active[h])
+        icao_filter_active[h] = addr;
+}
+
+int icaoFilterTest(uint32_t addr)
+{
+    uint32_t h, h0;
+
+    h0 = h = icaoHash(addr);
+    while (icao_filter_a[h] && icao_filter_a[h] != addr)
+        h = (h+1) & (ICAO_FILTER_SIZE-1);
+    if (icao_filter_a[h])
+        return 1;
+
+    h = h0;
+    while (icao_filter_b[h] && icao_filter_b[h] != addr)
+        h = (h+1) & (ICAO_FILTER_SIZE-1);
+    if (icao_filter_b[h])
+        return 1;
+
+    return 0;
+}
+
+uint32_t icaoFilterTestFuzzy(uint32_t partial)
+{
+    uint32_t h, h0;
+
+    partial &= 0x00ffff;
+    h0 = h = icaoHash(partial);
+    while (icao_filter_a[h] && (icao_filter_a[h] & 0x00ffff) != partial)
+        h = (h+1) & (ICAO_FILTER_SIZE-1);
+    if (icao_filter_a[h])
+        return icao_filter_a[h];
+
+    h = h0;
+    while (icao_filter_b[h] && (icao_filter_b[h] & 0x00ffff) != partial)
+        h = (h+1) & (ICAO_FILTER_SIZE-1);
+    if (icao_filter_b[h])
+        return icao_filter_b[h];
+
+    return 0;
+}
+
+// call this periodically:
+void icaoFilterExpire()
+{
+    static time_t next_flip = 0;
+    time_t now = time(NULL);
+
+    if (now >= next_flip) {
+        if (icao_filter_active == icao_filter_a) {
+            memset(icao_filter_b, 0, sizeof(icao_filter_b));
+            icao_filter_active = icao_filter_b;
+        } else {
+            memset(icao_filter_a, 0, sizeof(icao_filter_a));
+            icao_filter_active = icao_filter_a;
+        }
+        next_flip = now + MODES_ICAO_FILTER_TTL;
+    }
+}

icao_filter.h

@@ -0,0 +1,41 @@
+// Part of dump1090, a Mode S message decoder for RTLSDR devices.
+//
+// icao_filter.c: prototypes for ICAO address hashtable
+//
+// Copyright (c) 2014,2015 Oliver Jowett <oliver@mutability.co.uk>
+//
+// This file is free software: you may copy, redistribute and/or modify it  
+// under the terms of the GNU General Public License as published by the
+// Free Software Foundation, either version 2 of the License, or (at your  
+// option) any later version.  
+//
+// This file is distributed in the hope that it will be useful, but  
+// WITHOUT ANY WARRANTY; without even the implied warranty of  
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU  
+// General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License  
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+#ifndef DUMP1090_ICAO_FILTER_H
+#define DUMP1090_ICAO_FILTER_H
+
+// Call once:
+void icaoFilterInit();
+
+// Add an address to the filter
+void icaoFilterAdd(uint32_t addr);
+
+// Test if the given address matches the filter
+int icaoFilterTest(uint32_t addr);
+
+// Test if the top 16 bits match any previously added address.
+// If they do, returns an arbitrary one of the matched
+// addresses. Returns 0 on failure.
+uint32_t icaoFilterTestFuzzy(uint32_t partial);
+
+// Call this periodically to allow the filter to expire
+// old entries.
+void icaoFilterExpire();
+
+#endif

mode_s.c

@@ -67,46 +67,6 @@ int modesMessageLenByType(int type) {
     return (type & 0x10) ? MODES_LONG_MSG_BITS : MODES_SHORT_MSG_BITS ;
 }
 
-//=========================================================================
-//
-// Hash the ICAO address to index our cache of MODES_ICAO_CACHE_LEN
-// elements, that is assumed to be a power of two
-//
-uint32_t ICAOCacheHashAddress(uint32_t a) {
-    // The following three rounds wil make sure that every bit affects
-    // every output bit with ~ 50% of probability.
-    a = ((a >> 16) ^ a) * 0x45d9f3b;
-    a = ((a >> 16) ^ a) * 0x45d9f3b;
-    a = ((a >> 16) ^ a);
-    return a & (MODES_ICAO_CACHE_LEN-1);
-}
-//
-//=========================================================================
-//
-// Add the specified entry to the cache of recently seen ICAO addresses.
-// Note that we also add a timestamp so that we can make sure that the
-// entry is only valid for MODES_ICAO_CACHE_TTL seconds.
-//
-void addRecentlySeenICAOAddr(uint32_t addr) {
-    uint32_t h = ICAOCacheHashAddress(addr);
-    Modes.icao_cache[h*2] = addr;
-    Modes.icao_cache[h*2+1] = (uint32_t) time(NULL);
-}
-//
-//=========================================================================
-//
-// Returns 1 if the specified ICAO address was seen in a DF format with
-// proper checksum (not xored with address) no more than * MODES_ICAO_CACHE_TTL
-// seconds ago. Otherwise returns 0.
-//
-int ICAOAddressWasRecentlySeen(uint32_t addr) {
-    uint32_t h = ICAOCacheHashAddress(addr);
-    uint32_t a = Modes.icao_cache[h*2];
-    uint32_t t = Modes.icao_cache[h*2+1];
-    uint64_t tn = time(NULL);
-
-    return ( (a) && (a == addr) && ( (tn - t) <= MODES_ICAO_CACHE_TTL) );
-}
 //
 //=========================================================================
 //
@@ -342,7 +302,7 @@ void decodeModesMessage(struct modesMessage *mm, unsigned char *msg) {
         // If we correct, validate ICAO addr to help filter birthday paradox solutions.
         if (mm->correctedbits) {
             uint32_t ulAddr = (msg[1] << 16) | (msg[2] << 8) | (msg[3]); 
-            if (!ICAOAddressWasRecentlySeen(ulAddr))
+            if (!icaoFilterTest(ulAddr))
                 mm->correctedbits = 0;
         }
     }
@@ -357,11 +317,11 @@ void decodeModesMessage(struct modesMessage *mm, unsigned char *msg) {
 
         if ((mm->crcok = (0 == mm->crc))) {
             // DF 11 : if crc == 0 try to populate our ICAO addresses whitelist.
-            addRecentlySeenICAOAddr(mm->addr);
+            icaoFilterAdd(mm->addr);
         } else if (mm->crc < 80) {
-            mm->crcok = ICAOAddressWasRecentlySeen(mm->addr);
+            mm->crcok = icaoFilterTest(mm->addr);
             if (mm->crcok) {
-                addRecentlySeenICAOAddr(mm->addr);
+                icaoFilterAdd(mm->addr);
             }
         }
 
@@ -371,7 +331,7 @@ void decodeModesMessage(struct modesMessage *mm, unsigned char *msg) {
 
         if ((mm->crcok = (0 == mm->crc))) {
             // DF 17 : if crc == 0 try to populate our ICAO addresses whitelist.
-            addRecentlySeenICAOAddr(mm->addr);
+            icaoFilterAdd(mm->addr);
         }
 
     } else if (mm->msgtype == 18) { // DF 18
@@ -380,13 +340,13 @@ void decodeModesMessage(struct modesMessage *mm, unsigned char *msg) {
 
         if ((mm->crcok = (0 == mm->crc))) {
             // DF 18 : if crc == 0 try to populate our ICAO addresses whitelist.
-            addRecentlySeenICAOAddr(mm->addr);
+            icaoFilterAdd(mm->addr);
         }
 
     } else { // All other DF's
         // Compare the checksum with the whitelist of recently seen ICAO 
         // addresses. If it matches one, then declare the message as valid
-        mm->crcok = ICAOAddressWasRecentlySeen(mm->addr = mm->crc);
+        mm->crcok = icaoFilterTest(mm->addr = mm->crc);
     }
 
     // If we're checking CRC and the CRC is invalid, then we can't trust any 

view1090.c

@@ -96,16 +96,6 @@ void view1090Init(void) {
       }
 #endif
 
-    // Allocate the various buffers used by Modes
-    if ( NULL == (Modes.icao_cache = (uint32_t *) malloc(sizeof(uint32_t) * MODES_ICAO_CACHE_LEN * 2)))
-    {
-        fprintf(stderr, "Out of memory allocating data buffer.\n");
-        exit(1);
-    }
-
-    // Clear the buffers that have just been allocated, just in-case
-    memset(Modes.icao_cache, 0,   sizeof(uint32_t) * MODES_ICAO_CACHE_LEN * 2);
-
     // Validate the users Lat/Lon home location inputs
     if ( (Modes.fUserLat >   90.0)  // Latitude must be -90 to +90
       || (Modes.fUserLat <  -90.0)  // and 
@@ -127,6 +117,7 @@ void view1090Init(void) {
 
     // Prepare error correction tables
     modesChecksumInit(Modes.nfix_crc);
+    icaoFilterInit();
 }
 
 // Set up data connection
@@ -296,6 +287,7 @@ int main(int argc, char **argv) {
 
     // Keep going till the user does something that stops us
     while (!Modes.exit) {
+        icaoFilterExpire();
         interactiveRemoveStaleAircrafts();
         interactiveShowData();
         if ((fd == ANET_ERR) || (recv(c->fd, pk_buf, sizeof(pk_buf), MSG_PEEK | MSG_DONTWAIT) == 0)) {