NUCp tracking and CPR position sanity checks.

Track NUCp when we compute positions.

Do speed checks when we have an updated position with the same or worse NUCp
before accepting the new position. Don't do speed checks on new postions if
they improve NUCp - assume that the new position is better.

Include NUCp in aircraft.json

Gather stats on reasons for rejecting CPR data due to range/speed check failures.

Expire old positions if we have had no updates for 60 seconds.

Closes #16, closes #17.

README-json.md

@@ -41,6 +41,7 @@ This file contains dump1090's list of recently seen aircraft. The keys are:
    * squawk: the 4-digit squawk (octal representation)
    * flight: the flight name / callsign
    * lat, lon: the aircraft position in decimal degrees
+   * nucp: the NUCp (navigational uncertainty category) reported for the position
    * seen_pos: how long ago (in seconds before "now") the position was last updated
    * altitude: the aircraft altitude in feet, or "ground" if it is reporting it is on the ground
    * vert_rate: vertical rate in feet/minute
@@ -104,10 +105,14 @@ Each period has the following subkeys:
    * background: milliseconds spent doing network I/O, processing received network messages, and periodic tasks.
  * cpr: statistics about Compact Position Report message decoding. Has subkeys:
    * global_ok: global positions successfuly derived
-   * global_bad: global positions that were rejected because they were out of range
+   * global_bad: global positions that were rejected because they were inconsistent
+     * global_range: global positions that were rejected because they exceeded the receiver max range
+     * global_speed: global positions that were rejected because they failed the inter-position speed check
    * global_skipped: global position attempts skipped because we did not have the right data (e.g. even/odd messages crossed a zone boundary)
    * local_ok: local (relative) positions successfully found
-   * local_skipped: local (relative) positions not used because we did not have the right data (e.g. position was ambiguous given the receiver range)
+   * local_skipped: local (relative) positions not used because we did not have the right data
+     * local_range: local positions not used because they exceeded the receiver max range or fell into the ambiguous part of the receiver range
+     * local_speed: local positions not used because they failed the inter-position speed check
    * filtered: number of CPR messages ignored because they matched one of the heuristics for faulty transponder output
  * tracks: statistics on aircraft tracks. Each track represents a unique aircraft and persists for up to 5 minutes after the last message
    from the aircraft is heard. If messages from the same aircraft are subsequently heard after the 5 minute period, this will be counted

net_io.c

@@ -776,7 +776,7 @@ char *generateAircraftJson(const char *url_path, int *len) {
         if (a->bFlags & MODES_ACFLAGS_CALLSIGN_VALID)
             p += snprintf(p, end-p, ",\"flight\":\"%s\"", jsonEscapeString(a->flight));
         if (a->bFlags & MODES_ACFLAGS_LATLON_VALID)
-            p += snprintf(p, end-p, ",\"lat\":%f,\"lon\":%f,\"seen_pos\":%.1f", a->lat, a->lon, (now - a->seenLatLon)/1000.0);
+            p += snprintf(p, end-p, ",\"lat\":%f,\"lon\":%f,\"nucp\":%u,\"seen_pos\":%.1f", a->lat, a->lon, a->pos_nuc, (now - a->seenLatLon)/1000.0);
         if ((a->bFlags & MODES_ACFLAGS_AOG_VALID) && (a->bFlags & MODES_ACFLAGS_AOG))
             p += snprintf(p, end-p, ",\"altitude\":\"ground\"");
         else if (a->bFlags & MODES_ACFLAGS_ALTITUDE_VALID)
@@ -882,9 +882,13 @@ static char * appendStatsJson(char *p,
         p += snprintf(p, end-p,
                       ",\"cpr\":{\"global_ok\":%u"
                       ",\"global_bad\":%u"
+                      ",\"global_range\":%u"
+                      ",\"global_speed\":%u"
                       ",\"global_skipped\":%u"
                       ",\"local_ok\":%u"
                       ",\"local_skipped\":%u"
+                      ",\"local_range\":%u"
+                      ",\"local_speed\":%u"
                       ",\"filtered\":%u}"
                       ",\"cpu\":{\"demod\":%llu,\"reader\":%llu,\"background\":%llu}"
                       ",\"tracks\":{\"all\":%u"
@@ -892,9 +896,13 @@ static char * appendStatsJson(char *p,
                       ",\"messages\":%u}",
                       st->cpr_global_ok,
                       st->cpr_global_bad,
+                      st->cpr_global_range_checks,
+                      st->cpr_global_speed_checks,
                       st->cpr_global_skipped,
                       st->cpr_local_ok,
                       st->cpr_local_skipped,
+                      st->cpr_local_range_checks,
+                      st->cpr_local_speed_checks,
                       st->cpr_filtered,
                       (unsigned long long)demod_cpu_millis,
                       (unsigned long long)reader_cpu_millis,

stats.c

@@ -124,15 +124,23 @@ void display_stats(struct stats *st) {
 
     printf("%u global CPR attempts with valid positions\n"
            "%u global CPR attempts with bad data\n"
+           "  %u global CPR attempts that failed the range check\n"
+           "  %u global CPR attempts that failed the speed check\n"
            "%u global CPR attempts with insufficient data\n"
            "%u local CPR attempts with valid positions\n"
            "%u local CPR attempts with insufficient data\n"
+           "  %u local CPR attempts that failed the range check\n"
+           "  %u local CPR attempts that failed the speed check\n"
            "%u CPR messages that look like transponder failures filtered\n",
            st->cpr_global_ok,
            st->cpr_global_bad,
+           st->cpr_global_range_checks,
+           st->cpr_global_speed_checks,
            st->cpr_global_skipped,
            st->cpr_local_ok,
            st->cpr_local_skipped,
+           st->cpr_local_range_checks,
+           st->cpr_local_speed_checks,
            st->cpr_filtered);
 
     printf("%u unique aircraft tracks\n", st->unique_aircraft);
@@ -228,8 +236,12 @@ void add_stats(const struct stats *st1, const struct stats *st2, struct stats *t
     target->cpr_global_ok = st1->cpr_global_ok + st2->cpr_global_ok;
     target->cpr_global_bad = st1->cpr_global_bad + st2->cpr_global_bad;
     target->cpr_global_skipped = st1->cpr_global_skipped + st2->cpr_global_skipped;
+    target->cpr_global_range_checks = st1->cpr_global_range_checks + st2->cpr_global_range_checks;
+    target->cpr_global_speed_checks = st1->cpr_global_speed_checks + st2->cpr_global_speed_checks;
     target->cpr_local_ok = st1->cpr_local_ok + st2->cpr_local_ok;
     target->cpr_local_skipped = st1->cpr_local_skipped + st2->cpr_local_skipped;
+    target->cpr_local_range_checks = st1->cpr_local_range_checks + st2->cpr_local_range_checks;
+    target->cpr_local_speed_checks = st1->cpr_local_speed_checks + st2->cpr_local_speed_checks;
     target->cpr_filtered = st1->cpr_filtered + st2->cpr_filtered;
 
     // aircraft

stats.h

@@ -102,8 +102,12 @@ struct stats {
     unsigned int cpr_global_ok;
     unsigned int cpr_global_bad;
     unsigned int cpr_global_skipped;
+    unsigned int cpr_global_range_checks;
+    unsigned int cpr_global_speed_checks;
     unsigned int cpr_local_ok;
     unsigned int cpr_local_skipped;
+    unsigned int cpr_local_range_checks;
+    unsigned int cpr_local_speed_checks;
     unsigned int cpr_filtered;
 
     // aircraft:

track.c

@@ -49,6 +49,8 @@
 
 #include "dump1090.h"
 
+/* #define DEBUG_CPR_CHECKS */
+
 //
 // Return a new aircraft structure for the linked list of tracked
 // aircraft
@@ -121,10 +123,62 @@ static double greatcircle(double lat0, double lon0, double lat1, double lon1)
     return 6371e3 * acos(sin(lat0) * sin(lat1) + cos(lat0) * cos(lat1) * cos(fabs(lon0 - lon1)));
 }
 
-static int doGlobalCPR(struct aircraft *a, int fflag, int surface)
+// return true if it's OK for the aircraft to have travelled from its last known position
+// to a new position at (lat,lon,surface) at a time of now.
+static int speed_check(struct aircraft *a, double lat, double lon, uint64_t now, int surface)
+{
+    uint64_t elapsed;
+    double distance;
+    double range;
+    int speed;
+
+    if (!(a->bFlags & MODES_ACFLAGS_LATLON_VALID))
+        return 1; // no reference, assume OK
+
+    elapsed = now - a->seenLatLon;
+
+    // Work out a reasonable speed to use:
+    //  current speed + 50%
+    //  surface speed min 30kt, max 150kt
+    //  airborne speed min 300kt, no max
+    if ((a->bFlags & MODES_ACFLAGS_SPEED_VALID)) {
+        speed = a->speed * 3 / 2;
+        if (surface) {
+            if (speed < 30)
+                speed = 30;
+            if (speed > 150)
+                speed = 150;
+        } else {
+            if (speed < 300)
+                speed = 300;
+        }
+    } else {
+        // Guess.
+        speed = surface ? 150 : 1000;
+    }
+
+    // 100m (surface) or 500m (airborne) base distance to allow for minor errors,
+    // plus distance covered at the given speed for the elapsed time + 1 second.
+    range = (surface ? 0.1e3 : 0.5e3) + ((elapsed + 1000.0) / 1000.0) * (speed * 1852.0 / 3600.0);
+
+    // find actual distance
+    distance = greatcircle(a->lat, a->lon, lat, lon);
+
+#ifdef DEBUG_CPR_CHECKS
+    if (distance >= range) {
+        fprintf(stderr, "Speed check failed: %06x: %.3f,%.3f -> %.3f,%.3f in %.1f seconds, max speed %d kt, range %.1fkm, actual %.1fkm\n",
+                a->addr, a->lat, a->lon, lat, lon, elapsed/1000.0, speed, range/1000.0, distance/1000.0);
+    }
+#endif
+
+    return (distance < range);
+}
+
+static int doGlobalCPR(struct aircraft *a, int fflag, int surface, uint64_t now, double *lat, double *lon, unsigned *nuc)
 {
     int result;
-    double lat=0, lon=0;
+
+    *nuc = (a->even_cprnuc < a->odd_cprnuc ? a->even_cprnuc : a->odd_cprnuc); // worst of the two positions
 
     if (surface) {
         // surface global CPR
@@ -134,6 +188,8 @@ static int doGlobalCPR(struct aircraft *a, int fflag, int surface)
         if (a->bFlags & MODES_ACFLAGS_LATLON_REL_OK) { // Ok to try aircraft relative first
             reflat = a->lat;
             reflon = a->lon;
+            if (a->pos_nuc < *nuc)
+                *nuc = a->pos_nuc;
         } else if (Modes.bUserFlags & MODES_USER_LATLON_VALID) {
             reflat = Modes.fUserLat;
             reflon = Modes.fUserLon;
@@ -146,46 +202,54 @@ static int doGlobalCPR(struct aircraft *a, int fflag, int surface)
                                   a->even_cprlat, a->even_cprlon,
                                   a->odd_cprlat, a->odd_cprlon,
                                   fflag,
-                                  &lat, &lon);
+                                  lat, lon);
     } else {
         // airborne global CPR
         result = decodeCPRairborne(a->even_cprlat, a->even_cprlon,
                                    a->odd_cprlat, a->odd_cprlon,
                                    fflag,
-                                   &lat, &lon);
+                                   lat, lon);
     }
 
-    if (result < 0)
-        return result;
-
     // check max range
     if (Modes.maxRange > 0 && (Modes.bUserFlags & MODES_USER_LATLON_VALID)) {
-        double range = greatcircle(Modes.fUserLat, Modes.fUserLon, lat, lon);
+        double range = greatcircle(Modes.fUserLat, Modes.fUserLon, *lat, *lon);
-        if (range > Modes.maxRange)
+        if (range > Modes.maxRange) {
+#ifdef DEBUG_CPR_CHECKS
+            fprintf(stderr, "Global range check failed: %06x: %.3f,%.3f, max range %.1fkm, actual %.1fkm\n",
+                    a->addr, *lat, *lon, Modes.maxRange/1000.0, range/1000.0);
+#endif
+
+            Modes.stats_current.cpr_global_range_checks++;
             return (-2); // we consider an out-of-range value to be bad data
         }
-
-    a->lat = lat;
-    a->lon = lon;
-    return 0;
     }
 
-static int doLocalCPR(struct aircraft *a, int fflag, int surface, uint64_t now)
+    // check speed limit
+    if ((a->bFlags & MODES_ACFLAGS_LATLON_VALID) && a->pos_nuc >= *nuc && !speed_check(a, *lat, *lon, now, surface)) {
+        Modes.stats_current.cpr_global_speed_checks++;
+        return -2;
+    }
+
+    return result;
+}
+
+static int doLocalCPR(struct aircraft *a, int fflag, int surface, uint64_t now, double *lat, double *lon, unsigned *nuc)
 {
     // relative CPR
     // find reference location
-    double reflat, reflon, lat=0, lon=0;
+    double reflat, reflon;
     double range_limit = 0;
     int result;
 
-    if (a->bFlags & MODES_ACFLAGS_LATLON_REL_OK) {
+    *nuc = (fflag ? a->odd_cprnuc : a->even_cprnuc);
-        uint64_t elapsed = (now - a->seenLatLon);
 
+    if (a->bFlags & MODES_ACFLAGS_LATLON_REL_OK) {
         reflat = a->lat;
         reflon = a->lon;
 
-        // impose a range limit based on 2000km/h speed
+        if (a->pos_nuc < *nuc)
-        range_limit = 5e3 + (2000e3 * elapsed / 3600 / 1000); // 5km + 2000km/h
+            *nuc = a->pos_nuc;
     } else if (!surface && (Modes.bUserFlags & MODES_USER_LATLON_VALID)) {
         reflat = Modes.fUserLat;
         reflon = Modes.fUserLon;
@@ -199,9 +263,11 @@ static int doLocalCPR(struct aircraft *a, int fflag, int surface, uint64_t now)
         // at 200NM distance, this may resolve to a position
         // at (200-360) = 160NM in the wrong direction)
 
-        if (Modes.maxRange > 1852*180) {
+        if (Modes.maxRange <= 1852*180) {
+            range_limit = Modes.maxRange;
+        } else if (Modes.maxRange <= 1852*360) {
             range_limit = (1852*360) - Modes.maxRange;
-            if (range_limit <= 0)
+        } else {
             return (-1); // Can't do receiver-centered checks at all
         }
     } else {
@@ -213,25 +279,29 @@ static int doLocalCPR(struct aircraft *a, int fflag, int surface, uint64_t now)
                                fflag ? a->odd_cprlat : a->even_cprlat,
                                fflag ? a->odd_cprlon : a->even_cprlon,
                                fflag, surface,
-                               &lat, &lon);
+                               lat, lon);
     if (result < 0)
         return result;
+
     // check range limit
     if (range_limit > 0) {
-        double range = greatcircle(reflat, reflon, lat, lon);
+        double range = greatcircle(reflat, reflon, *lat, *lon);
-        if (range > range_limit)
+        if (range > range_limit) {
+#ifdef DEBUG_CPR_CHECKS
+            fprintf(stderr, "Local position ambiguous: %06x: %.3f,%.3f -> %.3f,%.3f, max range %.1fkm, actual %.1fkm\n",
+                    a->addr, reflat, reflon, *lat, *lon, range_limit/1000.0, range/1000.0);
+#endif
+            Modes.stats_current.cpr_local_range_checks++;
             return (-1);
         }
-
-    // check max range
-    if (Modes.maxRange > 0 && (Modes.bUserFlags & MODES_USER_LATLON_VALID)) {
-        double range = greatcircle(Modes.fUserLat, Modes.fUserLon, lat, lon);
-        if (range > Modes.maxRange)
-            return (-2); // we consider an out-of-range value to be bad data
     }
 
-    a->lat = lat;
+    // check speed limit
-    a->lon = lon;
+    if ((a->bFlags & MODES_ACFLAGS_LATLON_VALID) && a->pos_nuc >= *nuc && !speed_check(a, *lat, *lon, now, surface)) {
+        Modes.stats_current.cpr_local_speed_checks++;
+        return -1;
+    }
+
     return 0;
 }
 
@@ -239,6 +309,8 @@ static void updatePosition(struct aircraft *a, struct modesMessage *mm, uint64_t
 {
     int location_result = -1;
     int max_elapsed;
+    double new_lat = 0, new_lon = 0;
+    unsigned new_nuc = 0;
 
     if (mm->bFlags & MODES_ACFLAGS_AOG) {
         // Surface: 25 seconds if >25kt or speed unknown, 50 seconds otherwise
@@ -253,18 +325,22 @@ static void updatePosition(struct aircraft *a, struct modesMessage *mm, uint64_t
     }
 
     if (mm->bFlags & MODES_ACFLAGS_LLODD_VALID) {
+        a->odd_cprnuc  = mm->nuc_p;
         a->odd_cprlat  = mm->raw_latitude;
         a->odd_cprlon  = mm->raw_longitude;
-        a->odd_cprtime = mstime();
+        a->odd_cprtime = now;
     } else {
+        a->even_cprnuc  = mm->nuc_p;
         a->even_cprlat  = mm->raw_latitude;
         a->even_cprlon  = mm->raw_longitude;
-        a->even_cprtime = mstime();
+        a->even_cprtime = now;
     }
 
     // If we have enough recent data, try global CPR
     if (((mm->bFlags | a->bFlags) & MODES_ACFLAGS_LLEITHER_VALID) == MODES_ACFLAGS_LLBOTH_VALID && abs((int)(a->even_cprtime - a->odd_cprtime)) <= max_elapsed) {
-        location_result = doGlobalCPR(a, (mm->bFlags & MODES_ACFLAGS_LLODD_VALID), (mm->bFlags & MODES_ACFLAGS_AOG));
+        location_result = doGlobalCPR(a, (mm->bFlags & MODES_ACFLAGS_LLODD_VALID), (mm->bFlags & MODES_ACFLAGS_AOG), now, 
+                                      &new_lat, &new_lon, &new_nuc);
+
         if (location_result == -2) {
             // Global CPR failed because an airborne position produced implausible results.
             // This is bad data. Discard both odd and even messages and wait for a fresh pair.
@@ -285,7 +361,8 @@ static void updatePosition(struct aircraft *a, struct modesMessage *mm, uint64_t
 
     // Otherwise try relative CPR.
     if (location_result == -1) {
-        location_result = doLocalCPR(a, (mm->bFlags & MODES_ACFLAGS_LLODD_VALID), (mm->bFlags & MODES_ACFLAGS_AOG), now);
+        location_result = doLocalCPR(a, (mm->bFlags & MODES_ACFLAGS_LLODD_VALID), (mm->bFlags & MODES_ACFLAGS_AOG), now, &new_lat, &new_lon, &new_nuc);
+
         if (location_result == -1) {
             Modes.stats_current.cpr_local_skipped++;
         } else {
@@ -297,12 +374,16 @@ static void updatePosition(struct aircraft *a, struct modesMessage *mm, uint64_t
     if (location_result == 0) {
         // If we sucessfully decoded, back copy the results to mm so that we can print them in list output
         mm->bFlags |= MODES_ACFLAGS_LATLON_VALID;
-        mm->fLat    = a->lat;
+        mm->fLat    = new_lat;
-        mm->fLon    = a->lon;
+        mm->fLon    = new_lon;
 
         // Update aircraft state
         a->bFlags |= (MODES_ACFLAGS_LATLON_VALID | MODES_ACFLAGS_LATLON_REL_OK);
+        a->lat = new_lat;
+        a->lon = new_lon;
+        a->pos_nuc = new_nuc;
         a->seenLatLon      = a->seen;
+
     }
 }
 
@@ -518,6 +599,9 @@ static void trackRemoveStaleAircraft(uint64_t now)
             } else {
                 prev->next = a->next; free(a); a = prev->next;
             }
+        } else if ((a->bFlags & MODES_ACFLAGS_LATLON_VALID) && (now - a->seenLatLon) > TRACK_AIRCRAFT_POSITION_TTL) {
+            /* Position is too old and no longer valid */
+            a->bFlags &= ~(MODES_ACFLAGS_LATLON_VALID | MODES_ACFLAGS_LATLON_REL_OK);
         } else {
             prev = a; a = a->next;
         }

track.h

@@ -56,6 +56,9 @@
 /* Maximum age of a tracked aircraft with only 1 message received, in milliseconds */
 #define TRACK_AIRCRAFT_ONEHIT_TTL 60000
 
+/* Maximum validity of an aircraft position */
+#define TRACK_AIRCRAFT_POSITION_TTL 60000
+
 /* Structure used to describe the state of one tracked aircraft */
 struct aircraft {
     uint32_t      addr;           // ICAO address
@@ -79,13 +82,19 @@ struct aircraft {
     uint64_t      fatsv_last_emitted;      // time (millis) aircraft was last FA emitted
 
     // Encoded latitude and longitude as extracted by odd and even CPR encoded messages
+    uint64_t      odd_cprtime;
     int           odd_cprlat;
     int           odd_cprlon;
+    unsigned      odd_cprnuc;
+
+    uint64_t      even_cprtime;
     int           even_cprlat;
     int           even_cprlon;
-    uint64_t      odd_cprtime;
+    unsigned      even_cprnuc;
-    uint64_t      even_cprtime;
+
     double        lat, lon;       // Coordinated obtained from CPR encoded data
+    unsigned      pos_nuc;        // NUCp of last computed position
+
     int           bFlags;         // Flags related to valid fields in this structure
     struct aircraft *next;        // Next aircraft in our linked list