Support merging of arbitrary apt.dat[.gz] files

It is now allowed to have the same airport appear in several apt.dat
files ($scenery_path/NavData/apt/*.dat[.gz] for each scenery path, plus
the default $FG_ROOT/Airports/apt.dat.gz, coming last). Airports found
in earlier files(*) take precedence over those found later, in case
several apt.dat files define the same airports. Airports that are
skipped due to this mechanism are logged with
SG_LOG(SG_GENERAL, SG_INFO, ...).

(*) using 1) FG_SCENERY order (followed by $FG_ROOT/Airports/apt.dat.gz)
    and   2) lexicographic order inside each $scenery_path/NavData/apt
             folder

With this commit, APTLoader::parseAPT() is replaced by two methods:
readAptDatFile() and loadAirports():
  - APTLoader::readAptDatFile() reads airport definitions from an
    apt.dat file into APTLoader's 'airportInfoMap' member variable,
    discarding duplicate definitions due to overlapping apt.dat files
    ('airportInfoMap' is an std::unordered_map instance in C++11 and
    later, an std::map otherwise);
  - APTLoader::loadAirports() reads each airport definition from
    'airportInfoMap' and loads it into the NavCache, the same way as
    APTLoader::parseAPT() used to do.

The airportDBLoad() function is not useful anymore, and is thus removed
(in NavDataCache::doRebuild(), APTLoader::readAptDatFile() is now called
once per apt.dat file, but APTLoader::loadAirports() is only called
once at the end, after duplicate airports have been discarded; the class
interface is much better suited to this scheme, because it can cleanly
retain the state between these calls).

By the way, this commit fixes an old bug: APTLoader's member variable
'last_apt_id' was used in several places but never assigned to, except
in APTLoader::APTLoader() as the empty string.

Thanks to Alan Teeder for his feedback and testing.

src/Airports/apt_loader.cxx

@@ -44,6 +44,7 @@
 
 #include <string>
 #include <vector>
+#include <utility>              // std::pair, std::move()
 
 #include "airport.hxx"
 #include "runways.hxx"
@@ -85,7 +86,7 @@ APTLoader::APTLoader()
 
 APTLoader::~APTLoader() { }
 
-void APTLoader::parseAPT(const SGPath &aptdb_file)
+void APTLoader::readAptDatFile(const SGPath &aptdb_file)
 {
   string apt_dat = aptdb_file.utf8Str(); // full path to the file being parsed
   sg_gzifstream in(aptdb_file);
@@ -103,6 +104,15 @@ void APTLoader::parseAPT(const SGPath &aptdb_file)
 
   unsigned int line_id = 0;
   unsigned int line_num = 0;
+  // "airport identifier": terminology used in the apt.dat format spec. It is
+  // often an ICAO code, but not always.
+  string currentAirportId;
+  // Boolean used to make sure we don't try to load the same airport several
+  // times. Defaults to true only to ensure we don't add garbage to
+  // 'airportInfoMap' under the key "" (empty airport identifier) in case the
+  // apt.dat file doesn't have a start-of-airport row code (1, 16 or 17) after
+  // its header---which would be invalid, anyway.
+  bool skipAirport = true;
 
   // Read the apt.dat header (two lines)
   while ( line_num < 2 && std::getline(in, line) ) {
@@ -150,65 +160,136 @@ void APTLoader::parseAPT(const SGPath &aptdb_file)
     if ( line_id == 1  /* Airport */ ||
          line_id == 16 /* Seaplane base */ ||
          line_id == 17 /* Heliport */ ) {
-      parseAirportLine(apt_dat, simgear::strutils::split(line));
+      vector<string> tokens(simgear::strutils::split(line));
-    } else if ( line_id == 10 ) { // Runway v810
+      if (tokens.size() < 6) {
-      parseRunwayLine810(simgear::strutils::split(line));
+        SG_LOG( SG_GENERAL, SG_WARN,
-    } else if ( line_id == 100 ) { // Runway v850
+                apt_dat << ":"  << line_num << ": invalid airport header "
-      parseRunwayLine850(simgear::strutils::split(line));
+                "(at least 6 fields are required)" );
-    } else if ( line_id == 101 ) { // Water Runway v850
+        skipAirport = true; // discard everything until the next airport header
-      parseWaterRunwayLine850(simgear::strutils::split(line));
+        continue;
-    } else if ( line_id == 102 ) { // Helipad v850
+      }
-      parseHelipadLine850(simgear::strutils::split(line));
-    } else if ( line_id == 18 ) {
-      // beacon entry (ignore)
-    } else if ( line_id == 14 ) {
-      // control tower entry
-      vector<string> token(simgear::strutils::split(line));
 
-      double lat = atof( token[1].c_str() );
+      currentAirportId = tokens[4]; // often an ICAO, but not always
-      double lon = atof( token[2].c_str() );
+      // Check if the airport is already in 'airportInfoMap'; get the
-      double elev = atof( token[3].c_str() );
+      // existing entry, if any, otherwise insert a new one.
-      tower = SGGeod::fromDegFt(lon, lat, elev + last_apt_elev);
+      std::pair<AirportInfoMapType::iterator, bool>
-      cache->insertTower(currentAirportID, tower);
+        insertRetval = airportInfoMap.insert(
-    } else if ( line_id == 19 ) {
+          AirportInfoMapType::value_type(currentAirportId, RawAirportInfo()));
-      // windsock entry (ignore)
+      skipAirport = !insertRetval.second;
-    } else if ( line_id == 20 ) {
+
-      // Taxiway sign (ignore)
+      if ( skipAirport ) {
-    } else if ( line_id == 21 ) {
+        SG_LOG( SG_GENERAL, SG_INFO,
-      // lighting objects (ignore)
+                apt_dat << ":"  << line_num << ": skipping airport " <<
-    } else if ( line_id == 15 ) {
+                currentAirportId << " (already defined earlier)" );
-      // custom startup locations (ignore)
+      } else {
-    } else if ( line_id == 0 ) {
+        // We haven't seen this airport yet in any apt.dat file
-      // ??
+        RawAirportInfo& airportInfo = insertRetval.first->second;
-    } else if ( line_id >= 50 && line_id <= 56) {
+        airportInfo.file = aptdb_file;
-      parseCommLine(apt_dat, line_id, simgear::strutils::split(line));
+        airportInfo.rowCode = line_id;
-    } else if ( line_id == 110 ) {
+        airportInfo.firstLineNum = line_num;
-      pavement = true;
+        airportInfo.firstLineTokens =
-      parsePavementLine850(simgear::strutils::split(line, 0, 4));
+#if __cplusplus >= 201103L
-    } else if ( line_id >= 111 && line_id <= 114 ) {
+          std::move(tokens);    // requires C++11, untested
-      if ( pavement )
+#else
-        parsePavementNodeLine850(line_id, simgear::strutils::split(line));
+          tokens;
-    } else if ( line_id >= 115 && line_id <= 116 ) {
+#endif
-      // other pavement nodes (ignore)
+      }
-    } else if ( line_id == 120 ) {
-      pavement = false;
-    } else if ( line_id == 130 ) {
-      pavement = false;
-    } else if ( line_id >= 1000 ) {
-      // airport traffic flow (ignore)
     } else if ( line_id == 99 ) {
       SG_LOG( SG_GENERAL, SG_DEBUG,
-              apt_dat << ": code 99 found (normally at end of file)" );
+              apt_dat << ":"  << line_num << ": code 99 found "
-    } else {
+              "(normally at end of file)" );
-      std::ostringstream oss;
+    } else if ( !skipAirport ) {
-      oss << apt_dat << ":"  << line_num << ": unknown row code " << line_id;
+      // Line belonging to an already started, and not skipped airport entry;
-      SG_LOG( SG_GENERAL, SG_ALERT, oss.str() << " (" << line << ")" );
+      // just append it.
-      throw sg_format_exception(oss.str(), line);
+      airportInfoMap[currentAirportId].otherLines.
+#if __cplusplus >= 201103L
+        emplace_back(line_num, line_id, line); // requires C++11, untested
+#else
+        push_back(Line(line_num, line_id, line));
+#endif
     }
-  }
+  } // of file reading loop
 
   throwExceptionIfStreamError(in, aptdb_file);
-  finishAirport(apt_dat);
+}
+
+void APTLoader::loadAirports()
+{
+  // Loop over all airports found in all apt.dat files
+  for (AirportInfoMapType::const_iterator it = airportInfoMap.begin();
+       it != airportInfoMap.end(); it++) {
+    // Full path to the apt.dat file this airport info comes from
+    const string aptDat = it->second.file.utf8Str();
+    last_apt_id = it->first;    // this is just the current airport identifier
+    // The first line for this airport was already split over whitespace, but
+    // remains to be parsed for the most part.
+    parseAirportLine(it->second.rowCode, it->second.firstLineTokens);
+    const LinesList& lines = it->second.otherLines;
+
+    // Loop over the second and subsequent lines
+    for (LinesList::const_iterator linesIt = lines.begin();
+         linesIt != lines.end(); linesIt++) {
+      // Beware that linesIt->str may end with an '\r' character, see above!
+      unsigned int line_id = linesIt->rowCode;
+
+      if ( line_id == 10 ) { // Runway v810
+        parseRunwayLine810(simgear::strutils::split(linesIt->str));
+      } else if ( line_id == 100 ) { // Runway v850
+        parseRunwayLine850(simgear::strutils::split(linesIt->str));
+      } else if ( line_id == 101 ) { // Water Runway v850
+        parseWaterRunwayLine850(simgear::strutils::split(linesIt->str));
+      } else if ( line_id == 102 ) { // Helipad v850
+        parseHelipadLine850(simgear::strutils::split(linesIt->str));
+      } else if ( line_id == 18 ) {
+        // beacon entry (ignore)
+      } else if ( line_id == 14 ) {
+        // control tower entry
+        vector<string> token(simgear::strutils::split(linesIt->str));
+
+        double lat = atof( token[1].c_str() );
+        double lon = atof( token[2].c_str() );
+        double elev = atof( token[3].c_str() );
+        tower = SGGeod::fromDegFt(lon, lat, elev + last_apt_elev);
+        cache->insertTower(currentAirportID, tower);
+      } else if ( line_id == 19 ) {
+        // windsock entry (ignore)
+      } else if ( line_id == 20 ) {
+        // Taxiway sign (ignore)
+      } else if ( line_id == 21 ) {
+        // lighting objects (ignore)
+      } else if ( line_id == 15 ) {
+        // custom startup locations (ignore)
+      } else if ( line_id == 0 ) {
+        // ??
+      } else if ( line_id >= 50 && line_id <= 56) {
+        parseCommLine(aptDat, line_id, simgear::strutils::split(linesIt->str));
+      } else if ( line_id == 110 ) {
+        pavement = true;
+        parsePavementLine850(simgear::strutils::split(linesIt->str, 0, 4));
+      } else if ( line_id >= 111 && line_id <= 114 ) {
+        if ( pavement )
+          parsePavementNodeLine850(line_id,
+                                   simgear::strutils::split(linesIt->str));
+      } else if ( line_id >= 115 && line_id <= 116 ) {
+        // other pavement nodes (ignore)
+      } else if ( line_id == 120 ) {
+        pavement = false;
+      } else if ( line_id == 130 ) {
+        pavement = false;
+      } else if ( line_id >= 1000 ) {
+        // airport traffic flow (ignore)
+      } else {
+        std::ostringstream oss;
+        oss << aptDat << ":" << linesIt->number << ": unknown row code " <<
+          line_id;
+        SG_LOG( SG_GENERAL, SG_ALERT,
+                oss.str() << " (" << linesIt->str << ")" );
+        throw sg_format_exception(oss.str(), linesIt->str);
+      }
+    } // of loop over the second and subsequent apt.dat lines for the airport
+
+    finishAirport(aptDat);
+  } // of loop over 'airportInfoMap'
 }
 
 // Tell whether an apt.dat line is blank or a comment line
@@ -253,15 +334,13 @@ void APTLoader::finishAirport(const string& aptDat)
   currentAirportID = 0;
 }
 
-void APTLoader::parseAirportLine(const string& aptDat,
+// 'rowCode' is passed to avoid decoding it twice, since that work was already
+// done in order to detect the start of the new airport.
+void APTLoader::parseAirportLine(unsigned int rowCode,
                                  const vector<string>& token)
 {
   const string& id(token[4]);
   double elev = atof( token[1].c_str() );
-
-  // finish the previous airport
-  finishAirport(aptDat);
-
   last_apt_elev = elev;
 
   string name;
@@ -276,8 +355,7 @@ void APTLoader::parseAirportLine(const string& aptDat,
   rwy_lat_accum = 0.0;
   rwy_count = 0;
 
-  int robinType = atoi(token[0].c_str());
+  currentAirportID = cache->insertAirport(fptypeFromRobinType(rowCode),
-  currentAirportID = cache->insertAirport(fptypeFromRobinType(robinType),
                                           id, name);
 }
 
@@ -553,16 +631,7 @@ void APTLoader::parseCommLine(const string& aptDat, int lineId,
                "), skipping" );
 }
 
-// Load the airport data base from the specified aptdb file.  The
+// The 'metar.dat' file lists the airports that have METAR available.
-// metar file is used to mark the airports as having metar available
-// or not.
-bool airportDBLoad( const SGPath &aptdb_file )
-{
-  APTLoader ld;
-  ld.parseAPT(aptdb_file);
-  return true;
-}
-
 bool metarDataLoad(const SGPath& metar_file)
 {
   sg_gzifstream metar_in( metar_file );

src/Airports/apt_loader.hxx

@@ -28,13 +28,18 @@
 #include <string>
 #include <vector>
 
+#if __cplusplus >= 201103L
+  #include <unordered_map>
+#else
+  #include <map>
+#endif
+
 #include <simgear/compiler.h>
 #include <simgear/structure/SGSharedPtr.hxx>
 #include <simgear/math/SGGeod.hxx>
+#include <simgear/misc/sg_path.hxx>
 #include <Navaids/positioned.hxx>
 
-// Forward declarations
-class SGPath;
 class NavDataCache;
 class sg_gzifstream;
 class FGPavement;
@@ -48,9 +53,47 @@ public:
   APTLoader();
   ~APTLoader();
 
-  void parseAPT(const SGPath &aptdb_file);
+  // Read the specified apt.dat file into 'airportInfoMap'
+  void readAptDatFile(const SGPath& aptdb_file);
+  // Read all airports gathered in 'airportInfoMap' and load them into the
+  // navdata cache (even in case of overlapping apt.dat files,
+  // 'airportInfoMap' has only one entry per airport).
+  void loadAirports();
 
 private:
+  struct Line
+  {
+    Line(unsigned int number_, unsigned int rowCode_, std::string str_)
+      : number(number_), rowCode(rowCode_), str(str_) { }
+
+    unsigned int number;
+    unsigned int rowCode;         // Terminology of the apt.dat spec
+    std::string str;
+  };
+
+  typedef std::vector<Line> LinesList;
+
+  struct RawAirportInfo
+  {
+    // apt.dat file where the airport was defined
+    SGPath file;
+    // Row code for the airport (1, 16 or 17)
+    unsigned int rowCode;
+    // Line number in the apt.dat file where the airport definition starts
+    unsigned int firstLineNum;
+    // The whitespace-separated strings comprising the first line of the airport
+    // definition
+    std::vector<std::string> firstLineTokens;
+    // Subsequent lines of the airport definition (one element per line)
+    LinesList otherLines;
+  };
+
+#if __cplusplus >= 201103L
+  typedef std::unordered_map<std::string, RawAirportInfo> AirportInfoMapType;
+#else
+  typedef           std::map<std::string, RawAirportInfo> AirportInfoMapType;
+#endif
+
   typedef SGSharedPtr<FGPavement> FGPavementPtr;
 
   APTLoader(const APTLoader&);            // disable copy constructor
@@ -60,7 +103,7 @@ private:
   bool isBlankOrCommentLine(const std::string& line);
   void throwExceptionIfStreamError(const sg_gzifstream& input_stream,
                                    const SGPath& path);
-  void parseAirportLine(const std::string& aptDat,
+  void parseAirportLine(unsigned int rowCode,
                         const std::vector<std::string>& token);
   void finishAirport(const std::string& aptDat);
   void parseRunwayLine810(const std::vector<std::string>& token);
@@ -73,6 +116,8 @@ private:
   void parseCommLine(const std::string& aptDat, int lineId,
                      const std::vector<std::string>& token);
 
+  std::vector<std::string> token;
+  AirportInfoMapType airportInfoMap;
   double rwy_lat_accum;
   double rwy_lon_accum;
   double last_rwy_heading;
@@ -90,12 +135,6 @@ private:
   PositionedID currentAirportID;
 };
 
-// Load the airport data base from the specified aptdb file.  The
-// metar file is used to mark the airports as having metar available
-// or not.
-
-bool airportDBLoad(const SGPath& path);
-
 bool metarDataLoad(const SGPath& path);
 
 } // of namespace flighgear

src/Navaids/NavDataCache.cxx

@@ -1339,16 +1339,18 @@ void NavDataCache::doRebuild()
     SGTimeStamp st;
     {
         Transaction txn(this);
+        APTLoader aptLoader;
         string_list aptDatFiles;
 
         st.stamp();
         for (PathList::const_iterator it = d->aptDatPaths.begin();
              it != d->aptDatPaths.end(); it++) {
             aptDatFiles.push_back(it->realpath().utf8Str());
-            airportDBLoad(*it);
+            aptLoader.readAptDatFile(*it);
             stampCacheFile(*it); // this uses the realpath() of the file
         }
 
+        aptLoader.loadAirports(); // load airport data into the NavCache
         // Store the list of apt.dat files we have loaded
         writeOrderedStringListProperty(
           datTypeStr[DATFILETYPE_APT] + ".dat files", aptDatFiles,