Support multiple nav.dat and fix.dat files

src/Airports/apt_loader.cxx

@@ -99,7 +99,6 @@ void APTLoader::readAptDatFile(const SGPath &aptdb_file,
                           sg_location(aptdb_file));
   }
 
-  SG_LOG( SG_GENERAL, SG_INFO, "Opened apt.dat file: '" << apt_dat << "'" );
   string line;
 
   unsigned int rowCode = 0;     // terminology used in the apt.dat format spec

src/Main/fg_init.cxx

@@ -649,7 +649,7 @@ fgInitNav ()
   
   SGPath path(globals->get_fg_root());
   path.append( "Navaids/TACAN_freq.dat" );
-  flightgear::loadTacan(path, channellist);
+  flightgear::NavLoader().loadTacan(path, channellist);
   
   return true;
 }

src/Navaids/NavDataCache.cxx

@@ -351,10 +351,9 @@ public:
   }
 
   bool isCachedFileModified(const SGPath& path, bool verbose);
-  NavDataCache::DatFilesGroupInfo findDatFiles(
+  void findDatFiles(NavDataCache::DatFileType datFileType);
-    NavDataCache::DatFileType datFileType) const;
   bool areDatFilesModified(
-    const NavDataCache::DatFilesGroupInfo& datFilesGroupInfo,
+    NavDataCache::DatFileType datFileType,
     bool verbose);
 
   void callSqlite(int result, const string& sql)
@@ -872,8 +871,8 @@ public:
   bool transactionAborted;
   sqlite3_stmt_ptr beginTransactionStmt, commitTransactionStmt, rollbackTransactionStmt;
 
-  NavDataCache::DatFilesGroupInfo aptDatFilesInfo;
+  std::map<DatFileType, NavDataCache::DatFilesGroupInfo> datFilesInfo;
-  SGPath metarDatPath, navDatPath, fixDatPath, poiDatPath,
+  SGPath metarDatPath, poiDatPath,
   carrierDatPath, airwayDatPath;
 
   sqlite3_stmt_ptr readPropertyQuery, writePropertyQuery,
@@ -1039,8 +1038,9 @@ bool NavDataCache::NavDataCachePrivate::isCachedFileModified(const SGPath& path,
 // $FG_ROOT/Airports/apt.dat.gz for the 'apt' type).
 // Also compute the total size of all these files (in bytes), which is useful
 // for progress information.
-NavDataCache::DatFilesGroupInfo NavDataCache::NavDataCachePrivate::findDatFiles(
+// The result is stored in the datFilesInfo field.
-  NavDataCache::DatFileType datFileType) const
+void NavDataCache::NavDataCachePrivate::findDatFiles(
+  NavDataCache::DatFileType datFileType)
 {
   NavDataCache::DatFilesGroupInfo result;
   SGPath visitedPath;           // to avoid duplicates and time wasting
@@ -1093,7 +1093,7 @@ NavDataCache::DatFilesGroupInfo NavDataCache::NavDataCachePrivate::findDatFiles(
     result.totalSize += defaultDatFile.sizeInBytes();
   }
 
-  return result;
+  datFilesInfo[datFileType] = result;
 }
 
 // Compare:
@@ -1105,12 +1105,14 @@ NavDataCache::DatFilesGroupInfo NavDataCache::NavDataCachePrivate::findDatFiles(
 // This comparison is sensitive to the number and order of the files,
 // their respective SGPath::realpath() and SGPath::modTime().
 bool NavDataCache::NavDataCachePrivate::areDatFilesModified(
-  const NavDataCache::DatFilesGroupInfo& datFilesGroupInfo,
+  NavDataCache::DatFileType datFileType,
   bool verbose)
 {
   // 'apt' or 'metar' or 'fix' or...
+  const NavDataCache::DatFilesGroupInfo& datFilesGroupInfo =
+        datFilesInfo[datFileType];
   const string datTypeStr =
-                       NavDataCache::datTypeStr[datFilesGroupInfo.datFileType];
+                       NavDataCache::datTypeStr[datFileType];
   const string_list cachedFiles = outer->readOrderedStringListProperty(
     datTypeStr + ".dat files", &SGPath::pathListSep);
   const PathList& datFiles = datFilesGroupInfo.paths;
@@ -1279,23 +1281,18 @@ NavDataCache* NavDataCache::instance()
 void NavDataCache::updateListsOfDatFiles() {
   // All $scenery_path/NavData/apt/*.dat[.gz] files found inside scenery
   // paths, plus $FG_ROOT/Airports/apt.dat.gz (order matters).
-  d->aptDatFilesInfo = d->findDatFiles(DATFILETYPE_APT);
+  d->findDatFiles(DATFILETYPE_APT);
-  // Trivial extension to the other types of dat files:
+
-  // d->navDatFilesInfo = d->findDatFiles(DATFILETYPE_NAV);
+  // All $scenery_path/NavData/{nav,fix}/*.dat[.gz] files found inside scenery
-  // d->fixDatFilesInfo = d->findDatFiles(DATFILETYPE_FIX);
+  // paths, plus $FG_ROOT/Navaids/{nav,fix}.dat.gz (order matters).
-  // etc.
+  d->findDatFiles(DATFILETYPE_NAV);
-  //
+  d->findDatFiles(DATFILETYPE_FIX);
+
   // These ones are still managed the "old" way (no search through scenery
   // paths).
   d->metarDatPath = SGPath(globals->get_fg_root());
   d->metarDatPath.append("Airports/metar.dat.gz");
 
-  d->navDatPath = SGPath(globals->get_fg_root());
-  d->navDatPath.append("Navaids/nav.dat.gz");
-
-  d->fixDatPath = SGPath(globals->get_fg_root());
-  d->fixDatPath.append("Navaids/fix.dat.gz");
-
   d->poiDatPath = SGPath(globals->get_fg_root());
   d->poiDatPath.append("Navaids/poi.dat.gz");
 
@@ -1306,20 +1303,16 @@ void NavDataCache::updateListsOfDatFiles() {
   d->airwayDatPath.append("Navaids/awy.dat.gz");
 }
 
-NavDataCache::DatFilesGroupInfo
+const NavDataCache::DatFilesGroupInfo&
 NavDataCache::getDatFilesInfo(DatFileType datFileType) const
 {
-  switch (datFileType) {
+  auto iter = d->datFilesInfo.find(datFileType);
-    case DATFILETYPE_APT:
+  if (iter == d->datFilesInfo.end()) {
-      return d->aptDatFilesInfo;
+    throw sg_error("NavCache: requesting info about the list of " +
-    default:
+           datTypeStr[datFileType] + " dat files, however this is not "
-      SG_LOG(SG_NAVCACHE, SG_ALERT,
+           "implemented yet!");
-             "NavCache: requesting info about the list of " <<
-             datTypeStr[datFileType] << " dat files, however this is not "
-             "implemented yet!");
-      assert(false);
-      return DatFilesGroupInfo();
   }
+  return iter->second;
 }
 
 bool NavDataCache::isRebuildRequired()
@@ -1333,10 +1326,10 @@ bool NavDataCache::isRebuildRequired()
         return true;
     }
 
-  if (d->areDatFilesModified(d->aptDatFilesInfo, true) ||
+  if (d->areDatFilesModified(DATFILETYPE_APT, true) ||
       d->isCachedFileModified(d->metarDatPath, true) ||
-      d->isCachedFileModified(d->navDatPath, true) ||
+      d->areDatFilesModified(DATFILETYPE_NAV, true) ||
-      d->isCachedFileModified(d->fixDatPath, true) ||
+      d->areDatFilesModified(DATFILETYPE_FIX, true) ||
       d->isCachedFileModified(d->carrierDatPath, true) ||
 // since POI loading is disabled on Windows, don't check for it
 // this caused: https://code.google.com/p/flightgear-bugs/issues/detail?id=1227
@@ -1386,6 +1379,37 @@ void NavDataCache::setRebuildPhaseProgress(RebuildPhase ph, unsigned int percent
     d->rebuilder->setProgress(ph, percent);
 }
 
+void NavDataCache::loadDatFiles(
+    DatFileType type,
+    std::function<void(const SGPath&, std::size_t, std::size_t)> loader)
+{
+  SGTimeStamp st;
+  string typeStr = datTypeStr[type];
+  string_list datFiles;
+  NavDataCache::DatFilesGroupInfo datFilesInfo = getDatFilesInfo(type);
+  const PathList datPaths = datFilesInfo.paths;
+  std::size_t bytesReadSoFar = 0;
+
+  st.stamp();
+  for (PathList::const_iterator it = datPaths.begin();
+       it != datPaths.end(); it++) {
+    string path = it->realpath().utf8Str();
+    datFiles.push_back(path);
+    SG_LOG(SG_GENERAL, SG_INFO,
+           "Loading " + typeStr + ".dat file: '" << path << "'");
+    loader(*it, bytesReadSoFar, datFilesInfo.totalSize);
+    bytesReadSoFar += it->sizeInBytes();
+    stampCacheFile(*it); // this uses the realpath() of the file
+  }
+
+  // Store the list of .dat files we have loaded
+  writeOrderedStringListProperty(typeStr + ".dat files", datFiles,
+                                 &SGPath::pathListSep);
+  SG_LOG(SG_NAVCACHE, SG_INFO,
+         typeStr + ".dat files load took: " <<
+         st.elapsedMSec());
+}
+
 void NavDataCache::doRebuild()
 {
   try {
@@ -1400,45 +1424,30 @@ void NavDataCache::doRebuild()
     {
         Transaction txn(this);
         APTLoader aptLoader;
-        string_list aptDatFiles;
+        FixesLoader fixesLoader;
-        const PathList aptDatPaths = d->aptDatFilesInfo.paths;
+        NavLoader navLoader;
-        std::size_t bytesReadSoFar = 0;
+
+        using namespace std::placeholders;  // for _1, _2, _3...
+
+        loadDatFiles(DATFILETYPE_APT,
+                     std::bind(&APTLoader::readAptDatFile, &aptLoader, _1, _2, _3));
 
         st.stamp();
-        for (PathList::const_iterator it = aptDatPaths.begin();
-             it != aptDatPaths.end(); it++) {
-            aptDatFiles.push_back(it->realpath().utf8Str());
-            aptLoader.readAptDatFile(*it, bytesReadSoFar,
-                                     d->aptDatFilesInfo.totalSize);
-            bytesReadSoFar += it->sizeInBytes();
-            stampCacheFile(*it); // this uses the realpath() of the file
-        }
-
         setRebuildPhaseProgress(REBUILD_UNKNOWN);
-        SG_LOG(SG_NAVCACHE, SG_DEBUG, "Loading airports");
+        SG_LOG(SG_NAVCACHE, SG_DEBUG, "Processing airports");
-
         aptLoader.loadAirports(); // load airport data into the NavCache
-        // Store the list of apt.dat files we have loaded
-        writeOrderedStringListProperty(
-          datTypeStr[DATFILETYPE_APT] + ".dat files", aptDatFiles,
-          &SGPath::pathListSep);
         SG_LOG(SG_NAVCACHE, SG_INFO,
-               datTypeStr[DATFILETYPE_APT] + ".dat files load took:" <<
+               "processing airports took:" <<
                st.elapsedMSec());
 
         setRebuildPhaseProgress(REBUILD_UNKNOWN);
         metarDataLoad(d->metarDatPath);
         stampCacheFile(d->metarDatPath);
 
-        st.stamp();
+        loadDatFiles(DATFILETYPE_FIX,
-        FixesLoader().loadFixes(d->fixDatPath);
+                     std::bind(&FixesLoader::loadFixes, &fixesLoader, _1, _2, _3));
-        stampCacheFile(d->fixDatPath);
+        loadDatFiles(DATFILETYPE_NAV,
-        SG_LOG(SG_NAVCACHE, SG_INFO, "fix.dat load took:" << st.elapsedMSec());
+                     std::bind(&NavLoader::loadNav, &navLoader, _1, _2, _3));
-
-        st.stamp();
-        navDBInit(d->navDatPath);
-        stampCacheFile(d->navDatPath);
-        SG_LOG(SG_NAVCACHE, SG_INFO, "nav.dat load took:" << st.elapsedMSec());
 
         setRebuildPhaseProgress(REBUILD_UNKNOWN);
         st.stamp();
@@ -1466,7 +1475,8 @@ void NavDataCache::doRebuild()
 
       {
           Transaction txn(this);
-          loadCarrierNav(d->carrierDatPath);
+          NavLoader navLoader;
+          navLoader.loadCarrierNav(d->carrierDatPath);
           stampCacheFile(d->carrierDatPath);
 
           st.stamp();

src/Navaids/NavDataCache.hxx

@@ -26,6 +26,7 @@
 
 #include <memory>
 #include <cstddef>                   // for std::size_t
+#include <functional>
 
 #include <simgear/misc/strutils.hxx> // for string_list
 #include <Navaids/positioned.hxx>
@@ -73,7 +74,8 @@ public:
       DATFILETYPE_FIX,
       DATFILETYPE_POI,
       DATFILETYPE_CARRIER,
-      DATFILETYPE_TACAN_FREQ
+      DATFILETYPE_TACAN_FREQ,
+      DATFILETYPE_LAST
     };
 
     struct DatFilesGroupInfo {
@@ -82,12 +84,11 @@ public:
       std::size_t totalSize;        // total size of all these files, in bytes
     };
 
-    // Update d->aptDatFilesInfo, d->metarDatPath, d->navDatPath,
+    // Update d->datFilesInfo and legacy d->metarDatPath, d->poiDatPath,
-    // d->fixDatPath, d->poiDatPath, etc. by looking into
+    // etc. by looking into $scenery_path/NavData for each scenery path.
-    // $scenery_path/NavData for each scenery path.
     void updateListsOfDatFiles();
-    // Return d->aptDatFilesInfo if datFileType == DATFILETYPE_APT, etc.
+    // Returns datFilesInfo for the given type.
-    DatFilesGroupInfo getDatFilesInfo(DatFileType datFileType) const;
+    const DatFilesGroupInfo& getDatFilesInfo(DatFileType datFileType) const;
 
   /**
    * predicate - check if the cache needs to be rebuilt.
@@ -313,6 +314,12 @@ private:
   NavDataCache();
 
   friend class RebuildThread;
+
+  // A generic function for loading all navigation data files of the
+  // specified type (apt/fix/nav etc.) using the passed type-specific loader.
+  void loadDatFiles(DatFileType type,
+                    std::function<void(const SGPath&, std::size_t, std::size_t)> loader);
+
   void doRebuild();
 
   friend class Transaction;

src/Navaids/fixlist.cxx

@@ -58,8 +58,6 @@ FGFix::FGFix(PositionedID aGuid, const std::string& aIdent, const SGGeod& aPos)
 namespace flightgear
 {
 
-const unsigned int LINES_IN_FIX_DAT = 119724;
-
 FixesLoader::FixesLoader() : _cache(NavDataCache::instance())
 { }
 
@@ -67,7 +65,8 @@ FixesLoader::~FixesLoader()
 { }
 
 // Load fixes from the specified fix.dat (or fix.dat.gz) file
-void FixesLoader::loadFixes(const SGPath& path)
+void FixesLoader::loadFixes(const SGPath& path, std::size_t bytesReadSoFar,
+                            std::size_t totalSizeOfAllDatFiles)
 {
   sg_gzifstream in( path );
   const std::string utf8path = path.utf8Str();
@@ -150,7 +149,8 @@ void FixesLoader::loadFixes(const SGPath& path)
 
     if ((lineNumber % 100) == 0) {
       // every 100 lines
-      unsigned int percent = (lineNumber * 100) / LINES_IN_FIX_DAT;
+      unsigned int percent = ((bytesReadSoFar + in.approxOffset()) * 100)
+        / totalSizeOfAllDatFiles;
       _cache->setRebuildPhaseProgress(NavDataCache::REBUILD_FIXES, percent);
     }
   }

src/Navaids/fixlist.hxx

@@ -44,7 +44,8 @@ namespace flightgear
     ~FixesLoader();
 
     // Load fixes from the specified fix.dat (or fix.dat.gz) file
-    void loadFixes(const SGPath& path);
+    void loadFixes(const SGPath& path, std::size_t bytesReadSoFar,
+                   std::size_t totalSizeOfAllDatFiles);
 
   private:
     void throwExceptionIfStreamError(const sg_gzifstream& input_stream,

src/Navaids/navdb.cxx

@@ -31,7 +31,6 @@
 #include <cstddef>              // std::size_t
 #include <cerrno>
 #include <limits>
-#include <map>
 
 #include "navdb.hxx"
 
@@ -116,10 +115,10 @@ void alignLocaliserWithRunway(FGRunway* rwy, const string& ident, SGGeod& pos, d
   
   // back project that distance along the runway center line
   SGGeod newPos = rwy->pointOnCenterline(dist);
-  
+
   double hdg_diff = heading - rwy->headingDeg();
   SG_NORMALIZE_RANGE(hdg_diff, -180.0, 180.0);
-  
+
   if ( fabs(hdg_diff) <= autoAlignThreshold ) {
     pos = SGGeod::fromGeodFt(newPos, pos.getElevationFt());
     heading = rwy->headingDeg();
@@ -139,12 +138,12 @@ static double defaultNavRange(const string& ident, FGPositioned::Type type)
     case FGPositioned::NDB:
     case FGPositioned::VOR:
       return FG_NAV_DEFAULT_RANGE;
-      
+
     case FGPositioned::LOC:
     case FGPositioned::ILS:
     case FGPositioned::GS:
       return FG_LOC_DEFAULT_RANGE;
-      
+
     case FGPositioned::DME:
       return FG_DME_DEFAULT_RANGE;
 
@@ -186,9 +185,9 @@ static bool canBeDuplicate(FGPositionedRef ref, FGPositioned::Type type,
 
 // Parse a line from a file such as nav.dat or carrier_nav.dat. Load the
 // corresponding data into the NavDataCache.
-static PositionedID processNavLine(
+PositionedID NavLoader::processNavLine(
   const string& line, const string& utf8Path, unsigned int lineNum,
-  FGPositioned::Type type = FGPositioned::INVALID, unsigned long version = 810)
+  FGPositioned::Type type, unsigned long version)
 {
   NavDataCache* cache = NavDataCache::instance();
   int rowCode, elev_ft, freq, range;
@@ -325,11 +324,8 @@ static PositionedID processNavLine(
   //
 
   // First, eliminate nearby with the same name, type and ident.
-  static std::multimap<std::tuple<FGPositioned::Type, std::string,
-      std::string>, SGGeod> loadedNavs;  // Maps (type, ident, name) tuples
-                                         // already loaded to their locations.
   auto loadedNavsKey = std::make_tuple(type, ident, name);
-  auto matchingNavs = loadedNavs.equal_range(loadedNavsKey);
+  auto matchingNavs = _loadedNavs.equal_range(loadedNavsKey);
   for (auto it = matchingNavs.first; it != matchingNavs.second; ++it) {
     if (isNearby(pos, it->second)) {
       SG_LOG(SG_NAVAID, SG_INFO,
@@ -338,7 +334,7 @@ static PositionedID processNavLine(
       return 0;
     }
   }
-  loadedNavs.emplace(loadedNavsKey, pos);
+  _loadedNavs.emplace(loadedNavsKey, pos);
 
   // Then, eliminate nearby with the same type and ident.
   FGPositioned::TypeFilter dupTypeFilter(type);
@@ -374,11 +370,11 @@ static PositionedID processNavLine(
     return cache->insertNavaid(type, string(), name, pos, 0, 0, 0,
                                arp.first, arp.second);
   }
-  
+
   if (range < 1) {
     range = defaultNavRange(ident, type);
   }
-  
+
   AirportRunwayPair arp;
   FGRunwayRef runway;
   PositionedID navaid_dme = 0;
@@ -444,7 +440,7 @@ static PositionedID processNavLine(
 #endif
     } // of found runway in the DB
   } // of type is runway-related
-  
+
   bool isLoc = (type == FGPositioned::ILS) || (type == FGPositioned::LOC);
   if (runway && autoAlignLocalizers && isLoc) {
     alignLocaliserWithRunway(runway, ident, pos, multiuse);
@@ -452,7 +448,7 @@ static PositionedID processNavLine(
 
   PositionedID r = cache->insertNavaid(type, ident, name, pos, freq, range, multiuse,
                              arp.first, arp.second);
-  
+
   if (isLoc) {
     cache->setRunwayILS(arp.second, r);
   }
@@ -460,16 +456,16 @@ static PositionedID processNavLine(
   if (navaid_dme) {
     cache->setNavaidColocated(navaid_dme, r);
   }
-  
+
   return r;
 }
 
 // load and initialize the navigational databases
-void navDBInit(const SGPath& path)
+void NavLoader::loadNav(const SGPath& path, std::size_t bytesReadSoFar,
+                        std::size_t totalSizeOfAllDatFiles)
 {
   NavDataCache* cache = NavDataCache::instance();
   const string utf8Path = path.utf8Str();
-  const std::size_t fileSize = path.sizeInBytes();
   sg_gzifstream in(path);
 
   if ( !in.is_open() ) {
@@ -507,7 +503,8 @@ void navDBInit(const SGPath& path)
 
     if ((lineNumber % 100) == 0) {
       // every 100 lines
-      unsigned int percent = (in.approxOffset() * 100) / fileSize;
+      unsigned int percent = ((bytesReadSoFar + in.approxOffset()) * 100)
+        / totalSizeOfAllDatFiles;
       cache->setRebuildPhaseProgress(NavDataCache::REBUILD_NAVAIDS, percent);
     }
 
@@ -516,7 +513,7 @@ void navDBInit(const SGPath& path)
   throwExceptionIfStreamError(in, path);
 }
 
-void loadCarrierNav(const SGPath& path)
+void NavLoader::loadCarrierNav(const SGPath& path)
 {
   SG_LOG( SG_NAVAID, SG_DEBUG, "Opening file: " << path );
   const string utf8Path = path.utf8Str();
@@ -539,26 +536,26 @@ void loadCarrierNav(const SGPath& path)
   throwExceptionIfStreamError(in, path);
 }
 
-bool loadTacan(const SGPath& path, FGTACANList *channellist)
+bool NavLoader::loadTacan(const SGPath& path, FGTACANList *channellist)
 {
     SG_LOG( SG_NAVAID, SG_DEBUG, "opening file: " << path );
     sg_gzifstream inchannel( path );
-    
+
     if ( !inchannel.is_open() ) {
         SG_LOG( SG_NAVAID, SG_ALERT, "Cannot open file: " << path );
       return false;
     }
-    
+
     // skip first line
     inchannel >> skipeol;
     while ( ! inchannel.eof() ) {
         FGTACANRecord *r = new FGTACANRecord;
         inchannel >> (*r);
         channellist->add ( r );
- 	
+
     } // end while
 
     return true;
 }
-  
+
 } // of namespace flightgear

src/Navaids/navdb.hxx

@@ -26,23 +26,40 @@
 
 
 #include <simgear/compiler.h>
+#include <simgear/math/SGGeod.hxx>
 #include <string>
+#include <map>
+#include <tuple>
+#include <Navaids/positioned.hxx>
 
 // forward decls
 class FGTACANList;
 class SGPath;
-class SGPropertyNode;
-class FGRunway;
 
 namespace flightgear
 {
-  
+
-// load and initialize the navigational databases
+class NavLoader {
-void navDBInit(const SGPath& path);
+  public:
-  
+    // load and initialize the navigational databases
-void loadCarrierNav(const SGPath& path);
+    void loadNav(const SGPath& path, std::size_t bytesReadSoFar,
-  
+                 std::size_t totalSizeOfAllDatFiles);
-bool loadTacan(const SGPath& path, FGTACANList *channellist);
+
+    void loadCarrierNav(const SGPath& path);
+
+    bool loadTacan(const SGPath& path, FGTACANList *channellist);
+
+  private:
+    // Maps (type, ident, name) tuples already loaded to their locations.
+    std::multimap<std::tuple<FGPositioned::Type, std::string, std::string>,
+        SGGeod> _loadedNavs;
+
+    PositionedID processNavLine(const std::string& line,
+                                const std::string& utf8Path,
+                                unsigned int lineNum,
+                                FGPositioned::Type type = FGPositioned::INVALID,
+                                unsigned long version = 810);
+};
 
 } // of namespace flightgear