APTLoader: improve progress status info

It is not needed anymore to hardcode the number of lines of
$FG_ROOT/Airports/apt.dat.gz. The new method, which relies on
SGPath::sizeInBytes() and sg_gzifstream::approxOffset(), works as well
for the other dat files.

Add a new NavCache rebuild phase, REBUILD_READING_APT_DAT_FILES, since
the process is now made of two parts.

Rename NavDataCachePrivate::getDatFilesPaths() to
NavDataCachePrivate::findDatFiles(), because it now returns a
DatFilesGroupInfo (new struct) instead of a PathList. For the same
reason, rename NavDataCachePrivate::aptDatPaths to
NavDataCachePrivate::aptDatFilesInfo. Adapt signatures, etc.

This requires up-to-date SimGear and FGData.

src/Airports/apt_loader.cxx

@@ -42,6 +42,7 @@
 #include <simgear/structure/exception.hxx>
 #include <simgear/misc/sg_path.hxx>
 
+#include <cstddef>              // std::size_t
 #include <string>
 #include <vector>
 #include <utility>              // std::pair, std::move()
@@ -72,9 +73,6 @@ static FGPositioned::Type fptypeFromRobinType(unsigned int aType)
   }
 }
 
-// generated by 'wc -l' on the uncompressed file
-const unsigned int LINES_IN_APT_DAT = 2465648;
-
 namespace flightgear
 {
 APTLoader::APTLoader()
@@ -86,7 +84,9 @@ APTLoader::APTLoader()
 
 APTLoader::~APTLoader() { }
 
-void APTLoader::readAptDatFile(const SGPath &aptdb_file)
+void APTLoader::readAptDatFile(const SGPath &aptdb_file,
+                               std::size_t bytesReadSoFar,
+                               std::size_t totalSizeOfAllAptDatFiles)
 {
   string apt_dat = aptdb_file.utf8Str(); // full path to the file being parsed
   sg_gzifstream in(aptdb_file);
@@ -150,8 +150,10 @@ void APTLoader::readAptDatFile(const SGPath &aptdb_file)
 
     if ((line_num % 100) == 0) {
       // every 100 lines
-      unsigned int percent = (line_num * 100) / LINES_IN_APT_DAT;
+      unsigned int percent = ((bytesReadSoFar + in.approxOffset()) * 100)
-      cache->setRebuildPhaseProgress(NavDataCache::REBUILD_AIRPORTS, percent);
+                             / totalSizeOfAllAptDatFiles;
+      cache->setRebuildPhaseProgress(
+        NavDataCache::REBUILD_READING_APT_DAT_FILES, percent);
     }
 
     // Extract the first field into 'rowCode'
@@ -215,6 +217,9 @@ void APTLoader::readAptDatFile(const SGPath &aptdb_file)
 
 void APTLoader::loadAirports()
 {
+  AirportInfoMapType::size_type nbLoadedAirports = 0;
+  AirportInfoMapType::size_type nbAirports = airportInfoMap.size();
+
   // Loop over all airports found in all apt.dat files
   for (AirportInfoMapType::const_iterator it = airportInfoMap.begin();
        it != airportInfoMap.end(); it++) {
@@ -288,7 +293,18 @@ void APTLoader::loadAirports()
     } // of loop over the second and subsequent apt.dat lines for the airport
 
     finishAirport(aptDat);
+    nbLoadedAirports++;
+
+    if ((nbLoadedAirports % 300) == 0) {
+      // Every 300 airports
+      unsigned int percent = nbLoadedAirports * 100 / nbAirports;
+      cache->setRebuildPhaseProgress(NavDataCache::REBUILD_LOADING_AIRPORTS,
+                                     percent);
+    }
   } // of loop over 'airportInfoMap'
+
+  SG_LOG( SG_GENERAL, SG_INFO,
+          "Loaded data for " << nbAirports << " airports" );
 }
 
 // Tell whether an apt.dat line is blank or a comment line

src/Airports/apt_loader.hxx

@@ -53,8 +53,11 @@ public:
   APTLoader();
   ~APTLoader();
 
-  // Read the specified apt.dat file into 'airportInfoMap'
+  // Read the specified apt.dat file into 'airportInfoMap'.
-  void readAptDatFile(const SGPath& aptdb_file);
+  // 'bytesReadSoFar' and 'totalSizeOfAllAptDatFiles' are used for progress
+  // information.
+  void readAptDatFile(const SGPath& aptdb_file, std::size_t bytesReadSoFar,
+                      std::size_t totalSizeOfAllAptDatFiles);
   // Read all airports gathered in 'airportInfoMap' and load them into the
   // navdata cache (even in case of overlapping apt.dat files,
   // 'airportInfoMap' has only one entry per airport).

src/GUI/QtLauncher.cxx

@@ -111,8 +111,12 @@ void initNavCache()
             phase = cache->rebuild();
 
             switch (phase) {
-            case NavDataCache::REBUILD_AIRPORTS:
+            case NavDataCache::REBUILD_READING_APT_DAT_FILES:
-                rebuildProgress.setLabelText(QT_TR_NOOP("Loading airport data"));
+                rebuildProgress.setLabelText(QT_TR_NOOP("Reading airport data"));
+                break;
+
+            case NavDataCache::REBUILD_LOADING_AIRPORTS:
+                rebuildProgress.setLabelText(QT_TR_NOOP("Loading airports"));
                 break;
 
             case NavDataCache::REBUILD_FIXES:

src/Main/fg_init.cxx

@@ -589,7 +589,8 @@ fgInitNav ()
 
     static const char* splashIdentsByRebuildPhase[] = {
         "loading-nav-dat",
-        "navdata-airports",
+        "navdata-reading-apt-dat-files",
+        "navdata-loading-airports",
         "navdata-navaids",
         "navdata-fixes",
         "navdata-pois"

src/Navaids/NavDataCache.cxx

@@ -26,6 +26,7 @@
 #include "NavDataCache.hxx"
 
 // std
+#include <cstddef>  // for std::size_t
 #include <map>
 #include <cassert>
 #include <stdint.h> // for int64_t
@@ -234,6 +235,12 @@ public:
   }
 };
 
+struct DatFilesGroupInfo {
+  NavDataCache::DatFileType datFileType; // for instance, DATFILETYPE_APT
+  PathList paths;               // SGPath instances
+  std::size_t totalSize;        // total size of all these files, in bytes
+};
+
 class NavDataCache::NavDataCachePrivate
 {
 public:
@@ -333,9 +340,10 @@ public:
   }
 
   bool isCachedFileModified(const SGPath& path, bool verbose);
-  PathList getDatFilesPaths(NavDataCache::DatFileType datFileType) const;
+  DatFilesGroupInfo findDatFiles(NavDataCache::DatFileType datFileType)
-  bool areDatFilesModified(NavDataCache::DatFileType datFileType,
+    const;
-                           const PathList& datFiles, bool verbose);
+  bool areDatFilesModified(const DatFilesGroupInfo& datFilesGroupInfo,
+                           bool verbose);
 
   void callSqlite(int result, const string& sql)
   {
@@ -852,7 +860,7 @@ public:
   bool transactionAborted;
   sqlite3_stmt_ptr beginTransactionStmt, commitTransactionStmt, rollbackTransactionStmt;
 
-  PathList aptDatPaths;
+  DatFilesGroupInfo aptDatFilesInfo;
   SGPath metarDatPath, navDatPath, fixDatPath, poiDatPath,
   carrierDatPath, airwayDatPath;
 
@@ -1014,17 +1022,21 @@ bool NavDataCache::NavDataCachePrivate::isCachedFileModified(const SGPath& path,
   return isModified;
 }
 
-// Return the list of $scenery_path/NavData/<type>/*.dat[.gz] files found
+// Find the list of $scenery_path/NavData/<type>/*.dat[.gz] files found
 // inside scenery paths, plus the default file for the given type (e.g.,
 // $FG_ROOT/Airports/apt.dat.gz for the 'apt' type).
-PathList NavDataCache::NavDataCachePrivate::getDatFilesPaths(
+// Also compute the total size of all these files (in bytes), which is useful
+// for progress information.
+DatFilesGroupInfo NavDataCache::NavDataCachePrivate::findDatFiles(
   NavDataCache::DatFileType datFileType) const
 {
-  PathList result;
+  DatFilesGroupInfo result;
   SGPath visitedPath;           // to avoid duplicates and time wasting
   const string datFilesSubDir = "NavData/" +
                                NavDataCache::datTypeStr[datFileType];
   const PathList& sceneryPaths = globals->get_unmangled_fg_scenery();
+  result.datFileType = datFileType;
+  result.totalSize = 0;
 
   for (PathList::const_iterator dirsIt = sceneryPaths.begin();
        dirsIt != sceneryPaths.end(); dirsIt++) {
@@ -1051,8 +1063,10 @@ PathList NavDataCache::NavDataCachePrivate::getDatFilesPaths(
            filesIt != files.end(); filesIt++) {
         const std::string name = filesIt->file();
         if (simgear::strutils::ends_with(name, ".dat") ||
-            simgear::strutils::ends_with(name, ".dat.gz"))
+            simgear::strutils::ends_with(name, ".dat.gz")) {
-          result.push_back(*filesIt);
+          result.paths.push_back(*filesIt);
+          result.totalSize += filesIt->sizeInBytes();
+        }
       }
     }
   } // of loop over 'sceneryPaths'
@@ -1061,16 +1075,17 @@ PathList NavDataCache::NavDataCachePrivate::getDatFilesPaths(
   // now
   SGPath defaultDatFile(globals->get_fg_root());
   defaultDatFile.append(NavDataCache::defaultDatFile[datFileType]);
-  if ((result.empty() || result.back() != defaultDatFile) &&
+  if ((result.paths.empty() || result.paths.back() != defaultDatFile) &&
       defaultDatFile.isFile()) {
-    result.push_back(defaultDatFile);
+    result.paths.push_back(defaultDatFile);
+    result.totalSize += defaultDatFile.sizeInBytes();
   }
 
   return result;
 }
 
 // Compare:
-//   - the list of dat files given by 'datFiles';
+//   - the list of dat files given by 'datFilesGroupInfo';
 //   - the list obtained from the record with key '<type>.dat files' of the
 //     NavDataCache 'properties' table, where <type> is one of 'apt', 'metar',
 //     'fix', 'poi', etc..
@@ -1078,12 +1093,15 @@ PathList NavDataCache::NavDataCachePrivate::getDatFilesPaths(
 // This comparison is sensitive to the number and order of the files,
 // their respective SGPath::realpath() and SGPath::modTime().
 bool NavDataCache::NavDataCachePrivate::areDatFilesModified(
-  NavDataCache::DatFileType datFileType, const PathList& datFiles, bool verbose)
+  const DatFilesGroupInfo& datFilesGroupInfo,
+  bool verbose)
 {
   // 'apt' or 'metar' or 'fix' or...
-  const string datTypeStr = NavDataCache::datTypeStr[datFileType];
+  const string datTypeStr =
+                       NavDataCache::datTypeStr[datFilesGroupInfo.datFileType];
   const string_list cachedFiles = outer->readOrderedStringListProperty(
     datTypeStr + ".dat files", &SGPath::pathListSep);
+  const PathList& datFiles = datFilesGroupInfo.paths;
   PathList::const_iterator datFilesIt = datFiles.begin();
   string_list::const_iterator cachedFilesIt = cachedFiles.begin();
   // Same logic as in NavDataCachePrivate::isCachedFileModified()
@@ -1208,7 +1226,7 @@ NavDataCache::NavDataCache()
   Octree::global_spatialOctree =
     new Octree::Branch(SGBox<double>(-earthExtent, earthExtent), 1);
 
-  // Update d->aptDatPaths, d->metarDatPath, d->navDatPath, etc.
+  // Update d->aptDatFilesInfo, d->metarDatPath, d->navDatPath, etc.
   updateListsOfDatFiles();
 }
 
@@ -1235,10 +1253,10 @@ NavDataCache* NavDataCache::instance()
 void NavDataCache::updateListsOfDatFiles() {
   // All $scenery_path/NavData/apt/*.dat[.gz] files found inside scenery
   // paths, plus $FG_ROOT/Airports/apt.dat.gz (order matters).
-  d->aptDatPaths = d->getDatFilesPaths(DATFILETYPE_APT);
+  d->aptDatFilesInfo = d->findDatFiles(DATFILETYPE_APT);
   // Trivial extension to the other types of dat files:
-  // d->navDatPaths = d->getDatFilesPaths(DATFILETYPE_NAV);
+  // d->navDatFilesInfo = d->findDatFiles(DATFILETYPE_NAV);
-  // d->fixDatPaths = d->getDatFilesPaths(DATFILETYPE_FIX);
+  // d->fixDatFilesInfo = d->findDatFiles(DATFILETYPE_FIX);
   // etc.
   //
   // These ones are still managed the "old" way (no search through scenery
@@ -1273,7 +1291,7 @@ bool NavDataCache::isRebuildRequired()
         return true;
     }
 
-  if (d->areDatFilesModified(DATFILETYPE_APT, d->aptDatPaths, true) ||
+  if (d->areDatFilesModified(d->aptDatFilesInfo, true) ||
       d->isCachedFileModified(d->metarDatPath, true) ||
       d->isCachedFileModified(d->navDatPath, true) ||
       d->isCachedFileModified(d->fixDatPath, true) ||
@@ -1341,15 +1359,22 @@ void NavDataCache::doRebuild()
         Transaction txn(this);
         APTLoader aptLoader;
         string_list aptDatFiles;
+        const PathList aptDatPaths = d->aptDatFilesInfo.paths;
+        std::size_t bytesReadSoFar = 0;
 
         st.stamp();
-        for (PathList::const_iterator it = d->aptDatPaths.begin();
+        for (PathList::const_iterator it = aptDatPaths.begin();
-             it != d->aptDatPaths.end(); it++) {
+             it != aptDatPaths.end(); it++) {
             aptDatFiles.push_back(it->realpath().utf8Str());
-            aptLoader.readAptDatFile(*it);
+            aptLoader.readAptDatFile(*it, bytesReadSoFar,
+                                     d->aptDatFilesInfo.totalSize);
+            bytesReadSoFar += it->sizeInBytes();
             stampCacheFile(*it); // this uses the realpath() of the file
         }
 
+        setRebuildPhaseProgress(REBUILD_UNKNOWN);
+        SG_LOG(SG_NAVCACHE, SG_DEBUG, "Loading airports");
+
         aptLoader.loadAirports(); // load airport data into the NavCache
         // Store the list of apt.dat files we have loaded
         writeOrderedStringListProperty(

src/Navaids/NavDataCache.hxx

@@ -77,7 +77,8 @@ public:
     enum RebuildPhase
     {
         REBUILD_UNKNOWN = 0,
-        REBUILD_AIRPORTS,
+        REBUILD_READING_APT_DAT_FILES,
+        REBUILD_LOADING_AIRPORTS,
         REBUILD_NAVAIDS,
         REBUILD_FIXES,
         REBUILD_POIS,
@@ -298,6 +299,8 @@ private:
     void commitTransaction();
     void abortTransaction();
 
+  friend class DatFilesGroupInfo;
+
   enum DatFileType {
     DATFILETYPE_APT = 0,
     DATFILETYPE_METAR,