diff --git a/src/Airports/apt_loader.cxx b/src/Airports/apt_loader.cxx index 8a0acb15e..8ee3fab87 100644 --- a/src/Airports/apt_loader.cxx +++ b/src/Airports/apt_loader.cxx @@ -55,7 +55,8 @@ #include <iostream> #include <sstream> // std::istringstream -using namespace std; +using std::vector; +using std::string; static FGPositioned::Type fptypeFromRobinType(int aType) { @@ -86,13 +87,18 @@ APTLoader::~APTLoader() { } void APTLoader::parseAPT(const SGPath &aptdb_file) { + string apt_dat = aptdb_file.utf8Str(); // full path to the file being parsed sg_gzifstream in(aptdb_file); if ( !in.is_open() ) { - SG_LOG( SG_GENERAL, SG_ALERT, "Cannot open file: " << aptdb_file ); - throw sg_io_exception("cannot open apt.dat file", aptdb_file); + const std::string errMsg = simgear::strutils::error_string(errno); + SG_LOG( SG_GENERAL, SG_ALERT, + "Cannot open file '" << apt_dat << "': " << errMsg ); + throw sg_io_exception("Cannot open file (" + errMsg + ")", + sg_location(aptdb_file)); } + SG_LOG( SG_GENERAL, SG_INFO, "Opened apt.dat file: '" << apt_dat << "'" ); string line; unsigned int line_id = 0; @@ -110,15 +116,16 @@ void APTLoader::parseAPT(const SGPath &aptdb_file) if ( stripped_line != "I" && stripped_line != "A" ) { std::string pb = "invalid first line (neither 'I' nor 'A')"; SG_LOG( SG_GENERAL, SG_ALERT, aptdb_file << ": " << pb); - throw sg_format_exception("cannot parse apt.dat file: " + pb, - aptdb_file.utf8Str()); + throw sg_format_exception("cannot parse '" + apt_dat + "': " + pb, + stripped_line); } } else { // second line of the file std::istringstream s(line); int apt_dat_format_version; s >> apt_dat_format_version; SG_LOG( SG_GENERAL, SG_INFO, - "apt.dat format version: " << apt_dat_format_version ); + "apt.dat format version (" << apt_dat << "): " << + apt_dat_format_version ); } } // end of the apt.dat header @@ -143,7 +150,7 @@ void APTLoader::parseAPT(const SGPath &aptdb_file) if ( line_id == 1 /* Airport */ || line_id == 16 /* Seaplane base */ || line_id == 17 /* Heliport */ ) { - parseAirportLine(simgear::strutils::split(line)); + parseAirportLine(apt_dat, simgear::strutils::split(line)); } else if ( line_id == 10 ) { // Runway v810 parseRunwayLine810(simgear::strutils::split(line)); } else if ( line_id == 100 ) { // Runway v850 @@ -174,7 +181,7 @@ void APTLoader::parseAPT(const SGPath &aptdb_file) } else if ( line_id == 0 ) { // ?? } else if ( line_id >= 50 && line_id <= 56) { - parseCommLine(line_id, simgear::strutils::split(line)); + parseCommLine(apt_dat, line_id, simgear::strutils::split(line)); } else if ( line_id == 110 ) { pavement = true; parsePavementLine850(simgear::strutils::split(line, 0, 4)); @@ -190,16 +197,18 @@ void APTLoader::parseAPT(const SGPath &aptdb_file) } else if ( line_id >= 1000 ) { // airport traffic flow (ignore) } else if ( line_id == 99 ) { - SG_LOG( SG_GENERAL, SG_DEBUG, "End of file reached" ); + SG_LOG( SG_GENERAL, SG_DEBUG, + apt_dat << ": code 99 found (normally at end of file)" ); } else { - SG_LOG( SG_GENERAL, SG_ALERT, - "Unknown line(#" << line_num << ") in apt.dat file: " << line ); - throw sg_format_exception("malformed line in apt.dat:", line); + std::ostringstream oss; + oss << apt_dat << ":" << line_num << ": unknown row code " << line_id; + SG_LOG( SG_GENERAL, SG_ALERT, oss.str() << " (" << line << ")" ); + throw sg_format_exception(oss.str(), line); } } throwExceptionIfStreamError(in, aptdb_file); - finishAirport(); + finishAirport(apt_dat); } // Tell whether an apt.dat line is blank or a comment line @@ -222,7 +231,7 @@ void APTLoader::throwExceptionIfStreamError(const sg_gzifstream& input_stream, } } -void APTLoader::finishAirport() +void APTLoader::finishAirport(const string& aptDat) { if (currentAirportID == 0) { return; @@ -230,8 +239,8 @@ void APTLoader::finishAirport() if (!rwy_count) { currentAirportID = 0; - SG_LOG(SG_GENERAL, SG_ALERT, "ERROR: No runways for " << last_apt_id - << ", skipping." ); + SG_LOG( SG_GENERAL, SG_ALERT, "Error in '" << aptDat << + "': no runways for " << last_apt_id << ", skipping." ); return; } @@ -244,13 +253,14 @@ void APTLoader::finishAirport() currentAirportID = 0; } -void APTLoader::parseAirportLine(const vector<string>& token) +void APTLoader::parseAirportLine(const string& aptDat, + const vector<string>& token) { const string& id(token[4]); double elev = atof( token[1].c_str() ); // finish the previous airport - finishAirport(); + finishAirport(aptDat); last_apt_elev = elev; @@ -488,11 +498,12 @@ void APTLoader::parsePavementNodeLine850(int num, const vector<string>& token) } } -void APTLoader::parseCommLine(int lineId, const vector<string>& token) +void APTLoader::parseCommLine(const string& aptDat, int lineId, + const vector<string>& token) { if ( rwy_count <= 0 ) { SG_LOG( SG_GENERAL, SG_ALERT, - "No runways; skipping comm for " + last_apt_id); + aptDat << ": no runways, skipping comm for " << last_apt_id ); } SGGeod pos = SGGeod::fromDegFt(rwy_lon_accum / (double)rwy_count, @@ -538,7 +549,8 @@ void APTLoader::parseCommLine(int lineId, const vector<string>& token) cache->insertCommStation(ty, name, pos, freqKhz, rangeNm, currentAirportID); } else SG_LOG( SG_GENERAL, SG_DEBUG, - "Found unnamed comm. Skipping: " << lineId ); + aptDat << ": found unnamed comm (row ID " << lineId << + "), skipping" ); } // Load the airport data base from the specified aptdb file. The diff --git a/src/Airports/apt_loader.hxx b/src/Airports/apt_loader.hxx index a524be5a9..9193d7271 100644 --- a/src/Airports/apt_loader.hxx +++ b/src/Airports/apt_loader.hxx @@ -60,8 +60,9 @@ private: bool isBlankOrCommentLine(const std::string& line); void throwExceptionIfStreamError(const sg_gzifstream& input_stream, const SGPath& path); - void parseAirportLine(const std::vector<std::string>& token); - void finishAirport(); + void parseAirportLine(const std::string& aptDat, + const std::vector<std::string>& token); + void finishAirport(const std::string& aptDat); void parseRunwayLine810(const std::vector<std::string>& token); void parseRunwayLine850(const std::vector<std::string>& token); void parseWaterRunwayLine850(const std::vector<std::string>& token); @@ -69,7 +70,8 @@ private: void parsePavementLine850(const std::vector<std::string>& token); void parsePavementNodeLine850(int num, const std::vector<std::string>& token); - void parseCommLine(int lineId, const std::vector<std::string>& token); + void parseCommLine(const std::string& aptDat, int lineId, + const std::vector<std::string>& token); double rwy_lat_accum; double rwy_lon_accum; diff --git a/src/GUI/QtLauncher.cxx b/src/GUI/QtLauncher.cxx index 26f40176a..ac2ba5cd2 100644 --- a/src/GUI/QtLauncher.cxx +++ b/src/GUI/QtLauncher.cxx @@ -471,6 +471,8 @@ void initApp(int& argc, char** argv) bool runLauncherDialog() { + // Used for NavDataCache initialization: needed to find the apt.dat files + QtLauncher::setSceneryPaths(); // startup the nav-cache now. This pre-empts normal startup of // the cache, but no harm done. (Providing scenery paths are consistent) @@ -678,7 +680,7 @@ QtLauncher::~QtLauncher() globals->get_subsystem<FGHTTPClient>()->client()->cancelRequest(m_mpServerRequest); } -void QtLauncher::setSceneryPaths() +void QtLauncher::setSceneryPaths() // static method { globals->clear_fg_scenery(); diff --git a/src/GUI/QtLauncher_private.hxx b/src/GUI/QtLauncher_private.hxx index 301c7abd8..e8eae0e61 100644 --- a/src/GUI/QtLauncher_private.hxx +++ b/src/GUI/QtLauncher_private.hxx @@ -52,7 +52,7 @@ public: void setInAppMode(); - void setSceneryPaths(); + static void setSceneryPaths(); static void restartTheApp(QStringList fgArgs); protected: diff --git a/src/Navaids/CacheSchema.h b/src/Navaids/CacheSchema.h index e92c62f27..1a6773a17 100644 --- a/src/Navaids/CacheSchema.h +++ b/src/Navaids/CacheSchema.h @@ -1,7 +1,7 @@ #ifndef FG_NAVCACHE_SCHEMA_HXX #define FG_NAVCACHE_SCHEMA_HXX -const int SCHEMA_VERSION = 15; +const int SCHEMA_VERSION = 16; #define SCHEMA_SQL \ "CREATE TABLE properties (key VARCHAR, value VARCHAR);" \ diff --git a/src/Navaids/NavDataCache.cxx b/src/Navaids/NavDataCache.cxx index 91a701040..3f60ff9f6 100644 --- a/src/Navaids/NavDataCache.cxx +++ b/src/Navaids/NavDataCache.cxx @@ -54,6 +54,7 @@ #include <simgear/debug/logstream.hxx> #include <simgear/bucket/newbucket.hxx> #include <simgear/misc/sg_path.hxx> +#include <simgear/misc/sg_dir.hxx> #include <simgear/misc/strutils.hxx> #include <simgear/threads/SGThread.hxx> #include <simgear/threads/SGGuard.hxx> @@ -332,6 +333,9 @@ public: } bool isCachedFileModified(const SGPath& path, bool verbose); + PathList getDatFilesPaths(NavDataCache::DatFileType datFileType) const; + bool areDatFilesModified(NavDataCache::DatFileType datFileType, + const PathList& datFiles, bool verbose); void callSqlite(int result, const string& sql) { @@ -848,7 +852,8 @@ public: bool transactionAborted; sqlite3_stmt_ptr beginTransactionStmt, commitTransactionStmt, rollbackTransactionStmt; - SGPath aptDatPath, metarDatPath, navDatPath, fixDatPath, poiDatPath, + PathList aptDatPaths; + SGPath metarDatPath, navDatPath, fixDatPath, poiDatPath, carrierDatPath, airwayDatPath; sqlite3_stmt_ptr readPropertyQuery, writePropertyQuery, @@ -978,7 +983,9 @@ FGPositioned* NavDataCache::NavDataCachePrivate::loadById(sqlite3_int64 rowid, bool NavDataCache::NavDataCachePrivate::isCachedFileModified(const SGPath& path, bool verbose) { if (!path.exists()) { - throw sg_io_exception("isCachedFileModified: Missing file:", path); + throw sg_exception( + "NavCache: missing file '" + path.utf8Str() + "'", + string("NavDataCache::NavDataCachePrivate::isCachedFileModified()")); } sqlite_bind_temp_stdstring(statCacheCheck, 1, path.realpath().utf8Str()); @@ -999,15 +1006,166 @@ bool NavDataCache::NavDataCachePrivate::isCachedFileModified(const SGPath& path, isModified = (delta != 0); } else { - SG_LOG(SG_NAVCACHE, logLevel, "NavCache: initial build required for " << path); + SG_LOG(SG_NAVCACHE, logLevel, "NavCache: (re-)build required because '" << + path.utf8Str() << "' is not in the cache"); } reset(statCacheCheck); return isModified; } +// Return the list of $scenery_path/NavData/<type>/*.dat[.gz] files found +// inside scenery paths, plus the default file for the given type (e.g., +// $FG_ROOT/Airports/apt.dat.gz for the 'apt' type). +PathList NavDataCache::NavDataCachePrivate::getDatFilesPaths( + NavDataCache::DatFileType datFileType) const +{ + PathList result; + SGPath visitedPath; // to avoid duplicates and time wasting + const string datFilesSubDir = "NavData/" + + NavDataCache::datTypeStr[datFileType]; + const PathList& sceneryPaths = globals->get_unmangled_fg_scenery(); + + for (PathList::const_iterator dirsIt = sceneryPaths.begin(); + dirsIt != sceneryPaths.end(); dirsIt++) { + if (dirsIt->isNull() || *dirsIt == visitedPath) { + continue; // duplicate or empty path + } else if (! dirsIt->isDir()) { + SG_LOG(SG_NAVCACHE, SG_WARN, *dirsIt << + ": given as a scenery path, but is not a directory"); + visitedPath = *dirsIt; // to avoid duplicate log messages + continue; + } + + visitedPath = *dirsIt; + SGPath datFilesDir = visitedPath; + datFilesDir.append(datFilesSubDir); + + if (datFilesDir.isDir()) { + // Deterministic because the return value of simgear::Dir::children() is + // already sorted + const PathList files = simgear::Dir(datFilesDir).children( + simgear::Dir::TYPE_FILE | simgear::Dir::NO_DOT_OR_DOTDOT); + + for (PathList::const_iterator filesIt = files.begin(); + filesIt != files.end(); filesIt++) { + const std::string name = filesIt->file(); + if (simgear::strutils::ends_with(name, ".dat") || + simgear::strutils::ends_with(name, ".dat.gz")) + result.push_back(*filesIt); + } + } + } // of loop over 'sceneryPaths' + + // Add the default file (e.g., $FG_ROOT/Airports/apt.dat.gz), at least for + // now + SGPath defaultDatFile(globals->get_fg_root()); + defaultDatFile.append(NavDataCache::defaultDatFile[datFileType]); + if ((result.empty() || result.back() != defaultDatFile) && + defaultDatFile.isFile()) { + result.push_back(defaultDatFile); + } + + return result; +} + +// Compare: +// - the list of dat files given by 'datFiles'; +// - the list obtained from the record with key '<type>.dat files' of the +// NavDataCache 'properties' table, where <type> is one of 'apt', 'metar', +// 'fix', 'poi', etc.. +// +// This comparison is sensitive to the number and order of the files, +// their respective SGPath::realpath() and SGPath::modTime(). +bool NavDataCache::NavDataCachePrivate::areDatFilesModified( + NavDataCache::DatFileType datFileType, const PathList& datFiles, bool verbose) +{ + // 'apt' or 'metar' or 'fix' or... + const string datTypeStr = NavDataCache::datTypeStr[datFileType]; + const string_list cachedFiles = outer->readOrderedStringListProperty( + datTypeStr + ".dat files", &SGPath::pathListSep); + PathList::const_iterator datFilesIt = datFiles.begin(); + string_list::const_iterator cachedFilesIt = cachedFiles.begin(); + // Same logic as in NavDataCachePrivate::isCachedFileModified() + sgDebugPriority logLevel = verbose ? SG_WARN : SG_DEBUG; + + if (cachedFilesIt == cachedFiles.end() && datFilesIt != datFiles.end()) { + SG_LOG(SG_NAVCACHE, logLevel, + "NavCache: rebuild required for " << datTypeStr << ".dat files " + "(no file in cache, but " << datFiles.size() << " such file" << + (datFiles.size() > 1 ? "s" : "") << " found in scenery paths)"); + return true; + } + + while (datFilesIt != datFiles.end()) { + const SGPath& path = *(datFilesIt++); + + if (!path.exists()) { + throw sg_exception( + "NavCache: non-existent file '" + path.utf8Str() + "'", + string("NavDataCache::NavDataCachePrivate::areDatFilesModified()")); + } + + if (cachedFilesIt == cachedFiles.end()) { + SG_LOG(SG_NAVCACHE, logLevel, + "NavCache: rebuild required for " << datTypeStr << ".dat files " + "(less files in cache than in scenery paths)"); + return true; + } else { + string cachedFile = *(cachedFilesIt++); + string fileOnDisk = path.realpath().utf8Str(); + + if (cachedFile != fileOnDisk || isCachedFileModified(path, verbose)) { + // isCachedFileModified() does all the logging, so we only have to + // tell what is happening in case that method was not called. + if (cachedFile != fileOnDisk) { + SG_LOG(SG_NAVCACHE, logLevel, + "NavCache: rebuild required because '" << cachedFile << + "' (in cache) != '" << fileOnDisk << "' (on disk)"); + } + return true; + } + } + } // of loop over the elements of 'datFiles' + + if (cachedFilesIt != cachedFiles.end()) { + SG_LOG(SG_NAVCACHE, logLevel, + "NavCache: rebuild required for " << datTypeStr << ".dat files " + "(more files in cache than in scenery paths)"); + return true; + } + + SG_LOG(SG_NAVCACHE, SG_DEBUG, + "NavCache: no rebuild required for " << datTypeStr << ".dat files"); + return false; +} + + +// NavDataCache's static member variables static NavDataCache* static_instance = NULL; +const string_list NavDataCache::datTypeStr = { + string("apt"), + string("metar"), + string("awy"), + string("nav"), + string("fix"), + string("poi"), + string("carrier"), + string("TACAN_freq") +}; + +const string_list NavDataCache::defaultDatFile = { + string("Airports/apt.dat.gz"), + string("Airports/metar.dat.gz"), + string("Navaids/awy.dat.gz"), + string("Navaids/nav.dat.gz"), + string("Navaids/fix.dat.gz"), + string("Navaids/poi.dat.gz"), + string("Navaids/carrier.dat.gz"), + string("Navaids/TACAN_freq.dat.gz") +}; + NavDataCache::NavDataCache() { const int MAX_TRIES = 3; @@ -1050,26 +1208,8 @@ NavDataCache::NavDataCache() Octree::global_spatialOctree = new Octree::Branch(SGBox<double>(-earthExtent, earthExtent), 1); - d->aptDatPath = SGPath(globals->get_fg_root()); - d->aptDatPath.append("Airports/apt.dat.gz"); - - d->metarDatPath = SGPath(globals->get_fg_root()); - d->metarDatPath.append("Airports/metar.dat.gz"); - - d->navDatPath = SGPath(globals->get_fg_root()); - d->navDatPath.append("Navaids/nav.dat.gz"); - - d->fixDatPath = SGPath(globals->get_fg_root()); - d->fixDatPath.append("Navaids/fix.dat.gz"); - - d->poiDatPath = SGPath(globals->get_fg_root()); - d->poiDatPath.append("Navaids/poi.dat.gz"); - - d->carrierDatPath = SGPath(globals->get_fg_root()); - d->carrierDatPath.append("Navaids/carrier_nav.dat.gz"); - - d->airwayDatPath = SGPath(globals->get_fg_root()); - d->airwayDatPath.append("Navaids/awy.dat.gz"); + // Update d->aptDatPaths, d->metarDatPath, d->navDatPath, etc. + updateListsOfDatFiles(); } NavDataCache::~NavDataCache() @@ -1090,6 +1230,38 @@ NavDataCache* NavDataCache::instance() return static_instance; } +// Update the lists of dat files used for NavCache freshness checking and +// rebuilding. +void NavDataCache::updateListsOfDatFiles() { + // All $scenery_path/NavData/apt/*.dat[.gz] files found inside scenery + // paths, plus $FG_ROOT/Airports/apt.dat.gz (order matters). + d->aptDatPaths = d->getDatFilesPaths(DATFILETYPE_APT); + // Trivial extension to the other types of dat files: + // d->navDatPaths = d->getDatFilesPaths(DATFILETYPE_NAV); + // d->fixDatPaths = d->getDatFilesPaths(DATFILETYPE_FIX); + // etc. + // + // These ones are still managed the "old" way (no search through scenery + // paths). + d->metarDatPath = SGPath(globals->get_fg_root()); + d->metarDatPath.append("Airports/metar.dat.gz"); + + d->navDatPath = SGPath(globals->get_fg_root()); + d->navDatPath.append("Navaids/nav.dat.gz"); + + d->fixDatPath = SGPath(globals->get_fg_root()); + d->fixDatPath.append("Navaids/fix.dat.gz"); + + d->poiDatPath = SGPath(globals->get_fg_root()); + d->poiDatPath.append("Navaids/poi.dat.gz"); + + d->carrierDatPath = SGPath(globals->get_fg_root()); + d->carrierDatPath.append("Navaids/carrier_nav.dat.gz"); + + d->airwayDatPath = SGPath(globals->get_fg_root()); + d->airwayDatPath.append("Navaids/awy.dat.gz"); +} + bool NavDataCache::isRebuildRequired() { if (d->readOnly) { @@ -1101,7 +1273,7 @@ bool NavDataCache::isRebuildRequired() return true; } - if (d->isCachedFileModified(d->aptDatPath, true) || + if (d->areDatFilesModified(DATFILETYPE_APT, d->aptDatPaths, true) || d->isCachedFileModified(d->metarDatPath, true) || d->isCachedFileModified(d->navDatPath, true) || d->isCachedFileModified(d->fixDatPath, true) || @@ -1167,14 +1339,26 @@ void NavDataCache::doRebuild() SGTimeStamp st; { Transaction txn(this); + string_list aptDatFiles; st.stamp(); - airportDBLoad(d->aptDatPath); - SG_LOG(SG_NAVCACHE, SG_INFO, "apt.dat load took:" << st.elapsedMSec()); + for (PathList::const_iterator it = d->aptDatPaths.begin(); + it != d->aptDatPaths.end(); it++) { + aptDatFiles.push_back(it->realpath().utf8Str()); + airportDBLoad(*it); + stampCacheFile(*it); // this uses the realpath() of the file + } + + // Store the list of apt.dat files we have loaded + writeOrderedStringListProperty( + datTypeStr[DATFILETYPE_APT] + ".dat files", aptDatFiles, + &SGPath::pathListSep); + SG_LOG(SG_NAVCACHE, SG_INFO, + datTypeStr[DATFILETYPE_APT] + ".dat files load took:" << + st.elapsedMSec()); setRebuildPhaseProgress(REBUILD_UNKNOWN); metarDataLoad(d->metarDatPath); - stampCacheFile(d->aptDatPath); stampCacheFile(d->metarDatPath); st.stamp(); diff --git a/src/Navaids/NavDataCache.hxx b/src/Navaids/NavDataCache.hxx index 3b654eaee..1e24c8262 100644 --- a/src/Navaids/NavDataCache.hxx +++ b/src/Navaids/NavDataCache.hxx @@ -63,6 +63,10 @@ public: SGPath path() const; + // Update d->aptDatPaths, d->metarDatPath, d->navDatPath, d->fixDatPath, + // d->poiDatPath, etc. by looking into $scenery_path/NavData for each + // scenery path. + void updateListsOfDatFiles(); /** * predicate - check if the cache needs to be rebuilt. * This can happen is the cache file is missing or damaged, or one of the @@ -294,6 +298,25 @@ private: void commitTransaction(); void abortTransaction(); + enum DatFileType { + DATFILETYPE_APT = 0, + DATFILETYPE_METAR, + DATFILETYPE_AWY, + DATFILETYPE_NAV, + DATFILETYPE_FIX, + DATFILETYPE_POI, + DATFILETYPE_CARRIER, + DATFILETYPE_TACAN_FREQ + }; + + // datTypeStr[DATFILETYPE_APT] = "apt", etc. This gives, among other things, + // the subdirectory of $scenery_path/NavData where each type of dat file is + // looked for. + static const string_list datTypeStr; + // defaultDatFile[DATFILETYPE_APT] = std::string("Airports/apt.dat.gz"), etc. + // This tells where to find the historical dat files: those under $FG_ROOT. + static const string_list defaultDatFile; + class NavDataCachePrivate; std::auto_ptr<NavDataCachePrivate> d; };