fly/flightgear
1
0
Fork 0
Code Activity

NavCache: check SHA of files as well as modtime

Browse source 
If the modtime check fails on a file, do an SHA checksum to see if the contents
actually changed, and hence avoid unnecessary rebuilds, especially for
AppImage users.
This commit is contained in:
James Turner 2023-04-14 08:34:30 +01:00
parent 2915b6a394
commit ba578558d6
3 changed files with 86 additions and 105 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

68
src/Navaids/CacheSchema.h
View file

@ -1,38 +1,34 @@
#ifndef FG_NAVCACHE_SCHEMA_HXX
#define FG_NAVCACHE_SCHEMA_HXX
#pragma once
const int SCHEMA_VERSION = 20;
#define SCHEMA_SQL \
"CREATE TABLE properties (key VARCHAR, value VARCHAR);" \
"CREATE TABLE stat_cache (path VARCHAR unique, stamp INT);"\
\
"CREATE TABLE positioned (type INT, ident VARCHAR collate nocase," \
"name VARCHAR collate nocase, airport INT64, lon FLOAT, lat FLOAT," \
"elev_m FLOAT, octree_node INT, cart_x FLOAT, cart_y FLOAT, cart_z FLOAT);" \
\
"CREATE INDEX pos_octree ON positioned(octree_node);" \
"CREATE INDEX pos_ident ON positioned(ident collate nocase);" \
"CREATE INDEX pos_name ON positioned(name collate nocase);" \
"CREATE INDEX pos_apt_type ON positioned(airport, type);"\
\
"CREATE TABLE airport (has_metar BOOL);" \
"CREATE TABLE comm (freq_khz INT,range_nm INT);" \
"CREATE INDEX comm_freq ON comm(freq_khz);" \
\
"CREATE TABLE runway (heading FLOAT, length_ft FLOAT, width_m FLOAT," \
"surface INT, displaced_threshold FLOAT,stopway FLOAT,reciprocal INT64,ils INT64);" \
"CREATE TABLE navaid (freq INT,range_nm INT,multiuse FLOAT, runway INT64,colocated INT64);" \
"CREATE INDEX navaid_freq ON navaid(freq);" \
\
"CREATE TABLE octree (children INT);" \
\
"CREATE TABLE airway (ident VARCHAR collate nocase, network INT);" \
"CREATE INDEX airway_ident ON airway(ident);" \
\
"CREATE TABLE airway_edge (network INT,airway INT64,a INT64,b INT64);" \
"CREATE INDEX airway_edge_from ON airway_edge(a);" \
"CREATE INDEX airway_edge_to ON airway_edge(b);"
#endif
const int SCHEMA_VERSION = 21;
#define SCHEMA_SQL \
"CREATE TABLE properties (key VARCHAR, value VARCHAR);" \
"CREATE TABLE stat_cache (path VARCHAR unique, stamp INT, sha VARCHAR);" \
\
"CREATE TABLE positioned (type INT, ident VARCHAR collate nocase," \
"name VARCHAR collate nocase, airport INT64, lon FLOAT, lat FLOAT," \
"elev_m FLOAT, octree_node INT, cart_x FLOAT, cart_y FLOAT, cart_z FLOAT);" \
\
"CREATE INDEX pos_octree ON positioned(octree_node);" \
"CREATE INDEX pos_ident ON positioned(ident collate nocase);" \
"CREATE INDEX pos_name ON positioned(name collate nocase);" \
"CREATE INDEX pos_apt_type ON positioned(airport, type);" \
\
"CREATE TABLE airport (has_metar BOOL);" \
"CREATE TABLE comm (freq_khz INT,range_nm INT);" \
"CREATE INDEX comm_freq ON comm(freq_khz);" \
\
"CREATE TABLE runway (heading FLOAT, length_ft FLOAT, width_m FLOAT," \
"surface INT, displaced_threshold FLOAT,stopway FLOAT,reciprocal INT64,ils INT64);" \
"CREATE TABLE navaid (freq INT,range_nm INT,multiuse FLOAT, runway INT64,colocated INT64);" \
"CREATE INDEX navaid_freq ON navaid(freq);" \
\
"CREATE TABLE octree (children INT);" \
\
"CREATE TABLE airway (ident VARCHAR collate nocase, network INT);" \
"CREATE INDEX airway_ident ON airway(ident);" \
\
"CREATE TABLE airway_edge (network INT,airway INT64,a INT64,b INT64);" \
"CREATE INDEX airway_edge_from ON airway_edge(a);" \
"CREATE INDEX airway_edge_to ON airway_edge(b);"

95
src/Navaids/NavDataCache.cxx
View file

@ -1,23 +1,9 @@
// NavDataCache.cxx - defines a unified binary cache for navigation
// data, parsed from various text / XML sources.
// Written by James Turner, started 2012.
//
// Copyright (C) 2012 James Turner
//
// This program is free software; you can redistribute it and/or
// modify it under the terms of the GNU General Public License as
// published by the Free Software Foundation; either version 2 of the
// License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful, but
// WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
// General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program; if not, write to the Free Software
// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
/*
* SPDX-FileCopyrightText: (C) 2012 James Turner <james@flightgear.org>
* SPDX_FileComment: Defins a unified binary cache for navigation data, parsed from text/XMl sources
* SPDX-License-Identifier: GPL-2.0-or-later
*/
#include "config.h"
@ -55,13 +41,14 @@
#endif
// SimGear
#include <simgear/bucket/newbucket.hxx>
#include <simgear/debug/logstream.hxx>
#include <simgear/io/sg_file.hxx>
#include <simgear/misc/sg_dir.hxx>
#include <simgear/misc/sg_path.hxx>
#include <simgear/misc/strutils.hxx>
#include <simgear/sg_inlines.h>
#include <simgear/structure/exception.hxx>
#include <simgear/debug/logstream.hxx>
#include <simgear/bucket/newbucket.hxx>
#include <simgear/misc/sg_path.hxx>
#include <simgear/misc/sg_dir.hxx>
#include <simgear/misc/strutils.hxx>
#include <simgear/threads/SGThread.hxx>
#include "CacheSchema.h"
@ -571,9 +558,9 @@ public:
#define POSITIONED_COLS "rowid, type, ident, name, airport, lon, lat, elev_m, octree_node"
#define AND_TYPED "AND type>=?2 AND type <=?3"
statCacheCheck = prepare("SELECT stamp FROM stat_cache WHERE path=?");
statCacheCheck = prepare("SELECT stamp, sha FROM stat_cache WHERE path=?");
stampFileCache = prepare("INSERT OR REPLACE INTO stat_cache "
"(path, stamp) VALUES (?,?)");
"(path, stamp, sha) VALUES (?,?, ?)");
loadPositioned = prepare("SELECT " POSITIONED_COLS " FROM positioned WHERE rowid=?");
loadAirportStmt = prepare("SELECT has_metar FROM airport WHERE rowid=?");
@ -1091,28 +1078,35 @@ bool NavDataCache::NavDataCachePrivate::isCachedFileModified(const SGPath& path,
}
sqlite_bind_temp_stdstring(statCacheCheck, 1, path.realpath().utf8Str());
bool isModified = true;
sgDebugPriority logLevel = verbose ? SG_WARN : SG_DEBUG;
if (execSelect(statCacheCheck)) {
time_t modtime = sqlite3_column_int64(statCacheCheck, 0);
time_t delta = std::labs(modtime - path.modTime());
if (delta != 0)
{
SG_LOG(SG_NAVCACHE, logLevel, "NavCache: rebuild required for " << path <<
". Timestamps: " << modtime << " != " << path.modTime());
}
else
{
SG_LOG(SG_NAVCACHE, SG_DEBUG, "NavCache: no rebuild required for " << path);
}
isModified = (delta != 0);
} else {
if (!execSelect(statCacheCheck)) {
SG_LOG(SG_NAVCACHE, logLevel, "NavCache: (re-)build required because '" <<
path.utf8Str() << "' is not in the cache");
reset(statCacheCheck);
return true;
}
time_t modtime = sqlite3_column_int64(statCacheCheck, 0);
time_t delta = std::labs(modtime - path.modTime());
if (delta == 0) {
SG_LOG(SG_NAVCACHE, SG_DEBUG, "NavCache: modtime matches, no rebuild required for " << path);
reset(statCacheCheck);
return false;
}
const std::string shaHash{(char*)sqlite3_column_text(statCacheCheck, 1)};
SGFile f(path);
const auto fileHash = f.computeHash();
const auto isModified = (fileHash != shaHash);
reset(statCacheCheck);
if (!isModified) {
// the mode time check failed, but the hashes matched. Let's update our modtime so we
// don't compute the hash until the mod-time changes again.
SG_LOG(SG_NAVCACHE, logLevel, "NavCache: " << path << " has changed modtime but contents are unchanged, re-setting cahced mod-time");
outer->stampCacheFile(path, fileHash);
}
return isModified;
}
@ -1892,13 +1886,20 @@ bool NavDataCache::isCachedFileModified(const SGPath& path) const
return d->isCachedFileModified(path, false);
}
void NavDataCache::stampCacheFile(const SGPath& path)
void NavDataCache::stampCacheFile(const SGPath& path, const std::string& sha)
{
if (!isReadOnly()){
sqlite_bind_temp_stdstring(d->stampFileCache, 1, path.realpath().utf8Str());
sqlite3_bind_int64(d->stampFileCache, 2, path.modTime());
d->execInsert(d->stampFileCache);
if (!isReadOnly()){
sqlite_bind_temp_stdstring(d->stampFileCache, 1, path.realpath().utf8Str());
sqlite3_bind_int64(d->stampFileCache, 2, path.modTime());
if (sha.empty()) {
SGFile f(path);
sqlite_bind_temp_stdstring(d->stampFileCache, 3, f.computeHash());
} else {
sqlite_bind_temp_stdstring(d->stampFileCache, 3, sha);
}
d->execInsert(d->stampFileCache);
}
}
void NavDataCache::beginTransaction()

28
src/Navaids/NavDataCache.hxx
View file

@ -1,25 +1,9 @@
/**
* NavDataCache.hxx - defines a unified binary cache for navigation
* data, parsed from various text / XML sources.
*/
// Written by James Turner, started 2012.
//
// Copyright (C) 2012 James Turner
//
// This program is free software; you can redistribute it and/or
// modify it under the terms of the GNU General Public License as
// published by the Free Software Foundation; either version 2 of the
// License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful, but
// WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
// General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program; if not, write to the Free Software
// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
/*
* SPDX-FileCopyrightText: (C) 2012 James Turner <james@flightgear.org>
* SPDX_FileComment: Defins a unified binary cache for navigation data, parsed from text/XMl sources
* SPDX-License-Identifier: GPL-2.0-or-later
*/
#ifndef FG_NAVDATACACHE_HXX
#define FG_NAVDATACACHE_HXX
@ -133,7 +117,7 @@ public:
void setRebuildPhaseProgress(RebuildPhase ph, unsigned int percent = 0);
bool isCachedFileModified(const SGPath& path) const;
void stampCacheFile(const SGPath& path);
void stampCacheFile(const SGPath& path, const std::string& sha = {});
int readIntProperty(const std::string& key);
double readDoubleProperty(const std::string& key);