From 7d414886e00ebcb32038c775dc362580f32d79b9 Mon Sep 17 00:00:00 2001 From: Julian Smith Date: Thu, 10 Jun 2021 18:16:47 +0100 Subject: [PATCH] Added support for compressed continuous recordings. src/Aircraft/replay.*: If /sim/replay/record-continuous-compression is true, we compress each frame's data as a separate raw zlib stream. Requires latest simgear's simgear/io/iostreams/zlibstream.cxx for decompression with ZLibCompressionFormat::ZLIB_RAW. Haven't figured out how to extend simgear's code to provide a compressing ostream so for now we have our own local compression code. We open popup and set sim/replay/replay-error=true if we fail to read compressed data. scripts/python/recordreplay.py: Added test of compressed continuous recordings. docs-mini/README-recordings.md: Added information about compressed format. --- docs-mini/README-recordings.md | 19 +- scripts/python/recordreplay.py | 42 ++- src/Aircraft/replay.cxx | 462 ++++++++++++++++++++++++++------- src/Aircraft/replay.hxx | 3 + 4 files changed, 410 insertions(+), 116 deletions(-) diff --git a/docs-mini/README-recordings.md b/docs-mini/README-recordings.md index 49df002c2..3d1eb8981 100644 --- a/docs-mini/README-recordings.md +++ b/docs-mini/README-recordings.md @@ -14,7 +14,7 @@ As of 2020-12-22, there are three kinds of recordings: Normal recordings are compressed and contain frames at varying intervals with more recent frames being closer together in time. They are generated from the in-memory recording that Flightgear always maintains. They may contain multiplayer information. -Continuous recordings are uncompressed and written directly to a file while Flightgear runs, giving high resolution with near unlimited recording time. They may contain multiplayer information. As of 2020-12-23 they may contain information about extra properties, allowing replay of views and main window position/size. +Continuous recordings are written directly to a file while Flightgear runs, giving high resolution with near unlimited recording time. They may contain multiplayer information. As of 2020-12-23 they may contain information about extra properties, allowing replay of views and main window position/size. As of 2021-06-26 each frame's data can be compressed. Recovery recordings are essentially single-frame Continuous recordings. When enabled, Flightgear creates them periodically to allow recovery of a session if Flightgear crashes. @@ -32,6 +32,7 @@ Recovery recordings are essentially single-frame Continuous recordings. When ena * `/sim/replay/record-continuous` - if true, do continuous record to file. * `/sim/replay/record-signals` - if true (the default), include signals for user aircraft - these are the core values used to replay the user aircraft. * `/sim/replay/record-extra-properties` - if true, we include selected properties in recordings. + * `/sim/replay/record-continuous-compression` - if 1, we compress each frame's data. * Recovery recordings: * `/sim/replay/record-recovery-period` - if non-zero, we update recovery recording in specified interval. @@ -74,15 +75,13 @@ Normal recordings are written as a compressed gzip stream using `simgear::gzCont ### Continuous recordings -Continuous recordings do not use compression, in order to simplify random access when replaying. - * Header: * A zero-terminated magic string: `FlightGear Flight Recorder Tape` (variable `FlightRecorderFileMagic`). * A property tree represented as a `uint32` length followed by zero-terminated text. This contains: - * A `meta` node with various child nodes. + * A `meta` node with various child nodes. If this contains `continuous-compression` with value `1`, then each frame's data is compressed.. * `data[]` nodes describing the data items in each frame in the order in which they occur. Supported values are: @@ -98,7 +97,15 @@ Continuous recordings do not use compression, in order to simplify random access * Frame time as a binary double. - * A list of ordered `` items as described by the `data[]` nodes in the header. This format allows Flightgear to skip data items that it doesn't understand if loading a recording that was created by a newer version. + * If compression is used: + + * uint8_t flags. + * Bit 0: this frame has signals. + * Bit 1: this frame has multiplayer information. + * Bit 2: this frame has extrap properties. + * uint32_t compressed-size. + + * Frame data (can be compressed): a list of ordered `` items as described by the `data[]` nodes in the header. This format allows Flightgear to skip data items that it doesn't understand if loading a recording that was created by a newer version. * For `signals`, `` is binary data for the core aircraft properties. @@ -115,6 +122,8 @@ Continuous recordings do not use compression, in order to simplify random access When a Continuous recording is loaded, `FGReplay::loadTape()` first steps through the entire file, building up an index in memory that maps from frame times to a struct containing the offset of the frame in the file plus information on whether the frame has multiplayer and/or extra-properties information. This allows us to support the user jumping forwards and backwards in the recording. +If the recording uses compression, indexing uses the uint8_t flags and uint32_t compressed-size fields and does not need to decompress each frame's data. + If we are replaying from a URL, indexing takes place in the background (by requesting callbacks from the download's `simgear::HTTP::FileRequest`) and replay starts immediately. Thus we avoid having to wait until the entire recording has been downloaded before starting replay. diff --git a/scripts/python/recordreplay.py b/scripts/python/recordreplay.py index 01ec63b2d..4be8f4536 100755 --- a/scripts/python/recordreplay.py +++ b/scripts/python/recordreplay.py @@ -230,7 +230,7 @@ class Fg: def make_recording( fg, - continuous=0, + continuous=0, # 2 means continuous with compression. extra_properties=0, main_view=0, length=5, @@ -244,13 +244,15 @@ def make_recording( fg.fg['/sim/replay/record-signals'] = True # Just in case they are disabled by user. if continuous: assert not fg.fg['/sim/replay/record-continuous'] + if continuous == 2: + fg.fg['/sim/replay/record-continuous-compression'] = 1 fg.fg['/sim/replay/record-continuous'] = 1 t0 = time.time() while 1: if time.time() > t0 + length: break - fg.run_command('run view-step step=1') time.sleep(1) + fg.run_command('run view-step step=1') fg.fg['/sim/replay/record-continuous'] = 0 path = f'{g_tapedir}/{fg.aircraft}-continuous.fgtape' time.sleep(1) @@ -332,26 +334,44 @@ def test_record_replay( fg.waitfor('/sim/fdm-initialized', 1, timeout=45) fg.waitfor('/sim/replay/replay-state', 1) + t0 = time.time() + # Check replay time is ok. rtime_begin = fg.fg['/sim/replay/start-time'] rtime_end = fg.fg['/sim/replay/end-time'] rtime = rtime_end - rtime_begin log(f'rtime={rtime_begin}..{rtime_end}, recording length: {rtime}, length={length}') - assert rtime > length-1 and rtime < length+2, \ + assert rtime > length-1 and rtime <= length+2, \ f'length={length} rtime_begin={rtime_begin} rtime_end={rtime_end} rtime={rtime}' num_frames_extra_properties = fg.fg['/sim/replay/continuous-stats-num-frames-extra-properties'] log(f'num_frames_extra_properties={num_frames_extra_properties}') if continuous: if main_view: - assert num_frames_extra_properties > 1 + assert num_frames_extra_properties > 1, f'num_frames_extra_properties={num_frames_extra_properties}' else: assert num_frames_extra_properties == 0 else: assert num_frames_extra_properties in (0, None), \ f'num_frames_extra_properties={num_frames_extra_properties}' - time.sleep(length) + fg.run_command('run dialog-show dialog-name=replay') + + while 1: + t = time.time() + if t < t0 + length - 1: + pass + # Disabled because it seems that Flightgear starts replaying before + # we see replay-state set to 1 because scenery loading blocks + # things. + # + #assert not fg.fg['/sim/replay/replay-state-eof'], f'Replay has finished too early; lenth={length} t-t0={t-t0}' + if t > t0 + length + 1: + assert fg.fg['/sim/replay/replay-state-eof'], f'Replay has not finished on time; lenth={length} t-t0={t-t0}' + break + e = fg.fg['sim/replay/replay-error'] + assert not e, f'Replay failed: e={e}' + time.sleep(1) fg.close() @@ -676,7 +696,7 @@ if __name__ == '__main__': fgfs_old = None do_test = 'all' - continuous_s = [0, 1] + continuous_s = [0, 1, 2] # 2 is continuous with compression. extra_properties_s = [0, 1] main_view_s = [0, 1] multiplayer_s = [0, 1] @@ -698,19 +718,20 @@ if __name__ == '__main__': elif arg == '--carrier': do_test = 'carrier' elif arg == '--continuous': - continuous_s = map(int, next(args).split(',')) + continuous_s = [int(x) for x in next(args).split(',')] + log(f'continuous_s={continuous_s}') elif arg == '--tape-dir': g_tapedir = next(args) elif arg == '--extra-properties': - extra_properties_s = map(int, next(args).split(',')) + extra_properties_s = [int(x) for x in next(args).split(',')] elif arg == '--it-max': it_max = int(next(args)) elif arg == '--it-min': it_min = int(next(args)) elif arg == '--main-view': - main_view_s = map(int, next(args).split(',')) + main_view_s = [int(x) for x in next(args).split(',')] elif arg == '--multiplayer': - multiplayer_s = map(int, next(args).split(',')) + multiplayer_s = [int(x) for x in next(args).split(',')] elif arg == '-f': fgfs = next(args) elif arg == '--f-old': @@ -748,6 +769,7 @@ if __name__ == '__main__': length=10, ) else: + log(f'continuous_s={continuous_s}') its_max = len(multiplayer_s) * len(continuous_s) * len(extra_properties_s) * len(main_view_s) * len(fgfs_reverse_s) it = 0 for multiplayer in multiplayer_s: diff --git a/src/Aircraft/replay.cxx b/src/Aircraft/replay.cxx index bfa306cf8..a1abdd3fa 100644 --- a/src/Aircraft/replay.cxx +++ b/src/Aircraft/replay.cxx @@ -29,12 +29,20 @@ #include #include +#include +#include +#include +#include + +#include + #include #include #include #include #include +#include #include #include #include @@ -151,6 +159,7 @@ FGReplay::FGReplay() : last_lt_time(0.0), last_msg_time(0), last_replay_state(0), + replay_error(fgGetNode("sim/replay/replay-error", true)), m_sim_startup_xpos(fgGetNode("sim/startup/xpos", true)), m_sim_startup_ypos(fgGetNode("sim/startup/ypos", true)), m_sim_startup_xsize(fgGetNode("sim/startup/xsize", true)), @@ -299,7 +308,12 @@ void FGReplay::valueChanged(SGPropertyNode * node) SG_LOG(SG_SYSTEMS, SG_ALERT, "Failed to create link " << path_timeless.c_str() << " => " << path.file()); } SG_LOG(SG_SYSTEMS, SG_DEBUG, "Starting continuous recording to " << path); - popupTip("Continuous record to file started", 5 /*delay*/); + if (m_continuous_out_compression) { + popupTip("Continuous+compressed record to file started", 5 /*delay*/); + } + else { + popupTip("Continuous record to file started", 5 /*delay*/); + } } } @@ -683,6 +697,133 @@ saveRawReplayData(gzContainerWriter& output, const replay_list_type& ReplayData, } +/* streambuf that compresses using deflate(). */ +struct compression_streambuf : std::streambuf +{ + compression_streambuf( + std::ostream& out, + size_t buffer_uncompressed_size, + size_t buffer_compressed_size + ) + : + std::streambuf(), + out(out), + buffer_uncompressed(new char[buffer_uncompressed_size]), + buffer_uncompressed_size(buffer_uncompressed_size), + buffer_compressed(new char[buffer_compressed_size]), + buffer_compressed_size(buffer_compressed_size) + { + zstream.zalloc = nullptr; + zstream.zfree = nullptr; + zstream.opaque = nullptr; + + zstream.next_in = nullptr; + zstream.avail_in = 0; + + zstream.next_out = (unsigned char*) &buffer_compressed[0]; + zstream.avail_out = buffer_compressed_size; + + int e = deflateInit2( + &zstream, + Z_DEFAULT_COMPRESSION, + Z_DEFLATED, + -15 /*windowBits*/, + 8 /*memLevel*/, + Z_DEFAULT_STRATEGY + ); + if (e != Z_OK) + { + throw std::runtime_error("deflateInit2() failed"); + } + // We leave space for one character to simplify overflow(). + setp(&buffer_uncompressed[0], &buffer_uncompressed[0] + buffer_uncompressed_size - 1); + } + + // Flush compressed data to .out and reset zstream.next_out. + void _flush() + { + // Send all data in .buffer_compressed to .out. + size_t n = (char*) zstream.next_out - &buffer_compressed[0]; + out.write(&buffer_compressed[0], n); + zstream.next_out = (unsigned char*) &buffer_compressed[0]; + zstream.avail_out = buffer_compressed_size; + } + + /* Compresses specified bytes from buffer_uncompressed into + buffer_compressed, flushing to .out as necessary. Returns true if we get + EOF writing to .out. */ + bool _deflate(size_t n, bool flush) + { + assert(this->pbase() == &buffer_uncompressed[0]); + zstream.next_in = (unsigned char*) &buffer_uncompressed[0]; + zstream.avail_in = n; + for(;;) + { + if (!flush && !zstream.avail_in) break; + if (!zstream.avail_out) _flush(); + int e = deflate(&zstream, (!zstream.avail_in && flush) ? Z_FINISH : Z_NO_FLUSH); + if (e != Z_OK && e != Z_STREAM_END) + { + throw std::runtime_error("zip_deflate() failed"); + } + if (e == Z_STREAM_END) break; + } + if (flush) _flush(); + // We leave space for one character to simplify overflow(). + setp(&buffer_uncompressed[0], &buffer_uncompressed[0] + buffer_uncompressed_size - 1); + if (!out) return true; // EOF. + return false; + } + + int overflow(int c) override + { + // We've deliberately left space for one character, into which we write . + assert(this->pptr() == &buffer_uncompressed[0] + buffer_uncompressed_size - 1); + *this->pptr() = (char) c; + if (_deflate(buffer_uncompressed_size, false /*flush*/)) return EOF; + return c; + } + + int sync() override + { + _deflate(pptr() - &buffer_uncompressed[0], true /*flush*/); + return 0; + } + + ~compression_streambuf() + { + deflateEnd(&zstream); + } + + std::ostream& out; + z_stream zstream; + std::unique_ptr buffer_uncompressed; + size_t buffer_uncompressed_size; + std::unique_ptr buffer_compressed; + size_t buffer_compressed_size; +}; + + +/* Accepts uncompressed data via .write(), operator<< etc, and writes +compressed data to the supplied std::ostream. */ +struct compression_ostream : std::ostream +{ + compression_ostream( + std::ostream& out, + size_t buffer_uncompressed_size, + size_t buffer_compressed_size + ) + : + std::ostream(&streambuf), + streambuf(out, buffer_uncompressed_size, buffer_compressed_size) + { + } + + compression_streambuf streambuf; +}; + + + // Sets things up for writing to a normal or continuous fgtape file. // // extra: @@ -702,7 +843,8 @@ static SGPropertyNode_ptr saveSetup( const SGPropertyNode* extra, const SGPath& path, double duration, - FGTapeType tape_type + FGTapeType tape_type, + int continuous_compression=0 ) { SGPropertyNode_ptr config; @@ -722,7 +864,9 @@ static SGPropertyNode_ptr saveSetup( meta->setStringValue("aircraft-fdm", fgGetString("/sim/flight-model", "")); meta->setStringValue("closest-airport-id", fgGetString("/sim/airport/closest-airport-id", "")); meta->setStringValue("aircraft-version", fgGetString("/sim/aircraft-version", "(undefined)")); - + if (tape_type == FGTapeType_CONTINUOUS) { + meta->setIntValue("continuous-compression", continuous_compression); + } // add information on the tape's recording duration meta->setDoubleValue("tape-duration", duration); char StrBuffer[30]; @@ -783,7 +927,9 @@ SGPropertyNode_ptr FGReplay::continuousWriteHeader( FGTapeType tape_type ) { - SGPropertyNode_ptr config = saveSetup(NULL /*Extra*/, path, 0 /*Duration*/, tape_type); + m_continuous_out_compression = fgGetInt("/sim/replay/record-continuous-compression"); + SGPropertyNode_ptr config = saveSetup(NULL /*Extra*/, path, 0 /*Duration*/, + tape_type, m_continuous_out_compression); SGPropertyNode* signals = config->getNode("signals", true /*create*/); m_pRecorder->getConfig(signals); @@ -804,48 +950,8 @@ SGPropertyNode_ptr FGReplay::continuousWriteHeader( return config; } - -// Writes one frame of continuous record information. -// -bool -FGReplay::continuousWriteFrame(FGReplayData* r, std::ostream& out, SGPropertyNode_ptr config) +static void writeFrame2(FGReplayData* r, std::ostream& out, SGPropertyNode_ptr config) { - SG_LOG(SG_SYSTEMS, SG_BULK, "writing frame." - << " out.tellp()=" << out.tellp() - << " r->sim_time=" << r->sim_time - ); - // Don't write frame if no data to write. - bool r_has_data = false; - for (auto data: config->getChildren("data")) { - const char* data_type = data->getStringValue(); - if (!strcmp(data_type, "signals")) { - r_has_data = true; - break; - } - else if (!strcmp(data_type, "multiplayer")) { - if (!r->multiplayer_messages.empty()) { - r_has_data = true; - break; - } - } - else if (!strcmp(data_type, "extra-properties")) { - if (!r->extra_properties.empty()) { - r_has_data = true; - break; - } - } - else { - SG_LOG(SG_SYSTEMS, SG_ALERT, "unrecognised data_type=" << data_type); - assert(0); - } - } - if (!r_has_data) { - SG_LOG(SG_SYSTEMS, SG_DEBUG, "Not writing frame because no data to write"); - return true; - } - - out.write(reinterpret_cast(&r->sim_time), sizeof(r->sim_time)); - for (auto data: config->getChildren("data")) { const char* data_type = data->getStringValue(); if (!strcmp(data_type, "signals")) { @@ -880,6 +986,71 @@ FGReplay::continuousWriteFrame(FGReplayData* r, std::ostream& out, SGPropertyNod } } +} + +// Writes one frame of continuous record information. +// +bool +FGReplay::continuousWriteFrame(FGReplayData* r, std::ostream& out, SGPropertyNode_ptr config) +{ + SG_LOG(SG_SYSTEMS, SG_BULK, "writing frame." + << " out.tellp()=" << out.tellp() + << " r->sim_time=" << r->sim_time + ); + // Don't write frame if no data to write. + //bool r_has_data = false; + bool has_signals = false; + bool has_multiplayer = false; + bool has_extra_properties = false; + for (auto data: config->getChildren("data")) { + const char* data_type = data->getStringValue(); + if (!strcmp(data_type, "signals")) { + has_signals = true; + } + else if (!strcmp(data_type, "multiplayer")) { + if (!r->multiplayer_messages.empty()) { + has_multiplayer = true; + } + } + else if (!strcmp(data_type, "extra-properties")) { + if (!r->extra_properties.empty()) { + has_extra_properties = true; + } + } + else { + SG_LOG(SG_SYSTEMS, SG_ALERT, "unrecognised data_type=" << data_type); + assert(0); + } + } + if (!has_signals && !has_multiplayer && !has_extra_properties) { + SG_LOG(SG_SYSTEMS, SG_DEBUG, "Not writing frame because no data to write"); + return true; + } + + out.write(reinterpret_cast(&r->sim_time), sizeof(r->sim_time)); + + if (m_continuous_out_compression) { + uint8_t flags = 0; + if (has_signals) flags |= 1; + if (has_multiplayer) flags |= 2; + if (has_extra_properties) flags |= 4; + out.write((char*) &flags, sizeof(flags)); + + /* We need to first write the size of the compressed data so compress + to a temporary ostringstream first. */ + std::ostringstream compressed; + compression_ostream out_compressing(compressed, 1024, 1024); + writeFrame2(r, out_compressing, config); + out_compressing.flush(); + + uint32_t compressed_size = compressed.str().size(); + out.write((char*) &compressed_size, sizeof(compressed_size)); + out.write((char*) compressed.str().c_str(), compressed.str().size()); + } + else + { + writeFrame2(r, out, config); + } bool ok = true; if (!out) ok = false; return ok; @@ -1553,14 +1724,14 @@ FGReplay::replay(double time, FGReplayData* pCurrentFrame, FGReplayData* pOldFra m_pRecorder->replay(time, pCurrentFrame, pOldFrame, xpos, ypos, xsize, ysize); } -static int16_t read_int16(std::ifstream& in, size_t& pos) +static int16_t read_int16(std::istream& in, size_t& pos) { int16_t a; in.read(reinterpret_cast(&a), sizeof(a)); pos += sizeof(a); return a; } -static std::string read_string(std::ifstream& in, size_t& pos) +static std::string read_string(std::istream& in, size_t& pos) { int16_t length = read_int16(in, pos); std::vector path(length); @@ -1572,7 +1743,7 @@ static std::string read_string(std::ifstream& in, size_t& pos) /* Reads extra-property change items in next bytes. Throws if we don't exactly read bytes. */ -static void ReadFGReplayDataExtraProperties(std::ifstream& in, FGReplayData* replay_data, uint32_t length) +static void ReadFGReplayDataExtraProperties(std::istream& in, FGReplayData* replay_data, uint32_t length) { SG_LOG(SG_SYSTEMS, SG_BULK, "reading extra-properties. length=" << length); size_t pos=0; @@ -1600,34 +1771,23 @@ static void ReadFGReplayDataExtraProperties(std::ifstream& in, FGReplayData* rep } } -/* Reads all or part of a FGReplayData from uncompressed file. */ -static std::unique_ptr ReadFGReplayData( - std::ifstream& in, - size_t pos, +static bool ReadFGReplayData2( + std::istream& in, SGPropertyNode* config, bool load_signals, bool load_multiplayer, - bool load_extra_properties + bool load_extra_properties, + FGReplayData* ret ) { - /* Need to clear any eof bit, otherwise seekg() will not work (which is - pretty unhelpful). E.g. see: - https://stackoverflow.com/questions/16364301/whats-wrong-with-the-ifstream-seekg - */ - SG_LOG(SG_SYSTEMS, SG_BULK, "reading frame. pos=" << pos); - in.clear(); - in.seekg(pos); - - std::unique_ptr ret(new FGReplayData); - - in.read(reinterpret_cast(&ret->sim_time), sizeof(ret->sim_time)); ret->raw_data.resize(0); for (auto data: config->getChildren("data")) { const char* data_type = data->getStringValue(); - SG_LOG(SG_SYSTEMS, SG_DEBUG, "in.tellg()=" << in.tellg() << " data_type=" << data_type); + SG_LOG(SG_SYSTEMS, SG_BULK, "in.tellg()=" << in.tellg() << " data_type=" << data_type); uint32_t length; in.read(reinterpret_cast(&length), sizeof(length)); SG_LOG(SG_SYSTEMS, SG_DEBUG, "length=" << length); + if (!in) break; if (load_signals && !strcmp(data_type, "signals")) { ret->raw_data.resize(length); in.read(&ret->raw_data.front(), ret->raw_data.size()); @@ -1653,14 +1813,64 @@ static std::unique_ptr ReadFGReplayData( } } else if (load_extra_properties && !strcmp(data_type, "extra-properties")) { - ReadFGReplayDataExtraProperties(in, ret.get(), length); + ReadFGReplayDataExtraProperties(in, ret, length); } else { - SG_LOG(SG_GENERAL, SG_BULK, "Skipping unrecognised data: " << data_type); + SG_LOG(SG_GENERAL, SG_BULK, "Skipping unrecognised/unwanted data: " << data_type); in.seekg(length, std::ios_base::cur); } } + if (!in) { + SG_LOG(SG_SYSTEMS, SG_DEBUG, "Failed to read fgtape data"); + return false; + } + return true; +} + +/* Reads all or part of a FGReplayData from uncompressed file. */ +static std::unique_ptr ReadFGReplayData( + std::ifstream& in, + size_t pos, + SGPropertyNode* config, + bool load_signals, + bool load_multiplayer, + bool load_extra_properties, + int m_continuous_in_compression + ) +{ + /* Need to clear any eof bit, otherwise seekg() will not work (which is + pretty unhelpful). E.g. see: + https://stackoverflow.com/questions/16364301/whats-wrong-with-the-ifstream-seekg + */ + SG_LOG(SG_SYSTEMS, SG_BULK, "reading frame. pos=" << pos); + in.clear(); + in.seekg(pos); + std::unique_ptr ret(new FGReplayData); + + in.read(reinterpret_cast(&ret->sim_time), sizeof(ret->sim_time)); + if (!in) { + SG_LOG(SG_SYSTEMS, SG_DEBUG, "Failed to read fgtape frame at offset " << pos); + return nullptr; + } + bool ok; + if (m_continuous_in_compression) + { + uint8_t flags; + uint32_t compressed_size; + in.read((char*) &flags, sizeof(flags)); + in.read((char*) &compressed_size, sizeof(compressed_size)); + simgear::ZlibDecompressorIStream in_decompress(in, SGPath(), simgear::ZLibCompressionFormat::ZLIB_RAW); + ok = ReadFGReplayData2(in_decompress, config, load_signals, load_multiplayer, load_extra_properties, ret.get()); + } + else + { + ok = ReadFGReplayData2(in, config, load_signals, load_multiplayer, load_extra_properties, ret.get()); + } + if (!ok) { + SG_LOG(SG_SYSTEMS, SG_DEBUG, "Failed to read fgtape frame at offset " << pos); + return nullptr; + } return ret; } @@ -1691,8 +1901,17 @@ void FGReplay::replay( m_continuous_in_config, replay_signals, replay_multiplayer, - replay_extra_properties + replay_extra_properties, + m_continuous_in_compression ); + if (!replay_data) { + if (!replay_error->getBoolValue()) { + SG_LOG(SG_SYSTEMS, SG_ALERT, "Failed to read fgtape frame at offset=" << offset << " time=" << time); + popupTip("Replay failed: cannot read fgtape data", 10); + replay_error->setBoolValue(true); + } + return; + } assert(replay_data.get()); std::unique_ptr replay_data_old; if (offset_old) { @@ -1702,7 +1921,8 @@ void FGReplay::replay( m_continuous_in_config, replay_signals, replay_multiplayer, - replay_extra_properties + replay_extra_properties, + m_continuous_in_compression ); } if (replay_extra_properties) SG_LOG(SG_SYSTEMS, SG_BULK, @@ -1964,7 +2184,7 @@ int FGReplay::loadContinuousHeader(const std::string& path, std::istream* in, SG ok = true; } catch (std::exception& e) { - SG_LOG(SG_SYSTEMS, SG_DEBUG, "Failed to read Config properties in: " << path); + SG_LOG(SG_SYSTEMS, SG_DEBUG, "Failed to read Config properties in: " << path << ": " << e.what()); } if (!ok) { // Failed to read properties, so indicate that further download is needed. @@ -1981,6 +2201,7 @@ void FGReplay::indexContinuousRecording(const void* data, size_t numbytes) << " data=" << data << " numbytes=" << numbytes << " m_continuous_indexing_pos=" << m_continuous_indexing_pos + << " m_continuous_in_compression=" << m_continuous_in_compression << " m_continuous_in_time_to_frameinfo.size()=" << m_continuous_in_time_to_frameinfo.size() ); time_t t0 = time(NULL); @@ -2011,40 +2232,76 @@ void FGReplay::indexContinuousRecording(const void* data, size_t numbytes) << " m_continuous_indexing_in.tellg()=" << m_continuous_indexing_in.tellg() << " sim_time=" << sim_time ); + FGFrameInfo frameinfo; frameinfo.offset = m_continuous_indexing_pos; - - auto datas = m_continuous_in_config->getChildren("data"); - SG_LOG(SG_SYSTEMS, SG_BULK, "datas.size()=" << datas.size()); - for (auto data: datas) { - uint32_t length; - m_continuous_indexing_in.read(reinterpret_cast(&length), sizeof(length)); - SG_LOG(SG_SYSTEMS, SG_BULK, - "m_continuous_in.tellg()=" << m_continuous_indexing_in.tellg() - << " Skipping data_type=" << data->getStringValue() - << " length=" << length - ); - // Move forward bytes. - m_continuous_indexing_in.seekg(length, std::ios_base::cur); - if (!m_continuous_indexing_in) { - // Dont add bogus info to . - break; + if (m_continuous_in_compression) + { + // Skip compressed frame data without decompressing it. + uint8_t flags; + m_continuous_indexing_in.read((char*) &flags, sizeof(flags)); + frameinfo.has_signals = flags & 1; + frameinfo.has_multiplayer = flags & 2; + frameinfo.has_extra_properties = flags & 4; + + if (frameinfo.has_signals) + { + stats["signals"].num_frames += 1; } - if (length) { - stats[data->getStringValue()].num_frames += 1; - stats[data->getStringValue()].bytes += length; - if (!strcmp(data->getStringValue(), "signals")) { - frameinfo.has_signals = true; + if (frameinfo.has_multiplayer) + { + stats["multiplayer"].num_frames += 1; + ++m_num_frames_multiplayer; + m_continuous_in_multiplayer = true; + } + if (frameinfo.has_extra_properties) + { + stats["extra-properties"].num_frames += 1; + ++m_num_frames_extra_properties; + m_continuous_in_extra_properties = true; + } + + uint32_t compressed_size; + m_continuous_indexing_in.read((char*) &compressed_size, sizeof(compressed_size)); + SG_LOG(SG_SYSTEMS, SG_BULK, "compressed_size=" << compressed_size); + + m_continuous_indexing_in.seekg(compressed_size, std::ios_base::cur); + } + else + { + // Skip frame data. + auto datas = m_continuous_in_config->getChildren("data"); + SG_LOG(SG_SYSTEMS, SG_BULK, "datas.size()=" << datas.size()); + for (auto data: datas) { + uint32_t length; + m_continuous_indexing_in.read(reinterpret_cast(&length), sizeof(length)); + SG_LOG(SG_SYSTEMS, SG_BULK, + "m_continuous_in.tellg()=" << m_continuous_indexing_in.tellg() + << " Skipping data_type=" << data->getStringValue() + << " length=" << length + ); + // Move forward bytes. + m_continuous_indexing_in.seekg(length, std::ios_base::cur); + if (!m_continuous_indexing_in) { + // Dont add bogus info to . + break; } - else if (!strcmp(data->getStringValue(), "multiplayer")) { - frameinfo.has_multiplayer = true; - ++m_num_frames_multiplayer; - m_continuous_in_multiplayer = true; - } - else if (!strcmp(data->getStringValue(), "extra-properties")) { - frameinfo.has_extra_properties = true; - ++m_num_frames_extra_properties; - m_continuous_in_extra_properties = true; + if (length) { + stats[data->getStringValue()].num_frames += 1; + stats[data->getStringValue()].bytes += length; + if (!strcmp(data->getStringValue(), "signals")) { + frameinfo.has_signals = true; + } + else if (!strcmp(data->getStringValue(), "multiplayer")) { + frameinfo.has_multiplayer = true; + ++m_num_frames_multiplayer; + m_continuous_in_multiplayer = true; + } + else if (!strcmp(data->getStringValue(), "extra-properties")) { + frameinfo.has_extra_properties = true; + ++m_num_frames_extra_properties; + m_continuous_in_extra_properties = true; + } } } } @@ -2134,7 +2391,8 @@ FGReplay::loadTape(const SGPath& Filename, bool Preview, SGPropertyNode& MetaMet { SG_LOG(SG_SYSTEMS, SG_DEBUG, "loading Preview=" << Preview << " Filename=" << Filename); - /* Try to load as uncompressed Continuous recording first. */ + /* Try to load a Continuous recording first. */ + replay_error->setBoolValue(false); std::ifstream in_preview; std::ifstream& in(Preview ? in_preview : m_continuous_in); in.open(Filename.str()); @@ -2165,6 +2423,8 @@ FGReplay::loadTape(const SGPath& Filename, bool Preview, SGPropertyNode& MetaMet m_num_frames_multiplayer = 0; m_continuous_indexing_in.open(Filename.str()); m_continuous_indexing_pos = in.tellg(); + m_continuous_in_compression = m_continuous_in_config->getNode("meta/continuous-compression", true /*create*/)->getIntValue(); + SG_LOG(SG_SYSTEMS, SG_DEBUG, "m_continuous_in_compression=" << m_continuous_in_compression); SG_LOG(SG_SYSTEMS, SG_DEBUG, "filerequest=" << filerequest.get()); // Make an in-memory index of the recording. diff --git a/src/Aircraft/replay.hxx b/src/Aircraft/replay.hxx index 05addeecd..a56788e5c 100644 --- a/src/Aircraft/replay.hxx +++ b/src/Aircraft/replay.hxx @@ -231,6 +231,7 @@ private: SGPropertyNode_ptr speed_up; SGPropertyNode_ptr replay_multiplayer; SGPropertyNode_ptr recovery_period; + SGPropertyNode_ptr replay_error; SGPropertyNode_ptr m_sim_startup_xpos; SGPropertyNode_ptr m_sim_startup_ypos; @@ -274,6 +275,8 @@ private: // For writing uncompressed fgtape file. SGPropertyNode_ptr m_continuous_out_config; std::ofstream m_continuous_out; + int m_continuous_out_compression; + int m_continuous_in_compression; SGPropertyNode_ptr m_simple_time_enabled; };