TestSuite: Custom output formatting.

A large number of improvements have been made by subclassing the CppUnit classes
and overriding the base functions, including CompilerOutputter, TestListener,
and TextTestRunner.  All IO has also been captured by the test runner.

The result is that during the test suite run, only the characters '.', 'F', and
'E' are output for a pass, failure and error state.  At the end of each test
suite category, all failures and errors are reported in full detail, including
the different captured IO streams.  A final synopsis is printed out, improving
the overview in the case of too many tests failing.

For the fgCompilerOutputter class, the printSuccess(), printFailureReport(),
printFailureDetail(), and printSuiteStats() functions have been replaced to
implement the above printout design.  The class also stores the std::vector of
TestIOCapt structures for the final printouts.

The fgTestListener class handles the events from the running of the test suite.
The startTest() and endTest() functions are used for IO capture.  The IO is
placed into a TestIOCapt data structure, with one std::string for holding the
combined STDOUT and STDERR IO, and another for the SG_LOG IO.  If failures
occur, the TestIOCapt structure is appended to the fgCompilerOutputter vector.
The startTest() and endTest() functions are also used for starting and stopping
a timer to allow the full test suite to be timed.  And the addFailure() function
simply registers test failures or errors.

The fgTestRunner class overrides the CppUnit::TextTestRunner::run() function,
simply to prevent the base method from spawning a second test listener, causing
the test output to be duplicated.

Some auxiliary formatting functions have been added to print out titles,
sections, and synopsis summary lines.

test_suite/CMakeLists.txt

@@ -12,12 +12,18 @@ endforeach(test_category)
 set(TESTSUITE_SOURCES
     ${TESTSUITE_SOURCES}
     bootstrap.cxx
+    fgCompilerOutputter.cxx
+    fgTestListener.cxx
     fgTestRunner.cxx
+    formatting.cxx
     testSuite.cxx
 )
 set(TESTSUITE_HEADERS
     ${TESTSUITE_HEADERS}
+    fgCompilerOutputter.hxx
+    fgTestListener.hxx
     fgTestRunner.hxx
+    formatting.hxx
 )
 
 # The test suite output directory.

test_suite/fgCompilerOutputter.cxx

@@ -0,0 +1,119 @@
+/*
+ * Copyright (C) 2016 Edward d'Auvergne
+ *
+ * This file is part of the program FlightGear.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+
+#include <algorithm>
+#include <iomanip>
+
+#include <cppunit/SourceLine.h>
+#include <cppunit/TestResultCollector.h>
+
+#include "fgCompilerOutputter.hxx"
+#include "formatting.hxx"
+
+using namespace std;
+
+
+// Create a new class instance.
+fgCompilerOutputter * fgCompilerOutputter::defaultOutputter(CppUnit::TestResultCollector *result, vector<TestIOCapt> *capt, const clock_t *clock, CppUnit::OStream &stream)
+{
+    return new fgCompilerOutputter(result, capt, clock, stream);
+}
+
+
+// Printout for each failed test.
+void fgCompilerOutputter::printFailureDetail(CppUnit::TestFailure *failure)
+{
+    // Declarations.
+    TestIOCapt test_io;
+    vector<TestIOCapt>::iterator test_iter;
+
+    // Initial separator.
+    fg_stream << string(WIDTH_DIVIDER, '=') << endl;
+
+    // Test info.
+    fg_stream << (failure->isError() ? "ERROR: " : "FAIL: ") << failure->failedTestName() << endl;
+    fg_stream << string(WIDTH_DIVIDER, '-') << endl;
+    fg_stream << (failure->isError() ? "Error" : "Assertion") << ": ";
+    printFailureLocation(failure->sourceLine());
+    printFailureMessage(failure);
+
+    // The captured IO for this test.
+    test_iter = find_if(io_capt->begin(), io_capt->end(), matchTestName(failure->failedTestName()));
+    if (test_iter != io_capt->end())
+        test_io = *test_iter;
+
+    // Default IO streams.
+    fg_stream << string(WIDTH_DIVIDER, '-') << endl;
+    fg_stream << "STDOUT and STDERR:" << endl;
+    fg_stream << test_io.stdio << endl;
+
+    // SG_LOG IO streams.
+    fg_stream << string(WIDTH_DIVIDER, '-') << endl;
+    fg_stream << "SG_LOG:" << endl;
+    fg_stream << test_io.sg_log << endl;
+}
+
+
+// Detailed printout after a failed run of the test suite.
+void fgCompilerOutputter::printFailureReport()
+{
+    // Custom printouts for each failed test.
+    printFailuresList();
+
+    // A summary with timing info.
+    printSuiteStats();
+
+    // Final summary.
+    fg_stream << endl << "[ FAILED ]" << endl << endl;
+    fg_stream.flush();
+}
+
+
+// Printout of the test suite stats.
+void fgCompilerOutputter::printSuiteStats()
+{
+    // A divider.
+    fg_stream << string(WIDTH_DIVIDER, '-') << endl;
+
+    // Timing and test count line.
+    fg_stream << "Ran " << fg_result->runTests() << " tests";
+    streamsize prec = fg_stream.precision();
+    fg_stream << setprecision(3);
+    fg_stream << " in " << ((double)*suite_timer)/CLOCKS_PER_SEC << " seconds." << endl;
+    fg_stream << setprecision(prec);
+
+    // Failure line.
+    if (!fg_result->wasSuccessful()) {
+        fg_stream << endl << "Failures = " << fg_result->testFailures() << endl;
+        fg_stream << "Errors   = " << fg_result->testErrors() << endl;
+    }
+}
+
+
+// Print a summary after a successful run of the test suite.
+void fgCompilerOutputter::printSuccess()
+{
+    // A summary with timing info.
+    printSuiteStats();
+
+    // Final summary.
+    fg_stream << endl << "[ OK ]" << endl << endl;
+    fg_stream.flush();
+}

test_suite/fgCompilerOutputter.hxx

@@ -0,0 +1,77 @@
+/*
+ * Copyright (C) 2016 Edward d'Auvergne
+ *
+ * This file is part of the program FlightGear.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+
+#ifndef _FG_COMPILER_OUTPUTTER_HXX
+#define _FG_COMPILER_OUTPUTTER_HXX
+
+
+#include <cppunit/CompilerOutputter.h>
+#include <cppunit/TestFailure.h>
+
+#include "fgTestListener.hxx"
+
+
+// The custom outputter for the FlightGear test suite.
+class fgCompilerOutputter : public CppUnit::CompilerOutputter
+{
+    public:
+        // Constructor.
+        fgCompilerOutputter(CppUnit::TestResultCollector *result,
+            std::vector<TestIOCapt> *capt,
+            const clock_t *clock,
+            CppUnit::OStream &stream,
+            const std::string &locationFormat = CPPUNIT_COMPILER_LOCATION_FORMAT)
+        : CppUnit::CompilerOutputter(result, stream, locationFormat)
+        , io_capt(capt)
+        , fg_result(result)
+        , fg_stream(stream)
+        , suite_timer(clock)
+        {
+        }
+
+        // Create a new class instance.
+        static fgCompilerOutputter *defaultOutputter(CppUnit::TestResultCollector *result, std::vector<TestIOCapt> *capt, const clock_t *clock, CppUnit::OStream &stream);
+
+        // Print a summary after a successful run of the test suite.
+        void printSuccess();
+
+        // Detailed printout after a failed run of the test suite.
+        void printFailureReport();
+
+        // Printout for each failed test.
+        void printFailureDetail(CppUnit::TestFailure *failure);
+
+        // Printout of the test suite stats.
+        void printSuiteStats();
+
+        // The captured IO for each failed test.
+        std::vector<TestIOCapt> *io_capt;
+
+    private:
+        // Store copies of the base class objects.
+        CppUnit::TestResultCollector *fg_result;
+        CppUnit::OStream &fg_stream;
+
+        // The test suite time, in clock ticks.
+        const clock_t *suite_timer;
+};
+
+
+#endif    // _FG_COMPILER_OUTPUTTER_HXX

test_suite/fgTestListener.cxx

@@ -0,0 +1,87 @@
+/*
+ * Copyright (C) 2016 Edward d'Auvergne
+ *
+ * This file is part of the program FlightGear.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+
+#include <cppunit/Test.h>
+#include <cppunit/TestFailure.h>
+#include <cppunit/TestListener.h>
+
+#include "fgTestListener.hxx"
+
+using namespace std;
+
+
+// Handle failures.
+void fgTestListener::addFailure(const CppUnit::TestFailure &failure)
+{
+    m_failure = true;
+    if (failure.isError())
+        m_error = true;
+}
+
+
+// Override the base class function to restore IO streams.
+void fgTestListener::endTest(CppUnit::Test *test)
+{
+    // Test timing.
+    sum_time += clock() - m_time;
+
+    // Restore the IO streams.
+    cout.rdbuf(orig_cout);
+    cerr.rdbuf(orig_cerr);
+
+    // Per-test single character status feedback.
+    if (m_failure)
+        cerr << (m_error ? "E" : "F");
+    else
+        cerr << '.';
+    cerr.flush();
+
+    // Store the captured IO for any failed tests.
+    if (m_failure) {
+        // Set up the data structure.
+        TestIOCapt test_io;
+        test_io.name = test->getName();
+
+        // Standard IO.
+        test_io.stdio = capt.str();
+
+        // Add the test's IO to the list.
+        io_capt.push_back(test_io);
+    }
+}
+
+
+// Override the base class function to capture IO streams.
+void fgTestListener::startTest(CppUnit::Test *test)
+{
+    // Store the original STDOUT and STDERR for restoring later on.
+    orig_cout = cout.rdbuf();
+    orig_cerr = cerr.rdbuf();
+
+    // Clear the captured stream and then catch stdout and stderr.
+    capt.str(string());
+    cout.rdbuf(capt.rdbuf());
+    cerr.rdbuf(capt.rdbuf());
+
+    // Reset the test status.
+    m_failure = false;
+    m_error = false;
+    m_time = clock();
+}

test_suite/fgTestListener.hxx

@@ -0,0 +1,88 @@
+/*
+ * Copyright (C) 2016 Edward d'Auvergne
+ *
+ * This file is part of the program FlightGear.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+
+#ifndef _FG_TEST_LISTENER_HXX
+#define _FG_TEST_LISTENER_HXX
+
+
+#include <time.h>
+#include <vector>
+
+
+// Data structure for holding the captured IO for a failed test.
+struct TestIOCapt {
+    std::string name;
+    std::string stdio;
+    std::string sg_log;
+};
+
+
+// Match the test by name for std:find using a vector<TestIOCapt>.
+class matchTestName
+{
+    std::string _name;
+
+public:
+    matchTestName(const std::string &name) : _name(name) {}
+
+    bool operator()(const TestIOCapt &item) const {
+        return item.name == _name;
+    }
+};
+
+
+// The custom test runner for the FlightGear test suite.
+class fgTestListener : public CppUnit::TestListener
+{
+    public:
+        // Constructor.
+        fgTestListener(): m_failure(false), m_error(false), sum_time(0) { };
+
+        // Override the base class function to capture IO. streams
+        void startTest(CppUnit::Test *test);
+
+        // Override the base class function to restore IO streams.
+        void endTest(CppUnit::Test *test);
+
+        // Handle failures.
+        void addFailure(const CppUnit::TestFailure &failure);
+
+        // Test suite timing.
+        clock_t sum_time;
+
+        // IO capture for all failed tests.
+        std::vector<TestIOCapt> io_capt;
+
+    private:
+        // The original IO streams.
+        std::streambuf *orig_cerr, *orig_cout;
+
+        // Captured IO streams.
+        std::stringstream capt;
+
+        // Test timings.
+        clock_t m_time;
+
+        // Failure state.
+        bool m_failure, m_error;
+};
+
+
+#endif  // _FG_TEST_LISTENER_HXX

test_suite/fgTestRunner.cxx

@@ -18,12 +18,15 @@
  */
 
 
-#include <cppunit/CompilerOutputter.h>
 #include <cppunit/TestResult.h>
 #include <cppunit/TestResultCollector.h>
 #include <cppunit/TextTestRunner.h>
 #include <cppunit/extensions/TestFactoryRegistry.h>
 
+#include "fgCompilerOutputter.hxx"
+#include "fgTestListener.hxx"
+#include "formatting.hxx"
+
 
 // Execute all test suites for the given test category.
 int testRunner(const std::string& title)
@@ -31,15 +34,26 @@ int testRunner(const std::string& title)
     // Declarations.
     CppUnit::TextTestRunner runner;
 
+    // Print out a title.
+    printTitle(std::cerr, title);
+
     // Get all tests.
     runner.addTest(CppUnit::TestFactoryRegistry::getRegistry(title).makeTest());
 
+    // Set up the test listener.
+    fgTestListener *testListener;
+    testListener = new fgTestListener;
+    runner.eventManager().addListener(testListener);
+
     // Set the test suite output IO stream.
-    runner.setOutputter(CppUnit::CompilerOutputter::defaultOutputter(&runner.result(), std::cerr));
+    runner.setOutputter(new fgCompilerOutputter(&runner.result(), &testListener->io_capt, &testListener->sum_time, std::cerr));
 
     // Execute the tests.
     runner.run("", false, true, false);
 
+    // Clean up.
+    delete testListener;
+
     // Return the status of the tests.
     CppUnit::TestResultCollector &status = runner.result();
     return status.testFailures();

test_suite/formatting.cxx

@@ -0,0 +1,82 @@
+/*
+ * Copyright (C) 2016 Edward d'Auvergne
+ *
+ * This file is part of the program FlightGear.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+
+#include <string>
+
+#include "formatting.hxx"
+
+using namespace std;
+
+
+// Printout of a section label.
+void printSection(CppUnit::OStream &stream, const string &text)
+{
+    // Declarations.
+    string::size_type width = text.size();
+
+    // Format the text.
+    stream << endl;
+    stream << text << endl;
+    stream << string(width, '=') << endl << endl;
+}
+
+
+// Print a summary line.
+void printSummaryLine(CppUnit::OStream &stream, const string &name, int passed)
+{
+    // Declarations.
+    int n;
+    string dots, state;
+
+    // Passed.
+    if (!passed)
+        state = "OK";
+
+    // Skipped.
+    else if (passed == -1)
+        state = "Skipped";
+
+    // Failed.
+    else
+        state = "Failed";
+
+    // Dots.
+    n = WIDTH_DIVIDER - name.size() - state.size() - 6;
+    dots = string(n, '.');
+
+    // Write out the line.
+    stream << name;
+    stream << " " << dots << " ";
+    stream << "[ " << state << " ]" << endl;
+}
+
+
+// Printout of a title label.
+void printTitle(CppUnit::OStream &stream, const string &text)
+{
+    // Declarations.
+    string::size_type width = text.size() + 4;
+
+    // Format the text.
+    stream << endl << endl;
+    stream << string(width, '#') << endl;
+    stream << "# " << text << " #" << endl;
+    stream << string(width, '#') << endl << endl;
+}

test_suite/formatting.hxx

@@ -0,0 +1,40 @@
+/*
+ * Copyright (C) 2016-2018 Edward d'Auvergne
+ *
+ * This file is part of the program FlightGear.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+
+#ifndef _FG_TEST_SUITE_FORMATTING_HXX
+#define _FG_TEST_SUITE_FORMATTING_HXX
+
+#include <cppunit/portability/Stream.h>
+
+
+// Divider width.
+const int WIDTH_DIVIDER = 70;
+
+// Printout of a section label.
+void printSection(CppUnit::OStream &stream, const std::string &text);
+
+// Print a summary line.
+void printSummaryLine(CppUnit::OStream &stream, const std::string &name, int passed);
+
+// Printout of a title label.
+void printTitle(CppUnit::OStream &stream, const std::string &text);
+
+
+#endif  // _FG_TEST_SUITE_FORMATTING_HXX

test_suite/testSuite.cxx

@@ -20,6 +20,46 @@
 #include <iostream>
 
 #include "fgTestRunner.hxx"
+#include "formatting.hxx"
+
+using namespace std;
+
+
+// Print out a summary of the relax test suite.
+void summary(CppUnit::OStream &stream, int system_result, int unit_result, int gui_result, int simgear_result)
+{
+    // Title.
+    string text = "Summary of the FlightGear test suite";
+    printTitle(stream, text);
+
+    // Subtitle.
+    text = "Synopsis";
+    printSection(stream, text);
+
+    // System/functional test summary.
+    text = "System/functional tests";
+    printSummaryLine(stream, text, system_result);
+
+    // Unit test summary.
+    text = "Unit tests";
+    printSummaryLine(stream, text, unit_result);
+
+    // GUI test summary.
+    text = "GUI tests";
+    printSummaryLine(stream, text, gui_result);
+
+    // Simgear unit test summary.
+    text = "Simgear unit tests";
+    printSummaryLine(stream, text, simgear_result);
+
+    // Synopsis.
+    text ="Synopsis";
+    int synopsis = system_result + unit_result + gui_result + simgear_result;
+    printSummaryLine(stream, text, synopsis);
+
+    // End.
+    stream << endl << endl;
+}
 
 
 int main(void)
@@ -33,6 +73,9 @@ int main(void)
     status_gui = testRunner("GUI tests");
     status_simgear = testRunner("Simgear unit tests");
 
+    // Summary printout.
+    summary(cerr, status_system, status_unit, status_gui, status_simgear);
+
     // Failure.
     if (status_system > 0)
         return 1;