diff --git a/test_suite/CMakeLists.txt b/test_suite/CMakeLists.txt index db1452d12..566930197 100644 --- a/test_suite/CMakeLists.txt +++ b/test_suite/CMakeLists.txt @@ -12,12 +12,18 @@ endforeach(test_category) set(TESTSUITE_SOURCES ${TESTSUITE_SOURCES} bootstrap.cxx + fgCompilerOutputter.cxx + fgTestListener.cxx fgTestRunner.cxx + formatting.cxx testSuite.cxx ) set(TESTSUITE_HEADERS ${TESTSUITE_HEADERS} + fgCompilerOutputter.hxx + fgTestListener.hxx fgTestRunner.hxx + formatting.hxx ) # The test suite output directory. diff --git a/test_suite/fgCompilerOutputter.cxx b/test_suite/fgCompilerOutputter.cxx new file mode 100644 index 000000000..26006d24c --- /dev/null +++ b/test_suite/fgCompilerOutputter.cxx @@ -0,0 +1,119 @@ +/* + * Copyright (C) 2016 Edward d'Auvergne + * + * This file is part of the program FlightGear. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + + +#include +#include + +#include +#include + +#include "fgCompilerOutputter.hxx" +#include "formatting.hxx" + +using namespace std; + + +// Create a new class instance. +fgCompilerOutputter * fgCompilerOutputter::defaultOutputter(CppUnit::TestResultCollector *result, vector *capt, const clock_t *clock, CppUnit::OStream &stream) +{ + return new fgCompilerOutputter(result, capt, clock, stream); +} + + +// Printout for each failed test. +void fgCompilerOutputter::printFailureDetail(CppUnit::TestFailure *failure) +{ + // Declarations. + TestIOCapt test_io; + vector::iterator test_iter; + + // Initial separator. + fg_stream << string(WIDTH_DIVIDER, '=') << endl; + + // Test info. + fg_stream << (failure->isError() ? "ERROR: " : "FAIL: ") << failure->failedTestName() << endl; + fg_stream << string(WIDTH_DIVIDER, '-') << endl; + fg_stream << (failure->isError() ? "Error" : "Assertion") << ": "; + printFailureLocation(failure->sourceLine()); + printFailureMessage(failure); + + // The captured IO for this test. + test_iter = find_if(io_capt->begin(), io_capt->end(), matchTestName(failure->failedTestName())); + if (test_iter != io_capt->end()) + test_io = *test_iter; + + // Default IO streams. + fg_stream << string(WIDTH_DIVIDER, '-') << endl; + fg_stream << "STDOUT and STDERR:" << endl; + fg_stream << test_io.stdio << endl; + + // SG_LOG IO streams. + fg_stream << string(WIDTH_DIVIDER, '-') << endl; + fg_stream << "SG_LOG:" << endl; + fg_stream << test_io.sg_log << endl; +} + + +// Detailed printout after a failed run of the test suite. +void fgCompilerOutputter::printFailureReport() +{ + // Custom printouts for each failed test. + printFailuresList(); + + // A summary with timing info. + printSuiteStats(); + + // Final summary. + fg_stream << endl << "[ FAILED ]" << endl << endl; + fg_stream.flush(); +} + + +// Printout of the test suite stats. +void fgCompilerOutputter::printSuiteStats() +{ + // A divider. + fg_stream << string(WIDTH_DIVIDER, '-') << endl; + + // Timing and test count line. + fg_stream << "Ran " << fg_result->runTests() << " tests"; + streamsize prec = fg_stream.precision(); + fg_stream << setprecision(3); + fg_stream << " in " << ((double)*suite_timer)/CLOCKS_PER_SEC << " seconds." << endl; + fg_stream << setprecision(prec); + + // Failure line. + if (!fg_result->wasSuccessful()) { + fg_stream << endl << "Failures = " << fg_result->testFailures() << endl; + fg_stream << "Errors = " << fg_result->testErrors() << endl; + } +} + + +// Print a summary after a successful run of the test suite. +void fgCompilerOutputter::printSuccess() +{ + // A summary with timing info. + printSuiteStats(); + + // Final summary. + fg_stream << endl << "[ OK ]" << endl << endl; + fg_stream.flush(); +} diff --git a/test_suite/fgCompilerOutputter.hxx b/test_suite/fgCompilerOutputter.hxx new file mode 100644 index 000000000..650f4ce8e --- /dev/null +++ b/test_suite/fgCompilerOutputter.hxx @@ -0,0 +1,77 @@ +/* + * Copyright (C) 2016 Edward d'Auvergne + * + * This file is part of the program FlightGear. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + + +#ifndef _FG_COMPILER_OUTPUTTER_HXX +#define _FG_COMPILER_OUTPUTTER_HXX + + +#include +#include + +#include "fgTestListener.hxx" + + +// The custom outputter for the FlightGear test suite. +class fgCompilerOutputter : public CppUnit::CompilerOutputter +{ + public: + // Constructor. + fgCompilerOutputter(CppUnit::TestResultCollector *result, + std::vector *capt, + const clock_t *clock, + CppUnit::OStream &stream, + const std::string &locationFormat = CPPUNIT_COMPILER_LOCATION_FORMAT) + : CppUnit::CompilerOutputter(result, stream, locationFormat) + , io_capt(capt) + , fg_result(result) + , fg_stream(stream) + , suite_timer(clock) + { + } + + // Create a new class instance. + static fgCompilerOutputter *defaultOutputter(CppUnit::TestResultCollector *result, std::vector *capt, const clock_t *clock, CppUnit::OStream &stream); + + // Print a summary after a successful run of the test suite. + void printSuccess(); + + // Detailed printout after a failed run of the test suite. + void printFailureReport(); + + // Printout for each failed test. + void printFailureDetail(CppUnit::TestFailure *failure); + + // Printout of the test suite stats. + void printSuiteStats(); + + // The captured IO for each failed test. + std::vector *io_capt; + + private: + // Store copies of the base class objects. + CppUnit::TestResultCollector *fg_result; + CppUnit::OStream &fg_stream; + + // The test suite time, in clock ticks. + const clock_t *suite_timer; +}; + + +#endif // _FG_COMPILER_OUTPUTTER_HXX diff --git a/test_suite/fgTestListener.cxx b/test_suite/fgTestListener.cxx new file mode 100644 index 000000000..0a8dbc734 --- /dev/null +++ b/test_suite/fgTestListener.cxx @@ -0,0 +1,87 @@ +/* + * Copyright (C) 2016 Edward d'Auvergne + * + * This file is part of the program FlightGear. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + + +#include +#include +#include + +#include "fgTestListener.hxx" + +using namespace std; + + +// Handle failures. +void fgTestListener::addFailure(const CppUnit::TestFailure &failure) +{ + m_failure = true; + if (failure.isError()) + m_error = true; +} + + +// Override the base class function to restore IO streams. +void fgTestListener::endTest(CppUnit::Test *test) +{ + // Test timing. + sum_time += clock() - m_time; + + // Restore the IO streams. + cout.rdbuf(orig_cout); + cerr.rdbuf(orig_cerr); + + // Per-test single character status feedback. + if (m_failure) + cerr << (m_error ? "E" : "F"); + else + cerr << '.'; + cerr.flush(); + + // Store the captured IO for any failed tests. + if (m_failure) { + // Set up the data structure. + TestIOCapt test_io; + test_io.name = test->getName(); + + // Standard IO. + test_io.stdio = capt.str(); + + // Add the test's IO to the list. + io_capt.push_back(test_io); + } +} + + +// Override the base class function to capture IO streams. +void fgTestListener::startTest(CppUnit::Test *test) +{ + // Store the original STDOUT and STDERR for restoring later on. + orig_cout = cout.rdbuf(); + orig_cerr = cerr.rdbuf(); + + // Clear the captured stream and then catch stdout and stderr. + capt.str(string()); + cout.rdbuf(capt.rdbuf()); + cerr.rdbuf(capt.rdbuf()); + + // Reset the test status. + m_failure = false; + m_error = false; + m_time = clock(); +} diff --git a/test_suite/fgTestListener.hxx b/test_suite/fgTestListener.hxx new file mode 100644 index 000000000..e8a1b8a40 --- /dev/null +++ b/test_suite/fgTestListener.hxx @@ -0,0 +1,88 @@ +/* + * Copyright (C) 2016 Edward d'Auvergne + * + * This file is part of the program FlightGear. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + + +#ifndef _FG_TEST_LISTENER_HXX +#define _FG_TEST_LISTENER_HXX + + +#include +#include + + +// Data structure for holding the captured IO for a failed test. +struct TestIOCapt { + std::string name; + std::string stdio; + std::string sg_log; +}; + + +// Match the test by name for std:find using a vector. +class matchTestName +{ + std::string _name; + +public: + matchTestName(const std::string &name) : _name(name) {} + + bool operator()(const TestIOCapt &item) const { + return item.name == _name; + } +}; + + +// The custom test runner for the FlightGear test suite. +class fgTestListener : public CppUnit::TestListener +{ + public: + // Constructor. + fgTestListener(): m_failure(false), m_error(false), sum_time(0) { }; + + // Override the base class function to capture IO. streams + void startTest(CppUnit::Test *test); + + // Override the base class function to restore IO streams. + void endTest(CppUnit::Test *test); + + // Handle failures. + void addFailure(const CppUnit::TestFailure &failure); + + // Test suite timing. + clock_t sum_time; + + // IO capture for all failed tests. + std::vector io_capt; + + private: + // The original IO streams. + std::streambuf *orig_cerr, *orig_cout; + + // Captured IO streams. + std::stringstream capt; + + // Test timings. + clock_t m_time; + + // Failure state. + bool m_failure, m_error; +}; + + +#endif // _FG_TEST_LISTENER_HXX diff --git a/test_suite/fgTestRunner.cxx b/test_suite/fgTestRunner.cxx index cf4d6f921..9f0f0c9d2 100644 --- a/test_suite/fgTestRunner.cxx +++ b/test_suite/fgTestRunner.cxx @@ -18,12 +18,15 @@ */ -#include #include #include #include #include +#include "fgCompilerOutputter.hxx" +#include "fgTestListener.hxx" +#include "formatting.hxx" + // Execute all test suites for the given test category. int testRunner(const std::string& title) @@ -31,15 +34,26 @@ int testRunner(const std::string& title) // Declarations. CppUnit::TextTestRunner runner; + // Print out a title. + printTitle(std::cerr, title); + // Get all tests. runner.addTest(CppUnit::TestFactoryRegistry::getRegistry(title).makeTest()); + // Set up the test listener. + fgTestListener *testListener; + testListener = new fgTestListener; + runner.eventManager().addListener(testListener); + // Set the test suite output IO stream. - runner.setOutputter(CppUnit::CompilerOutputter::defaultOutputter(&runner.result(), std::cerr)); + runner.setOutputter(new fgCompilerOutputter(&runner.result(), &testListener->io_capt, &testListener->sum_time, std::cerr)); // Execute the tests. runner.run("", false, true, false); + // Clean up. + delete testListener; + // Return the status of the tests. CppUnit::TestResultCollector &status = runner.result(); return status.testFailures(); diff --git a/test_suite/formatting.cxx b/test_suite/formatting.cxx new file mode 100644 index 000000000..332576caa --- /dev/null +++ b/test_suite/formatting.cxx @@ -0,0 +1,82 @@ +/* + * Copyright (C) 2016 Edward d'Auvergne + * + * This file is part of the program FlightGear. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + + +#include + +#include "formatting.hxx" + +using namespace std; + + +// Printout of a section label. +void printSection(CppUnit::OStream &stream, const string &text) +{ + // Declarations. + string::size_type width = text.size(); + + // Format the text. + stream << endl; + stream << text << endl; + stream << string(width, '=') << endl << endl; +} + + +// Print a summary line. +void printSummaryLine(CppUnit::OStream &stream, const string &name, int passed) +{ + // Declarations. + int n; + string dots, state; + + // Passed. + if (!passed) + state = "OK"; + + // Skipped. + else if (passed == -1) + state = "Skipped"; + + // Failed. + else + state = "Failed"; + + // Dots. + n = WIDTH_DIVIDER - name.size() - state.size() - 6; + dots = string(n, '.'); + + // Write out the line. + stream << name; + stream << " " << dots << " "; + stream << "[ " << state << " ]" << endl; +} + + +// Printout of a title label. +void printTitle(CppUnit::OStream &stream, const string &text) +{ + // Declarations. + string::size_type width = text.size() + 4; + + // Format the text. + stream << endl << endl; + stream << string(width, '#') << endl; + stream << "# " << text << " #" << endl; + stream << string(width, '#') << endl << endl; +} diff --git a/test_suite/formatting.hxx b/test_suite/formatting.hxx new file mode 100644 index 000000000..ebbd85239 --- /dev/null +++ b/test_suite/formatting.hxx @@ -0,0 +1,40 @@ +/* + * Copyright (C) 2016-2018 Edward d'Auvergne + * + * This file is part of the program FlightGear. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + + +#ifndef _FG_TEST_SUITE_FORMATTING_HXX +#define _FG_TEST_SUITE_FORMATTING_HXX + +#include + + +// Divider width. +const int WIDTH_DIVIDER = 70; + +// Printout of a section label. +void printSection(CppUnit::OStream &stream, const std::string &text); + +// Print a summary line. +void printSummaryLine(CppUnit::OStream &stream, const std::string &name, int passed); + +// Printout of a title label. +void printTitle(CppUnit::OStream &stream, const std::string &text); + + +#endif // _FG_TEST_SUITE_FORMATTING_HXX diff --git a/test_suite/testSuite.cxx b/test_suite/testSuite.cxx index 43be60349..462e35fa4 100644 --- a/test_suite/testSuite.cxx +++ b/test_suite/testSuite.cxx @@ -20,6 +20,46 @@ #include #include "fgTestRunner.hxx" +#include "formatting.hxx" + +using namespace std; + + +// Print out a summary of the relax test suite. +void summary(CppUnit::OStream &stream, int system_result, int unit_result, int gui_result, int simgear_result) +{ + // Title. + string text = "Summary of the FlightGear test suite"; + printTitle(stream, text); + + // Subtitle. + text = "Synopsis"; + printSection(stream, text); + + // System/functional test summary. + text = "System/functional tests"; + printSummaryLine(stream, text, system_result); + + // Unit test summary. + text = "Unit tests"; + printSummaryLine(stream, text, unit_result); + + // GUI test summary. + text = "GUI tests"; + printSummaryLine(stream, text, gui_result); + + // Simgear unit test summary. + text = "Simgear unit tests"; + printSummaryLine(stream, text, simgear_result); + + // Synopsis. + text ="Synopsis"; + int synopsis = system_result + unit_result + gui_result + simgear_result; + printSummaryLine(stream, text, synopsis); + + // End. + stream << endl << endl; +} int main(void) @@ -33,6 +73,9 @@ int main(void) status_gui = testRunner("GUI tests"); status_simgear = testRunner("Simgear unit tests"); + // Summary printout. + summary(cerr, status_system, status_unit, status_gui, status_simgear); + // Failure. if (status_system > 0) return 1;