diff options
Diffstat (limited to 'utils/google-benchmark/test/complexity_test.cc')
| -rw-r--r-- | utils/google-benchmark/test/complexity_test.cc | 224 |
1 files changed, 35 insertions, 189 deletions
diff --git a/utils/google-benchmark/test/complexity_test.cc b/utils/google-benchmark/test/complexity_test.cc index db04183da..d2e5e194d 100644 --- a/utils/google-benchmark/test/complexity_test.cc +++ b/utils/google-benchmark/test/complexity_test.cc @@ -1,153 +1,46 @@ - #undef NDEBUG #include "benchmark/benchmark.h" -#include "../src/check.h" // NOTE: check.h is for internal use only! -#include "../src/re.h" // NOTE: re.h is for internal use only +#include "output_test.h" #include <cassert> -#include <cstring> -#include <iostream> -#include <sstream> #include <vector> -#include <utility> #include <algorithm> +#include <cstdlib> #include <cmath> namespace { -// ========================================================================= // -// -------------------------- Testing Case --------------------------------- // -// ========================================================================= // - -enum MatchRules { - MR_Default, // Skip non-matching lines until a match is found. - MR_Next // Match must occur on the next line. -}; - -struct TestCase { - std::string regex; - int match_rule; - - TestCase(std::string re, int rule = MR_Default) : regex(re), match_rule(rule) {} - - void Check(std::stringstream& remaining_output) const { - benchmark::Regex r; - std::string err_str; - r.Init(regex, &err_str); - CHECK(err_str.empty()) << "Could not construct regex \"" << regex << "\"" - << " got Error: " << err_str; - - std::string near = "<EOF>"; - std::string line; - bool first = true; - while (remaining_output.eof() == false) { - CHECK(remaining_output.good()); - std::getline(remaining_output, line); - // Keep the first line as context. - if (first) { - near = line; - first = false; - } - if (r.Match(line)) return; - CHECK(match_rule != MR_Next) << "Expected line \"" << line - << "\" to match regex \"" << regex << "\"" - << "\nstarted matching at line: \"" << near << "\""; - } - - CHECK(remaining_output.eof() == false) - << "End of output reached before match for regex \"" << regex - << "\" was found" - << "\nstarted matching at line: \"" << near << "\""; - } -}; - -std::vector<TestCase> ConsoleOutputTests; -std::vector<TestCase> JSONOutputTests; -std::vector<TestCase> CSVOutputTests; - -// ========================================================================= // -// -------------------------- Test Helpers --------------------------------- // -// ========================================================================= // - -class TestReporter : public benchmark::BenchmarkReporter { -public: - TestReporter(std::vector<benchmark::BenchmarkReporter*> reps) - : reporters_(reps) {} - - virtual bool ReportContext(const Context& context) { - bool last_ret = false; - bool first = true; - for (auto rep : reporters_) { - bool new_ret = rep->ReportContext(context); - CHECK(first || new_ret == last_ret) - << "Reports return different values for ReportContext"; - first = false; - last_ret = new_ret; - } - return last_ret; - } - - virtual void ReportRuns(const std::vector<Run>& report) { - for (auto rep : reporters_) - rep->ReportRuns(report); - } - - virtual void Finalize() { - for (auto rep : reporters_) - rep->Finalize(); - } - -private: - std::vector<benchmark::BenchmarkReporter*> reporters_; -}; - - -#define CONCAT2(x, y) x##y -#define CONCAT(x, y) CONCAT2(x, y) - -#define ADD_CASES(...) \ - int CONCAT(dummy, __LINE__) = AddCases(__VA_ARGS__) - -int AddCases(std::vector<TestCase>* out, std::initializer_list<TestCase> const& v) { - for (auto const& TC : v) - out->push_back(TC); - return 0; -} - -template <class First> -std::string join(First f) { return f; } - -template <class First, class ...Args> -std::string join(First f, Args&&... args) { - return std::string(std::move(f)) + "[ ]+" + join(std::forward<Args>(args)...); -} - -std::string dec_re = "[0-9]*[.]?[0-9]+([eE][-+][0-9]+)?"; - #define ADD_COMPLEXITY_CASES(...) \ int CONCAT(dummy, __LINE__) = AddComplexityTest(__VA_ARGS__) -int AddComplexityTest(std::vector<TestCase>* console_out, std::vector<TestCase>* json_out, - std::vector<TestCase>* csv_out, std::string big_o_test_name, +int AddComplexityTest(std::string big_o_test_name, std::string rms_test_name, std::string big_o) { - std::string big_o_str = dec_re + " " + big_o; - AddCases(console_out, { - {join("^" + big_o_test_name + "", big_o_str, big_o_str) + "[ ]*$"}, - {join("^" + rms_test_name + "", "[0-9]+ %", "[0-9]+ %") + "[ ]*$"} + SetSubstitutions({ + {"%bigo_name", big_o_test_name}, + {"%rms_name", rms_test_name}, + {"%bigo_str", "[ ]*" + std::string(dec_re) + " " + big_o}, + {"%bigo", big_o}, + {"%rms", "[ ]*[0-9]+ %"} }); - AddCases(json_out, { - {"\"name\": \"" + big_o_test_name + "\",$"}, + AddCases(TC_ConsoleOut, { + {"^%bigo_name %bigo_str %bigo_str[ ]*$"}, + {"^%bigo_name", MR_Not}, // Assert we we didn't only matched a name. + {"^%rms_name %rms %rms[ ]*$", MR_Next} + }); + AddCases(TC_JSONOut, { + {"\"name\": \"%bigo_name\",$"}, {"\"cpu_coefficient\": [0-9]+,$", MR_Next}, {"\"real_coefficient\": [0-9]{1,5},$", MR_Next}, - {"\"big_o\": \"" + big_o + "\",$", MR_Next}, + {"\"big_o\": \"%bigo\",$", MR_Next}, {"\"time_unit\": \"ns\"$", MR_Next}, {"}", MR_Next}, - {"\"name\": \"" + rms_test_name + "\",$"}, + {"\"name\": \"%rms_name\",$"}, {"\"rms\": [0-9]+%$", MR_Next}, {"}", MR_Next} }); - AddCases(csv_out, { - {"^\"" + big_o_test_name + "\",," + dec_re + "," + dec_re + "," + big_o + ",,,,,$"}, - {"^\"" + rms_test_name + "\",," + dec_re + "," + dec_re + ",,,,,,$", MR_Next} + AddCases(TC_CSVOut, { + {"^\"%bigo_name\",,%float,%float,%bigo,,,,,$"}, + {"^\"%bigo_name\"", MR_Not}, + {"^\"%rms_name\",,%float,%float,,,,,,$", MR_Next} }); return 0; } @@ -172,20 +65,20 @@ BENCHMARK(BM_Complexity_O1) -> Range(1, 1<<18) -> Complexity([](int){return 1.0; const char* big_o_1_test_name = "BM_Complexity_O1_BigO"; const char* rms_o_1_test_name = "BM_Complexity_O1_RMS"; -const char* enum_auto_big_o_1 = "\\([0-9]+\\)"; +const char* enum_big_o_1 = "\\([0-9]+\\)"; +// FIXME: Tolerate both '(1)' and 'lgN' as output when the complexity is auto deduced. +// See https://github.com/google/benchmark/issues/272 +const char* auto_big_o_1 = "(\\([0-9]+\\))|(lgN)"; const char* lambda_big_o_1 = "f\\(N\\)"; // Add enum tests -ADD_COMPLEXITY_CASES(&ConsoleOutputTests, &JSONOutputTests, &CSVOutputTests, - big_o_1_test_name, rms_o_1_test_name, enum_auto_big_o_1); +ADD_COMPLEXITY_CASES(big_o_1_test_name, rms_o_1_test_name, enum_big_o_1); // Add auto enum tests -ADD_COMPLEXITY_CASES(&ConsoleOutputTests, &JSONOutputTests, &CSVOutputTests, - big_o_1_test_name, rms_o_1_test_name, enum_auto_big_o_1); +ADD_COMPLEXITY_CASES(big_o_1_test_name, rms_o_1_test_name, auto_big_o_1); // Add lambda tests -ADD_COMPLEXITY_CASES(&ConsoleOutputTests, &JSONOutputTests, &CSVOutputTests, - big_o_1_test_name, rms_o_1_test_name, lambda_big_o_1); +ADD_COMPLEXITY_CASES(big_o_1_test_name, rms_o_1_test_name, lambda_big_o_1); // ========================================================================= // // --------------------------- Testing BigO O(N) --------------------------- // @@ -195,7 +88,7 @@ std::vector<int> ConstructRandomVector(int size) { std::vector<int> v; v.reserve(size); for (int i = 0; i < size; ++i) { - v.push_back(rand() % size); + v.push_back(std::rand() % size); } return v; } @@ -218,12 +111,10 @@ const char* enum_auto_big_o_n = "N"; const char* lambda_big_o_n = "f\\(N\\)"; // Add enum tests -ADD_COMPLEXITY_CASES(&ConsoleOutputTests, &JSONOutputTests, &CSVOutputTests, - big_o_n_test_name, rms_o_n_test_name, enum_auto_big_o_n); +ADD_COMPLEXITY_CASES(big_o_n_test_name, rms_o_n_test_name, enum_auto_big_o_n); // Add lambda tests -ADD_COMPLEXITY_CASES(&ConsoleOutputTests, &JSONOutputTests, &CSVOutputTests, - big_o_n_test_name, rms_o_n_test_name, lambda_big_o_n); +ADD_COMPLEXITY_CASES(big_o_n_test_name, rms_o_n_test_name, lambda_big_o_n); // ========================================================================= // // ------------------------- Testing BigO O(N*lgN) ------------------------- // @@ -246,62 +137,17 @@ const char* enum_auto_big_o_n_lg_n = "NlgN"; const char* lambda_big_o_n_lg_n = "f\\(N\\)"; // Add enum tests -ADD_COMPLEXITY_CASES(&ConsoleOutputTests, &JSONOutputTests, &CSVOutputTests, - big_o_n_lg_n_test_name, rms_o_n_lg_n_test_name, enum_auto_big_o_n_lg_n); +ADD_COMPLEXITY_CASES(big_o_n_lg_n_test_name, rms_o_n_lg_n_test_name, enum_auto_big_o_n_lg_n); // Add lambda tests -ADD_COMPLEXITY_CASES(&ConsoleOutputTests, &JSONOutputTests, &CSVOutputTests, - big_o_n_lg_n_test_name, rms_o_n_lg_n_test_name, lambda_big_o_n_lg_n); +ADD_COMPLEXITY_CASES(big_o_n_lg_n_test_name, rms_o_n_lg_n_test_name, lambda_big_o_n_lg_n); // ========================================================================= // // --------------------------- TEST CASES END ------------------------------ // // ========================================================================= // - int main(int argc, char* argv[]) { - benchmark::Initialize(&argc, argv); - benchmark::ConsoleReporter CR(benchmark::ConsoleReporter::OO_None); - benchmark::JSONReporter JR; - benchmark::CSVReporter CSVR; - struct ReporterTest { - const char* name; - std::vector<TestCase>& output_cases; - benchmark::BenchmarkReporter& reporter; - std::stringstream out_stream; - std::stringstream err_stream; - - ReporterTest(const char* n, - std::vector<TestCase>& out_tc, - benchmark::BenchmarkReporter& br) - : name(n), output_cases(out_tc), reporter(br) { - reporter.SetOutputStream(&out_stream); - reporter.SetErrorStream(&err_stream); - } - } TestCases[] = { - {"ConsoleReporter", ConsoleOutputTests, CR}, - {"JSONReporter", JSONOutputTests, JR}, - {"CSVReporter", CSVOutputTests, CSVR} - }; - - // Create the test reporter and run the benchmarks. - std::cout << "Running benchmarks...\n"; - TestReporter test_rep({&CR, &JR, &CSVR}); - benchmark::RunSpecifiedBenchmarks(&test_rep); - - for (auto& rep_test : TestCases) { - std::string msg = std::string("\nTesting ") + rep_test.name + " Output\n"; - std::string banner(msg.size() - 1, '-'); - std::cout << banner << msg << banner << "\n"; - - std::cerr << rep_test.err_stream.str(); - std::cout << rep_test.out_stream.str(); - - for (const auto& TC : rep_test.output_cases) - TC.Check(rep_test.out_stream); - - std::cout << "\n"; - } - return 0; + RunOutputTests(argc, argv); } |