Brian Silverman | f59fe3f | 2020-09-22 21:04:09 -0700 | [diff] [blame] | 1 | #include "aos/events/logging/lzma_encoder.h" |
| 2 | |
Austin Schuh | 99f7c6a | 2024-06-25 22:07:44 -0700 | [diff] [blame^] | 3 | #include "absl/flags/declare.h" |
| 4 | #include "absl/flags/flag.h" |
Austin Schuh | 3bd4c40 | 2020-11-06 18:19:06 -0800 | [diff] [blame] | 5 | #include "gmock/gmock.h" |
Brian Silverman | f59fe3f | 2020-09-22 21:04:09 -0700 | [diff] [blame] | 6 | #include "gtest/gtest.h" |
| 7 | |
Philipp Schrader | 790cb54 | 2023-07-05 21:06:52 -0700 | [diff] [blame] | 8 | #include "aos/events/logging/buffer_encoder_param_test.h" |
Austin Schuh | 6bdcc37 | 2024-06-27 14:49:11 -0700 | [diff] [blame] | 9 | #include "aos/testing/tmpdir.h" |
Philipp Schrader | 790cb54 | 2023-07-05 21:06:52 -0700 | [diff] [blame] | 10 | #include "aos/util/file.h" |
| 11 | |
Austin Schuh | 99f7c6a | 2024-06-25 22:07:44 -0700 | [diff] [blame^] | 12 | ABSL_DECLARE_FLAG(int32_t, lzma_threads); |
Austin Schuh | be91b34 | 2022-06-27 00:53:45 -0700 | [diff] [blame] | 13 | |
Brian Silverman | f59fe3f | 2020-09-22 21:04:09 -0700 | [diff] [blame] | 14 | namespace aos::logger::testing { |
| 15 | |
James Kuszmaul | f4bf9fe | 2021-05-10 22:58:24 -0700 | [diff] [blame] | 16 | INSTANTIATE_TEST_SUITE_P( |
Austin Schuh | be91b34 | 2022-06-27 00:53:45 -0700 | [diff] [blame] | 17 | MtLzma, BufferEncoderTest, |
Austin Schuh | 48d10d6 | 2022-10-16 22:19:23 -0700 | [diff] [blame] | 18 | ::testing::Combine(::testing::Values([](size_t max_message_size) { |
Austin Schuh | 99f7c6a | 2024-06-25 22:07:44 -0700 | [diff] [blame^] | 19 | absl::SetFlag(&FLAGS_lzma_threads, 3); |
Austin Schuh | 48d10d6 | 2022-10-16 22:19:23 -0700 | [diff] [blame] | 20 | return std::make_unique<LzmaEncoder>(max_message_size, |
| 21 | 2, 4096); |
Austin Schuh | be91b34 | 2022-06-27 00:53:45 -0700 | [diff] [blame] | 22 | }), |
| 23 | ::testing::Values([](std::string_view filename) { |
| 24 | return std::make_unique<LzmaDecoder>(filename); |
| 25 | }), |
| 26 | ::testing::Range(0, 100))); |
| 27 | |
| 28 | INSTANTIATE_TEST_SUITE_P( |
| 29 | MtLzmaThreaded, BufferEncoderTest, |
Austin Schuh | 48d10d6 | 2022-10-16 22:19:23 -0700 | [diff] [blame] | 30 | ::testing::Combine(::testing::Values([](size_t max_message_size) { |
Austin Schuh | 99f7c6a | 2024-06-25 22:07:44 -0700 | [diff] [blame^] | 31 | absl::SetFlag(&FLAGS_lzma_threads, 3); |
Austin Schuh | 48d10d6 | 2022-10-16 22:19:23 -0700 | [diff] [blame] | 32 | return std::make_unique<LzmaEncoder>(max_message_size, |
| 33 | 5, 4096); |
Austin Schuh | be91b34 | 2022-06-27 00:53:45 -0700 | [diff] [blame] | 34 | }), |
| 35 | ::testing::Values([](std::string_view filename) { |
| 36 | return std::make_unique<ThreadedLzmaDecoder>(filename); |
| 37 | }), |
| 38 | ::testing::Range(0, 100))); |
| 39 | |
| 40 | INSTANTIATE_TEST_SUITE_P( |
Brian Silverman | f59fe3f | 2020-09-22 21:04:09 -0700 | [diff] [blame] | 41 | Lzma, BufferEncoderTest, |
Austin Schuh | 48d10d6 | 2022-10-16 22:19:23 -0700 | [diff] [blame] | 42 | ::testing::Combine(::testing::Values([](size_t max_message_size) { |
Austin Schuh | 99f7c6a | 2024-06-25 22:07:44 -0700 | [diff] [blame^] | 43 | absl::SetFlag(&FLAGS_lzma_threads, 1); |
Austin Schuh | 48d10d6 | 2022-10-16 22:19:23 -0700 | [diff] [blame] | 44 | return std::make_unique<LzmaEncoder>(max_message_size, |
| 45 | 2, 4096); |
Brian Silverman | f59fe3f | 2020-09-22 21:04:09 -0700 | [diff] [blame] | 46 | }), |
| 47 | ::testing::Values([](std::string_view filename) { |
| 48 | return std::make_unique<LzmaDecoder>(filename); |
| 49 | }), |
| 50 | ::testing::Range(0, 100))); |
| 51 | |
Tyler Chatow | 7df6083 | 2021-07-15 21:18:36 -0700 | [diff] [blame] | 52 | INSTANTIATE_TEST_SUITE_P( |
| 53 | LzmaThreaded, BufferEncoderTest, |
Austin Schuh | 48d10d6 | 2022-10-16 22:19:23 -0700 | [diff] [blame] | 54 | ::testing::Combine(::testing::Values([](size_t max_message_size) { |
Austin Schuh | 99f7c6a | 2024-06-25 22:07:44 -0700 | [diff] [blame^] | 55 | absl::SetFlag(&FLAGS_lzma_threads, 1); |
Austin Schuh | 48d10d6 | 2022-10-16 22:19:23 -0700 | [diff] [blame] | 56 | return std::make_unique<LzmaEncoder>(max_message_size, |
| 57 | 5, 4096); |
Tyler Chatow | 7df6083 | 2021-07-15 21:18:36 -0700 | [diff] [blame] | 58 | }), |
| 59 | ::testing::Values([](std::string_view filename) { |
| 60 | return std::make_unique<ThreadedLzmaDecoder>(filename); |
| 61 | }), |
| 62 | ::testing::Range(0, 100))); |
| 63 | |
Austin Schuh | 3bd4c40 | 2020-11-06 18:19:06 -0800 | [diff] [blame] | 64 | // Tests that we return as much of the file as we can read if the end is |
| 65 | // corrupted. |
| 66 | TEST_F(BufferEncoderBaseTest, CorruptedBuffer) { |
| 67 | std::uniform_int_distribution<int> quantity_distribution(20, 60); |
Austin Schuh | 6bdcc37 | 2024-06-27 14:49:11 -0700 | [diff] [blame] | 68 | const std::string file_path = aos::testing::TestTmpDir() + "/foo"; |
Austin Schuh | 3bd4c40 | 2020-11-06 18:19:06 -0800 | [diff] [blame] | 69 | |
| 70 | std::vector<std::vector<uint8_t>> encoded_buffers; |
| 71 | { |
| 72 | const int encode_chunks = quantity_distribution(*random_number_generator()); |
Austin Schuh | 48d10d6 | 2022-10-16 22:19:23 -0700 | [diff] [blame] | 73 | const auto encoder = std::make_unique<LzmaEncoder>( |
| 74 | BufferEncoderBaseTest::kMaxMessageSize, 2); |
Austin Schuh | 3bd4c40 | 2020-11-06 18:19:06 -0800 | [diff] [blame] | 75 | encoded_buffers = CreateAndEncode(encode_chunks, encoder.get()); |
| 76 | encoder->Finish(); |
| 77 | |
| 78 | std::string contents = ""; |
| 79 | for (auto span : encoder->queue()) { |
| 80 | absl::StrAppend( |
| 81 | &contents, |
| 82 | std::string_view(reinterpret_cast<const char *>(span.data()), |
| 83 | span.size())); |
| 84 | } |
| 85 | aos::util::WriteStringToFileOrDie( |
| 86 | file_path, contents.substr(0, contents.size() - 200)); |
| 87 | } |
| 88 | |
| 89 | const size_t total_encoded_size = TotalSize(encoded_buffers); |
| 90 | |
| 91 | // Try decoding in multiple random chunkings. |
| 92 | for (int i = 0; i < 20; ++i) { |
| 93 | const auto decoder = std::make_unique<LzmaDecoder>(file_path); |
| 94 | std::vector<std::vector<uint8_t>> decoded_buffers; |
| 95 | size_t total_decoded_size = 0; |
| 96 | while (true) { |
| 97 | const int chunk_size = quantity_distribution(*random_number_generator()); |
| 98 | std::vector<uint8_t> chunk(chunk_size); |
| 99 | const size_t read_result = |
| 100 | decoder->Read(chunk.data(), chunk.data() + chunk_size); |
| 101 | // Eventually we'll get here, once the decoder is really sure it's done. |
| 102 | if (read_result == 0) { |
| 103 | // Sanity check the math in the test code. |
| 104 | LOG(INFO) << "Decoded " << total_decoded_size << " encoded " |
| 105 | << total_encoded_size; |
| 106 | CHECK_EQ(total_decoded_size, TotalSize(decoded_buffers)); |
| 107 | break; |
| 108 | } |
| 109 | // If we're at the end, trim off the 0s so our comparison later works out. |
| 110 | chunk.resize(read_result); |
| 111 | total_decoded_size += read_result; |
| 112 | decoded_buffers.emplace_back(std::move(chunk)); |
| 113 | } |
| 114 | auto flattened_encoded = Flatten(encoded_buffers); |
| 115 | auto flattened_decoded = Flatten(decoded_buffers); |
| 116 | |
| 117 | ASSERT_LE(flattened_decoded.size(), flattened_encoded.size()); |
| 118 | flattened_encoded.resize(flattened_decoded.size()); |
| 119 | |
| 120 | ASSERT_THAT(flattened_decoded, ::testing::Eq(flattened_encoded)); |
| 121 | } |
| 122 | } |
| 123 | |
Brian Silverman | f59fe3f | 2020-09-22 21:04:09 -0700 | [diff] [blame] | 124 | } // namespace aos::logger::testing |