commit a5f9d48469b70675d2a06abf231ed4c4e41834f8
author Mithun Bharadwaj <mithun.bharadwaj@bluerivertech.com> Wed Aug 02 16:10:40 2023 -0700
committer James Kuszmaul <james.kuszmaul@bluerivertech.com> Wed Sep 13 10:24:04 2023 -0700
tree 49153da2ac5944d0a3bbac53ddc5db7ff7e5dd67
parent 99aec9e298f1f64ab5a28f80807336642dbfc9a3

Track which types of data are stored in which files

This sets us up to pre-load timestamps but not data when sorting log
files by recording the types of messages in each log file.  The goal is
to be able to load all timestamps and remote timestamps completely into
RAM at the beginning if they are in separate files, and then reduce our
data buffers to something small.  By loading all timestamps into RAM at
the beginning efficiently, we should be able to solve the entire class
of frozen bugs.

Change-Id: Ib21f9d3b39356abad6b912dd5108f971a9ed12ce
Signed-off-by: James Kuszmaul <james.kuszmaul@bluerivertech.com>

aos/events/logging/multinode_logger_test.cc

diff --git a/aos/events/logging/multinode_logger_test.cc b/aos/events/logging/multinode_logger_test.cc
index 2073a5a..f6e5447 100644
--- a/aos/events/logging/multinode_logger_test.cc
+++ b/aos/events/logging/multinode_logger_test.cc
@@ -124,6 +124,49 @@
 
     EXPECT_EQ(log_header[15].message().parts_index(), 0);
     EXPECT_EQ(log_header[16].message().parts_index(), 1);
+
+    // And that the data_stored field is right.
+    EXPECT_THAT(*log_header[2].message().data_stored(),
+                ::testing::ElementsAre(StoredDataType::DATA,
+                                       StoredDataType::TIMESTAMPS));
+    EXPECT_THAT(*log_header[3].message().data_stored(),
+                ::testing::ElementsAre(StoredDataType::DATA,
+                                       StoredDataType::TIMESTAMPS));
+    EXPECT_THAT(*log_header[4].message().data_stored(),
+                ::testing::ElementsAre(StoredDataType::DATA,
+                                       StoredDataType::TIMESTAMPS));
+
+    EXPECT_THAT(*log_header[5].message().data_stored(),
+                ::testing::ElementsAre(StoredDataType::DATA,
+                                       StoredDataType::TIMESTAMPS));
+    EXPECT_THAT(*log_header[6].message().data_stored(),
+                ::testing::ElementsAre(StoredDataType::DATA,
+                                       StoredDataType::TIMESTAMPS));
+    EXPECT_THAT(*log_header[7].message().data_stored(),
+                ::testing::ElementsAre(StoredDataType::DATA,
+                                       StoredDataType::TIMESTAMPS));
+
+    EXPECT_THAT(*log_header[8].message().data_stored(),
+                ::testing::ElementsAre(StoredDataType::DATA));
+    EXPECT_THAT(*log_header[9].message().data_stored(),
+                ::testing::ElementsAre(StoredDataType::DATA));
+
+    EXPECT_THAT(*log_header[10].message().data_stored(),
+                ::testing::ElementsAre(StoredDataType::DATA));
+    EXPECT_THAT(*log_header[11].message().data_stored(),
+                ::testing::ElementsAre(StoredDataType::DATA));
+
+    EXPECT_THAT(*log_header[12].message().data_stored(),
+                ::testing::ElementsAre(StoredDataType::REMOTE_TIMESTAMPS));
+    EXPECT_THAT(*log_header[13].message().data_stored(),
+                ::testing::ElementsAre(StoredDataType::REMOTE_TIMESTAMPS));
+    EXPECT_THAT(*log_header[14].message().data_stored(),
+                ::testing::ElementsAre(StoredDataType::REMOTE_TIMESTAMPS));
+
+    EXPECT_THAT(*log_header[15].message().data_stored(),
+                ::testing::ElementsAre(StoredDataType::REMOTE_TIMESTAMPS));
+    EXPECT_THAT(*log_header[16].message().data_stored(),
+                ::testing::ElementsAre(StoredDataType::REMOTE_TIMESTAMPS));
   }
 
   const std::vector<LogFile> sorted_parts = SortParts(logfiles_);