Merge "Support sorting parts files across reboots"
diff --git a/aos/events/logging/logfile_sorting.cc b/aos/events/logging/logfile_sorting.cc
index 2db9230..86d3930 100644
--- a/aos/events/logging/logfile_sorting.cc
+++ b/aos/events/logging/logfile_sorting.cc
@@ -139,7 +139,10 @@
   // Name from a log.  All logs below have been confirmed to match.
   std::string name;
 
-  std::map<std::string, UnsortedLogParts> unsorted_parts;
+  // Mapping from parts_uuid, source_boot_uuid -> parts.  We have log files
+  // where the parts_uuid stays constant across reboots.
+  std::map<std::pair<std::string, std::string>, UnsortedLogParts>
+      unsorted_parts;
 };
 
 // Sort part files without UUIDs and part indexes as well.  Extract everything
@@ -167,6 +170,19 @@
   std::set<std::string> boots;
 };
 
+// This is an intermediate representation of Boots.  The data we have when we
+// are iterating through the log file headers isn't enough to create Boots as we
+// go, so we need to save an intermediate and then rewrite it into the final
+// result once we know everything.
+struct MapBoots {
+  // Maps the boot UUID to the boot count.  Since boot UUIDs are unique, we
+  // don't need to be node specific and can do this for all nodes.
+  std::map<std::string, int> boot_count_map;
+
+  // Maps the node name to a set of all boots for that node.
+  std::map<std::string, std::vector<std::string>> boots;
+};
+
 // Helper class to make it easier to sort a list of log files into
 // std::vector<LogFile>
 struct PartsSorter {
@@ -189,7 +205,7 @@
   void PopulateFromFiles(const std::vector<std::string> &parts);
 
   // Wrangle parts_list into a map of boot uuids -> boot counts.
-  std::map<std::string, int> ComputeBootCounts();
+  MapBoots ComputeBootCounts();
 
   // Reformats old_parts into a list of logfiles and returns it.  This destroys
   // state in PartsSorter.
@@ -266,6 +282,8 @@
       continue;
     }
 
+    VLOG(1) << "Header " << FlatbufferToJson(log_header.value()) << " " << part;
+
     if (configuration_sha256.empty()) {
       CHECK(log_header->message().has_configuration())
           << ": Failed to find header on " << part;
@@ -351,10 +369,15 @@
       }
     }
 
-    auto it = log_it->second.unsorted_parts.find(parts_uuid);
+    VLOG(1) << "Parts: " << parts_uuid << ", source boot uuid "
+            << source_boot_uuid;
+    auto it = log_it->second.unsorted_parts.find(
+        std::pair(parts_uuid, std::string(source_boot_uuid)));
     if (it == log_it->second.unsorted_parts.end()) {
       it = log_it->second.unsorted_parts
-               .insert(std::make_pair(parts_uuid, UnsortedLogParts()))
+               .insert(std::make_pair(
+                   std::make_pair(parts_uuid, std::string(source_boot_uuid)),
+                   UnsortedLogParts()))
                .first;
       it->second.monotonic_start_time = monotonic_start_time;
       it->second.realtime_start_time = realtime_start_time;
@@ -364,7 +387,6 @@
       it->second.source_boot_uuid = source_boot_uuid;
       it->second.config_sha256 = configuration_sha256;
     } else {
-      CHECK_EQ(it->second.source_boot_uuid, source_boot_uuid);
       CHECK_EQ(it->second.config_sha256, configuration_sha256);
     }
 
@@ -450,7 +472,7 @@
   return result;
 }
 
-std::map<std::string, int> PartsSorter::ComputeBootCounts() {
+MapBoots PartsSorter::ComputeBootCounts() {
   std::map<std::string, NodeBootState> boot_constraints;
 
   // TODO(austin): This is the "old" way.  Once we have better ordering info in
@@ -478,8 +500,8 @@
              std::vector<std::pair<std::string, std::pair<int, int>>>>
         node_boot_parts_index_ranges;
 
-    for (std::pair<const std::string, UnsortedLogParts> &parts :
-         logs.second.unsorted_parts) {
+    for (std::pair<const std::pair<std::string, std::string>, UnsortedLogParts>
+             &parts : logs.second.unsorted_parts) {
       CHECK_GT(parts.second.parts.size(), 0u);
 
       // Track that this boot exists so we know the overall set of boots we need
@@ -627,15 +649,17 @@
   // And now walk the constraint graph we have generated to order the boots.
   // This doesn't need to catch all the cases, it just needs to report when it
   // fails.
-  std::map<std::string, int> boot_count_map;
-
+  MapBoots boots;
   for (std::pair<const std::string, NodeBootState> &node_state :
        boot_constraints) {
     CHECK_GT(node_state.second.boots.size(), 0u)
         << ": Need a boot from each node.";
     if (node_state.second.boots.size() == 1u) {
-      boot_count_map.insert(
+      boots.boot_count_map.insert(
           std::make_pair(*node_state.second.boots.begin(), 0));
+      boots.boots.insert(std::make_pair(
+          node_state.first,
+          std::vector<std::string>{*node_state.second.boots.begin()}));
       continue;
     }
 
@@ -717,18 +741,22 @@
 
     VLOG(1) << "Node " << node_state.first;
     size_t boot_count = 0;
+    boots.boots.insert(std::make_pair(node_state.first, sorted_boots));
     for (const std::string &boot : sorted_boots) {
       VLOG(1) << "  Boot " << boot;
-      boot_count_map.insert(std::make_pair(std::move(boot), boot_count));
+      boots.boot_count_map.insert(std::make_pair(std::move(boot), boot_count));
       ++boot_count;
     }
   }
 
-  return boot_count_map;
+  return boots;
 }
 
 std::vector<LogFile> PartsSorter::FormatNewParts() {
-  const std::map<std::string, int> boot_counts = ComputeBootCounts();
+  // Rewrite MapBoots to Boots since we have enough information here.
+  std::shared_ptr<Boots> boot_counts = std::make_shared<Boots>();
+  MapBoots map_boot_counts = ComputeBootCounts();
+  boot_counts->boot_count_map = std::move(map_boot_counts.boot_count_map);
 
   std::map<std::string, std::shared_ptr<const Configuration>>
       copied_config_sha256;
@@ -741,8 +769,9 @@
     new_file.logger_node = logs.second.logger_node;
     new_file.logger_boot_uuid = logs.second.logger_boot_uuid;
     {
-      auto boot_count_it = boot_counts.find(new_file.logger_boot_uuid);
-      CHECK(boot_count_it != boot_counts.end());
+      auto boot_count_it =
+          boot_counts->boot_count_map.find(new_file.logger_boot_uuid);
+      CHECK(boot_count_it != boot_counts->boot_count_map.end());
       new_file.logger_boot_count = boot_count_it->second;
     }
     new_file.monotonic_start_time = logs.second.monotonic_start_time;
@@ -751,8 +780,8 @@
     new_file.corrupted = corrupted;
     bool seen_part = false;
     std::string config_sha256;
-    for (std::pair<const std::string, UnsortedLogParts> &parts :
-         logs.second.unsorted_parts) {
+    for (std::pair<const std::pair<std::string, std::string>, UnsortedLogParts>
+             &parts : logs.second.unsorted_parts) {
       LogParts new_parts;
       new_parts.monotonic_start_time = parts.second.monotonic_start_time;
       new_parts.realtime_start_time = parts.second.realtime_start_time;
@@ -762,14 +791,17 @@
           parts.second.logger_realtime_start_time;
       new_parts.log_event_uuid = logs.first;
       new_parts.source_boot_uuid = parts.second.source_boot_uuid;
-      new_parts.parts_uuid = parts.first;
+      new_parts.parts_uuid = parts.first.first;
       new_parts.node = std::move(parts.second.node);
+      new_parts.boots = boot_counts;
 
       {
-        auto boot_count_it = boot_counts.find(new_parts.source_boot_uuid);
-        CHECK(boot_count_it != boot_counts.end());
+        auto boot_count_it =
+            boot_counts->boot_count_map.find(new_parts.source_boot_uuid);
+        CHECK(boot_count_it != boot_counts->boot_count_map.end());
         new_parts.boot_count = boot_count_it->second;
       }
+      new_parts.logger_boot_count = new_file.logger_boot_count;
 
       std::sort(parts.second.parts.begin(), parts.second.parts.end(),
                 [](const std::pair<std::string, int> &a,
@@ -841,6 +873,26 @@
     }
     result.emplace_back(std::move(new_file));
   }
+
+  {
+    CHECK_EQ(config_sha256_lookup.size() + copied_config_sha256.size(), 1u)
+        << ": We only support log files with 1 config in them.";
+    std::shared_ptr<const aos::Configuration> config =
+        config_sha256_lookup.empty() ? copied_config_sha256.begin()->second
+                                     : config_sha256_lookup.begin()->second;
+
+    boot_counts->boots.resize(configuration::NodesCount(config.get()));
+    for (std::pair<const std::string, std::vector<std::string>> &boots :
+         map_boot_counts.boots) {
+      size_t node_index = 0;
+      if (configuration::MultiNode(config.get())) {
+        node_index = configuration::GetNodeIndex(config.get(), boots.first);
+      }
+
+      boot_counts->boots[node_index] = std::move(boots.second);
+    }
+  }
+
   return result;
 }
 
diff --git a/aos/events/logging/logfile_sorting.h b/aos/events/logging/logfile_sorting.h
index 1dd0cb3..1d05dc7 100644
--- a/aos/events/logging/logfile_sorting.h
+++ b/aos/events/logging/logfile_sorting.h
@@ -2,6 +2,7 @@
 #define AOS_EVENTS_LOGGING_LOGFILE_SORTING_H_
 
 #include <iostream>
+#include <map>
 #include <string>
 #include <vector>
 
@@ -12,6 +13,15 @@
 namespace aos {
 namespace logger {
 
+struct Boots {
+  // Maps the boot UUID to the boot count.  Since boot UUIDs are unique, we
+  // don't need to be node specific and can do this for all nodes.
+  std::map<std::string, int> boot_count_map;
+
+  // Maps the node index to a set of all boots for that node.
+  std::vector<std::vector<std::string>> boots;
+};
+
 // Datastructure to hold ordered parts.
 struct LogParts {
   // Monotonic and realtime start times for this set of log files.  For log
@@ -42,6 +52,11 @@
   // source_boot_uuid's for a node.
   size_t boot_count = 0;
 
+  // Boot number for the node where this data was logged.
+  // This is theoretically redundant with LogFile, except that we quickly end up
+  // using LogParts without the corresponding LogFile datastructure.
+  size_t logger_boot_count = 0;
+
   // Pre-sorted list of parts.
   std::vector<std::string> parts;
 
@@ -49,6 +64,9 @@
   // log files with the same config.
   std::string config_sha256;
   std::shared_ptr<const aos::Configuration> config;
+
+  // Information about all the boots that the system has observed.
+  std::shared_ptr<const Boots> boots;
 };
 
 // Datastructure to hold parts from the same run of the logger which have no