Refactor analysis.py in to separate classes and files
No code changes other than adding a class hierarchy.
This is the first of a set of restructurings to support
adding additional tools that require log parsing.
Change-Id: Ifafd02cb3e755e146e21d146e12482ec799de702
diff --git a/frc971/analysis/analysis.py b/frc971/analysis/logentry.py
old mode 100755
new mode 100644
similarity index 69%
rename from frc971/analysis/analysis.py
rename to frc971/analysis/logentry.py
index 8817a26..5b9a2ed
--- a/frc971/analysis/analysis.py
+++ b/frc971/analysis/logentry.py
@@ -1,142 +1,8 @@
#!/usr/bin/python3
-import collections
-import matplotlib
-from matplotlib import pylab
-from matplotlib.font_manager import FontProperties
import re
import sys
-class Dataset(object):
- def __init__(self):
- self.time = []
- self.data = []
-
- def Add(self, time, data):
- self.time.append(time)
- self.data.append(data)
-
-
-class Plotter(object):
- def __init__(self):
- self.signal = collections.OrderedDict()
-
- def Add(self, binary, struct_instance_name, *data_search_path):
- """
- Specifies a specific piece of data to plot
-
- Args:
- binary: str, The name of the executable that generated the log.
- struct_instance_name: str, The name of the struct instance whose data
- contents should be plotted.
- data_search_path: [str], The path into the struct of the exact piece of
- data to plot.
-
- Returns:
- None
- """
- self.signal[(binary, struct_instance_name, data_search_path)] = Dataset()
-
- def HandleLine(self, line):
- """
- Parses a line from a log file and adds the data to the plot data.
-
- Args:
- line: str, The line from the log file to parse
-
- Returns:
- None
- """
- pline = ParseLine(line)
- pline_data = None
-
- for key in self.signal:
- value = self.signal[key]
- binary = key[0]
- struct_instance_name = key[1]
- data_search_path = key[2]
- boolean_multiplier = None
-
- # If the plot definition line ends with a "-b X" where X is a number then
- # that number gets drawn when the value is True. Zero gets drawn when the
- # value is False.
- if len(data_search_path) >= 2 and data_search_path[-2] == '-b':
- boolean_multiplier = float(data_search_path[-1])
- data_search_path = data_search_path[:-2]
-
- # Make sure that we're looking at the right binary structure instance.
- if binary == pline.name:
- if pline.msg.startswith(struct_instance_name + ': '):
- # Parse the structure once.
- if pline_data is None:
- _, _, pline_data = pline.ParseStruct()
- # Traverse the structure as specified in `data_search_path`.
- # This lets the user access very deeply nested structures.
- data = pline_data
- for path in data_search_path:
- data = data[path]
-
- if boolean_multiplier is not None:
- if data == 'T':
- value.Add(pline.time, boolean_multiplier)
- else:
- value.Add(pline.time, 0)
- else:
- value.Add(pline.time, data)
-
- def Plot(self, no_binary_in_legend):
- """
- Plots all the data after it's parsed.
-
- This should only be called after `HandleFile` has been called so that there
- is actual data to plot.
- """
- for key in self.signal:
- value = self.signal[key]
-
- # Create a legend label using the binary name (optional), the structure
- # name and the data search path.
- label = key[1] + '.' + '.'.join(key[2])
- if not no_binary_in_legend:
- label = key[0] + ' ' + label
-
- pylab.plot(value.time, value.data, label=label)
-
- # Set legend font size to small and move it to the top center.
- fontP = FontProperties()
- fontP.set_size('small')
- pylab.legend(bbox_to_anchor=(0.5, 1.05), prop=fontP)
-
- pylab.show()
-
- def PlotFile(self, f, no_binary_in_legend=False):
- """
- Parses and plots all the data.
-
- Args:
- f: str, The filename of the log whose data to parse and plot.
-
- Returns:
- None
- """
- self.HandleFile(f)
- self.Plot(no_binary_in_legend)
-
- def HandleFile(self, f):
- """
- Parses the specified log file.
-
- Args:
- f: str, The filename of the log whose data to parse.
-
- Returns:
- None
- """
- with open(f, 'r') as fd:
- for line in fd:
- self.HandleLine(line)
-
-
"""
A regular expression to match the envelope part of the log entry.
Parsing of the JSON msg is handled elsewhere.
@@ -158,7 +24,11 @@
""", re.VERBOSE)
class LogEntry:
- """This class provides a way to parse log entries."""
+ """
+ This class provides a way to parse log entries.
+ The header portion of the log entry is parsed eagerly.
+ The structured portion of a log entry is parsed on demand.
+ """
def __init__(self, line):
"""Populates a LogEntry from a line."""
@@ -177,56 +47,69 @@
def __str__(self):
"""Formats the data cleanly."""
- return '%s(%d)(%d): %s at %fs: %s: %d: %s' % (self.name, self.pid, self.msg_index, self.level, self.time, self.filename, self.linenumber, self.msg)
+ return '%s(%d)(%d): %s at %fs: %s: %d: %s' % (
+ self.name, self.pid, self.msg_index, self.level, self.time, self.filename, self.linenumber, self.msg)
- def __JsonizeTokenArray(self, sub_array, tokens, token_index):
- """Parses an array from the provided tokens.
-
- Args:
- sub_array: list, The list to stick the elements in.
- tokens: list of strings, The list with all the tokens in it.
- token_index: int, Where to start in the token list.
+ def ParseStruct(self):
+ """Parses the message as a structure.
Returns:
- int, The last token used.
+ struct_name, struct_type, json dict.
"""
- # Make sure the data starts with a '['
- if tokens[token_index] != '[':
- print(tokens)
- print('Expected [ at beginning, found', tokens[token_index + 1])
- return None
+ struct_name_index = self.msg.find(':')
+ struct_name = self.msg[0:struct_name_index]
- # Eat the '['
- token_index += 1
-
- # Loop through the tokens.
- while token_index < len(tokens):
- if tokens[token_index + 1] == ',':
- # Next item is a comma, so we should just add the element.
- sub_array.append(tokens[token_index])
- token_index += 2
- elif tokens[token_index + 1] == ']':
- # Next item is a ']', so we should just add the element and finish.
- sub_array.append(tokens[token_index])
- token_index += 1
- return token_index
+ struct_body = self.msg[struct_name_index+2:]
+ tokens = []
+ this_token = ''
+ # For the various deliminators, append what we have found so far to the
+ # list and the token.
+ for char in struct_body:
+ if char == '{':
+ if this_token:
+ tokens.append(this_token)
+ this_token = ''
+ tokens.append('{')
+ elif char == '}':
+ if this_token:
+ tokens.append(this_token)
+ this_token = ''
+ tokens.append('}')
+ elif char == '[':
+ if this_token:
+ tokens.append(this_token)
+ this_token = ''
+ tokens.append('[')
+ elif char == ']':
+ if this_token:
+ tokens.append(this_token)
+ this_token = ''
+ tokens.append(']')
+ elif char == ':':
+ if this_token:
+ tokens.append(this_token)
+ this_token = ''
+ tokens.append(':')
+ elif char == ',':
+ if this_token:
+ tokens.append(this_token)
+ this_token = ''
+ tokens.append(',')
+ elif char == ' ':
+ if this_token:
+ tokens.append(this_token)
+ this_token = ''
else:
- # Otherwise, it must be a sub-message.
- sub_json = dict()
- token_index = self.JsonizeTokens(sub_json, tokens, token_index + 1)
- sub_array.append(sub_json)
- if tokens[token_index] == ',':
- # Handle there either being another data element.
- token_index += 1
- elif tokens[token_index] == ']':
- # Handle the end of the array.
- return token_index
- else:
- print('Unexpected ', tokens[token_index])
- return None
+ this_token += char
+ if this_token:
+ tokens.append(this_token)
- print('Unexpected end')
- return None
+ struct_type = tokens[0]
+ json = dict()
+ # Now that we have tokens, parse them.
+ self.JsonizeTokens(json, tokens, 1)
+
+ return (struct_name, struct_type, json)
def JsonizeTokens(self, json, tokens, token_index):
"""Creates a json-like dictionary from the provided tokens.
@@ -315,72 +198,60 @@
print('Unexpected end')
return None
- def ParseStruct(self):
- """Parses the message as a structure.
+ def __JsonizeTokenArray(self, sub_array, tokens, token_index):
+ """Parses an array from the provided tokens.
+
+ Args:
+ sub_array: list, The list to stick the elements in.
+ tokens: list of strings, The list with all the tokens in it.
+ token_index: int, Where to start in the token list.
Returns:
- struct_name, struct_type, json dict.
+ int, The last token used.
"""
- struct_name_index = self.msg.find(':')
- struct_name = self.msg[0:struct_name_index]
+ # Make sure the data starts with a '['
+ if tokens[token_index] != '[':
+ print(tokens)
+ print('Expected [ at beginning, found', tokens[token_index + 1])
+ return None
- struct_body = self.msg[struct_name_index+2:]
- tokens = []
- this_token = ''
- # For the various deliminators, append what we have found so far to the
- # list and the token.
- for char in struct_body:
- if char == '{':
- if this_token:
- tokens.append(this_token)
- this_token = ''
- tokens.append('{')
- elif char == '}':
- if this_token:
- tokens.append(this_token)
- this_token = ''
- tokens.append('}')
- elif char == '[':
- if this_token:
- tokens.append(this_token)
- this_token = ''
- tokens.append('[')
- elif char == ']':
- if this_token:
- tokens.append(this_token)
- this_token = ''
- tokens.append(']')
- elif char == ':':
- if this_token:
- tokens.append(this_token)
- this_token = ''
- tokens.append(':')
- elif char == ',':
- if this_token:
- tokens.append(this_token)
- this_token = ''
- tokens.append(',')
- elif char == ' ':
- if this_token:
- tokens.append(this_token)
- this_token = ''
+ # Eat the '['
+ token_index += 1
+
+ # Loop through the tokens.
+ while token_index < len(tokens):
+ if tokens[token_index + 1] == ',':
+ # Next item is a comma, so we should just add the element.
+ sub_array.append(tokens[token_index])
+ token_index += 2
+ elif tokens[token_index + 1] == ']':
+ # Next item is a ']', so we should just add the element and finish.
+ sub_array.append(tokens[token_index])
+ token_index += 1
+ return token_index
else:
- this_token += char
- if this_token:
- tokens.append(this_token)
+ # Otherwise, it must be a sub-message.
+ sub_json = dict()
+ token_index = self.JsonizeTokens(sub_json, tokens, token_index + 1)
+ sub_array.append(sub_json)
+ if tokens[token_index] == ',':
+ # Handle there either being another data element.
+ token_index += 1
+ elif tokens[token_index] == ']':
+ # Handle the end of the array.
+ return token_index
+ else:
+ print('Unexpected ', tokens[token_index])
+ return None
- struct_type = tokens[0]
- json = dict()
- # Now that we have tokens, parse them.
- self.JsonizeTokens(json, tokens, 1)
+ print('Unexpected end')
+ return None
- return (struct_name, struct_type, json)
-
-
-def ParseLine(line):
- return LogEntry(line)
if __name__ == '__main__':
+ def ParseLine(line):
+ return LogEntry(line)
+
print('motor_writer(2240)(07421): DEBUG at 0000000819.99620s: ../../frc971/output/motor_writer.cc: 105: sending: .aos.controls.OutputCheck{pwm_value:221, pulse_length:2.233333}')
line = ParseLine('motor_writer(2240)(07421): DEBUG at 0000000819.99620s: ../../frc971/output/motor_writer.cc: 105: sending: .aos.controls.OutputCheck{pwm_value:221, pulse_length:2.233333}')
if '.aos.controls.OutputCheck' in line.msg:
diff --git a/frc971/analysis/logreader.py b/frc971/analysis/logreader.py
new file mode 100644
index 0000000..34f1b01
--- /dev/null
+++ b/frc971/analysis/logreader.py
@@ -0,0 +1,97 @@
+#!/usr/bin/python3
+
+import collections
+from logentry import LogEntry
+
+class Dataset(object):
+ def __init__(self):
+ self.time = []
+ self.data = []
+
+ def Add(self, time, data):
+ self.time.append(time)
+ self.data.append(data)
+
+class CollectingLogReader(object):
+ """
+ Reads log files and collected requested data.
+ """
+ def __init__(self):
+ self.signal = collections.OrderedDict()
+
+ def Add(self, binary, struct_instance_name, *data_search_path):
+ """
+ Specifies a specific piece of data to collect
+
+ Args:
+ binary: str, The name of the executable that generated the log.
+ struct_instance_name: str, The name of the struct instance whose data
+ contents should be collected.
+ data_search_path: [str], The path into the struct of the exact piece of
+ data to collect.
+
+ Returns:
+ None
+ """
+ self.signal[(binary, struct_instance_name, data_search_path)] = Dataset()
+
+ def HandleFile(self, f):
+ """
+ Parses the specified log file.
+
+ Args:
+ f: str, The filename of the log whose data to parse.
+
+ Returns:
+ None
+ """
+ with open(f, 'r') as fd:
+ for line in fd:
+ self.HandleLine(line)
+
+ def HandleLine(self, line):
+ """
+ Parses a line from a log file and adds the data to the plot data.
+
+ Args:
+ line: str, The line from the log file to parse
+
+ Returns:
+ None
+ """
+ pline = LogEntry(line)
+ pline_data = None
+
+ for key in self.signal:
+ value = self.signal[key]
+ binary = key[0]
+ struct_instance_name = key[1]
+ data_search_path = key[2]
+ boolean_multiplier = None
+
+ # If the plot definition line ends with a "-b X" where X is a number then
+ # that number gets drawn when the value is True. Zero gets drawn when the
+ # value is False.
+ if len(data_search_path) >= 2 and data_search_path[-2] == '-b':
+ boolean_multiplier = float(data_search_path[-1])
+ data_search_path = data_search_path[:-2]
+
+ # Make sure that we're looking at the right binary structure instance.
+ if binary == pline.name:
+ if pline.msg.startswith(struct_instance_name + ': '):
+ # Parse the structure once.
+ if pline_data is None:
+ _, _, pline_data = pline.ParseStruct()
+ # Traverse the structure as specified in `data_search_path`.
+ # This lets the user access very deeply nested structures.
+ data = pline_data
+ for path in data_search_path:
+ data = data[path]
+
+ if boolean_multiplier is not None:
+ if data == 'T':
+ value.Add(pline.time, boolean_multiplier)
+ else:
+ value.Add(pline.time, 0)
+ else:
+ value.Add(pline.time, data)
diff --git a/frc971/analysis/plot_action.py b/frc971/analysis/plot_action.py
index 2f50ffd..d7dd48d 100755
--- a/frc971/analysis/plot_action.py
+++ b/frc971/analysis/plot_action.py
@@ -2,7 +2,7 @@
import sys
import numpy
-import analysis
+from plotter import Plotter
import argparse
def ReadPlotDefinitions(filename):
@@ -62,7 +62,7 @@
args = arg_parser.parse_args(sys.argv[1:])
- p = analysis.Plotter()
+ p = Plotter()
# If the user defines the list of data to plot in a file, read it from there.
if args.plot_defs:
diff --git a/frc971/analysis/plotter.py b/frc971/analysis/plotter.py
new file mode 100755
index 0000000..0f032ef
--- /dev/null
+++ b/frc971/analysis/plotter.py
@@ -0,0 +1,50 @@
+#!/usr/bin/python3
+
+from logreader import CollectingLogReader
+import matplotlib
+from matplotlib import pylab
+from matplotlib.font_manager import FontProperties
+
+class Plotter(CollectingLogReader):
+ """
+ A CollectingLogReader that plots collected data.
+ """
+
+ def PlotFile(self, f, no_binary_in_legend=False):
+ """
+ Parses and plots all the data.
+
+ Args:
+ f: str, The filename of the log whose data to parse and plot.
+
+ Returns:
+ None
+ """
+ self.HandleFile(f)
+ self.Plot(no_binary_in_legend)
+
+ def Plot(self, no_binary_in_legend):
+ """
+ Plots all the data after it's parsed.
+
+ This should only be called after `HandleFile` has been called so that there
+ is actual data to plot.
+ """
+ for key in self.signal:
+ value = self.signal[key]
+
+ # Create a legend label using the binary name (optional), the structure
+ # name and the data search path.
+ label = key[1] + '.' + '.'.join(key[2])
+ if not no_binary_in_legend:
+ label = key[0] + ' ' + label
+
+ pylab.plot(value.time, value.data, label=label)
+
+ # Set legend font size to small and move it to the top center.
+ fontP = FontProperties()
+ fontP.set_size('small')
+ pylab.legend(bbox_to_anchor=(0.5, 1.05), prop=fontP)
+
+ pylab.show()
+