frc971/analysis/analysis.py

#!/usr/bin/python3

import collections
import matplotlib
from matplotlib import pylab
from matplotlib.font_manager import FontProperties
import re
import sys

class Dataset(object):
  def __init__(self):
    self.time = []
    self.data = []

  def Add(self, time, data):
    self.time.append(time)
    self.data.append(data)


class Plotter(object):
  def __init__(self):
    self.signal = collections.OrderedDict()

  def Add(self, binary, struct_instance_name, *data_search_path):
    """
    Specifies a specific piece of data to plot

    Args:
      binary: str, The name of the executable that generated the log.
      struct_instance_name: str, The name of the struct instance whose data
                            contents should be plotted.
      data_search_path: [str], The path into the struct of the exact piece of
                        data to plot.

    Returns:
      None
    """
    self.signal[(binary, struct_instance_name, data_search_path)] = Dataset()

  def HandleLine(self, line):
    """
    Parses a line from a log file and adds the data to the plot data.

    Args:
      line: str, The line from the log file to parse

    Returns:
      None
    """
    pline = ParseLine(line)
    pline_data = None

    for key in self.signal:
      value = self.signal[key]
      binary = key[0]
      struct_instance_name = key[1]
      data_search_path = key[2]
      boolean_multiplier = None

      # If the plot definition line ends with a "-b X" where X is a number then
      # that number gets drawn when the value is True. Zero gets drawn when the
      # value is False.
      if len(data_search_path) >= 2 and data_search_path[-2] == '-b':
        boolean_multiplier = float(data_search_path[-1])
        data_search_path = data_search_path[:-2]

      # Make sure that we're looking at the right binary structure instance.
      if binary == pline.name:
        if pline.msg.startswith(struct_instance_name + ': '):
          # Parse the structure once.
          if pline_data is None:
              _, _, pline_data = pline.ParseStruct()
          # Traverse the structure as specified in `data_search_path`.
          # This lets the user access very deeply nested structures.
          data = pline_data
          for path in data_search_path:
            data = data[path]

          if boolean_multiplier is not None:
            if data == 'T':
              value.Add(pline.time, boolean_multiplier)
            else:
              value.Add(pline.time, 0)
          else:
            value.Add(pline.time, data)

  def Plot(self, no_binary_in_legend):
    """
    Plots all the data after it's parsed.

    This should only be called after `HandleFile` has been called so that there
    is actual data to plot.
    """
    for key in self.signal:
      value = self.signal[key]

      # Create a legend label using the binary name (optional), the structure
      # name and the data search path.
      label = key[1] + '.' + '.'.join(key[2])
      if not no_binary_in_legend:
        label = key[0] + ' ' + label

      pylab.plot(value.time, value.data, label=label)

    # Set legend font size to small and move it to the top center.
    fontP = FontProperties()
    fontP.set_size('small')
    pylab.legend(bbox_to_anchor=(0.5, 1.05), prop=fontP)

    pylab.show()

  def PlotFile(self, f, no_binary_in_legend=False):
    """
    Parses and plots all the data.

    Args:
      f: str, The filename of the log whose data to parse and plot.

    Returns:
      None
    """
    self.HandleFile(f)
    self.Plot(no_binary_in_legend)

  def HandleFile(self, f):
    """
    Parses the specified log file.

    Args:
      f: str, The filename of the log whose data to parse.

    Returns:
      None
    """
    with open(f, 'r') as fd:
      for line in fd:
        self.HandleLine(line)


"""
A regular expression to match the envelope part of the log entry.
Parsing of the JSON msg is handled elsewhere.
"""
LOG_RE = re.compile("""
  (.*?)              # 1 name
  \((\d+)\)          # 2 pid
  \((\d+)\)          # 3 message_index
  :\s
  (\w+?)             # 4 level
  \s+at\s+
  (\d+\.\d+)s        # 5 time
  :\s
  ([A-Za-z0-9_./-]+) # 6 filename
  :\s
  (\d+)              # 7 linenumber
  :\s
  (.*)               # 8 msg
  """, re.VERBOSE)

class LogEntry:
  """This class provides a way to parse log entries."""

  def __init__(self, line):
    """Populates a LogEntry from a line."""
    m = LOG_RE.match(line)
    if m is None:
        print("LOG_RE failed on", line)
        sys.exit(1)
    self.name = m.group(1)
    self.pid_index = int(m.group(2))
    self.msg_index = int(m.group(3))
    self.level = m.group(4)
    self.time = float(m.group(5))
    self.filename = m.group(6)
    self.linenumber = m.group(7)
    self.msg = m.group(8)

  def __str__(self):
    """Formats the data cleanly."""
    return '%s(%d)(%d): %s at %fs: %s: %d: %s' % (self.name, self.pid, self.msg_index, self.level, self.time, self.filename, self.linenumber, self.msg)

  def __JsonizeTokenArray(self, sub_array, tokens, token_index):
    """Parses an array from the provided tokens.

    Args:
      sub_array: list, The list to stick the elements in.
      tokens: list of strings, The list with all the tokens in it.
      token_index: int, Where to start in the token list.

    Returns:
      int, The last token used.
    """
    # Make sure the data starts with a '['
    if tokens[token_index] != '[':
      print(tokens)
      print('Expected [ at beginning, found', tokens[token_index + 1])
      return None

    # Eat the '['
    token_index += 1

    # Loop through the tokens.
    while token_index < len(tokens):
      if tokens[token_index + 1] == ',':
        # Next item is a comma, so we should just add the element.
        sub_array.append(tokens[token_index])
        token_index += 2
      elif tokens[token_index + 1] == ']':
        # Next item is a ']', so we should just add the element and finish.
        sub_array.append(tokens[token_index])
        token_index += 1
        return token_index
      else:
        # Otherwise, it must be a sub-message.
        sub_json = dict()
        token_index = self.JsonizeTokens(sub_json, tokens, token_index + 1)
        sub_array.append(sub_json)
        if tokens[token_index] == ',':
          # Handle there either being another data element.
          token_index += 1
        elif tokens[token_index] == ']':
          # Handle the end of the array.
          return token_index
        else:
          print('Unexpected ', tokens[token_index])
          return None

    print('Unexpected end')
    return None

  def JsonizeTokens(self, json, tokens, token_index):
    """Creates a json-like dictionary from the provided tokens.

    Args:
      json: dict, The dict to stick the elements in.
      tokens: list of strings, The list with all the tokens in it.
      token_index: int, Where to start in the token list.

    Returns:
      int, The last token used.
    """
    # Check that the message starts with a {
    if tokens[token_index] != '{':
      print(tokens)
      print('Expected { at beginning, found', tokens[token_index])
      return None

    # Eat the {
    token_index += 1

    # States and state variable for parsing elements.
    STATE_INIT = 'init'
    STATE_HAS_NAME = 'name'
    STATE_HAS_COLON = 'colon'
    STATE_EXPECTING_SUBMSG = 'submsg'
    STATE_EXPECTING_COMMA = 'comma'
    parser_state = STATE_INIT

    while token_index < len(tokens):
      if tokens[token_index] == '}':
        # Finish if there is a }
        return token_index + 1
      elif tokens[token_index] == '{':
        if parser_state != STATE_EXPECTING_SUBMSG:
          print(tokens)
          print(parser_state)
          print('Bad input, was not expecting {')
          return None
        # Found a submessage, parse it.
        sub_json = dict()
        token_index = self.JsonizeTokens(sub_json, tokens, token_index)
        json[token_name] = sub_json
        parser_state = STATE_EXPECTING_COMMA
      else:
        if parser_state == STATE_INIT:
          # This token is the name.
          token_name = tokens[token_index]
          parser_state = STATE_HAS_NAME
        elif parser_state == STATE_HAS_NAME:
          if tokens[token_index] != ':':
            print(tokens)
            print(parser_state)
            print('Bad input, found', tokens[token_index], 'expected :')
            return None
          # After a name, comes a :
          parser_state = STATE_HAS_COLON
        elif parser_state == STATE_HAS_COLON:
          # After the colon, figure out what is next.
          if tokens[token_index] == '[':
            # Found a sub-array!
            sub_array = []
            token_index = self.__JsonizeTokenArray(sub_array, tokens, token_index)
            json[token_name] = sub_array
            parser_state = STATE_EXPECTING_COMMA
          elif tokens[token_index + 1] == '{':
            # Found a sub-message, trigger parsing it.
            parser_state = STATE_EXPECTING_SUBMSG
          else:
            # This is just an element, move on.
            json[token_name] = tokens[token_index]
            parser_state = STATE_EXPECTING_COMMA
        elif parser_state == STATE_EXPECTING_COMMA:
          # Complain if there isn't a comma here.
          if tokens[token_index] != ',':
            print(tokens)
            print(parser_state)
            print('Bad input, found', tokens[token_index], 'expected ,')
            return None
          parser_state = STATE_INIT
        else:
          print('Bad parser state')
          return None
        token_index += 1

    print('Unexpected end')
    return None

  def ParseStruct(self):
    """Parses the message as a structure.

    Returns:
      struct_name, struct_type, json dict.
    """
    struct_name_index = self.msg.find(':')
    struct_name = self.msg[0:struct_name_index]

    struct_body = self.msg[struct_name_index+2:]
    tokens = []
    this_token = ''
    # For the various deliminators, append what we have found so far to the
    # list and the token.
    for char in struct_body:
      if char == '{':
        if this_token:
          tokens.append(this_token)
          this_token = ''
        tokens.append('{')
      elif char == '}':
        if this_token:
          tokens.append(this_token)
          this_token = ''
        tokens.append('}')
      elif char == '[':
        if this_token:
          tokens.append(this_token)
          this_token = ''
        tokens.append('[')
      elif char == ']':
        if this_token:
          tokens.append(this_token)
          this_token = ''
        tokens.append(']')
      elif char == ':':
        if this_token:
          tokens.append(this_token)
          this_token = ''
        tokens.append(':')
      elif char == ',':
        if this_token:
          tokens.append(this_token)
          this_token = ''
        tokens.append(',')
      elif char == ' ':
        if this_token:
          tokens.append(this_token)
          this_token = ''
      else:
        this_token += char
    if this_token:
      tokens.append(this_token)

    struct_type = tokens[0]
    json = dict()
    # Now that we have tokens, parse them.
    self.JsonizeTokens(json, tokens, 1)

    return (struct_name, struct_type, json)


def ParseLine(line):
  return LogEntry(line)

if __name__ == '__main__':
  print('motor_writer(2240)(07421): DEBUG   at 0000000819.99620s: ../../frc971/output/motor_writer.cc: 105: sending: .aos.controls.OutputCheck{pwm_value:221, pulse_length:2.233333}')
  line = ParseLine('motor_writer(2240)(07421): DEBUG   at 0000000819.99620s: ../../frc971/output/motor_writer.cc: 105: sending: .aos.controls.OutputCheck{pwm_value:221, pulse_length:2.233333}')
  if '.aos.controls.OutputCheck' in line.msg:
    print(line)
    print(line.ParseStruct())

  line = ParseLine('claw(2263)(19404): DEBUG   at 0000000820.00000s: ../../aos/common/controls/control_loop-tmpl.h: 104: position: .frc971.control_loops.ClawGroup.Position{top:.frc971.control_loops.HalfClawPosition{position:1.672153, front:.frc971.HallEffectStruct{current:f, posedge_count:0, negedge_count:52}, calibration:.frc971.HallEffectStruct{current:f, posedge_count:6, negedge_count:13}, back:.frc971.HallEffectStruct{current:f, posedge_count:0, negedge_count:62}, posedge_value:0.642681, negedge_value:0.922207}, bottom:.frc971.control_loops.HalfClawPosition{position:1.353539, front:.frc971.HallEffectStruct{current:f, posedge_count:2, negedge_count:150}, calibration:.frc971.HallEffectStruct{current:f, posedge_count:8, negedge_count:18}, back:.frc971.HallEffectStruct{current:f, posedge_count:0, negedge_count:6}, posedge_value:0.434514, negedge_value:0.759491}}')
  print(line.ParseStruct())

  line = ParseLine('joystick_proxy(2255)(39560): DEBUG   at 0000000820.00730s: ../../aos/prime/input/joystick_input.cc: 61: sending: .aos.RobotState{joysticks:[.aos.Joystick{buttons:0, axis:[0.000000, 1.000000, 1.000000, 0.000000]}, .aos.Joystick{buttons:0, axis:[-0.401575, 1.000000, -1.007874, 0.000000]}, .aos.Joystick{buttons:0, axis:[0.007874, 0.000000, 1.000000, -1.007874]}, .aos.Joystick{buttons:0, axis:[0.000000, 0.000000, 0.000000, 0.000000]}], test_mode:f, fms_attached:f, enabled:T, autonomous:f, team_id:971, fake:f}')
  print(line.ParseStruct())