Speed up plotter flatbuffer parsing
We had been lazily doing all the reflection work on every single
message. Do the work to look up the field metadata and the such only once
prior to iterating over all the messages.
Change-Id: I2bdcfdd6f71b819b162c1e602a99aedfb6924f02
Signed-off-by: James Kuszmaul <jabukuszmaul@gmail.com>
diff --git a/aos/network/www/aos_plotter.ts b/aos/network/www/aos_plotter.ts
index 0145c07..c91a430 100644
--- a/aos/network/www/aos_plotter.ts
+++ b/aos/network/www/aos_plotter.ts
@@ -60,31 +60,50 @@
this.messages.push(
new TimestampedMessage(Table.getRootTable(new ByteBuffer(data)), time));
}
- private readField<T>(
- message: Table, fieldName: string,
- normalReader: (message: Table, name: string) => T | null,
- vectorReader: (message: Table, name: string) => T[] | null): T[]|null {
- // Typescript handles bindings in non-obvious ways that aren't caught well
- // by the compiler.
- normalReader = normalReader.bind(this.parser);
- vectorReader = vectorReader.bind(this.parser);
+ private parseFieldName(rawName: string): [string, boolean, number|null] {
+ // If the fieldName includes an array index at the end, attempt to read a
+ // vector.
+ // The indices can either be in the form [X] for some index X, or be an
+ // empty [], which is a request to return all values.
const regex = /(.*)\[([0-9]*)\]/;
- const match = fieldName.match(regex);
+ const match = rawName.match(regex);
if (match) {
const name = match[1];
- const vector = vectorReader(message, name);
- if (vector === null) {
- return null;
- }
- if (match[2] === "") {
- return vector;
- } else {
- const index = parseInt(match[2]);
- return (index < vector.length) ? [vector[index]] : null;
- }
+ const requestFullVector = match[2] === '';
+ const requestedIndex = requestFullVector ? null : parseInt(match[2]);
+ return [name, true, requestedIndex];
+ } else {
+ return [rawName, false, null];
}
- const singleResult = normalReader(message, fieldName);
- return singleResult ? [singleResult] : null;
+ }
+ private readField<T>(
+ typeIndex: number, fieldName: string,
+ normalReader: (typeIndex: number, name: string) => (t: Table) => T | null,
+ vectorReader: (typeIndex: number, name: string) => (t: Table) => T[] |
+ null): (t: Table) => T[] | null {
+ const [name, isVector, vectorIndex] = this.parseFieldName(fieldName);
+ if (isVector) {
+ const vectorLambda = vectorReader(typeIndex, name);
+ const requestFullVector = vectorIndex === null;
+ return (message: Table) => {
+ const vector = vectorLambda(message);
+ if (vector === null) {
+ return null;
+ }
+ if (requestFullVector) {
+ return vector;
+ } else {
+ return (vectorIndex < vector.length) ? [vector[vectorIndex]] : null;
+ }
+ };
+ }
+ const singleLambda = normalReader(typeIndex, fieldName);
+ return (message: Table) => {
+ // For single values, return as a 1-vector so that the type is
+ // consistent with that used for vectors.
+ const singleValue = singleLambda(message);
+ return (singleValue === null) ? null : [singleValue];
+ };
}
// Returns a time-series of every single instance of the given field. Format
// of the return value is [time0, value0, time1, value1,... timeN, valueN],
@@ -94,17 +113,40 @@
// If you want to retrieve a single signal from a vector, you can specify it
// as "field_name[index]".
getField(field: string[]): Point[] {
+ if (this.messages.length == 0) {
+ return [];
+ }
+ const rootType = this.messages[0].message.typeIndex;
const fieldName = field[field.length - 1];
const subMessage = field.slice(0, field.length - 1);
+
+ const lambdas = [];
+ let currentType = rootType;
+ for (const subMessageName of subMessage) {
+ lambdas.push(this.readField(
+ currentType, subMessageName,
+ (typeIndex: number, name: string) =>
+ this.parser.readTableLambda(typeIndex, name),
+ (typeIndex: number, name: string) =>
+ this.parser.readVectorOfTablesLambda(typeIndex, name)));
+ const [name, isVector, vectorIndex] = this.parseFieldName(subMessageName);
+ currentType =
+ this.parser.getField(name, currentType).type().index();
+ }
+ const fieldLambda = this.readField(
+ currentType, fieldName,
+ (typeIndex: number, name: string) =>
+ this.parser.readScalarLambda(typeIndex, name),
+ (typeIndex: number, name: string) =>
+ this.parser.readVectorOfScalarsLambda(typeIndex, name));
+
const results = [];
for (let ii = 0; ii < this.messages.length; ++ii) {
let tables = [this.messages[ii].message];
- for (const subMessageName of subMessage) {
+ for (const lambda of lambdas) {
let nextTables = [];
for (const table of tables) {
- const nextTable = this.readField(
- table, subMessageName, Parser.prototype.readTable,
- Parser.prototype.readVectorOfTables);
+ const nextTable = lambda(table);
if (nextTable === null) {
continue;
}
@@ -112,20 +154,21 @@
}
tables = nextTables;
}
+
+ const values = [];
+ for (const table of tables) {
+ const value = fieldLambda(table);
+ if (value !== null) {
+ values.push(value);
+ }
+ }
const time = this.messages[ii].time;
if (tables.length === 0) {
results.push(new Point(time, NaN));
} else {
- for (const table of tables) {
- const values = this.readField(
- table, fieldName, Parser.prototype.readScalar,
- Parser.prototype.readVectorOfScalars);
- if (values === null) {
- results.push(new Point(time, NaN));
- } else {
- for (const value of values) {
- results.push(new Point(time, (value === null) ? NaN : value));
- }
+ for (const valueVector of values) {
+ for (const value of valueVector) {
+ results.push(new Point(time, (value === null) ? NaN : value));
}
}
}
diff --git a/aos/network/www/reflection.ts b/aos/network/www/reflection.ts
index 09206e0..4362806 100644
--- a/aos/network/www/reflection.ts
+++ b/aos/network/www/reflection.ts
@@ -8,6 +8,7 @@
import {reflection, aos} from 'org_frc971/aos/configuration_generated';
import {ByteBuffer} from 'org_frc971/external/com_github_google_flatbuffers/ts/byte-buffer';
+import {Long} from 'org_frc971/external/com_github_google_flatbuffers/ts/long';
// Returns the size, in bytes, of the given type. For vectors/strings/etc.
// returns the size of the offset.
@@ -170,6 +171,17 @@
// Table.getRootTable() on the received buffer to construct the Table, and
// then access the members using the various methods of the Parser (or just
// convert the entire object to a javascript Object/JSON using toObject()).
+// There are three basic ways to access fields in a Table:
+// 1) Call toObject(), which turns the entire table into a javascript object.
+// This is not meant to be particularly fast, but is useful to, e.g.,
+// convert something to JSON, or as a debugging tool.
+// 2) Use the read*Lambda() accessors: These return a function that lets you
+// access the specified field given a table. This is used by the plotter
+// to repeatedly access the same field on a bunch of tables of the same type,
+// without having to redo all the reflection-related work on every access.
+// 3) Use the read*() accessors: These just call the lambda returned by
+// read*Lambda() for you, as a convenience. This is cleaner to use, but for
+// repeated lookups on tables of the same type, this may be inefficient.
export class Parser {
constructor(private readonly schema: reflection.Schema) {}
@@ -256,92 +268,121 @@
// readDefaults is true and the field is unset, we will look-up the default
// value for the field and return that.
// For 64-bit fields, returns a flatbuffer Long rather than a standard number.
- // TODO(james): For this and other accessors, determine if there is a
- // significant performance gain to be had by using readScalar to construct
- // an accessor method rather than having to redo the schema inspection on
- // every call.
- readScalar(table: Table, fieldName: string, readDefaults: boolean = false) {
- const field = this.getField(fieldName, table.typeIndex);
+ readScalar(table: Table, fieldName: string, readDefaults: boolean = false):
+ number|Long|null {
+ return this.readScalarLambda(
+ table.typeIndex, fieldName, readDefaults)(table);
+ }
+ // Like readScalar(), except that this returns an accessor for the specified
+ // field, rather than the value of the field itself.
+ // Note that the *Lambda() methods take a typeIndex instead of a Table, which
+ // can be obtained using table.typeIndex.
+ readScalarLambda(
+ typeIndex: number, fieldName: string,
+ readDefaults: boolean = false): (t: Table) => number | Long | null {
+ const field = this.getField(fieldName, typeIndex);
const fieldType = field.type();
- const isStruct = this.getType(table.typeIndex).isStruct();
+ const isStruct = this.getType(typeIndex).isStruct();
if (!isScalar(fieldType.baseType())) {
throw new Error('Field ' + fieldName + ' is not a scalar type.');
}
if (isStruct) {
- return table.readScalar(
- fieldType.baseType(), table.offset + field.offset());
+ const baseType = fieldType.baseType();
+ return (t: Table) => {
+ return t.readScalar(baseType, t.offset + field.offset());
+ };
}
- const offset =
- table.offset + table.bb.__offset(table.offset, field.offset());
- if (offset === table.offset) {
- if (!readDefaults) {
- return null;
- }
- if (isInteger(fieldType.baseType())) {
- if (isLong(fieldType.baseType())) {
- return field.defaultInteger();
- } else {
- if (field.defaultInteger().high != 0) {
- throw new Error(
- '<=4 byte integer types should not use 64-bit default values.');
- }
- return field.defaultInteger().low;
+ return (t: Table) => {
+ const offset = t.offset + t.bb.__offset(t.offset, field.offset());
+ if (offset === t.offset) {
+ if (!readDefaults) {
+ return null;
}
- } else {
- return field.defaultReal();
+ if (isInteger(fieldType.baseType())) {
+ if (isLong(fieldType.baseType())) {
+ return field.defaultInteger();
+ } else {
+ if (field.defaultInteger().high != 0) {
+ throw new Error(
+ '<=4 byte integer types should not use 64-bit default values.');
+ }
+ return field.defaultInteger().low;
+ }
+ } else {
+ return field.defaultReal();
+ }
}
- }
- return table.readScalar(fieldType.baseType(), offset);
+ return t.readScalar(fieldType.baseType(), offset);
+ };
}
// Reads a string with the given field name from the provided Table.
// If the field is unset, returns null.
readString(table: Table, fieldName: string): string|null {
- const field = this.getField(fieldName, table.typeIndex);
+ return this.readStringLambda(table.typeIndex, fieldName)(table);
+ }
+
+ readStringLambda(typeIndex: number, fieldName: string):
+ (t: Table) => string | null {
+ const field = this.getField(fieldName, typeIndex);
const fieldType = field.type();
if (fieldType.baseType() !== reflection.BaseType.String) {
throw new Error('Field ' + fieldName + ' is not a string.');
}
- const offsetToOffset =
- table.offset + table.bb.__offset(table.offset, field.offset());
- if (offsetToOffset === table.offset) {
- return null;
- }
- return table.bb.__string(offsetToOffset) as string;
+
+ return (t: Table) => {
+ const offsetToOffset =
+ t.offset + t.bb.__offset(t.offset, field.offset());
+ if (offsetToOffset === t.offset) {
+ return null;
+ }
+ return t.bb.__string(offsetToOffset) as string;
+ };
}
// Reads a sub-message from the given Table. The sub-message may either be
// a struct or a Table. Returns null if the sub-message is not set.
readTable(table: Table, fieldName: string): Table|null {
- const field = this.getField(fieldName, table.typeIndex);
+ return this.readTableLambda(table.typeIndex, fieldName)(table);
+ }
+ readTableLambda(typeIndex: number, fieldName: string): (t: Table) => Table|null {
+ const field = this.getField(fieldName, typeIndex);
const fieldType = field.type();
- const parentIsStruct = this.getType(table.typeIndex).isStruct();
+ const parentIsStruct = this.getType(typeIndex).isStruct();
if (fieldType.baseType() !== reflection.BaseType.Obj) {
throw new Error('Field ' + fieldName + ' is not an object type.');
}
if (parentIsStruct) {
- return new Table(
- table.bb, fieldType.index(), table.offset + field.offset());
- }
-
- const offsetToOffset =
- table.offset + table.bb.__offset(table.offset, field.offset());
- if (offsetToOffset === table.offset) {
- return null;
+ return (t: Table) => {
+ return new Table(t.bb, fieldType.index(), t.offset + field.offset());
+ };
}
const elementIsStruct = this.getType(fieldType.index()).isStruct();
- const objectStart =
- elementIsStruct ? offsetToOffset : table.bb.__indirect(offsetToOffset);
- return new Table(table.bb, fieldType.index(), objectStart);
+ return (table: Table) => {
+ const offsetToOffset =
+ table.offset + table.bb.__offset(table.offset, field.offset());
+ if (offsetToOffset === table.offset) {
+ return null;
+ }
+
+ const objectStart = elementIsStruct ? offsetToOffset :
+ table.bb.__indirect(offsetToOffset);
+ return new Table(table.bb, fieldType.index(), objectStart);
+ };
}
// Reads a vector of scalars (like readScalar, may return a vector of Long's
// instead). Also, will return null if the vector is not set.
- readVectorOfScalars(table: Table, fieldName: string): number[]|null {
- const field = this.getField(fieldName, table.typeIndex);
+ readVectorOfScalars(table: Table, fieldName: string): number[]|Long[]|null {
+ return this.readVectorOfScalarsLambda(table.typeIndex, fieldName)(table);
+ }
+
+ readVectorOfScalarsLambda(typeIndex: number, fieldName: string):
+ (t: Table) => number[] | Long[] | null {
+ const field = this.getField(fieldName, typeIndex);
const fieldType = field.type();
if (fieldType.baseType() !== reflection.BaseType.Vector) {
throw new Error('Field ' + fieldName + ' is not an vector.');
@@ -350,24 +391,30 @@
throw new Error('Field ' + fieldName + ' is not an vector of scalars.');
}
- const offsetToOffset =
- table.offset + table.bb.__offset(table.offset, field.offset());
- if (offsetToOffset === table.offset) {
- return null;
- }
- const numElements = table.bb.__vector_len(offsetToOffset);
- const result = [];
- const baseOffset = table.bb.__vector(offsetToOffset);
- const scalarSize = typeSize(fieldType.element());
- for (let ii = 0; ii < numElements; ++ii) {
- result.push(
- table.readScalar(fieldType.element(), baseOffset + scalarSize * ii));
- }
- return result;
+ return (table: Table) => {
+ const offsetToOffset =
+ table.offset + table.bb.__offset(table.offset, field.offset());
+ if (offsetToOffset === table.offset) {
+ return null;
+ }
+ const numElements = table.bb.__vector_len(offsetToOffset);
+ const result = [];
+ const baseOffset = table.bb.__vector(offsetToOffset);
+ const scalarSize = typeSize(fieldType.element());
+ for (let ii = 0; ii < numElements; ++ii) {
+ result.push(table.readScalar(
+ fieldType.element(), baseOffset + scalarSize * ii));
+ }
+ return result;
+ };
}
// Reads a vector of tables. Returns null if vector is not set.
- readVectorOfTables(table: Table, fieldName: string) {
- const field = this.getField(fieldName, table.typeIndex);
+ readVectorOfTables(table: Table, fieldName: string): Table[]|null {
+ return this.readVectorOfTablesLambda(table.typeIndex, fieldName)(table);
+ }
+ readVectorOfTablesLambda(typeIndex: number, fieldName: string):
+ (t: Table) => Table[] | null {
+ const field = this.getField(fieldName, typeIndex);
const fieldType = field.type();
if (fieldType.baseType() !== reflection.BaseType.Vector) {
throw new Error('Field ' + fieldName + ' is not an vector.');
@@ -376,30 +423,37 @@
throw new Error('Field ' + fieldName + ' is not an vector of objects.');
}
- const offsetToOffset =
- table.offset + table.bb.__offset(table.offset, field.offset());
- if (offsetToOffset === table.offset) {
- return null;
- }
- const numElements = table.bb.__vector_len(offsetToOffset);
- const result = [];
- const baseOffset = table.bb.__vector(offsetToOffset);
const elementSchema = this.getType(fieldType.index());
const elementIsStruct = elementSchema.isStruct();
const elementSize = elementIsStruct ? elementSchema.bytesize() :
typeSize(fieldType.element());
- for (let ii = 0; ii < numElements; ++ii) {
- const elementOffset = baseOffset + elementSize * ii;
- result.push(new Table(
- table.bb, fieldType.index(),
- elementIsStruct ? elementOffset :
- table.bb.__indirect(elementOffset)));
- }
- return result;
+
+ return (table: Table) => {
+ const offsetToOffset =
+ table.offset + table.bb.__offset(table.offset, field.offset());
+ if (offsetToOffset === table.offset) {
+ return null;
+ }
+ const numElements = table.bb.__vector_len(offsetToOffset);
+ const result = [];
+ const baseOffset = table.bb.__vector(offsetToOffset);
+ for (let ii = 0; ii < numElements; ++ii) {
+ const elementOffset = baseOffset + elementSize * ii;
+ result.push(new Table(
+ table.bb, fieldType.index(),
+ elementIsStruct ? elementOffset :
+ table.bb.__indirect(elementOffset)));
+ }
+ return result;
+ };
}
// Reads a vector of strings. Returns null if not set.
readVectorOfStrings(table: Table, fieldName: string): string[]|null {
- const field = this.getField(fieldName, table.typeIndex);
+ return this.readVectorOfStringsLambda(table.typeIndex, fieldName)(table);
+ }
+ readVectorOfStringsLambda(typeIndex: number, fieldName: string):
+ (t: Table) => string[] | null {
+ const field = this.getField(fieldName, typeIndex);
const fieldType = field.type();
if (fieldType.baseType() !== reflection.BaseType.Vector) {
throw new Error('Field ' + fieldName + ' is not an vector.');
@@ -408,18 +462,20 @@
throw new Error('Field ' + fieldName + ' is not an vector of strings.');
}
- const offsetToOffset =
- table.offset + table.bb.__offset(table.offset, field.offset());
- if (offsetToOffset === table.offset) {
- return null;
- }
- const numElements = table.bb.__vector_len(offsetToOffset);
- const result = [];
- const baseOffset = table.bb.__vector(offsetToOffset);
- const offsetSize = typeSize(fieldType.element());
- for (let ii = 0; ii < numElements; ++ii) {
- result.push(table.bb.__string(baseOffset + offsetSize * ii));
- }
- return result;
+ return (table: Table) => {
+ const offsetToOffset =
+ table.offset + table.bb.__offset(table.offset, field.offset());
+ if (offsetToOffset === table.offset) {
+ return null;
+ }
+ const numElements = table.bb.__vector_len(offsetToOffset);
+ const result = [];
+ const baseOffset = table.bb.__vector(offsetToOffset);
+ const offsetSize = typeSize(fieldType.element());
+ for (let ii = 0; ii < numElements; ++ii) {
+ result.push(table.bb.__string(baseOffset + offsetSize * ii));
+ }
+ return result;
+ };
}
}
diff --git a/frc971/wpilib/imu_plot_utils.ts b/frc971/wpilib/imu_plot_utils.ts
index ee1a190..7657d37 100644
--- a/frc971/wpilib/imu_plot_utils.ts
+++ b/frc971/wpilib/imu_plot_utils.ts
@@ -6,6 +6,7 @@
import {Point} from 'org_frc971/aos/network/www/plotter';
import {Table} from 'org_frc971/aos/network/www/reflection';
import {ByteBuffer} from 'org_frc971/external/com_github_google_flatbuffers/ts/byte-buffer';
+import {Long} from 'org_frc971/external/com_github_google_flatbuffers/ts/long';
import Schema = configuration.reflection.Schema;
import IMUValuesBatch = imu.frc971.IMUValuesBatch;
@@ -19,7 +20,7 @@
constructor(private readonly schema: Schema) {
super(schema);
}
- private readScalar(table: Table, fieldName: string): number {
+ private readScalar(table: Table, fieldName: string): number|Long|null {
return this.parser.readScalar(table, fieldName);
}
addMessage(data: Uint8Array, time: number): void {
diff --git a/y2020/control_loops/superstructure/turret_plotter.ts b/y2020/control_loops/superstructure/turret_plotter.ts
index 34279ba..488aed9 100644
--- a/y2020/control_loops/superstructure/turret_plotter.ts
+++ b/y2020/control_loops/superstructure/turret_plotter.ts
@@ -7,6 +7,7 @@
import {Point} from 'org_frc971/aos/network/www/plotter';
import {Table} from 'org_frc971/aos/network/www/reflection';
import {ByteBuffer} from 'org_frc971/external/com_github_google_flatbuffers/ts/byte-buffer';
+import {Long} from 'org_frc971/external/com_github_google_flatbuffers/ts/long';
import Connection = proxy.Connection;
import Schema = configuration.reflection.Schema;
@@ -19,7 +20,7 @@
constructor(private readonly schema: Schema) {
super(schema);
}
- private readScalar(table: Table, fieldName: string): number {
+ private readScalar(table: Table, fieldName: string): number|Long|null {
return this.parser.readScalar(table, fieldName);
}