Handle subscription of messages in the webapp.
Messages larger than a threshold are split and reassembled due to the
size limit in webrtc. Threshold may have to be adjusted somewhere between
64KiB and 256KiB.
This also includes a basic handler for a ping message and a more
advanced image handler.
Change-Id: If66acfb1bb84e9d3ff686994a94b1480cb70b2aa
diff --git a/y2020/BUILD b/y2020/BUILD
index 1d281bb..c80550b 100644
--- a/y2020/BUILD
+++ b/y2020/BUILD
@@ -125,3 +125,15 @@
srcs = ["__init__.py"],
visibility = ["//visibility:public"],
)
+
+sh_binary(
+ name = "web_proxy",
+ srcs = ["web_proxy.sh"],
+ data = [
+ ":config.json",
+ "//aos/network:web_proxy_main",
+ "//y2020/www:main_bundle",
+ "//y2020/www:files",
+ "//y2020/www:flatbuffers",
+ ],
+)
diff --git a/y2020/vision/BUILD b/y2020/vision/BUILD
index 2e3723d..fa30531 100644
--- a/y2020/vision/BUILD
+++ b/y2020/vision/BUILD
@@ -1,4 +1,4 @@
-load("@com_github_google_flatbuffers//:build_defs.bzl", "flatbuffer_cc_library")
+load("@com_github_google_flatbuffers//:build_defs.bzl", "flatbuffer_cc_library", "flatbuffer_ts_library")
load("//aos:config.bzl", "aos_config")
flatbuffer_cc_library(
@@ -36,3 +36,9 @@
"//aos/events:shm_event_loop",
],
)
+
+flatbuffer_ts_library(
+ name = "vision_ts_fbs",
+ srcs = ["vision.fbs"],
+ visibility = ["//y2020:__subpackages__"],
+)
diff --git a/y2020/vision/vision.fbs b/y2020/vision/vision.fbs
index 66f695b..b8d8bd9 100644
--- a/y2020/vision/vision.fbs
+++ b/y2020/vision/vision.fbs
@@ -14,7 +14,7 @@
// The number of columns in the image.
cols:int;
// The image data.
- data:[byte];
+ data:[ubyte];
// Timestamp when the frame was captured.
monotonic_timestamp_ns:long;
realtime_timestamp_ns:long;
diff --git a/y2020/web_proxy.sh b/y2020/web_proxy.sh
new file mode 100755
index 0000000..8e1a570
--- /dev/null
+++ b/y2020/web_proxy.sh
@@ -0,0 +1 @@
+./aos/network/web_proxy_main --config=y2020/config.json --data_dir=y2020/www
diff --git a/y2020/www/BUILD b/y2020/www/BUILD
new file mode 100644
index 0000000..146456f
--- /dev/null
+++ b/y2020/www/BUILD
@@ -0,0 +1,45 @@
+load("@build_bazel_rules_typescript//:defs.bzl", "ts_library")
+load("@build_bazel_rules_nodejs//:defs.bzl", "rollup_bundle")
+
+ts_library(
+ name = "main",
+ srcs = [
+ "main.ts",
+ "image_handler.ts",
+ ],
+ deps = [
+ "//aos/network/www:proxy",
+ "//y2020/vision:vision_ts_fbs",
+ ],
+ visibility = ["//y2020:__subpackages__"],
+)
+
+rollup_bundle(
+ name = "main_bundle",
+ entry_point = "y2020/www/main",
+ deps = [
+ "main",
+ ],
+ visibility = ["//y2020:__subpackages__"],
+)
+
+filegroup(
+ name = "files",
+ srcs = glob([
+ "**/*.html",
+ "**/*.css",
+ ]),
+ visibility=["//visibility:public"],
+)
+
+genrule(
+ name = "flatbuffers",
+ srcs = [
+ "@com_github_google_flatbuffers//:flatjs",
+ ],
+ outs = [
+ "flatbuffers.js",
+ ],
+ cmd = "cp $(location @com_github_google_flatbuffers//:flatjs) $@",
+ visibility=["//y2020:__subpackages__"],
+)
diff --git a/y2020/www/image_handler.ts b/y2020/www/image_handler.ts
new file mode 100644
index 0000000..abaf831
--- /dev/null
+++ b/y2020/www/image_handler.ts
@@ -0,0 +1,61 @@
+import {frc971} from 'y2020/vision/vision_generated';
+
+export class ImageHandler {
+ private canvas = document.createElement('canvas');
+
+ constructor() {
+ document.body.appendChild(this.canvas);
+ }
+
+ handleImage(data: Uint8Array) {
+ const fbBuffer = new flatbuffers.ByteBuffer(data);
+ const image = frc971.vision.CameraImage.getRootAsCameraImage(fbBuffer);
+
+ const width = image.cols();
+ const height = image.rows();
+ if (width === 0 || height === 0) {
+ return;
+ }
+ const imageBuffer = new Uint8ClampedArray(width * height * 4); // RGBA
+
+ // Read four bytes (YUYV) from the data and transform into two pixels of
+ // RGBA for canvas
+ for (const j = 0; j < height; j++) {
+ for (const i = 0; i < width; i += 2) {
+ const y1 = image.data((j * width + i) * 2);
+ const u = image.data((j * width + i) * 2 + 1);
+ const y2 = image.data((j * width + i + 1) * 2);
+ const v = image.data((j * width + i + 1) * 2 + 1);
+
+ // Based on https://en.wikipedia.org/wiki/YUV#Converting_between_Y%E2%80%B2UV_and_RGB
+ const c1 = y1 - 16;
+ const c2 = y2 - 16;
+ const d = u - 128;
+ const e = v - 128;
+
+ imageBuffer[(j * width + i) * 4 + 0] = (298 * c1 + 409 * e + 128) >> 8;
+ imageBuffer[(j * width + i) * 4 + 1] =
+ (298 * c1 - 100 * d - 208 * e + 128) >> 8;
+ imageBuffer[(j * width + i) * 4 + 2] = (298 * c1 + 516 * d + 128) >> 8;
+ imageBuffer[(j * width + i) * 4 + 3] = 255;
+ imageBuffer[(j * width + i) * 4 + 4] = (298 * c2 + 409 * e + 128) >> 8;
+ imageBuffer[(j * width + i) * 4 + 5] =
+ (298 * c2 - 100 * d - 208 * e + 128) >> 8;
+ imageBuffer[(j * width + i) * 4 + 6] = (298 * c2 + 516 * d + 128) >> 8;
+ imageBuffer[(j * width + i) * 4 + 7] = 255;
+ }
+ }
+
+ const ctx = this.canvas.getContext('2d');
+
+ this.canvas.width = width;
+ this.canvas.height = height;
+ const idata = ctx.createImageData(width, height);
+ idata.data.set(imageBuffer);
+ ctx.putImageData(idata, 0, 0);
+ }
+
+ getId() {
+ return frc971.vision.CameraImage.getFullyQualifiedName();
+ }
+}
diff --git a/y2020/www/index.html b/y2020/www/index.html
new file mode 100644
index 0000000..97e16d4
--- /dev/null
+++ b/y2020/www/index.html
@@ -0,0 +1,11 @@
+<html>
+ <head>
+ <script src="flatbuffers.js"></script>
+ <script src="main_bundle.min.js" defer></script>
+ <link rel="stylesheet" href="styles.css">
+ </head>
+ <body>
+ <div id="config">
+ </div>
+ </body>
+</html>
diff --git a/y2020/www/main.ts b/y2020/www/main.ts
new file mode 100644
index 0000000..7831713
--- /dev/null
+++ b/y2020/www/main.ts
@@ -0,0 +1,11 @@
+import {Connection} from 'aos/network/www/proxy';
+
+import {ImageHandler} from './image_handler';
+
+const conn = new Connection();
+
+conn.connect();
+
+const iHandler = new ImageHandler();
+
+conn.addHandler(iHandler.getId(), (data) => iHandler.handleImage(data));
diff --git a/y2020/www/styles.css b/y2020/www/styles.css
new file mode 100644
index 0000000..23ceb21
--- /dev/null
+++ b/y2020/www/styles.css
@@ -0,0 +1,5 @@
+.channel {
+ display: flex;
+ border-bottom: 1px solid;
+ font-size: 24px;
+}