Add y2022 WebRTC image_streamer.
Change-Id: I262fc87f5ae17d4f3a16cc7a153dcf816c118038
Signed-off-by: Tyler Chatow <tchatow@gmail.com>
diff --git a/y2022/image_streamer/www/BUILD b/y2022/image_streamer/www/BUILD
new file mode 100644
index 0000000..908c5b4
--- /dev/null
+++ b/y2022/image_streamer/www/BUILD
@@ -0,0 +1,52 @@
+load("@npm//@bazel/typescript:index.bzl", "ts_library")
+load("//tools/build_rules:js.bzl", "rollup_bundle")
+load("//frc971/downloader:downloader.bzl", "aos_downloader_dir")
+
+package(default_visibility = ["//visibility:public"])
+
+filegroup(
+ name = "files",
+ srcs = glob([
+ "**/*.html",
+ "**/*.css",
+ ]),
+)
+
+ts_library(
+ name = "proxy",
+ srcs = [
+ "proxy.ts",
+ ],
+ deps = [
+ "//aos/network:web_proxy_ts_fbs",
+ "@com_github_google_flatbuffers//ts:flatbuffers_ts",
+ ],
+)
+
+ts_library(
+ name = "main",
+ srcs = [
+ "main.ts",
+ ],
+ deps = [
+ ":proxy",
+ ],
+)
+
+rollup_bundle(
+ name = "main_bundle",
+ entry_point = "main.ts",
+ deps = [
+ "main",
+ ],
+)
+
+aos_downloader_dir(
+ name = "www_files",
+ srcs = [
+ ":files",
+ ":main_bundle.min.js",
+ ],
+ dir = "image_streamer_www",
+ visibility = ["//visibility:public"],
+)
diff --git a/y2022/image_streamer/www/index.html b/y2022/image_streamer/www/index.html
new file mode 100644
index 0000000..f71750d
--- /dev/null
+++ b/y2022/image_streamer/www/index.html
@@ -0,0 +1,89 @@
+<html>
+ <head>
+ <script type="text/javascript"></script>
+ <script src="main_bundle.min.js" defer></script>
+ <style>
+ code {
+ background-color: #eee;
+ }
+ </style>
+ </head>
+ <body>
+ <div>
+ <video id="stream" autoplay playsinline muted style="height: 95%">
+ Your browser does not support video
+ </video>
+ </div>
+ <p>
+ Stats: <span id="stats_protocol">Not connected</span>, <span id="stats_bps"></span> Kibit/s, <span id="stats_fps"></span> fps
+ </p>
+ <span>
+ <div id="bad_remote_port_error" style="display: none">
+ <p>
+ Remote emitted a port <span id="bad_remote_port_port"></span> which may not be supported over the FMS.
+ Try running <code>image_streamer</code> with <code>-min_port=</code> and <code>-max_port=</code>
+ </p>
+ </div>
+ <div id="bad_local_port_error" style="display: none">
+ <p>
+ Local emitted a port <span id="bad_local_port_port"></span> which may not be supported over the FMS, or may fallback to TCP.
+ </p>
+ <p>To fix:</p>
+ <ul>
+ <li>
+ Firefox: Not supported
+ </li>
+ <li>Chrome:
+ <ul>
+ <li>
+ Windows:
+ <br>
+ Add registry entry <code>Software\Policies\Google\Chrome\WebRtcUdpPortRange = "5800-5810"</code>
+ <br>
+ (For Chromium, <code>Software\Policies\Chromium\WebRtcUdpPortRange</code>)
+ </li>
+ <li>
+ Linux:
+ <br>
+ <code>mkdir -p /etc/opt/chrome/policies/managed</code> OR <code>mkdir -p /etc/chromium/policies/managed</code> OR, ON SOME DISTROS <code>mkdir -p /etc/chromium-browser/policies/managed</code>
+ <br>
+ <code>echo '{"WebRtcUdpPortRange": "5800-5810"}' > /etc/<>/policies/managed/webrtc_port_policy.json</code>
+ </li>
+ </ul>
+ </li>
+ </ul>
+ </div>
+ <div id="bad_remote_address_error" style="display: none">
+ <p>Remote emitted an address <span id="bad_remote_address_address"></span> which requires mDNS resolution. This may not be supported.</p>
+ </div>
+ <div id="bad_local_address_error" style="display: none">
+ <p>Local emitted an address <span id="bad_local_address_address"></span> which requires mDNS resolution. This may not be supported.</p>
+ <p>To fix:</p>
+ <ul>
+ <li>
+ Firefox: Navigate to <a href="about:config">about:config</a> (May need to select and copy into new tab).
+ Set <code>media.peerconnection.ice.obfuscate_host_addresses</code> to <code>false</code>.
+ </li>
+ <li>Chrome:
+ <ul>
+ <li>
+ Windows:
+ <br>
+ Add registry entry <code>Software\Policies\Google\Chrome\WebRtcLocalIpsAllowedUrls\1 = "*<span class="page_hostname"></span>*"</code>
+ <br>
+ (For Chromium, <code>Software\Policies\Chromium\WebRtcLocalIpsAllowedUrls\1</code>)
+ </li>
+ <li>
+ Linux:
+ <br>
+ <code>mkdir -p /etc/opt/chrome/policies/managed</code> OR <code>mkdir -p /etc/chromium/policies/managed</code> OR, ON SOME DISTROS <code>mkdir -p /etc/chromium-browser/policies/managed</code>
+ <br>
+ <code>echo '{"WebRtcLocalIpsAllowedUrls": ["*<span class="page_hostname"></span>*"]}' > /etc/<>/policies/managed/webrtc_policy.json</code>
+ </li>
+ </ul>
+ </li>
+ </ul>
+ </div>
+ </span>
+ </body>
+</html>
diff --git a/y2022/image_streamer/www/main.ts b/y2022/image_streamer/www/main.ts
new file mode 100644
index 0000000..5a3165e
--- /dev/null
+++ b/y2022/image_streamer/www/main.ts
@@ -0,0 +1,5 @@
+import {Connection} from './proxy';
+
+const conn = new Connection();
+
+conn.connect();
diff --git a/y2022/image_streamer/www/proxy.ts b/y2022/image_streamer/www/proxy.ts
new file mode 100644
index 0000000..bfe8135
--- /dev/null
+++ b/y2022/image_streamer/www/proxy.ts
@@ -0,0 +1,248 @@
+import {Builder, ByteBuffer} from 'flatbuffers';
+import {Payload, SdpType, WebSocketIce, WebSocketMessage, WebSocketSdp} from 'org_frc971/aos/network/web_proxy_generated';
+
+// Port 9 is used to indicate an active (outgoing) TCP connection. The server
+// would send a corresponding candidate with the actual TCP port it is
+// listening on. Ignore NaN since it doesn't tell us anything about whether the
+// port selected will have issues through the FMS firewall.
+function validPort(port: number): boolean {
+ return Number.isNaN(port) || port == 9 || (port >= 1180 && port <= 1190) ||
+ (port >= 5800 && port <= 5810);
+}
+
+// Some browsers don't support the port property so provide our own function
+// to get it.
+function getIcePort(candidate: RTCIceCandidate): number {
+ if (candidate.port === undefined) {
+ return Number(candidate.candidate.split(' ')[5]);
+ } else {
+ return candidate.port;
+ }
+}
+
+function isMdnsAddress(address: string): boolean {
+ return address.includes('.local');
+}
+
+function getIceAddress(candidate: RTCIceCandidate): string {
+ if (candidate.address === undefined) {
+ return candidate.candidate.split(' ')[4];
+ } else {
+ return candidate.address;
+ }
+}
+
+export class Connection {
+ private webSocketConnection: WebSocket|null = null;
+ private rtcPeerConnection: RTCPeerConnection|null = null;
+ private html5VideoElement: HTMLMediaElement|null = null;
+ private webSocketUrl: string;
+ private statsInterval: number;
+
+ private candidateNominatedId: string;
+ private lastRtpTimestamp: number = 0;
+ private lastBytesReceived: number = 0;
+ private lastFramesDecoded: number = 0;
+
+
+ constructor() {
+ const server = location.host;
+ this.webSocketUrl = `ws://${server}/ws`;
+
+ for (let elem of document.getElementsByClassName('page_hostname')) {
+ (elem as HTMLElement).innerText = location.hostname;
+ }
+ }
+
+ connect(): void {
+ this.html5VideoElement =
+ (document.getElementById('stream') as HTMLMediaElement);
+
+ this.webSocketConnection = new WebSocket(this.webSocketUrl);
+ this.webSocketConnection.binaryType = 'arraybuffer';
+ this.webSocketConnection.addEventListener(
+ 'message', (e) => this.onWebSocketMessage(e));
+ }
+
+
+ checkRemoteCandidate(candidate: RTCIceCandidate) {
+ const port = getIcePort(candidate);
+ if (!validPort(port)) {
+ document.getElementById('bad_remote_port_port').innerText =
+ port.toString();
+ document.getElementById('bad_remote_port_error').style['display'] =
+ 'inherit';
+ }
+ const address = getIceAddress(candidate);
+ if (isMdnsAddress(address)) {
+ document.getElementById('bad_remote_address_address').innerText = address;
+ document.getElementById('bad_remote_address_error').style['display'] =
+ 'inherit';
+ }
+ }
+
+ checkLocalCandidate(candidate: RTCIceCandidate) {
+ const port = getIcePort(candidate);
+ if (!validPort(port)) {
+ document.getElementById('bad_local_port_port').innerText =
+ port.toString();
+ document.getElementById('bad_local_port_error').style['display'] =
+ 'inherit';
+ }
+ const address = getIceAddress(candidate);
+ if (isMdnsAddress(address)) {
+ document.getElementById('bad_local_address_address').innerText = address;
+ document.getElementById('bad_local_address_error').style['display'] =
+ 'inherit';
+ }
+ }
+
+ onLocalDescription(desc: RTCSessionDescriptionInit): void {
+ console.log('Local description: ' + JSON.stringify(desc));
+ this.rtcPeerConnection.setLocalDescription(desc).then(() => {
+ const builder = new Builder(512);
+ const sdpFb = WebSocketSdp.createWebSocketSdp(
+ builder, SdpType.ANSWER, builder.createString(desc.sdp));
+ const message = WebSocketMessage.createWebSocketMessage(
+ builder, Payload.WebSocketSdp, sdpFb);
+ builder.finish(message);
+ const array = builder.asUint8Array();
+
+ this.webSocketConnection.send(array.buffer.slice(array.byteOffset));
+ });
+ }
+
+ onIncomingSDP(sdp: RTCSessionDescriptionInit): void {
+ console.log('Incoming SDP: ' + JSON.stringify(sdp));
+ this.rtcPeerConnection.setRemoteDescription(sdp);
+ this.rtcPeerConnection.createAnswer().then(
+ (e) => this.onLocalDescription(e));
+ }
+
+ onIncomingICE(ice: RTCIceCandidateInit): void {
+ let candidate = new RTCIceCandidate(ice);
+ console.log('Incoming ICE: ' + JSON.stringify(ice));
+ this.rtcPeerConnection.addIceCandidate(candidate);
+
+ // If end of candidates, won't have a port.
+ if (candidate.candidate !== '') {
+ this.checkRemoteCandidate(candidate);
+ }
+ }
+
+ onRequestStats(track: MediaStreamTrack): void {
+ this.rtcPeerConnection.getStats(track).then((stats) => {
+ // getStats returns a list of stats of various types in an implementation
+ // defined order. We would like to get the protocol in use. This is found
+ // in remote-candidate. However, (again, implementation defined), some
+ // browsers return only remote-candidate's in use, while others return all
+ // of them that attempted negotiation. To figure this out, look at the
+ // currently nominated candidate-pair, then match up it's remote with a
+ // remote-candidate we see later. Since the order isn't defined, store the
+ // id in this in case the remote-candidate comes before candidate-pair.
+
+ for (let dict of stats.values()) {
+ if (dict.type === 'candidate-pair' && dict.nominated) {
+ this.candidateNominatedId = dict.remoteCandidateId;
+ }
+ if (dict.type === 'remote-candidate' &&
+ dict.id === this.candidateNominatedId) {
+ document.getElementById('stats_protocol').innerText = dict.protocol;
+ }
+ if (dict.type === 'inbound-rtp') {
+ const timestamp = dict.timestamp;
+ const bytes_now = dict.bytesReceived;
+ const frames_decoded = dict.framesDecoded;
+
+ document.getElementById('stats_bps').innerText =
+ Math.round(
+ (bytes_now - this.lastBytesReceived) * 8 /* bits */ /
+ 1024 /* kbits */ / (timestamp - this.lastRtpTimestamp) *
+ 1000 /* ms */)
+ .toString();
+
+ document.getElementById('stats_fps').innerText =
+ (Math.round(
+ (frames_decoded - this.lastFramesDecoded) /
+ (timestamp - this.lastRtpTimestamp) * 1000 /* ms */ * 10) /
+ 10).toString();
+
+
+ this.lastRtpTimestamp = timestamp;
+ this.lastBytesReceived = bytes_now;
+ this.lastFramesDecoded = frames_decoded;
+ }
+ }
+ });
+ }
+
+ onAddRemoteStream(event: RTCTrackEvent): void {
+ const stream = event.streams[0];
+ this.html5VideoElement.srcObject = stream;
+
+ const track = stream.getTracks()[0];
+ this.statsInterval =
+ window.setInterval(() => this.onRequestStats(track), 1000);
+ }
+
+ onIceCandidate(event: RTCPeerConnectionIceEvent): void {
+ if (event.candidate == null) {
+ return;
+ }
+
+ console.log(
+ 'Sending ICE candidate out: ' + JSON.stringify(event.candidate));
+
+ const builder = new Builder(512);
+ const iceFb = WebSocketIce.createWebSocketIce(
+ builder, builder.createString(event.candidate.candidate), null,
+ event.candidate.sdpMLineIndex);
+ const message = WebSocketMessage.createWebSocketMessage(
+ builder, Payload.WebSocketIce, iceFb);
+ builder.finish(message);
+ const array = builder.asUint8Array();
+
+ this.webSocketConnection.send(array.buffer.slice(array.byteOffset));
+
+ // If end of candidates, won't have a port.
+ if (event.candidate.candidate !== '') {
+ this.checkLocalCandidate(event.candidate);
+ }
+ }
+
+ // When we receive a websocket message, we need to determine what type it is
+ // and handle appropriately. Either by setting the remote description or
+ // adding the remote ice candidate.
+ onWebSocketMessage(e: MessageEvent): void {
+ const buffer = new Uint8Array(e.data)
+ const fbBuffer = new ByteBuffer(buffer);
+ const message = WebSocketMessage.getRootAsWebSocketMessage(fbBuffer);
+
+ if (!this.rtcPeerConnection) {
+ this.rtcPeerConnection = new RTCPeerConnection();
+ this.rtcPeerConnection.ontrack = (e) => this.onAddRemoteStream(e);
+ this.rtcPeerConnection.onicecandidate = (e) => this.onIceCandidate(e);
+ }
+
+ switch (message.payloadType()) {
+ case Payload.WebSocketSdp:
+ const sdpFb = message.payload(new WebSocketSdp());
+ const sdp:
+ RTCSessionDescriptionInit = {type: 'offer', sdp: sdpFb.payload()};
+
+ this.onIncomingSDP(sdp);
+ break;
+ case Payload.WebSocketIce:
+ const iceFb = message.payload(new WebSocketIce());
+ const candidate = {} as RTCIceCandidateInit;
+ candidate.candidate = iceFb.candidate();
+ candidate.sdpMLineIndex = iceFb.sdpMLineIndex();
+ candidate.sdpMid = iceFb.sdpMid();
+ this.onIncomingICE(candidate);
+ break;
+ default:
+ console.log('got an unknown message');
+ break;
+ }
+ }
+}