Rest of 2016 vision code.
Vision2016Debug:
- Added live debug (debug_reciever);
- Added file replay (blob_stream_replay).
- Add gtk event code.
- Updated code and fixed compile errors after rebase.
- Added useful tools for reference. As per Austins directions.
Change-Id: I7c5e7df01eb09057178bcb99dd3e302ca274ac76
diff --git a/aos/vision/blob/stream_view.h b/aos/vision/blob/stream_view.h
index 9d63e8d..bee1dc9 100644
--- a/aos/vision/blob/stream_view.h
+++ b/aos/vision/blob/stream_view.h
@@ -10,30 +10,32 @@
namespace aos {
namespace vision {
-class BlobStreamViewer {
+class BlobStreamViewer : public DebugViewer {
public:
- BlobStreamViewer() : view_(false) {}
+ BlobStreamViewer() : DebugViewer(false) {}
+ explicit BlobStreamViewer(bool flip) : DebugViewer(flip) {}
+
void Submit(ImageFormat fmt, const BlobList &blob_list) {
SetFormatAndClear(fmt);
DrawBlobList(blob_list, {255, 255, 255});
}
inline void SetFormatAndClear(ImageFormat fmt) {
- if (fmt.w != ptr_.fmt().w || fmt.h != ptr_.fmt().h) {
+ if (!image_.fmt().Equals(fmt)) {
printf("resizing data: %d, %d\n", fmt.w, fmt.h);
- outbuf_.reset(new PixelRef[fmt.ImgSize()]);
- ptr_ = ImagePtr{fmt, outbuf_.get()};
- view_.UpdateImage(ptr_);
+ image_ = ImageValue(fmt);
+ UpdateImage(image_.get());
}
- memset(ptr_.data(), 0, fmt.ImgSize() * sizeof(PixelRef));
+ memset(image_.data(), 0, fmt.ImgSize() * sizeof(PixelRef));
}
inline void DrawBlobList(const BlobList &blob_list, PixelRef color) {
- for (const auto &blob : blob_list) {
- for (int i = 0; i < (int)blob.ranges.size(); ++i) {
- for (const auto &range : blob.ranges[i]) {
+ ImagePtr ptr = img();
+ for (const RangeImage &blob : blob_list) {
+ for (int i = 0; i < (int)blob.ranges().size(); ++i) {
+ for (const auto &range : blob.ranges()[i]) {
for (int j = range.st; j < range.ed; ++j) {
- ptr_.get_px(j, i + blob.min_y) = color;
+ ptr.get_px(j, i + blob.min_y()) = color;
}
}
}
@@ -42,15 +44,16 @@
inline void DrawSecondBlobList(const BlobList &blob_list, PixelRef color1,
PixelRef color2) {
+ ImagePtr ptr = img();
for (const auto &blob : blob_list) {
- for (int i = 0; i < (int)blob.ranges.size(); ++i) {
- for (const auto &range : blob.ranges[i]) {
+ for (int i = 0; i < (int)blob.ranges().size(); ++i) {
+ for (const auto &range : blob.ranges()[i]) {
for (int j = range.st; j < range.ed; ++j) {
- auto px = ptr_.get_px(j, i + blob.min_y);
+ auto px = ptr.get_px(j, i + blob.min_y());
if (px.r == 0 && px.g == 0 && px.b == 0) {
- ptr_.get_px(j, i + blob.min_y) = color1;
+ ptr.get_px(j, i + blob.min_y()) = color1;
} else {
- ptr_.get_px(j, i + blob.min_y) = color2;
+ ptr.get_px(j, i + blob.min_y()) = color2;
}
}
}
@@ -58,13 +61,13 @@
}
}
- DebugViewer *view() { return &view_; }
+ // Backwards compatible.
+ DebugViewer *view() { return this; }
+
+ ImagePtr img() { return image_.get(); }
private:
- std::unique_ptr<PixelRef[]> outbuf_;
- ImagePtr ptr_;
-
- DebugViewer view_;
+ ImageValue image_;
};
} // namespace vision
diff --git a/aos/vision/events/BUILD b/aos/vision/events/BUILD
index e003dc9..2b5b9e7 100644
--- a/aos/vision/events/BUILD
+++ b/aos/vision/events/BUILD
@@ -1,3 +1,4 @@
+load('/tools/build_rules/gtk_dependent', 'gtk_dependent_cc_binary', 'gtk_dependent_cc_library')
package(default_visibility = ["//visibility:public"])
cc_library(
@@ -56,3 +57,12 @@
'//aos/testing:googletest',
],
)
+
+gtk_dependent_cc_library(
+ name = "gtk_event",
+ srcs = ["gtk_event.cc"],
+ deps = [
+ ":epoll_events",
+ '@usr_repo//:gtk+-3.0',
+ ],
+)
diff --git a/aos/vision/events/epoll_events.cc b/aos/vision/events/epoll_events.cc
index f191259..e4f789f 100644
--- a/aos/vision/events/epoll_events.cc
+++ b/aos/vision/events/epoll_events.cc
@@ -42,17 +42,10 @@
}
event->ReadEvent();
}
-
- for (EpollWatcher *watcher : watchers_) {
- watcher->Wake();
- }
}
}
void EpollLoop::AddWait(EpollWait *wait) { waits_.push_back(wait); }
-void EpollLoop::AddWatcher(EpollWatcher *watcher) {
- watchers_.push_back(watcher);
-}
// Calculates the new timeout value to pass to epoll_wait.
int EpollLoop::CalculateTimeout() {
diff --git a/aos/vision/events/epoll_events.h b/aos/vision/events/epoll_events.h
index d7574f9..ee288c4 100644
--- a/aos/vision/events/epoll_events.h
+++ b/aos/vision/events/epoll_events.h
@@ -17,6 +17,7 @@
// Performs an asychronous wait using an EpollLoop.
//
// Note: this does not have very high resolution (sub-millisecond).
+// TODO(parker): This is mostly broken.
class EpollWait {
public:
virtual ~EpollWait() {}
@@ -37,10 +38,15 @@
int Recalculate(const monotonic_clock::time_point now) {
if (time_ < monotonic_clock::epoch()) return -1;
if (time_ <= now) {
- Done();
time_ = monotonic_clock::time_point(::std::chrono::seconds(-1));
- return -1;
+ Done();
}
+ // Duplicate above to allow Done to change itself.
+ if (time_ < monotonic_clock::epoch()) return -1;
+ if (time_ <= now) {
+ return -1;// Recalculate(now);
+ }
+
if (time_ - now > ::std::chrono::milliseconds(INT_MAX)) {
return INT_MAX;
} else {
@@ -75,17 +81,6 @@
EpollLoop *loop_ = nullptr;
};
-// Provides a way for code to be notified every time after events are handled by
-// an EpollLoop. This is mainly a hack for the GTK integration and testing;
-// think very carefully before using it anywhere else.
-class EpollWatcher {
- public:
- virtual ~EpollWatcher() {}
-
- // Called after events have been processed each time the event loop wakes up.
- virtual void Wake() = 0;
-};
-
// A file descriptor based event loop implemented with epoll.
class EpollLoop {
public:
@@ -95,7 +90,6 @@
// None of these take ownership of the passed-in objects.
void AddWait(EpollWait *wait);
void Add(EpollEvent *event);
- void AddWatcher(EpollWatcher *watcher);
// Delete event. Note that there are caveats here as this is
// not idiot proof.
@@ -111,6 +105,10 @@
// Loops forever, handling events.
void Run();
+ // Fuses with gtk_main().
+ // Note that the dep for this is separate: //aos/vision/events:gtk_event
+ void RunWithGtkMain();
+
private:
int epoll_fd() { return epoll_fd_.get(); }
@@ -118,7 +116,6 @@
::aos::ScopedFD epoll_fd_;
::std::vector<EpollWait *> waits_;
- ::std::vector<EpollWatcher *> watchers_;
};
} // namespace events
diff --git a/aos/vision/events/gtk_event.cc b/aos/vision/events/gtk_event.cc
new file mode 100644
index 0000000..e8b43aa
--- /dev/null
+++ b/aos/vision/events/gtk_event.cc
@@ -0,0 +1,68 @@
+#include <gtk/gtk.h>
+#include <gdk/gdk.h>
+#include <thread>
+#include <sys/epoll.h>
+#include <mutex>
+#include <condition_variable>
+
+#include "aos/vision/events/epoll_events.h"
+
+namespace aos {
+namespace events {
+
+void EpollLoop::RunWithGtkMain() {
+ int timeout;
+ static constexpr size_t kNumberOfEvents = 64;
+ epoll_event events[kNumberOfEvents];
+ int number_events = 0;
+
+ std::mutex m;
+ std::condition_variable cv;
+ bool all_events_handled = false;
+ auto handle_cb = [&]() {
+ {
+ std::unique_lock<std::mutex> lk(m);
+
+ for (int i = 0; i < number_events; i++) {
+ EpollEvent *event = static_cast<EpollEvent *>(events[i].data.ptr);
+ if ((events[i].events & ~(EPOLLIN | EPOLLPRI)) != 0) {
+ LOG(FATAL, "unexpected epoll events set in %x on %d\n",
+ events[i].events, event->fd());
+ }
+ event->ReadEvent();
+ }
+ timeout = CalculateTimeout();
+
+ all_events_handled = true;
+ }
+ cv.notify_one();
+ };
+ handle_cb();
+ using HandleCBType = decltype(handle_cb);
+
+ std::thread t([&]() {
+ std::unique_lock<std::mutex> lk(m);
+ while (true) {
+ cv.wait(lk, [&all_events_handled]{return all_events_handled;});
+ // Wait for handle_cb to be done.
+ number_events = PCHECK(epoll_wait(epoll_fd(), events, kNumberOfEvents, timeout));
+ all_events_handled = false;
+ // Trigger handle_cb on main_thread to avoid concurrency.
+ gdk_threads_add_idle(+[](gpointer user_data) -> gboolean {
+ auto& handle_cb = *reinterpret_cast<HandleCBType*>(user_data);
+ handle_cb();
+ return G_SOURCE_REMOVE;
+ }, &handle_cb);
+ }
+ });
+ gtk_main();
+
+ // TODO(parker): Allow concurrent proxy onto the normal thread just like Gtk
+ // in order to allow event loop fusion, and make event addition thread-safe.
+
+ // Avoid stack destructors (explicitly shutting down of the thread.)
+ exit(EXIT_SUCCESS);
+}
+
+} // namespace events
+} // namespace aos
diff --git a/aos/vision/events/tcp_client.h b/aos/vision/events/tcp_client.h
index e7b80a6..7ce01f7 100644
--- a/aos/vision/events/tcp_client.h
+++ b/aos/vision/events/tcp_client.h
@@ -1,5 +1,5 @@
-#ifndef _AOS_VISION_DEBUG_TCP_SERVER_H_
-#define _AOS_VISION_DEBUG_TCP_SERVER_H_
+#ifndef _AOS_VISION_DEBUG_TCP_CLIENT_H_
+#define _AOS_VISION_DEBUG_TCP_CLIENT_H_
#include "aos/vision/events/epoll_events.h"
@@ -19,4 +19,4 @@
} // namespace events
} // namespace aos
-#endif // _AOS_VISION_DEBUG_TCP_SERVER_H_
+#endif // _AOS_VISION_DEBUG_TCP_CLIENT_H_
diff --git a/aos/vision/image/jpeg_routines.cc b/aos/vision/image/jpeg_routines.cc
index e0c93dc..b6be41b 100644
--- a/aos/vision/image/jpeg_routines.cc
+++ b/aos/vision/image/jpeg_routines.cc
@@ -224,7 +224,7 @@
int step = cinfo.num_components * cinfo.image_width;
unsigned char *buffers[cinfo.image_height];
for (size_t i = 0; i < cinfo.image_height; ++i) {
- buffers[i] = reinterpret_cast<unsigned char *>(&out[offset]);
+ buffers[i] = &reinterpret_cast<unsigned char *>(out)[offset];
offset += step;
}
diff --git a/aos/vision/tools/BUILD b/aos/vision/tools/BUILD
new file mode 100644
index 0000000..56bbb5b
--- /dev/null
+++ b/aos/vision/tools/BUILD
@@ -0,0 +1,19 @@
+load('/tools/build_rules/gtk_dependent', 'gtk_dependent_cc_binary', 'gtk_dependent_cc_library')
+
+gtk_dependent_cc_binary(name = "jpeg_vision_test",
+ srcs = ["jpeg_vision_test.cc"],
+ deps = [
+ "//aos/common/logging:logging",
+ "//aos/common/logging:implementations",
+ "//aos/vision/math:vector",
+ "//aos/vision/image:reader",
+ "//aos/vision/image:jpeg_routines",
+ "//aos/vision/blob:threshold",
+ "//aos/vision/blob:range_image",
+ "//aos/vision/events:epoll_events",
+ "//aos/vision/blob:stream_view",
+ "//aos/vision/image:image_stream",
+ "//aos/vision/events:tcp_server",
+ "//aos/vision/events:gtk_event",
+ ],
+)
diff --git a/aos/vision/tools/jpeg_vision_test.cc b/aos/vision/tools/jpeg_vision_test.cc
new file mode 100644
index 0000000..37d9ffe
--- /dev/null
+++ b/aos/vision/tools/jpeg_vision_test.cc
@@ -0,0 +1,100 @@
+// This file is to collect data from a camera to use for debug and testing. This
+// should not placed on a robot. This is okay as it is a utility of limited use
+// only.
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <netdb.h>
+#include <poll.h>
+#include <string.h>
+#include <gtk/gtk.h>
+#include <vector>
+#include <memory>
+#include <fstream>
+
+#include "aos/common/logging/logging.h"
+#include "aos/common/logging/implementations.h"
+#include "aos/vision/math/vector.h"
+#include "aos/vision/image/reader.h"
+#include "aos/vision/image/jpeg_routines.h"
+#include "aos/vision/blob/threshold.h"
+#include "aos/vision/blob/range_image.h"
+#include "aos/vision/blob/stream_view.h"
+#include "aos/vision/events/epoll_events.h"
+#include "aos/vision/image/image_stream.h"
+#include "aos/vision/events/tcp_server.h"
+
+namespace aos {
+namespace vision {
+
+// Connects up a camera with our processing.
+class ChannelImageStream : public ImageStreamEvent {
+ public:
+ ChannelImageStream(const std::string &fname,
+ const camera::CameraParams ¶ms)
+ : ImageStreamEvent(fname, params), view_(true) {
+ // Lambda to record image data to a file on key press.
+ view_.view()->key_press_event = [this](uint32_t /*keyval*/) {
+ std::ofstream ofs("/tmp/test.jpg", std::ofstream::out);
+ ofs << prev_data_;
+ ofs.close();
+ };
+ }
+
+ // Handle an image from the camera.
+ void ProcessImage(DataRef data,
+ aos::monotonic_clock::time_point /*timestamp*/) {
+ ImageFormat fmt = GetFmt(data);
+ if (!fmt.Equals(view_.img().fmt())) view_.SetFormatAndClear(fmt);
+ if (!ProcessJpeg(data, view_.img().data())) return;
+
+ ImagePtr img_ptr = view_.img();
+ prev_data_ = data.to_string();
+
+
+ // Threshold the image with the given lambda.
+ RangeImage rimg = DoThreshold(img_ptr, [](PixelRef &px) {
+ if (px.g > 88) {
+ uint8_t min = std::min(px.b, px.r);
+ uint8_t max = std::max(px.b, px.r);
+ if (min >= px.g || max >= px.g) return false;
+ uint8_t a = px.g - min;
+ uint8_t b = px.g - max;
+ return (a > 10 && b > 10);
+ }
+ return false;
+ });
+
+ view_.DrawBlobList({rimg}, {255, 255, 255});
+
+ view_.Redraw();
+ }
+
+ private:
+ std::string prev_data_;
+
+ // responsible for handling drawing
+ BlobStreamViewer view_;
+};
+} // namespace aos
+} // namespace vision
+
+int main(int argc, char *argv[]) {
+ ::aos::logging::Init();
+ ::aos::logging::AddImplementation(
+ new ::aos::logging::StreamLogImplementation(stdout));
+ aos::events::EpollLoop loop;
+ gtk_init(&argc, &argv);
+
+ camera::CameraParams params = {.width = 640 * 2,
+ .height = 480 * 2,
+ .exposure = 10,
+ .brightness = 128,
+ .gain = 0,
+ .fps = 10};
+
+ aos::vision::ChannelImageStream strm1("/dev/video1", params);
+
+ loop.Add(&strm1);
+ loop.RunWithGtkMain();
+}