Make object detection work
Signed-off-by: Filip Kujawa <filip.j.kujawa@gmail.com>
Change-Id: Iff6978c55b5f2566c2891c979fed7a20a83bf690
diff --git a/y2023/vision/yolov5.cc b/y2023/vision/yolov5.cc
index 17d4ad5..473437c 100644
--- a/y2023/vision/yolov5.cc
+++ b/y2023/vision/yolov5.cc
@@ -7,8 +7,11 @@
#include <tflite/public/edgetpu.h>
#include <tflite/public/edgetpu_c.h>
+#include <chrono>
#include <opencv2/dnn.hpp>
+#include <string>
+#include "absl/types/span.h"
#include "gflags/gflags.h"
#include "glog/logging.h"
@@ -23,6 +26,8 @@
DEFINE_int32(nthreads, 6, "Number of threads to use during inference.");
+DEFINE_bool(visualize_detections, false, "Display inference output");
+
namespace y2023 {
namespace vision {
@@ -38,7 +43,7 @@
private:
// Convert an OpenCV Mat object to a tensor input
// that can be fed to the TensorFlow Lite model.
- void ConvertCVMatToTensor(cv::Mat src, uint8_t *in);
+ void ConvertCVMatToTensor(cv::Mat src, absl::Span<uint8_t> tensor);
// Resizes, converts color space, and converts
// image data type before inference.
@@ -50,10 +55,10 @@
const int columns);
// Performs non-maximum suppression to remove overlapping bounding boxes.
- void NonMaximumSupression(const std::vector<std::vector<float>> &orig_preds,
- const int rows, const int columns,
- std::vector<Detection> *detections,
- std::vector<int> *indices);
+ std::vector<Detection> NonMaximumSupression(
+ const std::vector<std::vector<float>> &orig_preds, const int rows,
+ const int columns, std::vector<Detection> *detections,
+ std::vector<int> *indices);
// Models
std::unique_ptr<tflite::FlatBufferModel> model_;
std::unique_ptr<tflite::Interpreter> interpreter_;
@@ -71,7 +76,7 @@
int img_width_;
// Input of the interpreter
- uint8_t *input_8_;
+ absl::Span<uint8_t> input_8_;
// Subtract this offset from class labels to get the actual label.
static constexpr int kClassIdOffset = 5;
@@ -80,94 +85,69 @@
std::unique_ptr<YOLOV5> MakeYOLOV5() { return std::make_unique<YOLOV5Impl>(); }
void YOLOV5Impl::LoadModel(const std::string path) {
- LOG(INFO) << "Load model: start";
-
+ VLOG(1) << "Load model: Start";
tflite::ops::builtin::BuiltinOpResolver resolver;
model_ = tflite::FlatBufferModel::VerifyAndBuildFromFile(path.c_str());
-
- /*
- auto model_impl = model_->GetModel();
- model_impl->subgraphs();
- LOG(INFO) << model_impl;
- LOG(INFO) << model_impl->subgraphs();
- auto subgraphs = model_impl->subgraphs();
- LOG(INFO) << subgraphs->size();
- LOG(INFO) << subgraphs->Get(0)->inputs()->size();
- LOG(INFO) << subgraphs->Get(0)->inputs()->Get(0);
- (void)subgraphs;
- */
-
- LOG(INFO) << "Load model: Build Model from file";
-
CHECK(model_);
CHECK(model_->initialized());
+ VLOG(1) << "Load model: Build model from file success";
+
CHECK_EQ(tflite::InterpreterBuilder(*model_, resolver)(&interpreter_),
kTfLiteOk);
- LOG(INFO) << "Load model: Interpreter builder done";
- /*
- LOG(INFO) << &interpreter_->primary_subgraph();
- LOG(INFO) << interpreter_->subgraph(0);
- LOG(INFO) << interpreter_->subgraphs_size();
- LOG(INFO) << interpreter_->subgraph(0)->inputs().size();
- LOG(INFO) << interpreter_->inputs().size();
- */
-
- //interpreter_->SetExternalContext(kTfLiteEdgeTpuContext, edgetpu_context.get());
- // LOG(INFO) << "After set external context";
+ VLOG(1) << "Load model: Interpreter builder success";
size_t num_devices;
std::unique_ptr<edgetpu_device, decltype(&edgetpu_free_devices)> devices(
edgetpu_list_devices(&num_devices), &edgetpu_free_devices);
- //const auto &available_tpus =
- // edgetpu::EdgeTpuManager::GetSingleton()->EnumerateEdgeTpu();
- //LOG(INFO) << "Available tpus: " << available_tpus.size();
-
- LOG(INFO) << "Load model: Getting devices";
CHECK_EQ(num_devices, 1ul);
const auto &device = devices.get()[0];
- (void )device;
- LOG(INFO) << "Load model: Got Device";
+ VLOG(1) << "Load model: Got Devices";
- auto *delegate = edgetpu_create_delegate(device.type, device.path, nullptr, 0);
+ auto *delegate =
+ edgetpu_create_delegate(device.type, device.path, nullptr, 0);
interpreter_->ModifyGraphWithDelegate(delegate);
+ VLOG(1) << "Load model: Modify graph with delegate complete";
TfLiteStatus status = interpreter_->AllocateTensors();
CHECK_EQ(status, kTfLiteOk);
CHECK(interpreter_);
- LOG(INFO) << "Load model: Allocate tensors success";
+ VLOG(1) << "Load model: Allocate tensors success";
+
input_ = interpreter_->inputs()[0];
- LOG(INFO) << "After set inputs";
- LOG(INFO) << input_;
TfLiteIntArray *dims = interpreter_->tensor(input_)->dims;
in_height_ = dims->data[1];
in_width_ = dims->data[2];
in_channels_ = dims->data[3];
in_type_ = interpreter_->tensor(input_)->type;
- input_8_ = interpreter_->typed_tensor<uint8_t>(input_);
-
+
+ int tensor_size = 1;
+ for (int i = 0; i < dims->size; i++) {
+ tensor_size *= dims->data[i];
+ }
+ input_8_ =
+ absl::Span(interpreter_->typed_tensor<uint8_t>(input_), tensor_size);
interpreter_->SetNumThreads(FLAGS_nthreads);
- LOG(INFO) << "End of load";
+ VLOG(1) << "Load model: Done";
}
-void YOLOV5Impl::ConvertCVMatToTensor(cv::Mat src, uint8_t *in) {
+void YOLOV5Impl::ConvertCVMatToTensor(cv::Mat src, absl::Span<uint8_t> tensor) {
CHECK(src.type() == CV_8UC3);
int n = 0, nc = src.channels(), ne = src.elemSize();
- LOG(INFO) << "ConvertCVMatToTensor - Rows " << src.rows;
- LOG(INFO) << "ConvertCVMatToTensor - Cols " << src.cols;
+ VLOG(2) << "ConvertCVMatToTensor: Rows " << src.rows;
+ VLOG(2) << "ConvertCVMatToTensor: Cols " << src.cols;
for (int y = 0; y < src.rows; ++y) {
+ auto *row_ptr = src.ptr<uint8_t>(y);
for (int x = 0; x < src.cols; ++x) {
for (int c = 0; c < nc; ++c) {
- (void)ne;
- (void)n;
- in[n++] = src.data[y * src.step + x * ne + c];
+ tensor[n++] = *(row_ptr + x * ne + c);
}
}
}
@@ -192,7 +172,7 @@
return result_vec;
}
-void YOLOV5Impl::NonMaximumSupression(
+std::vector<Detection> YOLOV5Impl::NonMaximumSupression(
const std::vector<std::vector<float>> &orig_preds, const int rows,
const int columns, std::vector<Detection> *detections,
std::vector<int> *indices)
@@ -204,19 +184,24 @@
for (int i = 0; i < rows; i++) {
if (orig_preds[i][4] > FLAGS_conf_threshold) {
- int left = (orig_preds[i][0] - orig_preds[i][2] / 2) * img_width_;
- int top = (orig_preds[i][1] - orig_preds[i][3] / 2) * img_height_;
- int w = orig_preds[i][2] * img_width_;
- int h = orig_preds[i][3] * img_height_;
+ float x = orig_preds[i][0];
+ float y = orig_preds[i][1];
+ float w = orig_preds[i][2];
+ float h = orig_preds[i][3];
+ int left = static_cast<int>((x - 0.5 * w) * img_width_);
+ int top = static_cast<int>((y - 0.5 * h) * img_height_);
+ int width = static_cast<int>(w * img_width_);
+ int height = static_cast<int>(h * img_height_);
for (int j = 5; j < columns; j++) {
scores.push_back(orig_preds[i][j] * orig_preds[i][4]);
}
cv::minMaxLoc(scores, nullptr, &confidence, nullptr, &class_id);
+ scores.clear();
if (confidence > FLAGS_conf_threshold) {
- Detection detection{cv::Rect(left, top, w, h), confidence,
- class_id.x - kClassIdOffset};
+ Detection detection{cv::Rect(left, top, width, height), confidence,
+ class_id.x};
detections->push_back(detection);
}
}
@@ -230,31 +215,36 @@
confidences.push_back(d.confidence);
}
- (void)indices;
- // TODO(FILIP): Fix linker error.
- // cv::dnn::NMSBoxes(boxes, confidences, FLAGS_conf_threshold,
- // FLAGS_nms_threshold, *indices);
+ cv::dnn::NMSBoxes(boxes, confidences, FLAGS_conf_threshold,
+ FLAGS_nms_threshold, *indices);
+
+ std::vector<Detection> filtered_detections;
+ for (size_t i = 0; i < indices->size(); i++) {
+ filtered_detections.push_back((*detections)[(*indices)[i]]);
+ }
+
+ VLOG(1) << "NonMaximumSupression: " << detections->size() - indices->size()
+ << " detections filtered out";
+
+ return filtered_detections;
}
std::vector<Detection> YOLOV5Impl::ProcessImage(cv::Mat frame) {
+ VLOG(1) << "\n";
+
+ auto start = std::chrono::high_resolution_clock::now();
img_height_ = frame.rows;
img_width_ = frame.cols;
- //Preprocess;
cv::resize(frame, frame, cv::Size(in_height_, in_width_), cv::INTER_CUBIC);
cv::cvtColor(frame, frame, cv::COLOR_BGR2RGB);
frame.convertTo(frame, CV_8U);
- LOG(INFO) << "After preprocess - Before convert to tensor";
ConvertCVMatToTensor(frame, input_8_);
- // Inference
- LOG(INFO) << "Before Invoke";
TfLiteStatus status = interpreter_->Invoke();
CHECK_EQ(status, kTfLiteOk);
- LOG(INFO) << "After invoke, status checked";
-
int output_tensor_index = interpreter_->outputs()[0];
TfLiteIntArray *out_dims = interpreter_->tensor(output_tensor_index)->dims;
int num_rows = out_dims->data[1];
@@ -264,22 +254,48 @@
std::vector<std::vector<float>> orig_preds =
TensorToVector2D(src_tensor, num_rows, num_columns);
- LOG(INFO) << "After tensor to vector 2D";
std::vector<int> indices;
std::vector<Detection> detections;
- NonMaximumSupression(orig_preds, num_rows, num_columns, &detections,
- &indices);
- LOG(INFO) << "After NMS";
- for (size_t i = 0; i < interpreter_->outputs().size(); i++) {
- LOG(INFO) << "Detection #" << i << " | " << interpreter_->outputs()[i];
+ std::vector<Detection> filtered_detections;
+ filtered_detections = NonMaximumSupression(orig_preds, num_rows, num_columns,
+ &detections, &indices);
+ VLOG(1) << "---";
+ for (size_t i = 0; i < filtered_detections.size(); i++) {
+ VLOG(1) << "Detection #" << i << " | Class ID #"
+ << filtered_detections[i].class_id << " @ "
+ << filtered_detections[i].confidence << " confidence";
}
- if (detections.size() > 0) {
- LOG(INFO) << "Detection ID: " << detections[0].class_id;
- LOG(INFO) << "Confidence" << detections[0].confidence;
+
+ VLOG(1) << "---";
+
+ auto stop = std::chrono::high_resolution_clock::now();
+
+ VLOG(1) << "Inference time: "
+ << std::chrono::duration_cast<std::chrono::milliseconds>(stop - start)
+ .count();
+
+ if (FLAGS_visualize_detections) {
+ cv::resize(frame, frame, cv::Size(img_width_, img_height_), 0, 0, true);
+ for (size_t i = 0; i < filtered_detections.size(); i++) {
+ VLOG(1) << "Bounding Box | X: " << filtered_detections[i].box.x
+ << " Y: " << filtered_detections[i].box.y
+ << " W: " << filtered_detections[i].box.width
+ << " H: " << filtered_detections[i].box.height;
+ cv::rectangle(frame, filtered_detections[i].box, cv::Scalar(255, 0, 0),
+ 2);
+ cv::putText(
+ frame, std::to_string(filtered_detections[i].class_id),
+ cv::Point(filtered_detections[i].box.x, filtered_detections[i].box.y),
+ cv::FONT_HERSHEY_COMPLEX, 1.0, cv::Scalar(0, 0, 255), 1, cv::LINE_AA);
+ }
+ cv::cvtColor(frame, frame, cv::COLOR_BGR2RGB);
+ cv::imshow("yolo", frame);
+ cv::waitKey(10);
}
- return detections;
+
+ return filtered_detections;
};
} // namespace vision