diff options
-rw-r--r-- | CMakeLists.txt | 59 | ||||
-rw-r--r-- | mobilenet.cpp | 212 | ||||
-rw-r--r-- | mobilenetssd.cpp | 210 |
3 files changed, 481 insertions, 0 deletions
diff --git a/CMakeLists.txt b/CMakeLists.txt new file mode 100644 index 0000000..f8c4ff5 --- /dev/null +++ b/CMakeLists.txt @@ -0,0 +1,59 @@ +cmake_minimum_required(VERSION 3.20) + +project( HexagonTFLiteDemos ) +find_package( OpenCV REQUIRED ) +set(CMAKE_SYSTEM_NAME Linux) +set(CMAKE_SYSTEM_PROCESSOR aarch64) +set(CMAKE_C_COMPILER aarch64-linux-gnu-gcc) +set(CMAKE_CXX_COMPILER aarch64-linux-gnu-g++) +set(GCC_COMPILER_VERSION "" CACHE STRING "GCC Compiler version") +set(GNU_MACHINE "aarch64-linux-gnu" CACHE STRING "GNU compiler triple") + + +if (NOT OPENCV_CROSS_INSTALL_DIR) + message(FATAL_ERROR + "Please specify path to OpenCV install directory using OPENCV_CROSS_INSTALL_DIR in cmdline.\n") +endif() + +if (NOT TFLITE_BUILD_DIR) + message(FATAL_ERROR + "Please specify path to TensorFlowLite build directory using TFLITE_BUILD_DIR in cmdline.\n") +endif() + +if (NOT TFLITE_SRC_DIR) + message(FATAL_ERROR + "Please specify path to TensorFlowLite build directory using TFLITE_SRC_DIR in cmdline.\n") +endif() + +include_directories(${OPENCV_CROSS_INSTALL_DIR}/include/opencv4/ ${TFLITE_SRC_DIR} ) +include_directories( ${TFLITE_BUILD_DIR}/bin/external/flatbuffers/_virtual_includes/flatbuffers/ ) +add_executable(MobileNetSSD mobilenetssd.cpp ) +add_executable(MobileNet mobilenet.cpp ) +link_directories(${OPENCV_CROSS_INSTALL_DIR}/lib) +link_directories(${TFLITE_BUILD_DIR}/bin/tensorflow/lite/) +set(CMAKE_LIBRARY_PATH ${TFLITE_BUILD_DIR}/bin/tensorflow/lite) +target_link_libraries( MobileNetSSD ${TFLITE_BUILD_DIR}/bin/tensorflow/lite/delegates/hexagon/libhexagon_delegate.a) +target_link_libraries( MobileNetSSD ${TFLITE_BUILD_DIR}/bin/tensorflow/lite/delegates/hexagon/libhexagon_implementation.a) +target_link_libraries( MobileNetSSD ${TFLITE_BUILD_DIR}/bin/tensorflow/lite/delegates/hexagon/libhexagon_delegate_kernel.a) +target_link_libraries( MobileNetSSD ${TFLITE_BUILD_DIR}/bin/tensorflow/lite/delegates/hexagon/libutils.a) +target_link_libraries( MobileNetSSD ${TFLITE_BUILD_DIR}/bin/tensorflow/lite/delegates/hexagon/builders/libop_builder.a) +target_link_libraries( MobileNetSSD ${TFLITE_BUILD_DIR}/bin/tensorflow/lite/delegates/libserialization.a) +target_link_libraries( MobileNetSSD ${TFLITE_BUILD_DIR}/bin/tensorflow/lite/delegates/utils/libsimple_delegate.a) +target_link_libraries( MobileNetSSD ${TFLITE_BUILD_DIR}/bin/tensorflow/lite/tools/evaluation/libutils.a) +target_link_libraries( MobileNetSSD ${TFLITE_BUILD_DIR}/bin/tensorflow/lite/delegates/libutils.a) +target_link_libraries( MobileNetSSD ${TFLITE_BUILD_DIR}/bin/external/farmhash_archive/libfarmhash.a) + +target_link_libraries( MobileNet ${TFLITE_BUILD_DIR}/bin/tensorflow/lite/delegates/hexagon/libhexagon_delegate.a) +target_link_libraries( MobileNet ${TFLITE_BUILD_DIR}/bin/tensorflow/lite/delegates/hexagon/libhexagon_implementation.a) +target_link_libraries( MobileNet ${TFLITE_BUILD_DIR}/bin/tensorflow/lite/delegates/hexagon/libhexagon_delegate_kernel.a) +target_link_libraries( MobileNet ${TFLITE_BUILD_DIR}/bin/tensorflow/lite/delegates/hexagon/libutils.a) +target_link_libraries( MobileNet ${TFLITE_BUILD_DIR}/bin/tensorflow/lite/delegates/hexagon/builders/libop_builder.a) +target_link_libraries( MobileNet ${TFLITE_BUILD_DIR}/bin/tensorflow/lite/delegates/libserialization.a) +target_link_libraries( MobileNet ${TFLITE_BUILD_DIR}/bin/tensorflow/lite/delegates/utils/libsimple_delegate.a) +target_link_libraries( MobileNet ${TFLITE_BUILD_DIR}/bin/tensorflow/lite/tools/evaluation/libutils.a) +target_link_libraries( MobileNet ${TFLITE_BUILD_DIR}/bin/tensorflow/lite/delegates/libutils.a) +target_link_libraries( MobileNet ${TFLITE_BUILD_DIR}/bin/external/farmhash_archive/libfarmhash.a) + + +target_link_libraries( MobileNetSSD -ldl -lstdc++ -lm -ltensorflowlite -L${TFLITE_BUILD_DIR}/bin/tensorflow/lite -lopencv_dnn -lopencv_highgui -lopencv_photo -lopencv_stitching -lopencv_core -lopencv_video -lopencv_videoio -ldl -lopencv_imgcodecs -lopencv_calib3d -lopencv_features2d -lopencv_flann -lopencv_imgproc -lopencv_core -L${OPENCV_CROSS_INSTALL_DIR}/lib) +target_link_libraries( MobileNet -ldl -lstdc++ -lm -ltensorflowlite -L${TFLITE_BUILD_DIR}/bin/tensorflow/lite -lopencv_dnn -lopencv_highgui -lopencv_photo -lopencv_stitching -lopencv_core -lopencv_video -lopencv_videoio -ldl -lopencv_imgcodecs -lopencv_calib3d -lopencv_features2d -lopencv_flann -lopencv_imgproc -lopencv_core -L${OPENCV_CROSS_INSTALL_DIR}/lib) diff --git a/mobilenet.cpp b/mobilenet.cpp new file mode 100644 index 0000000..621b09e --- /dev/null +++ b/mobilenet.cpp @@ -0,0 +1,212 @@ +#include <iostream> +#include <fstream> +#include <cstdlib> +#include <vector> +#include <string> +#include <iterator> +#include <unordered_map> +#include <algorithm> +#include <algorithm> +#include <functional> +#include <queue> +#include <stdlib.h> +#include <stdio.h> +#include <ctype.h> +#include <math.h> +#include <stdint.h> +#include <assert.h> +#include <dirent.h> + +#include <opencv2/core.hpp> +#include <opencv2/videoio.hpp> +#include <opencv2/highgui.hpp> +#include <opencv2/imgproc.hpp> + +#include "tensorflow/lite/kernels/register.h" +#include <tensorflow/lite/model.h> +#include <tensorflow/lite/interpreter.h> +#include <tensorflow/lite/delegates/hexagon/hexagon_delegate.h> + +#include <tensorflow/lite/kernels/register.h> +#include <tensorflow/lite/optional_debug_tools.h> +#include <tensorflow/lite/profiling/profiler.h> +#include <tensorflow/lite/string_util.h> +#include <tensorflow/lite/tools/command_line_flags.h> +#include <tensorflow/lite/tools/delegates/delegate_provider.h> +#include <tensorflow/lite/builtin_op_data.h> + +using namespace cv; +using std::cout; using std::cerr; using std::endl; + +#define IMG_HEIGHT 224 +#define IMG_WIDTH 224 +#define IMG_DEPTH 3 +#define IMG_ELEMENT_SIZE 1 +#define BUFFER_SIZE (IMG_ELEMENT_SIZE * IMG_WIDTH * IMG_HEIGHT * IMG_DEPTH) +#define NUM_CATEGORIES 1001 +#define IMG_DIR "/home/linaro/apps/256_ObjectCategories-jpegs/" +#define NUM_FRAMES 16 +#define MODEL_NAME "/home/linaro/apps/mobilenet_quant_v1_224.tflite" +#define LABELS_FILE "/home/linaro/apps/labels.txt" + +std::vector<std::string> labels; + +int read_labels_from_file(void) +{ + std::ifstream ifs(LABELS_FILE); + if (!ifs.is_open()) { + cout << "File " << LABELS_FILE << " not found" << endl; + return -1; + } + std::string line; + + while (std::getline(ifs, line)) + labels.push_back(line); + + return 0; +} + +struct indexed_data { + float f; + int i; +}; + +int indexed_data_sort(const void *va, const void *vb) +{ + const struct indexed_data *a = (const struct indexed_data *)va; + const struct indexed_data *b = (const struct indexed_data *)vb; + + if (a->f > b->f) return -1; + if (a->f < b->f) return 1; + return 0; +} + +struct indexed_data data[NUM_CATEGORIES]; + +static float get_top5_match(uint8_t *values, int n_entries, + Mat *frame) +{ + const float threshold = 0.001f; + int i, k; + char label[256]; + int fontface = cv::FONT_HERSHEY_PLAIN; + double scale = 1.0; + int thickness = 1; + int baseline = 0; + float fvalue; + int count = 0; + + for (i = 0; i < NUM_CATEGORIES; i++) { + float fvalue = values[i] / 255.0; + if (fvalue > threshold) { + data[count].f = fvalue; + data[count].i = i; + count++; + } + } + qsort(data, count, sizeof(struct indexed_data), indexed_data_sort); + for (i = 0; i < 5; i++) { + k = (i + 1)*20; + sprintf(label, "%02.2f%%, %s", + data[i].f * 100, labels[data[i].i].c_str()); + cv::Size text = cv::getTextSize(label, fontface, scale, + thickness, &baseline); + cv::rectangle(*frame, cv::Point(30, k) + cv::Point(0, baseline), + cv::Point(30, k) + cv::Point(text.width, -text.height), + CV_RGB(0,0,0), cv::FILLED); + cv::putText(*frame, label, cv::Point(30,k), cv::FONT_HERSHEY_PLAIN, + 1.0, cv::Scalar(57,255,20), 1, LINE_8); + + } + return 0; +} + +int main(int, char**) +{ + std::unique_ptr<tflite::FlatBufferModel> model; + tflite::ops::builtin::BuiltinOpResolver resolver; + std::unique_ptr<tflite::Interpreter> interpreter; + TfLiteHexagonDelegateOptions params = {0}; + const char* path = "/dsp/cdsp/"; + uint8_t *input_8, *output_8; + struct dirent *dentry; + Mat frame, rsz_frame; + size_t label_count; + char img_path[1024]; + int ret, idx = 0; + char fname[16]; + DIR *d; + + model = tflite::FlatBufferModel::BuildFromFile(MODEL_NAME); + if (!model) { + std::cout << "Failed to mmap model "; + exit(-1); + } + tflite::InterpreterBuilder(*model, resolver)(&interpreter); + if (!interpreter) { + std::cout << "Failed to construct interpreter"; + exit(-1); + } + interpreter->SetAllowFp16PrecisionForFp32(false); + interpreter->SetNumThreads(4); + + std::cout << "Enabling Hexagon Delegate!\n"; + + TfLiteHexagonInitWithPath(path); + auto *delegate = TfLiteHexagonDelegateCreate(¶ms); + + interpreter->ModifyGraphWithDelegate(delegate); + + if (interpreter->AllocateTensors() != kTfLiteOk) { + std::cout << "Failed to allocate tensors!" << endl; + exit(-1); + } + + const std::vector<int> inputs = interpreter->inputs(); + int input = interpreter->inputs()[0]; + int output = interpreter->outputs()[0]; + + TfLiteIntArray* output_dims = interpreter->tensor(output)->dims; + auto output_size = output_dims->data[output_dims->size - 1]; + TfLiteType input_type = interpreter->tensor(input)->type; + + /* We ONLY support UInt8 on this quantized model */ + input_8 = interpreter->typed_tensor<uint8_t>(input); + output_8 = interpreter->typed_output_tensor<uint8_t>(0); + + Mat rgb_mat(IMG_HEIGHT, IMG_WIDTH, CV_8UC3, input_8); + + if (read_labels_from_file()) + exit(-1); + + d = opendir(IMG_DIR); + if (!d) { + printf("Error opening %s directory\n", IMG_DIR); + return -1; + } + + while ((dentry = readdir(d)) != NULL) { + if (!strcmp(dentry->d_name, ".") || !strcmp(dentry->d_name, "..")) + continue; + idx = idx%NUM_FRAMES; + sprintf(fname, "Frame%d\n", idx++); + sprintf(img_path, "%s%s",IMG_DIR, dentry->d_name); + frame = imread(img_path, 1); + + cv::resize(frame, frame, cv::Size(IMG_HEIGHT, IMG_WIDTH)); + cv::cvtColor(frame, rgb_mat, cv::COLOR_BGR2RGB); + + if (interpreter->Invoke() != kTfLiteOk) { + std::cout << "Failed to invoke tflite!"; + exit(-1); + } + + get_top5_match(output_8, output_size, &rgb_mat); + imshow(fname, rgb_mat); + + waitKey(1); + } + + closedir(d); + return 0; +} diff --git a/mobilenetssd.cpp b/mobilenetssd.cpp new file mode 100644 index 0000000..2728f84 --- /dev/null +++ b/mobilenetssd.cpp @@ -0,0 +1,210 @@ +#include <iostream> +#include <fstream> +#include <cstdlib> +#include <vector> +#include <string> +#include <iterator> +#include <unordered_map> +#include <algorithm> +#include <algorithm> +#include <functional> +#include <queue> +#include <stdlib.h> +#include <stdio.h> +#include <ctype.h> +#include <math.h> +#include <stdint.h> +#include <assert.h> +#include <dirent.h> + +#include <opencv2/core.hpp> +#include <opencv2/videoio.hpp> +#include <opencv2/highgui.hpp> +#include <opencv2/imgproc.hpp> + +#include <tensorflow/lite/kernels/register.h> +#include <tensorflow/lite/model.h> +#include <tensorflow/lite/interpreter.h> +#include <tensorflow/lite/delegates/hexagon/hexagon_delegate.h> + +#include <tensorflow/lite/kernels/register.h> +#include <tensorflow/lite/optional_debug_tools.h> +#include <tensorflow/lite/profiling/profiler.h> +#include <tensorflow/lite/string_util.h> +#include <tensorflow/lite/tools/command_line_flags.h> +#include <tensorflow/lite/tools/delegates/delegate_provider.h> +#include <tensorflow/lite/builtin_op_data.h> + +using namespace cv; +using std::cout; using std::cerr; using std::endl; + +#define IMG_HEIGHT 300 +#define IMG_WIDTH 300 +#define IMG_DEPTH 3 +#define IMG_ELEMENT_SIZE 1 +#define BUFFER_SIZE (IMG_ELEMENT_SIZE * IMG_WIDTH * IMG_HEIGHT * IMG_DEPTH) +#define MODEL_NAME "/home/linaro/apps/coco_ssd_mobilenet_v1_1.0_quant_2018_06_29.tflite" +#define LABELS_FILE "/home/linaro/apps/labelmap.txt" + +std::vector<std::string> labels; + +int read_labels_from_file(void) +{ + std::ifstream ifs(LABELS_FILE); + if (!ifs.is_open()) { + cout << "File " << LABELS_FILE << " not found" << endl; + return -1; + } + std::string line; + + while (std::getline(ifs, line)) + labels.push_back(line); + + return 0; +} + +/* Max detection Items */ +#define MAX_ITEMS 10 + +/* Minimum Accuracy required in percentage */ +#define MIN_ACCURACY 40 + +struct SSDbox { + float top; + float left; + float bottom; + float right; + uint32_t id; + float score; +}; +static struct SSDbox SSDBoxes[MAX_ITEMS]; + +void SSDshowOutput(float *scores, float *mclasses, float *boxes, Mat *frame) +{ + int fontface = cv::FONT_HERSHEY_PLAIN; + double scale = 1.0; + int thickness = 1; + char label[256]; + int baseline = 0; + int i; + + for ( i = 0; i < MAX_ITEMS; i++) { + SSDBoxes[i].top = frame->rows * boxes[i * 4]; + SSDBoxes[i].left = frame->cols * boxes[i * 4 + 1]; + SSDBoxes[i].bottom = frame->rows * boxes[i * 4 + 2]; + SSDBoxes[i].right = frame->cols * boxes[i * 4 + 3]; + SSDBoxes[i].id = (uint32_t)mclasses[i] + 1; + SSDBoxes[i].score = scores[i] * 100.00; + } + + for ( i = 0; i < MAX_ITEMS; i++) { + if (SSDBoxes[i].score < MIN_ACCURACY) + break; +#if DEBUG + std::cout << i <<":\t" << SSDBoxes[i].score << ":\t" << + labels[(SSDBoxes[i].id) ] <<":\t(" << + (uint32_t) SSDBoxes[i].top << ", " << + (uint32_t)SSDBoxes[i].left << ", " << + (uint32_t) SSDBoxes[i].bottom << ", " << + (uint32_t) SSDBoxes[i].right << ")"<< std::endl; +#endif + sprintf(label, "%02.2f%%, %s", SSDBoxes[i].score, + labels[SSDBoxes[i].id].c_str()); + cv::rectangle(*frame, cv::Point((uint32_t) SSDBoxes[i].left, (uint32_t)SSDBoxes[i].top), + cv::Point( (uint32_t) SSDBoxes[i].right, (uint32_t) SSDBoxes[i].bottom), + cv::Scalar(0,255, 0), 1); + + cv::Size text = cv::getTextSize(label, fontface, scale, thickness, &baseline); + cv::rectangle(*frame, cv::Point((uint32_t) SSDBoxes[i].left, (uint32_t)SSDBoxes[i].top) + + cv::Point(0, baseline), cv::Point((uint32_t) SSDBoxes[i].left, (uint32_t)SSDBoxes[i].top) + + cv::Point(text.width, -text.height), CV_RGB(0,0,0), cv::FILLED); + cv::putText(*frame, label, cv::Point((uint32_t) SSDBoxes[i].left, (uint32_t)SSDBoxes[i].top), + cv::FONT_HERSHEY_PLAIN, 1.0, cv::Scalar(57,255,20), 0.1, LINE_8); + } +} + +int main(int, char**) +{ + std::unique_ptr<tflite::FlatBufferModel> model; + tflite::ops::builtin::BuiltinOpResolver resolver; + std::unique_ptr<tflite::Interpreter> interpreter; + TfLiteHexagonDelegateOptions params = {0}; + const char* path = "/dsp/cdsp/"; + Mat frame, frame1; + size_t label_count; + char img_path[1024]; + int ret, idx = 0; + char fname[16]; + DIR *d; + + model = tflite::FlatBufferModel::BuildFromFile(MODEL_NAME); + if (!model) { + std::cout << "Failed to mmap model "; + exit(-1); + } + tflite::InterpreterBuilder(*model, resolver)(&interpreter); + if (!interpreter) { + std::cout << "Failed to construct interpreter"; + exit(-1); + } + interpreter->SetAllowFp16PrecisionForFp32(false); + interpreter->SetNumThreads(4); + + std::cout << "Enabling Hexagon Delegate!\n"; + + TfLiteHexagonInitWithPath(path); + auto *delegate = TfLiteHexagonDelegateCreate(¶ms); + + interpreter->ModifyGraphWithDelegate(delegate); + + if (interpreter->AllocateTensors() != kTfLiteOk) { + std::cout << "Failed to allocate tensors!" << endl; + exit(-1); + } + + const std::vector<int> inputs = interpreter->inputs(); + const std::vector<int> outputs = interpreter->outputs(); + int input = interpreter->inputs()[0]; + /* We ONLY support UInt8 on this quantized model */ + uint8_t *finput = interpreter->typed_tensor<uint8_t>(input); + float *locations, *classes, *scores, *dcount; + + locations = interpreter->typed_output_tensor<float>(0); + classes = interpreter->typed_output_tensor<float>(1); + scores = interpreter->typed_output_tensor<float>(2); + dcount = interpreter->typed_output_tensor<float>(3); + + if (read_labels_from_file()) + exit(-1); + + Mat rgb_mat(IMG_HEIGHT, IMG_WIDTH, CV_8UC3, finput); + VideoCapture capture(0); + if (!capture.isOpened()) + { + cerr << "ERROR: Can't initialize camera capture" << endl; + return 1; + } + + for (;;) + { + capture >> frame; + if (frame.empty()) + { + cerr << "ERROR: Can't grab camera frame." << endl; + break; + } + /* resize to 300 x 300 */ + cv::resize(frame, frame1, cv::Size(IMG_HEIGHT, IMG_WIDTH)); + cv::cvtColor(frame1, rgb_mat, cv::COLOR_BGR2RGB); + + if (interpreter->Invoke() != kTfLiteOk) { + std::cout << "Failed to invoke tflite!"; + exit(-1); + } + + SSDshowOutput(scores, classes, locations, &frame); + imshow("MoblieNetSSD-Demo", frame); + waitKey(1); + } + return 0; +} |