# Object detection - Media Pipe

Sure, I can provide an overview of the steps involved in object detection with Mediapipe and C++.

Part 1: Setting up the environment

Before starting with the coding part, you need to set up the environment for using Mediapipe. Here are the steps:

Install the Mediapipe library by following the instructions given in the Mediapipe documentation.

Set up your development environment to link against the Mediapipe library.

Download the object detection model from the Mediapipe model zoo, which will be used for detecting objects in the input video frames.

Part 2: Loading the model

Once you have set up the environment, the next step is to load the object detection model into your C++ application. Here are the steps:

Create an instance of the Mediapipe graph, which will be used to run the object detection model.

Load the object detection model into the graph using the appropriate Mediapipe API function.

Part 3: Reading input frames

The next step is to read the input video frames from a video file or camera stream. Here are the steps:

Initialize a video reader object using OpenCV or another video processing library.

Read each frame from the video reader and convert it to a format that can be used as input to the Mediapipe graph.

Part 4: Running the model

The next step is to run the object detection model on each input frame. Here are the steps:

Pass each input frame through the Mediapipe graph using the appropriate API function.

Extract the output of the object detection model, which will contain information about the objects detected in the input frame.

Part 5: Displaying the output

The final step is to display the output of the object detection model. Here are the steps:

Draw the bounding boxes around the objects detected in the input frames using the output from the object detection model.

Display the input frames with the bounding boxes overlaid on top of the objects detected.

This is an overview of the steps involved in object detection with Mediapipe and C++. Let me know if you want me to provide you with more detailed code snippets or if you have any specific questions related to this process.

#include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/formats/image_frame.h"
#include "mediapipe/framework/formats/image_frame_opencv.h"
#include "mediapipe/framework/graph.h"
#include "mediapipe/framework/graph.pb.h"
#include "mediapipe/framework/port/file_helpers.h"
#include "mediapipe/framework/port/opencv_highgui_inc.h"
#include "mediapipe/framework/port/opencv_imgproc_inc.h"
#include "mediapipe/framework/port/parse_text_proto.h"
#include "mediapipe/framework/port/status.h"
#include "mediapipe/framework/tool/validate_type.h"
#include "mediapipe/util/resource_util.h"
#include <iostream>
#include <memory>
#include <string>
#include <vector>

namespace mp = mediapipe;

int main(int argc, char** argv) {
  // Initialize the Mediapipe framework.
  absl::Status status = mp::InitializeMediapipe();
  if (!status.ok()) {
    std::cerr << "Failed to initialize Mediapipe: " << status.message() << std::endl;
    return EXIT_FAILURE;
  }

  // Create a calculator graph.
  mp::CalculatorGraphConfig config;
  if (!mediapipe::file::GetContents("path/to/graph.pbtxt", &config)) {
    std::cerr << "Failed to read graph configuration file." << std::endl;
    return EXIT_FAILURE;
  }

  mp::CalculatorGraph graph;
  status = graph.Initialize(config);
  if (!status.ok()) {
    std::cerr << "Failed to initialize calculator graph: " << status.message() << std::endl;
    return EXIT_FAILURE;
  }

  // Load the object detection model.
  std::unique_ptr<mp::GraphDef> object_detection_model =
      mediapipe::tool::LoadBinaryProto<mp::GraphDef>("path/to/object_detection_model.pb");
  if (!object_detection_model) {
    std::cerr << "Failed to load object detection model." << std::endl;
    return EXIT_FAILURE;
  }

  return EXIT_SUCCESS;
}

Part 2

// Load the object detection model.
std::unique_ptr<mp::GraphDef> object_detection_model =
    mediapipe::tool::LoadBinaryProto<mp::GraphDef>("path/to/object_detection_model.pb");
if (!object_detection_model) {
  std::cerr << "Failed to load object detection model." << std::endl;
  return EXIT_FAILURE;
}

// Initialize the object detection calculator in the graph.
mp::CalculatorGraphConfig::Node* object_detection_node_config =
    config.mutable_node()->Mutable(0);
object_detection_node_config->mutable_calculator_options()->Clear();
object_detection_node_config->mutable_calculator()->set_type("ObjectDetectionCalculator");
object_detection_node_config->mutable_calculator_options()->MergeFrom(
    mp::ParseTextProtoOrDie<mediapipe::ObjectDetectionCalculatorOptions>(
        "score_threshold: 0.5\n"
        "use_regular_non_max_suppression: true\n"
        "min_suppression_overlap: 0.5\n"
        "max_num_detections: 10\n"));
object_detection_node_config->add_input_side_packet()->set_tag("MODEL_GRAPH");
object_detection_node_config->add_output_stream()->set_tag("DETECTIONS");

mp::StatusOr<CalculatorGraphConfig::Node> object_detection_node_status =
    mediapipe::ParseNodeConfigOrDie(*object_detection_node_config);
if (!object_detection_node_status.ok()) {
  std::cerr << "Failed to parse object detection node configuration: " << status.message()
            << std::endl;
  return EXIT_FAILURE;
}

mediapipe::CalculatorGraph graph;
status = graph.Initialize({*object_detection_node_status});
if (!status.ok()) {
  std::cerr << "Failed to initialize object detection calculator graph: " << status.message()
            << std::endl;
  return EXIT_FAILURE;
}

status = graph.StartRun({});
if (!status.ok()) {
  std::cerr << "Failed to start object detection calculator graph: " << status.message()
            << std::endl;
  return EXIT_FAILURE;
}

status = graph.AddPacketToInputStream(
    "MODEL_GRAPH", mediapipe::Adopt(object_detection_model.release()).AsUniquePtr());
if (!status.ok()) {
  std::cerr << "Failed to add object detection model packet to input stream: " << status.message()
            << std::endl;
  return EXIT_FAILURE;
}

part 3

// Open the input video file.
cv::VideoCapture cap("path/to/video_file");

if (!cap.isOpened()) {
  std::cerr << "Failed to open input video file." << std::endl;
  return EXIT_FAILURE;
}

// Read each frame from the video file and pass it through the graph.
cv::Mat frame;
while (cap.read(frame)) {
  // Convert the frame to a format that can be used as input to the graph.
  std::unique_ptr<mediapipe::ImageFrame> image_frame = absl::make_unique<mediapipe::ImageFrame>(
      mediapipe::ImageFormat::SRGB, frame.cols, frame.rows,
      mediapipe::ImageFrame::kDefaultAlignmentBoundary);
  cv::Mat input_mat = mediapipe::formats::MatView(image_frame.get());
  cv::cvtColor(frame, input_mat, cv::COLOR_BGR2RGB);

  // Create an input packet from the converted frame.
  mp::Packet input_packet = mp::MakePacket<mediapipe::ImageFrame>(std::move(image_frame));

  // Send the input packet to the graph.
  status = graph.AddPacketToInputStream("input_video", std::move(input_packet));
  if (!status.ok()) {
    std::cerr << "Failed to add input packet to stream: " << status.message() << std::endl;
    break;
  }

  // Get the output packets from the graph and process them.
  mp::Packet output_packet;
  while (graph.GetPacket(&output_packet)) {
    if (!output_packet.ValidateAsType<mediapipe::DetectionProtoList>().ok()) {
      std::cerr << "Invalid output packet type." << std::endl;
      break;
    }

    // Extract the detection results from the output packet and process them.
    const mediapipe::DetectionProtoList& detection_list =
        output_packet.Get<mediapipe::DetectionProtoList>();

    // Process the detection results...

    // Release the output packet.
    output_packet.Release();
  }
}

// Release the video capture object.
cap.release();

part 4

// Send the input packet to the graph.
status = graph.AddPacketToInputStream("input_video", std::move(input_packet));
if (!status.ok()) {
  std::cerr << "Failed to add input packet to stream: " << status.message() << std::endl;
  break;
}

// Get the output packets from the graph and process them.
mp::Packet output_packet;
while (graph.GetPacket(&output_packet)) {
  if (!output_packet.ValidateAsType<mediapipe::DetectionProtoList>().ok()) {
    std::cerr << "Invalid output packet type." << std::endl;
    break;
  }

  // Extract the detection results from the output packet and process them.
  const mediapipe::DetectionProtoList& detection_list =
      output_packet.Get<mediapipe::DetectionProtoList>();

  for (const auto& detection : detection_list.detection()) {
    std::cout << "Detected object: " << detection.label() << std::endl;
    std::cout << "  Confidence: " << detection.score() << std::endl;
    std::cout << "  Bounding box: (" << detection.location_data().relative_bounding_box().xmin() << ","
              << detection.location_data().relative_bounding_box().ymin() << ","
              << detection.location_data().relative_bounding_box().xmax() << ","
              << detection.location_data().relative_bounding_box().ymax() << ")" << std::endl;

    // Process the detection results...
  }

  // Release the output packet.
  output_packet.Release();
}

part 5

// Shut down the calculator graph.
status = graph.CloseAllPacketSources();
if (!status.ok()) {
  std::cerr << "Failed to close all packet sources: " << status.message() << std::endl;
}

status = graph.WaitUntilDone();
if (!status.ok()) {
  std::cerr << "Failed to wait for graph to finish: " << status.message() << std::endl;
}

// Shut down the Mediapipe framework.
mediapipe::ShutdownMediapipe();

return EXIT_SUCCESS;