1

I am currently working on a project that detects specific objects in a video stream. If a particular object is detected, an alarm of some sort will be rung. An HTML page will display the object detection output and the name of the object detected. I have used flask with python for the HTML page. The video from the ESP32 Camera is given to a python script through a URL; then, the output will be displayed on the HTML page. I am not using the camera webserver rather using a flask web app. I have completed the object detection part and the object detection output successfully. But I can't seem to figure out a way to control the alarm system attached to the ESP32-Cam through the python script. This is my python code

from pickletools import read_uint1
from flask import Flask, render_template, Response
import cv2
import cvzone


#url = 'C:/Users/Mukesh/Downloads/videoplayback.mp4'
url = '192.168.35.7:81/stream'

classNames = []
classFile = 'C:/Users/Mukesh/Desktop/Mini Project/Code/AIM/coco.names'
with open(classFile, 'rt') as f:
    classNames = f.read().split('\n')
configPath = 'C:/Users/Mukesh/Desktop/Mini Project/Code/AIM/ssd_mobilenet_v3_large_coco_2020_01_14.pbtxt'
weightsPath = "C:/Users/Mukesh/Desktop/Mini Project/Code/AIM/frozen_inference_graph.pb"
net = cv2.dnn_DetectionModel(weightsPath, configPath)
net.setInputSize(320, 320)
net.setInputScale(1.0 / 127.5)
net.setInputMean((127.5, 127.5, 127.5))
net.setInputSwapRB(True)
app = Flask(__name__)
thres = 0.55
nmsThres = 0.2
cap = cv2.VideoCapture(url)
cap.set(3, 640)
cap.set(4, 480)

def gen_frames():
    while 1:
        isTrue, img = cap.read()
        img = cv2.flip(img, 1)
        if img is not None:
            classIds, confs, bbox = net.detect(
                img, confThreshold=thres, nmsThreshold=nmsThres)
            try:
                for classId, conf, box in zip(classIds.flatten(), confs.flatten(), bbox):
                    id = classNames[classId - 1]
                    #Send ID to HTML page
                    print(id)
                    if id=='drone':
                        cvzone.cornerRect(img, box)
                        cv2.putText(img, f'{classNames[classId - 1].upper()} {round(conf * 100, 2)}',
                                (box[0] + 10, box[1] +
                                 30), cv2.FONT_HERSHEY_COMPLEX_SMALL,
                                1, (0, 255, 0), 2)
                frame = img
                ret, buffer = cv2.imencode('.jpg', frame)
                frame = buffer.tobytes()
                cv2.imshow('',img)
                if cv2.waitKey(20)&0xff==ord(' '):
                    break
                yield (b'--frame\r\n'
                       b'Content-Type: image/jpeg\r\n\r\n' + frame + b'\r\n')
            except:
                pass
            cv2.waitKey(1)
        else:
            break


@app.route('/video_feed')
def video_feed():
    return Response(gen_frames(), mimetype='multipart/x-mixed-replace; boundary=frame')


@app.route('/')
def home():
    """Video streaming home page."""
    return render_template('Home.html')


if __name__ == '__main__':
    app.run(debug=True)

This is the webpage

<!DOCTYPE html>
<html lang="en">

    <head>
        <title>Home</title>
        <meta charset="UTF-8">
        <meta http-equiv="X-UA-Compatible" content="IE=edge">
        <meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no">
        <link href="https://stackpath.bootstrapcdn.com/bootstrap/4.4.1/css/bootstrap.min.css" rel="stylesheet">
        <link rel="stylesheet" href="https://stackpath.bootstrapcdn.com/bootstrap/4.1.3/css/bootstrap.min.css"
        integrity="sha384-MCw98/SFnGE8fJT3GXwEOngsV7Zt27NXFoaoApmYm81iuXoPkFOJwJ8ERdknLPMO" crossorigin="anonymous">
        <script src="http://ajax.googleapis.com/ajax/libs/jquery/1.9.1/jquery.min.js" type="text/javascript"></script>
        <script src="https://stackpath.bootstrapcdn.com/bootstrap/4.4.1/js/bootstrap.bundle.min.js"></script>
        <style>
            body {
                font-family: 'Gill Sans', 'Gill Sans MT', Calibri, 'Trebuchet MS', sans-serif;
                background-image: linear-gradient(to right, #2E3192, #1BFFFF);
                background-position: center;
                background-repeat: no-repeat;
                background-size: cover;
                position: relative;
                min-height: 100vh;
                display: flex;
                flex-flow: row;
            }

            .booth {
                flex: 1 1 auto;
            }

            .f2 {
                flex: 1 1 auto;
                display: flex;
                flex-flow: column-reverse;
            }

            .f2 .text {
                flex: 0 1 auto;
            }

            .f2 .frame {
                flex: 1 1 auto;
                position: relative;
                display: flex;
                flex-flow: column;
            }

            .f2 .frame #click-photo {
                flex: 0 1 auto;
            }

            .f2 .frame #canvas {
                flex: 1 1 auto;
            }

            #canvas {
                background-color: red;
            }

        </style>

    </head>

    <body>
        <div class="booth">
            <!--video id="video" width="100%" height="100%" autoplay></video-->
            <img id="video" src="{{ url_for('video_feed') }}" width="100%">
        </div>
        <div class="f2">
            <div class="frame">
                <canvas id="canvas"></canvas>
                <button id="click-photo">Click Photo</button>
            </div>
            <div class="text">
                <h2 id="status" src="{{ url_for('obj')}}"></h2>
                <!---Drone status--->
                <h2 id="cam"></h2>
                <!--Camera location-->
            </div>
        </div>
        <script>
            let video = document.querySelector("#video");
            let click_button = document.querySelector("#click-photo");
            let canvas = document.querySelector("#canvas");

            click_button.addEventListener('click', function () {
                canvas.getContext('2d').drawImage(video, 0, 0, canvas.width, canvas.height);
                let image_data_url = canvas.toDataURL('image/jpeg');
                console.log(image_data_url);
                //document.getElementById("status").innerHTML = " Drone: detected";
                document.getElementById("cam").innerHTML = "Camera: 1";
            });
            var stop = function () {
                var stream = video.srcObject;
                var tracks = stream.getTracks();
                for (var i = 0; i < tracks.length; i++) {
                    var track = tracks[i];
                    track.stop();
                }
                video.srcObject = null;
            }
            var start = function () {
                var video = document.getElementById('video'),
                    vendorUrl = window.URL || window.webkitURL;
                if (navigator.mediaDevices.getUserMedia) {
                    navigator.mediaDevices.getUserMedia({ video: true })
                        .then(function (stream) {
                            video.srcObject = stream;
                        }).catch(function (error) {
                            console.log("Something went wrong!");
                        });
                }
            }
            $(function () {
                start();
            });
        </script>
    </body>

</html>

This is the ESP32 Camera code

#include "esp_camera.h"
#include <WiFi.h>
#include "esp_timer.h"
#include "img_converters.h"
#include "Arduino.h"
#include "fb_gfx.h"
#include "soc/soc.h" //disable brownout problems
#include "soc/rtc_cntl_reg.h"  //disable brownout problems
#include "esp_http_server.h"
#include <ESP32Servo.h>
//Replace with your network credentials
const char* ssid = "Mukesh";
const char* password = "qwertyuiop";

#define PART_BOUNDARY "123456789000000000000987654321"

// This project was tested with the AI Thinker Model, M5STACK PSRAM Model and M5STACK WITHOUT PSRAM
#define CAMERA_MODEL_AI_THINKER
//#define CAMERA_MODEL_M5STACK_PSRAM
//#define CAMERA_MODEL_M5STACK_WITHOUT_PSRAM

// Not tested with this model
//#define CAMERA_MODEL_WROVER_KIT

#if defined(CAMERA_MODEL_WROVER_KIT)
  #define PWDN_GPIO_NUM    -1
  #define RESET_GPIO_NUM   -1
  #define XCLK_GPIO_NUM    21
  #define SIOD_GPIO_NUM    26
  #define SIOC_GPIO_NUM    27

  #define Y9_GPIO_NUM      35
  #define Y8_GPIO_NUM      34
  #define Y7_GPIO_NUM      39
  #define Y6_GPIO_NUM      36
  #define Y5_GPIO_NUM      19
  #define Y4_GPIO_NUM      18
  #define Y3_GPIO_NUM       5
  #define Y2_GPIO_NUM       4
  #define VSYNC_GPIO_NUM   25
  #define HREF_GPIO_NUM    23
  #define PCLK_GPIO_NUM    22

#elif defined(CAMERA_MODEL_M5STACK_PSRAM)
  #define PWDN_GPIO_NUM     -1
  #define RESET_GPIO_NUM    15
  #define XCLK_GPIO_NUM     27
  #define SIOD_GPIO_NUM     25
  #define SIOC_GPIO_NUM     23

  #define Y9_GPIO_NUM       19
  #define Y8_GPIO_NUM       36
  #define Y7_GPIO_NUM       18
  #define Y6_GPIO_NUM       39
  #define Y5_GPIO_NUM        5
  #define Y4_GPIO_NUM       34
  #define Y3_GPIO_NUM       35
  #define Y2_GPIO_NUM       32
  #define VSYNC_GPIO_NUM    22
  #define HREF_GPIO_NUM     26
  #define PCLK_GPIO_NUM     21

#elif defined(CAMERA_MODEL_M5STACK_WITHOUT_PSRAM)
  #define PWDN_GPIO_NUM     -1
  #define RESET_GPIO_NUM    15
  #define XCLK_GPIO_NUM     27
  #define SIOD_GPIO_NUM     25
  #define SIOC_GPIO_NUM     23

  #define Y9_GPIO_NUM       19
  #define Y8_GPIO_NUM       36
  #define Y7_GPIO_NUM       18
  #define Y6_GPIO_NUM       39
  #define Y5_GPIO_NUM        5
  #define Y4_GPIO_NUM       34
  #define Y3_GPIO_NUM       35
  #define Y2_GPIO_NUM       17
  #define VSYNC_GPIO_NUM    22
  #define HREF_GPIO_NUM     26
  #define PCLK_GPIO_NUM     21

#elif defined(CAMERA_MODEL_AI_THINKER)
  #define PWDN_GPIO_NUM     32
  #define RESET_GPIO_NUM    -1
  #define XCLK_GPIO_NUM      0
  #define SIOD_GPIO_NUM     26
  #define SIOC_GPIO_NUM     27

  #define Y9_GPIO_NUM       35
  #define Y8_GPIO_NUM       34
  #define Y7_GPIO_NUM       39
  #define Y6_GPIO_NUM       36
  #define Y5_GPIO_NUM       21
  #define Y4_GPIO_NUM       19
  #define Y3_GPIO_NUM       18
  #define Y2_GPIO_NUM        5
  #define VSYNC_GPIO_NUM    25
  #define HREF_GPIO_NUM     23
  #define PCLK_GPIO_NUM     22
#else
  #error "Camera model not selected"
#endif
// #define Ser_1 14
// #define Ser_2 15
// #define ser_step 5
// Servo sn1;
// Servo sn2;
// Servo s1;
// Servo s2;
// int pos1=0;
// int pos2=0;

static const char* _STREAM_CONTENT_TYPE = "multipart/x-mixed-replace;boundary=" PART_BOUNDARY;
static const char* _STREAM_BOUNDARY = "\r\n--" PART_BOUNDARY "\r\n";
static const char* _STREAM_PART = "Content-Type: image/jpeg\r\nContent-Length: %u\r\n\r\n";

httpd_handle_t stream_httpd = NULL;

static esp_err_t stream_handler(httpd_req_t *req){
  camera_fb_t * fb = NULL;
  esp_err_t res = ESP_OK;
  size_t _jpg_buf_len = 0;
  uint8_t * _jpg_buf = NULL;
  char * part_buf[64];

  res = httpd_resp_set_type(req, _STREAM_CONTENT_TYPE);
  if(res != ESP_OK){
    return res;
  }

  while(true){
    fb = esp_camera_fb_get();
    if (!fb) {
      Serial.println("Camera capture failed");
      res = ESP_FAIL;
    } else {
      if(fb->width > 400){
        if(fb->format != PIXFORMAT_JPEG){
          bool jpeg_converted = frame2jpg(fb, 80, &_jpg_buf, &_jpg_buf_len);
          esp_camera_fb_return(fb);
          fb = NULL;
          if(!jpeg_converted){
            Serial.println("JPEG compression failed");
            res = ESP_FAIL;
          }
        } else {
          _jpg_buf_len = fb->len;
          _jpg_buf = fb->buf;
        }
      }
    }
    if(res == ESP_OK){
      size_t hlen = snprintf((char *)part_buf, 64, _STREAM_PART, _jpg_buf_len);
      res = httpd_resp_send_chunk(req, (const char *)part_buf, hlen);
    }
    if(res == ESP_OK){
      res = httpd_resp_send_chunk(req, (const char *)_jpg_buf, _jpg_buf_len);
    }
    if(res == ESP_OK){
      res = httpd_resp_send_chunk(req, _STREAM_BOUNDARY, strlen(_STREAM_BOUNDARY));
    }
    if(fb){
      esp_camera_fb_return(fb);
      fb = NULL;
      _jpg_buf = NULL;
    } else if(_jpg_buf){
      free(_jpg_buf);
      _jpg_buf = NULL;
    }
    if(res != ESP_OK){
      break;
    }
    //Serial.printf("MJPG: %uB\n",(uint32_t)(_jpg_buf_len));
  }
  return res;
}

void startCameraServer(){
  httpd_config_t config = HTTPD_DEFAULT_CONFIG();
  config.server_port = 80;

  httpd_uri_t index_uri = {
    .uri       = "/",
    .method    = HTTP_GET,
    .handler   = stream_handler,
    .user_ctx  = NULL
  };

  //Serial.printf("Starting web server on port: '%d'\n", config.server_port);
  if (httpd_start(&stream_httpd, &config) == ESP_OK) {
    httpd_register_uri_handler(stream_httpd, &index_uri);
  }
}

void setup() {
  WRITE_PERI_REG(RTC_CNTL_BROWN_OUT_REG, 0); //disable brownout detector
  // s1.setPeriodHertz(50);
  // s2.setPeriodHertz(50);
  // sn1.attach(2,1000,2000);
  // sn2.attach(13,1000,2000);
  // s1.attach(Ser_1,1000,2000);
  // s2.attach(Ser_2,1000,2000);
  // s1.write(pos1);
  // s2.write(pos2);
  Serial.begin(115200);
  Serial.setDebugOutput(false);

  camera_config_t config;
  config.ledc_channel = LEDC_CHANNEL_0;
  config.ledc_timer = LEDC_TIMER_0;
  config.pin_d0 = Y2_GPIO_NUM;
  config.pin_d1 = Y3_GPIO_NUM;
  config.pin_d2 = Y4_GPIO_NUM;
  config.pin_d3 = Y5_GPIO_NUM;
  config.pin_d4 = Y6_GPIO_NUM;
  config.pin_d5 = Y7_GPIO_NUM;
  config.pin_d6 = Y8_GPIO_NUM;
  config.pin_d7 = Y9_GPIO_NUM;
  config.pin_xclk = XCLK_GPIO_NUM;
  config.pin_pclk = PCLK_GPIO_NUM;
  config.pin_vsync = VSYNC_GPIO_NUM;
  config.pin_href = HREF_GPIO_NUM;
  config.pin_sscb_sda = SIOD_GPIO_NUM;
  config.pin_sscb_scl = SIOC_GPIO_NUM;
  config.pin_pwdn = PWDN_GPIO_NUM;
  config.pin_reset = RESET_GPIO_NUM;
  config.xclk_freq_hz = 20000000;
  config.pixel_format = PIXFORMAT_JPEG;

  if(psramFound()){
    config.frame_size = FRAMESIZE_UXGA;
    config.jpeg_quality = 10;
    config.fb_count = 2;
  } else {
    config.frame_size = FRAMESIZE_SVGA;
    config.jpeg_quality = 12;
    config.fb_count = 1;
  }

  // Camera init
  esp_err_t err = esp_camera_init(&config);
  if (err != ESP_OK) {
    Serial.printf("Camera init failed with error 0x%x", err);
    return;
  }
  // Wi-Fi connection
  WiFi.begin(ssid, password);
  while (WiFi.status() != WL_CONNECTED) {
    delay(500);
    Serial.print(".");
  }
  Serial.println("");
  Serial.println("WiFi connected");

  Serial.print("Camera Stream Ready! Go to: http://");
  Serial.print(WiFi.localIP());

  // Start streaming web server
  startCameraServer();
}

void loop() {
  delay(1);
}

Now my question is: How to control GPIO pins on ESP32 Camera, from the python script, so that when an object is detected, ESP32-Cam can activate an alarm system?

1 Answer 1

1

The easy way you can use http request when you detect something, while running the python app, you send a POST request, this link could help you:

https://stackoverflow.com/questions/10313001/is-it-possible-to-make-post-request-in-flask

in the ESP32 you will receive the request since you have esp_http_server you must add a condition of what to do when you receive the request.

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service and acknowledge you have read our privacy policy.

Not the answer you're looking for? Browse other questions tagged or ask your own question.