ESP32CAM – Digital Panning using a Joystick and ST7789 Display.

As with most ideas, this one came at the inconvenient time of 11:13pm – so I sent a Whatsapp message to myself to remind me in the morning. As you can see – the pretence is simple – have the camera take an image which is larger than the display, and control the window with a joystick.

#include "esp_camera.h"
#include <TFT_eSPI.h> // Hardware-specific library
#include <SPI.h>

#define CAMERA_MODEL_AI_THINKER
#define PWDN_GPIO_NUM     32
#define RESET_GPIO_NUM    -1
#define XCLK_GPIO_NUM      0
#define SIOD_GPIO_NUM     26
#define SIOC_GPIO_NUM     27
#define Y9_GPIO_NUM       35
#define Y8_GPIO_NUM       34
#define Y7_GPIO_NUM       39
#define Y6_GPIO_NUM       36
#define Y5_GPIO_NUM       21
#define Y4_GPIO_NUM       19
#define Y3_GPIO_NUM       18
#define Y2_GPIO_NUM        5
#define VSYNC_GPIO_NUM    25
#define HREF_GPIO_NUM     23
#define PCLK_GPIO_NUM     22

const int xPin = 2;
const int yPin = 4;
volatile int x = 200;   //200 for VGA
volatile int y = 120;   //120 for VGA

TFT_eSPI tft = TFT_eSPI();       // Invoke custom library
TFT_eSprite spr = TFT_eSprite(&tft);

camera_config_t config;

uint16_t *scr;
long initalTime = 0;
long frameTime = 1;
volatile bool screenRefreshFlag = true;

Libraries, Definitions, Globals and Constructors

This is the standard setup for the camera and display rigs that have been featured on the edge detection experiments.

The joystick input pins and intial x & y window offset are declared.

VGA resolution 640pix wide, so with a 240pix wide window gives 200pix padding. Similarly, the 480pix height of a VGA frame gives 120pix padding top & bottom.

void setup() {
  Serial.begin(115200);

  pinMode(xPin, INPUT);
  pinMode(yPin, INPUT);

  config.ledc_channel = LEDC_CHANNEL_0;
  config.ledc_timer = LEDC_TIMER_0;
  config.pin_d0 = Y2_GPIO_NUM;
  config.pin_d1 = Y3_GPIO_NUM;
  config.pin_d2 = Y4_GPIO_NUM;
  config.pin_d3 = Y5_GPIO_NUM;
  config.pin_d4 = Y6_GPIO_NUM;
  config.pin_d5 = Y7_GPIO_NUM;
  config.pin_d6 = Y8_GPIO_NUM;
  config.pin_d7 = Y9_GPIO_NUM;
  config.pin_xclk = XCLK_GPIO_NUM;
  config.pin_pclk = PCLK_GPIO_NUM;
  config.pin_vsync = VSYNC_GPIO_NUM;
  config.pin_href = HREF_GPIO_NUM;
  config.pin_sscb_sda = SIOD_GPIO_NUM;
  config.pin_sscb_scl = SIOC_GPIO_NUM;
  config.pin_pwdn = PWDN_GPIO_NUM;
  config.pin_reset = RESET_GPIO_NUM;
  config.xclk_freq_hz = 20000000;
  config.frame_size = FRAMESIZE_VGA;
  config.pixel_format = PIXFORMAT_RGB565;
  config.grab_mode = CAMERA_GRAB_LATEST;   
  config.fb_location = CAMERA_FB_IN_PSRAM;
  config.jpeg_quality = 12;
  config.fb_count = 2;       

  esp_err_t err = esp_camera_init(&config);
  sensor_t * s = esp_camera_sensor_get();
  s->set_brightness(s, 0);     // -2 to 2
  s->set_contrast(s, 0);       // -2 to 2
  s->set_saturation(s, 0);     // -2 to 2
  s->set_special_effect(s, 0); // 0 to 6 (0 - No Effect, 1 - Negative, 2 - Grayscale, 3 - Red Tint, 4 - Green Tint, 5 - Blue Tint, 6 - Sepia)
  s->set_whitebal(s, 1);       // 0=disable, 1=enable
  s->set_awb_gain(s, 1);       // 0=disable, 1=enable
  s->set_wb_mode(s, 0);        // 0 to 4 - if awb_gain enabled (0 - Auto, 1 - Sunny, 2 - Cloudy, 3 - Office, 4 - Home)
  s->set_exposure_ctrl(s, 1);  // 0=disable, 1=enable
  s->set_aec2(s, 0);           // 0=disable, 1=enable
  s->set_ae_level(s, 0);       // -2 to 2
  s->set_aec_value(s, 300);    // 0 to 1200
  s->set_gain_ctrl(s, 1);      // 0=disable, 1=enable
  s->set_agc_gain(s, 0);       // 0 to 30
  s->set_gainceiling(s, (gainceiling_t)0);  // 0 to 6
  s->set_bpc(s, 0);            // 0=disable, 1=enable
  s->set_wpc(s, 1);            // 0=disable, 1=enable
  s->set_raw_gma(s, 1);        // 0=disable, 1=enable
  s->set_lenc(s, 1);           // 0=disable, 1=enable
  s->set_hmirror(s, 0);        // 0=disable, 1=enable
  s->set_vflip(s, 0);          // 0=disable, 1=enable
  s->set_dcw(s, 1);            // 0=disable, 1=enable
  s->set_colorbar(s, 0);       // 0=disable, 1=enable

  tft.init();
  tft.setRotation(3);
  tft.fillScreen(TFT_BLACK);
  tft.setTextColor(TFT_BLACK, TFT_WHITE);
  scr = (uint16_t*)spr.createSprite(240, 240);
  tft.drawString("Loading...", 105, 105, 2);

  delay(1000);
}

The Setup( )

The Serial monitor was used to look at the timings.

As noted above, this uses VGA resolution, rather than the intended full UXGA. The reason is that the pixel format is set to RGB565, so each camera pixel required 2 bytes.

Therefore a single 1600×1200 UXGA image would occupy 3.84MB and the ESP32 would just shit a brick.

As always, the setup( ) ends with displaying a simple “Loading…” screen for a second to verify the display works.

void loop() {

  initalTime = millis();

  camera_fb_t  * fb = NULL;
  fb = esp_camera_fb_get();
  long split_1 = millis() - initalTime;

  for (int y_count = 0; y_count < 240; y_count++) {
    int yPart = (y_count + y) * 1280;
    for (int x_count = 0; x_count < 240; x_count++) {
      int xPart = (x_count + x) * 2;
      byte first_byte = fb->buf[yPart + xPart];
      byte second_byte = fb->buf[yPart + xPart + 1];
      scr[(y_count * 240) + x_count] = (second_byte << 8) + first_byte;
    }
  }

  esp_camera_fb_return(fb);
  long split_2 = millis() - initalTime;

  int xValue = analogRead(xPin);
  int yValue = analogRead(yPin);
  if ((yValue == 0) && (y > 0)) {
    //move up
    y = y + 5;
  }
  if ((yValue > 4000) && (y < 240)) {
    //move down
    y = y - 5;
  }
  if ((xValue == 0) && (x > 0)) {
    //move left
    x = x + 5;
  }
  if ((xValue > 4000) && (x < 400)) {
    //move right
    x = x - 5;
  }

  spr.drawLine(120, 0, 120, 240, TFT_BLACK);
  spr.drawLine(0, 120, 240, 120, TFT_BLACK);
  spr.drawString(String(frameTime), 100, 220, 2); //print frame time in milliseconds
  spr.drawString("ms", 125, 220, 2);
  spr.pushSprite(0, 0);
  long split_3 = millis() - initalTime;

  frameTime = millis() - initalTime;
  Serial.println(initalTime);
  Serial.println(split_1);
  Serial.println(split_2);
  Serial.println(split_3);

}

The Loop( )

The concept of the sketch is simple:

• Camera takes a photo and saves to frame buffer.
• Nested for( ) loops grab the necessary pixels from the frame buffer to create a 240×240 window.
• The joystick values are read and the window offset is adjusted accordingly.
• The display shows the contents of the window.
• Repeat ad infinitum.

The magic number of “1280” is the number of bytes in a single line of VGA frame (640pixels x 2bytes per pixel).

As we all know, the ADC of the ESP32 is dire at best, and not helped by the display and camera pulling current. Therefore the joystick control was kept simple: either fully off (reading 0) or convincingly on (reading over 4000 – max is 4096).

The “split” variables are for timing purposes. (More about this in the ‘conclusion’)