ESP32 Talking Face Assistant

#include <Arduino.h>
#include <SPI.h>
#include <Adafruit_GFX.h>
#include <Adafruit_GC9A01A.h>
#include <driver/i2s.h>
#include <esp_camera.h>
#include <math.h>

#define TFT_CS 10
#define TFT_DC 13
#define TFT_RST 14
#define I2S_BCLK 15
#define I2S_LRCLK 16
#define I2S_MIC_DATA 17
#define I2S_SPK_DATA 18
#define BUTTON_PIN 21
#define CAM_XCLK 1
#define CAM_PCLK 2
#define CAM_D0 4
#define CAM_D1 5
#define CAM_D2 6
#define CAM_D3 7
#define CAM_D4 38
#define CAM_D5 39
#define CAM_D6 40
#define CAM_D7 41
#define CAM_VSYNC 42
#define CAM_HREF 47
#define CAM_SDA 8
#define CAM_SCL 9

const uint16_t COLOR_BLACK = 0x0000;
const uint16_t COLOR_WHITE = 0xFFFF;
const uint16_t COLOR_NAVY = 0x000F;
const uint16_t COLOR_CYAN = 0x07FF;
const uint16_t COLOR_YELLOW = 0xFFE0;

Adafruit_GC9A01A tft(TFT_CS, TFT_DC, TFT_RST);
bool cameraReady = false;
bool eyesClosed = false;
bool listening = false;
unsigned long lastBlinkMs = 0;
unsigned long lastFaceMs = 0;
unsigned long lastButtonMs = 0;

void drawFace(bool closed, bool isListening) {
  const uint16_t eyeColor = isListening ? COLOR_YELLOW : COLOR_CYAN;
  tft.fillScreen(COLOR_BLACK);
  tft.fillCircle(120, 120, 116, COLOR_NAVY);
  tft.drawCircle(120, 120, 115, COLOR_CYAN);
  if (closed) {
    tft.fillRoundRect(48, 88, 48, 7, 3, eyeColor);
    tft.fillRoundRect(144, 88, 48, 7, 3, eyeColor);
  } else {
    tft.fillRoundRect(47, 61, 50, 64, 22, eyeColor);
    tft.fillRoundRect(143, 61, 50, 64, 22, eyeColor);
    tft.fillCircle(72, 93, 14, COLOR_BLACK);
    tft.fillCircle(168, 93, 14, COLOR_BLACK);
  }
  tft.drawLine(82, 146, 94, 157, COLOR_WHITE);
  tft.drawLine(94, 157, 120, 163, COLOR_WHITE);
  tft.drawLine(120, 163, 146, 157, COLOR_WHITE);
  tft.drawLine(146, 157, 158, 146, COLOR_WHITE);
  tft.setTextColor(COLOR_WHITE, COLOR_NAVY);
  tft.setTextSize(2);
  if (isListening) {
    tft.setCursor(48, 190);
    tft.print("Listening");
  } else if (cameraReady) {
    tft.setCursor(58, 190);
    tft.print("Hello!");
  } else {
    tft.setCursor(45, 190);
    tft.print("Camera?");
  }
}

bool startCamera() {
  camera_config_t config = {};
  config.ledc_channel = LEDC_CHANNEL_0;
  config.ledc_timer = LEDC_TIMER_0;
  config.pin_d0 = CAM_D0; config.pin_d1 = CAM_D1;
  config.pin_d2 = CAM_D2; config.pin_d3 = CAM_D3;
  config.pin_d4 = CAM_D4; config.pin_d5 = CAM_D5;
  config.pin_d6 = CAM_D6; config.pin_d7 = CAM_D7;
  config.pin_xclk = CAM_XCLK; config.pin_pclk = CAM_PCLK;
  config.pin_vsync = CAM_VSYNC; config.pin_href = CAM_HREF;
  config.pin_sscb_sda = CAM_SDA; config.pin_sscb_scl = CAM_SCL;
  config.pin_pwdn = -1; config.pin_reset = -1;
  config.xclk_freq_hz = 20000000;
  config.pixel_format = PIXFORMAT_GRAYSCALE;
  config.frame_size = FRAMESIZE_QQVGA;
  config.jpeg_quality = 12;
  config.fb_count = 1;
  config.grab_mode = CAMERA_GRAB_LATEST;
  return esp_camera_init(&config) == ESP_OK;
}

void startAudio() {
  i2s_config_t config = {};
  config.mode = (i2s_mode_t)(I2S_MODE_MASTER | I2S_MODE_TX | I2S_MODE_RX);
  config.sample_rate = 16000;
  config.bits_per_sample = I2S_BITS_PER_SAMPLE_32BIT;
  config.channel_format = I2S_CHANNEL_FMT_RIGHT_LEFT;
  config.communication_format = I2S_COMM_FORMAT_STAND_I2S;
  config.dma_buf_count = 6;
  config.dma_buf_len = 64;
  config.tx_desc_auto_clear = true;
  i2s_pin_config_t pins = {};
  pins.bck_io_num = I2S_BCLK;
  pins.ws_io_num = I2S_LRCLK;
  pins.data_out_num = I2S_SPK_DATA;
  pins.data_in_num = I2S_MIC_DATA;
  i2s_driver_install(I2S_NUM_0, &config, 0, nullptr);
  i2s_set_pin(I2S_NUM_0, &pins);
  i2s_zero_dma_buffer(I2S_NUM_0);
}

void playGreeting() {
  const int sampleRate = 16000;
  const int noteSamples = sampleRate / 5;
  const float notes[] = {523.25f, 659.25f, 783.99f};
  int32_t frame[2];
  for (float frequency : notes) {
    for (int i = 0; i < noteSamples; ++i) {
      const float wave = sinf(2.0f * PI * frequency * i / sampleRate);
      const int32_t sample = (int32_t)(wave * 180000000.0f);
      frame[0] = sample; frame[1] = sample;
      size_t written = 0;
      i2s_write(I2S_NUM_0, frame, sizeof(frame), &written, portMAX_DELAY);
    }
  }
}

int microphoneLevel() {
  int32_t frames[32];
  size_t bytesRead = 0;
  if (i2s_read(I2S_NUM_0, frames, sizeof(frames), &bytesRead, 5 / portTICK_PERIOD_MS) != ESP_OK || bytesRead == 0) return 0;
  int64_t total = 0;
  const size_t count = bytesRead / sizeof(int32_t);
  for (size_t i = 0; i < count; ++i) total += llabs((int64_t)frames[i]);
  return (int)(total / count / 1000000);
}

void setup() {
  pinMode(BUTTON_PIN, INPUT_PULLUP);
  SPI.begin(12, -1, 11, TFT_CS);
  tft.begin();
  tft.setRotation(0);
  drawFace(false, false);
  startAudio();
  cameraReady = startCamera();
  drawFace(false, false);
}

void loop() {
  const unsigned long now = millis();
  const bool shouldListen = microphoneLevel() > 25;
  if (shouldListen != listening && now - lastFaceMs > 200) {
    listening = shouldListen;
    drawFace(eyesClosed, listening);
    lastFaceMs = now;
  }
  if (now - lastBlinkMs > 3500) {
    eyesClosed = true;
    drawFace(true, listening);
    delay(120);
    eyesClosed = false;
    drawFace(false, listening);
    lastBlinkMs = now;
    lastFaceMs = now;
  }
  if (digitalRead(BUTTON_PIN) == LOW && now - lastButtonMs > 600) {
    lastButtonMs = now;
    listening = true;
    drawFace(false, true);
    if (cameraReady) {
      camera_fb_t *frame = esp_camera_fb_get();
      if (frame != nullptr) esp_camera_fb_return(frame);
    }
    playGreeting();
    listening = false;
    drawFace(false, false);
  }
  delay(10);
}