Telugu AI Voice Assistant

#include <Arduino.h>
#include <WiFi.h>
#include <HTTPClient.h>
#include <SPI.h>
#include <ArduinoJson.h>
#include <Adafruit_GFX.h>
#include <Adafruit_ST7735.h>
#include "driver/i2s.h"

#define TFT_DC 14
#define TFT_RST 4
#define MIC_LRCLK 27
#define MIC_DOUT 32
#define JOY_Y 35
#define JOY_SW 33

// CookMate backend contract (configure these after you provide your API details):
// POST /cookmate/voice: raw pcm_s16le body plus headers below.
// JSON reply: {"recipe":{"id","title","servings","steps":["..."]},
// "current_step":0,"assistant_text":"...","audio_pcm_url":"https://...",
// "dish_image_rgb565_url":"https://..."}
// audio_pcm_url returns signed 16-bit mono PCM, 16000 Hz.
// dish_image_rgb565_url returns exactly 160 x 72 RGB565 big-endian bytes.

// Hoisted type definitions
enum Screen { HOME, LISTENING, RECIPE, TIMER, ERROR_SCREEN };


// Forward declarations
void header(const char *subtitle);
void printWrap(const String &text, int x, int y, int maxChars, uint16_t color);
void tone(uint16_t hz, uint16_t ms);
void drawHome();
void drawRecipe();
void showError(const String &message);
void drawListening(bool speech);
void drawTimer();
void configureMic();
bool capturePushToTalk(int16_t *samples, size_t &count);
bool drawDishImage(const String &url);
void playPcmUrl(const String &url);
bool applyBackendReply(const String &json);
void sendCookingVoice();
void connectWiFi();
void handleJoystick();

const char *WIFI_SSID = "YOUR_WIFI_NAME";
const char *WIFI_PASSWORD = "YOUR_WIFI_PASSWORD";
const char *BACKEND_URL = "https://YOUR_BACKEND_HOST/cookmate/voice";
const char *BACKEND_API_KEY = "YOUR_COOKMATE_API_KEY";

constexpr int TFT_CS = 13, TFT_DC = 14, TFT_RST = 4;
constexpr int MIC_BCLK = 26, MIC_LRCLK = 27, MIC_DOUT = 32;
constexpr int AUDIO_DAC = 25;
constexpr int JOY_X = 34, JOY_Y = 35, JOY_SW = 33;
constexpr uint32_t SAMPLE_RATE = 16000;
constexpr size_t MAX_CAPTURE_SECONDS = 6;  // 192 kB, safe on a standard ESP32.
constexpr size_t MAX_SAMPLES = SAMPLE_RATE * MAX_CAPTURE_SECONDS;
constexpr uint16_t BG = ST77XX_BLACK, PANEL = 0x1082, ACCENT = ST77XX_CYAN;
constexpr uint16_t IMAGE_W = 160, IMAGE_H = 72;
constexpr int VAD_START_LEVEL = 650, VAD_STOP_LEVEL = 380;
constexpr uint32_t MIN_SPEECH_MS = 250, END_SILENCE_MS = 1200;


Adafruit_ST7735 tft(TFT_CS, TFT_DC, TFT_RST);
Screen screen = HOME;
int homeSelection = 0;
int currentStep = 0;
int stepCount = 0;
String recipeId, recipeTitle, assistantText, recipeSteps[12];
String audioUrl, imageUrl;
bool imageAvailable = false, lastButton = HIGH;
uint32_t lastJoyMove = 0, timerEndsAt = 0;

void header(const char *subtitle) {
  tft.fillRect(0, 0, 160, 21, ST77XX_BLUE);
  tft.setTextSize(1); tft.setTextColor(ST77XX_WHITE, ST77XX_BLUE);
  tft.setCursor(5, 4); tft.print("COOKMATE");
  tft.setCursor(76, 4); tft.print(subtitle);
}

void printWrap(const String &text, int x, int y, int maxChars, uint16_t color) {
  tft.setTextSize(1); tft.setTextColor(color, BG); tft.setCursor(x, y);
  int col = 0;
  for (size_t i = 0; i < text.length(); ++i) {
    char c = text[i];
    if (c == '\n' || col >= maxChars) { tft.println(); col = 0; if (c == '\n') continue; }
    tft.print(c); ++col;
  }
}

void tone(uint16_t hz, uint16_t ms) {
  uint32_t period = 1000000UL / hz, endAt = millis() + ms;
  while (millis() < endAt) { dacWrite(AUDIO_DAC, 176); delayMicroseconds(period / 2); dacWrite(AUDIO_DAC, 80); delayMicroseconds(period / 2); }
  dacWrite(AUDIO_DAC, 128);
}

void drawHome() {
  screen = HOME; tft.fillScreen(BG); header(WiFi.status() == WL_CONNECTED ? "ONLINE" : "OFFLINE");
  tft.fillRoundRect(7, 29, 146, 36, 4, PANEL);
  tft.setTextColor(ST77XX_WHITE, PANEL); tft.setTextSize(1); tft.setCursor(14, 38);
  tft.print("Hold button and ask a cooking"); tft.setCursor(14, 50); tft.print("question in Telugu or English.");
  const char *options[] = {"Cook by voice", "Current recipe", "Cooking timer"};
  for (int i = 0; i < 3; ++i) {
    int y = 74 + i * 19; uint16_t fill = i == homeSelection ? ACCENT : PANEL;
    uint16_t ink = i == homeSelection ? ST77XX_BLACK : ST77XX_WHITE;
    tft.fillRoundRect(9, y, 142, 16, 3, fill); tft.setTextColor(ink, fill); tft.setCursor(17, y + 4);
    tft.print(i == homeSelection ? "> " : "  "); tft.print(options[i]);
  }
  tft.setTextColor(ST77XX_YELLOW, BG); tft.setCursor(8, 136); tft.print("UP/DOWN: menu");
  tft.setCursor(8, 148); tft.print("HOLD: talk  PRESS: select");
}

void drawRecipe() {
  screen = RECIPE; tft.fillScreen(BG); header("RECIPE");
  tft.setTextColor(ST77XX_YELLOW, BG); tft.setTextSize(1); tft.setCursor(7, 27); tft.print(recipeTitle.length() ? recipeTitle : "No recipe yet");
  if (!recipeTitle.length()) { printWrap("Ask CookMate for a recipe first.", 8, 48, 25, ST77XX_WHITE); return; }
  if (imageAvailable) tft.drawRect(0, 34, 160, 74, ACCENT);
  int textY = imageAvailable ? 112 : 42;
  tft.setTextColor(ACCENT, BG); tft.setCursor(7, textY); tft.print("STEP "); tft.print(currentStep + 1); tft.print("/"); tft.print(stepCount);
  printWrap(recipeSteps[currentStep], 7, textY + 12, 25, ST77XX_WHITE);
  tft.setTextColor(ST77XX_YELLOW, BG); tft.setCursor(7, 148); tft.print("UP/DN STEP  PRESS HOME");
}

void showError(const String &message) { screen = ERROR_SCREEN; tft.fillScreen(BG); header("CHECK"); printWrap(message, 10, 43, 24, ST77XX_RED); tft.setTextColor(ST77XX_YELLOW, BG); tft.setCursor(10, 148); tft.print("PRESS TO RETURN"); }

void drawListening(bool speech) {
  screen = LISTENING; tft.fillScreen(BG); header("LISTENING");
  tft.fillRoundRect(10, 40, 140, 52, 5, speech ? ST77XX_RED : PANEL);
  tft.setTextColor(ST77XX_WHITE, speech ? ST77XX_RED : PANEL); tft.setTextSize(2); tft.setCursor(32, 53); tft.print(speech ? "SPEAKING" : "READY");
  tft.setTextSize(1); tft.setCursor(22, 108); tft.setTextColor(ST77XX_WHITE, BG); tft.print("Hold button while talking");
  tft.setCursor(18, 121); tft.print("Release to send to CookMate");
}

void drawTimer() {
  screen = TIMER; tft.fillScreen(BG); header("TIMER");
  if (timerEndsAt <= millis()) { tft.setTextColor(ST77XX_YELLOW, BG); tft.setTextSize(2); tft.setCursor(30, 56); tft.print("NO TIMER"); printWrap("Voice request: set a 10 minute timer", 12, 96, 24, ST77XX_WHITE); }
  else { uint32_t sec = (timerEndsAt - millis()) / 1000; tft.setTextColor(ACCENT, BG); tft.setTextSize(3); tft.setCursor(33, 56); tft.printf("%02lu:%02lu", sec / 60, sec % 60); printWrap("Timer is running. CookMate will alert you.", 12, 102, 24, ST77XX_WHITE); }
  tft.setTextSize(1); tft.setTextColor(ST77XX_YELLOW, BG); tft.setCursor(10, 148); tft.print("PRESS TO RETURN");
}

void configureMic() {
  i2s_config_t cfg = {}; cfg.mode = (i2s_mode_t)(I2S_MODE_MASTER | I2S_MODE_RX); cfg.sample_rate = SAMPLE_RATE;
  cfg.bits_per_sample = I2S_BITS_PER_SAMPLE_32BIT; cfg.channel_format = I2S_CHANNEL_FMT_ONLY_LEFT;
  cfg.communication_format = I2S_COMM_FORMAT_I2S; cfg.intr_alloc_flags = ESP_INTR_FLAG_LEVEL1; cfg.dma_buf_count = 8; cfg.dma_buf_len = 256;
  i2s_driver_install(I2S_NUM_0, &cfg, 0, nullptr);
  i2s_pin_config_t pins = {}; pins.bck_io_num = MIC_BCLK; pins.ws_io_num = MIC_LRCLK; pins.data_out_num = I2S_PIN_NO_CHANGE; pins.data_in_num = MIC_DOUT;
  i2s_set_pin(I2S_NUM_0, &pins); i2s_zero_dma_buffer(I2S_NUM_0);
}

bool capturePushToTalk(int16_t *samples, size_t &count) {
  count = 0; bool heardSpeech = false; uint32_t speechAt = 0, silenceAt = 0, lastDraw = 0;
  drawListening(false); tone(850, 45);
  while (digitalRead(JOY_SW) == LOW && count < MAX_SAMPLES) {
    int32_t raw[128]; size_t bytesRead = 0;
    if (i2s_read(I2S_NUM_0, raw, sizeof(raw), &bytesRead, pdMS_TO_TICKS(200)) != ESP_OK) return false;
    int peak = 0; size_t received = bytesRead / sizeof(int32_t);
    for (size_t i = 0; i < received && count < MAX_SAMPLES; ++i) { int16_t s = (int16_t)(raw[i] >> 14); samples[count++] = s; peak = max(peak, abs((int)s)); }
    bool nowSpeech = peak > VAD_START_LEVEL;
    if (nowSpeech && !heardSpeech) { heardSpeech = true; speechAt = millis(); }
    if (heardSpeech) { if (peak < VAD_STOP_LEVEL) { if (!silenceAt) silenceAt = millis(); } else silenceAt = 0; }
    if (millis() - lastDraw > 250) { drawListening(nowSpeech); lastDraw = millis(); }
    if (heardSpeech && silenceAt && millis() - silenceAt >= END_SILENCE_MS && millis() - speechAt >= MIN_SPEECH_MS) break;
  }
  tone(1200, 45); return heardSpeech && count >= SAMPLE_RATE / 4;
}

bool drawDishImage(const String &url) {
  if (!url.length()) return false;
  HTTPClient http; http.setTimeout(15000); http.begin(url); int status = http.GET();
  if (status != HTTP_CODE_OK) { http.end(); return false; }
  WiFiClient *stream = http.getStreamPtr(); uint8_t pair[2]; tft.startWrite();
  tft.setAddrWindow(0, 35, IMAGE_W, IMAGE_H);
  size_t expected = IMAGE_W * IMAGE_H * 2;
  for (size_t i = 0; i < expected; ++i) { if (stream->readBytes(pair, 2) != 2) { tft.endWrite(); http.end(); return false; } tft.writePixel((pair[0] << 8) | pair[1]); }
  tft.endWrite(); http.end(); return true;
}

void playPcmUrl(const String &url) {
  if (!url.length()) return;
  HTTPClient http; http.setTimeout(20000); http.begin(url); int status = http.GET();
  if (status != HTTP_CODE_OK) { http.end(); return; }
  WiFiClient *stream = http.getStreamPtr(); uint8_t bytes[256];
  while (http.connected() || stream->available()) {
    size_t n = stream->readBytes(bytes, sizeof(bytes));
    for (size_t i = 0; i + 1 < n; i += 2) { int16_t sample = (int16_t)(bytes[i] | (bytes[i + 1] << 8)); dacWrite(AUDIO_DAC, (uint8_t)((sample >> 8) + 128)); delayMicroseconds(62); }
  }
  dacWrite(AUDIO_DAC, 128); http.end();
}

bool applyBackendReply(const String &json) {
  JsonDocument doc; DeserializationError err = deserializeJson(doc, json); if (err) return false;
  JsonObject recipe = doc["recipe"]; if (!recipe.isNull()) {
    recipeId = recipe["id"] | ""; recipeTitle = recipe["title"] | "CookMate recipe"; stepCount = 0;
    for (JsonVariant step : recipe["steps"].as<JsonArray>()) if (stepCount < 12) recipeSteps[stepCount++] = step.as<String>();
  }
  currentStep = constrain(doc["current_step"] | currentStep, 0, max(0, stepCount - 1));
  assistantText = doc["assistant_text"] | "Recipe updated."; audioUrl = doc["audio_pcm_url"] | ""; imageUrl = doc["dish_image_rgb565_url"] | "";
  int timerSeconds = doc["timer_seconds"] | 0; if (timerSeconds > 0) timerEndsAt = millis() + (uint32_t)timerSeconds * 1000UL;
  return stepCount > 0;
}

void sendCookingVoice() {
  if (WiFi.status() != WL_CONNECTED) { showError("Wi-Fi is required for CookMate."); return; }
  int16_t *audio = (int16_t *)malloc(MAX_SAMPLES * sizeof(int16_t)); if (!audio) { showError("Not enough memory for audio capture."); return; }
  size_t count = 0; if (!capturePushToTalk(audio, count)) { free(audio); showError("I did not hear speech. Hold the button and speak closer to the mic."); return; }
  tft.fillScreen(BG); header("THINKING"); printWrap("CookMate is understanding your Telugu or English cooking request...", 12, 48, 24, ACCENT);
  HTTPClient http; http.setTimeout(35000); http.begin(BACKEND_URL); http.addHeader("Content-Type", "application/octet-stream");
  http.addHeader("Authorization", String("Bearer ") + BACKEND_API_KEY); http.addHeader("X-Audio-Format", "pcm_s16le"); http.addHeader("X-Sample-Rate", "16000");
  http.addHeader("X-STT-Mode", "telugu-code-mixed"); http.addHeader("X-CookMate-Recipe-Id", recipeId); http.addHeader("X-CookMate-Step", String(currentStep));
  int status = http.POST((uint8_t *)audio, count * sizeof(int16_t)); String reply = status == HTTP_CODE_OK ? http.getString() : ""; free(audio); http.end();
  if (status != HTTP_CODE_OK || !applyBackendReply(reply)) { showError("CookMate request failed. Check the backend URL, API key, and JSON response."); return; }
  imageAvailable = false; drawRecipe(); imageAvailable = drawDishImage(imageUrl); drawRecipe();
  Serial.println(assistantText); playPcmUrl(audioUrl);
}

void connectWiFi() { WiFi.mode(WIFI_STA); WiFi.begin(WIFI_SSID, WIFI_PASSWORD); uint32_t start = millis(); while (WiFi.status() != WL_CONNECTED && millis() - start < 15000) delay(250); }

void handleJoystick() {
  bool button = digitalRead(JOY_SW); bool justPressed = lastButton == HIGH && button == LOW; lastButton = button;
  if (screen == HOME && button == LOW && homeSelection == 0) { sendCookingVoice(); while (digitalRead(JOY_SW) == LOW) delay(10); return; }
  if (justPressed) { if (screen == HOME) { if (homeSelection == 1) drawRecipe(); else if (homeSelection == 2) drawTimer(); } else drawHome(); return; }
  if (screen == HOME && millis() - lastJoyMove > 220) { int y = analogRead(JOY_Y); if (y < 900 || y > 3200) { homeSelection = (homeSelection + (y < 900 ? 2 : 1)) % 3; lastJoyMove = millis(); drawHome(); } }
  if (screen == RECIPE && millis() - lastJoyMove > 220 && stepCount) { int y = analogRead(JOY_Y); if (y < 900 || y > 3200) { currentStep = constrain(currentStep + (y < 900 ? -1 : 1), 0, stepCount - 1); lastJoyMove = millis(); drawRecipe(); } }
}

void setup() {
  Serial.begin(115200); pinMode(JOY_SW, INPUT_PULLUP); pinMode(AUDIO_DAC, OUTPUT);
  tft.initR(INITR_BLACKTAB); tft.setRotation(1); configureMic(); tft.fillScreen(BG); header("STARTING"); connectWiFi(); drawHome();
}
void loop() {
  handleJoystick();
  if (timerEndsAt && millis() >= timerEndsAt) { timerEndsAt = 0; tone(1000, 250); tone(1400, 250); if (screen == TIMER) drawTimer(); }
  if (screen == TIMER && millis() % 1000 < 15) drawTimer();
}