Aijia Yao

MIT EECS • HTMAA 2025

Week 13 - UI and Programming

December 2025

This week was mainly about UI development.

What I Did

For group assignment, Anthony recommended an usefull software sketchbook called Processing.
Learning from the examples in "Processing Tutorials", I developed some "Time" related contents and the UI for my final project.

Time

"Time" is a crucial concept not only in real physical world but also in programming, when not properly managed can lead to crazy bugs in the system. See this article explaining how a single mistyped operator in GitHub actions triggered a $2,500 CI/CD meltdown.

What time is it now? Here is an analog clock to tell you: (implemented with HTML5 Canvas and JavaScript)

And digital clock --:--:-- is much easier to implement, see the code below:


  function pad2(n){ return String(n).padStart(2,'0'); }

  function updateDigitalClock(){
    const now = new Date();
    const h = pad2(now.getHours());
    const m = pad2(now.getMinutes());
    const s = pad2(now.getSeconds());
    document.getElementById('digitalClock').textContent = `${h}:${m}:${s}`;
  
    const y = now.getFullYear();
    const mo = pad2(now.getMonth() + 1); // months are 0-based
    const d = pad2(now.getDate());

    document.getElementById('digitalClock').textContent =
      `${h}:${m}:${s} • ${y}-${mo}-${d}`;
  }
  updateDigitalClock();
  setInterval(updateDigitalClock, 250);

With the help of ChatGPT5.2 (See prompts here), I also implemented an hourglass that can count down 30s with a flip animation:

30.0 s

(Click to flip and it will restart)

After debugging a few things, the JavaScript code for this hourglass is as followed:

    
  const hg = document.getElementById('hg');
  const timeEl = document.getElementById('time');

  const DURATION = 30_000; // ms
  let startTime = null;
  let raf = null;

  function updateTimer(now){
    if(!startTime) return;
    const elapsed = now - startTime;
    const remaining = Math.max(0, DURATION - elapsed);
    timeEl.textContent = (remaining / 1000).toFixed(1) + ' s';
    if(remaining > 0) raf = requestAnimationFrame(updateTimer);
  }

  function restartSand(){
    // stop CSS animations and clear "forwards" stuck state
    hg.classList.remove('running');

    const animEls = hg.querySelectorAll('.sand-top,.sand-bottom,.stream');
    animEls.forEach(el => { el.style.animation = 'none'; });

    // force reflow
    void hg.getBoundingClientRect();

    // restore animations
    animEls.forEach(el => { el.style.animation = ''; });

    // restart
    hg.classList.add('running');

    // restart timer
    startTime = performance.now();
    cancelAnimationFrame(raf);
    raf = requestAnimationFrame(updateTimer);
  }

  function flip(){
    hg.classList.toggle('flipped');
    restartSand();
  }

  hg.addEventListener('click', flip);

  // start once on load
  restartSand();

UI for Final Project

For my final project, I designed a web UI to monitor the keyword detection status and control the solenoid state (which serves as the electromagnet to control the hourglass flow rate) remotely. The UI is implemented with HTML, CSS and JavaScript, and communicates with the ESP32-S3 microcontroller via Server-Sent Events (SSE) for real-time updates.

The html page for the UI is linked here and below is a screenshot of the control pannel when not connected:

The code for this UI and control logic is in Arduino sketch below.


#include <Arduino.h>
#include <AudioTools.h>
#include <Adafruit_NeoPixel.h>

// ---- WiFi / Web ----
#include <WiFi.h>
#include <WebServer.h>

// ---- Edge Impulse ----
#include <FinalProj_inferencing.h>

// ---- I2S pins for XIAO ESP32S3 ----
#define I2S_WS_PIN   7
#define I2S_SCK_PIN  44
#define I2S_SD_PIN   8

I2SStream i2s;

// ---- NeoPixel ----
#define NEOPIXEL_PIN  6
#define NEOPIXEL_N    1
Adafruit_NeoPixel strip(NEOPIXEL_N, NEOPIXEL_PIN, NEO_GRB + NEO_KHZ800);

// ---- Solenoid (MOSFET gate) ----
#define SOLENOID_PIN  4
static inline void solenoidWrite(bool on) { digitalWrite(SOLENOID_PIN, on ? HIGH : LOW); }

// ---- Edge Impulse audio buffer ----
static float   ei_audio[EI_CLASSIFIER_DSP_INPUT_FRAME_SIZE];
static int32_t raw_i2s[256];

// ================ Web UI (HTML + SSE) ================
const char* WIFI_SSID = "MIT";
const char* WIFI_PASS = "password";

WebServer server(80);
WiFiClient sseClient;

// Current UI state (latched LED + latest score/label + solenoid status)
static char    g_label[64] = "boot";
static float   g_score = 0.0f;
static uint8_t g_led_r = 0, g_led_g = 0, g_led_b = 0;

// Solenoid UI: last keyword that triggered solenoid + status label
static char g_sol_last_kw[64] = "none";   // e.g. "Fantastic" or "Stupid" or "manual"
static char g_sol_status[16]  = "NORMAL"; // "STOP" / "GO" / "NORMAL"
static bool solManualOverride = false;

// Full score list as JSON array string: [{"l":"A","v":0.1234},...]
static char g_all_scores_json[1024] = "[]";

// Solenoid mode shared by EI logic + web control
enum SolMode : uint8_t { NORMAL, FORCE_OFF, FORCE_ON };
static SolMode solMode = NORMAL;

// Normal mode toggler state
static bool solState = false;
static uint32_t lastToggleMs = 0;

static void setUIState(const char* label, float score,
                       uint8_t r, uint8_t g, uint8_t b,
                       const char* solLastKw, const char* solStatus) {
  strncpy(g_label, label ? label : "", sizeof(g_label) - 1);
  g_label[sizeof(g_label) - 1] = '\0';
  g_score = score;

  g_led_r = r; g_led_g = g; g_led_b = b;

  if (solLastKw) {
    strncpy(g_sol_last_kw, solLastKw, sizeof(g_sol_last_kw) - 1);
    g_sol_last_kw[sizeof(g_sol_last_kw) - 1] = '\0';
  }
  if (solStatus) {
    strncpy(g_sol_status, solStatus, sizeof(g_sol_status) - 1);
    g_sol_status[sizeof(g_sol_status) - 1] = '\0';
  }
}

static inline void setPixelRGB(uint8_t r, uint8_t g, uint8_t b) {
  strip.setPixelColor(0, strip.Color(r, g, b));
  strip.show();
}

const char INDEX_HTML[] PROGMEM = R"HTML(



  
  
  Hourglass Keyword
  


  
    
      
      LED Keyword Indicator
    

    —
    score: —
    

    
      
        Hourglass status
        NORMAL
      
      
      
        Last trigger keyword: none
        Manual override: OFF
      

      
        
        
        
      
    

    
      All keyword scores
      waiting for data...
    
  

  


)HTML";

void handleRoot() {
  server.send(200, "text/html", INDEX_HTML);
}

void handleEvents() {
  WiFiClient client = server.client();
  client.print(
    "HTTP/1.1 200 OK\r\n"
    "Content-Type: text/event-stream\r\n"
    "Cache-Control: no-cache\r\n"
    "Connection: keep-alive\r\n"
    "Access-Control-Allow-Origin: *\r\n"
    "\r\n"
  );
  sseClient = client;
}

// Manual solenoid control endpoint:
//  /solenoid?mode=stop   => FORCE_ON, manual override ON, status STOP
//  /solenoid?mode=go     => FORCE_OFF, manual override ON, status GO
//  /solenoid?mode=normal => NORMAL, manual override OFF, status NORMAL
void handleSolenoidControl() {
  if (!server.hasArg("mode")) {
    server.send(400, "text/plain", "missing mode");
    return;
  }

  String mode = server.arg("mode");

  if (mode == "stop") {
    solMode = FORCE_ON;
    solManualOverride = true;
    strncpy(g_sol_last_kw, "manual", sizeof(g_sol_last_kw) - 1);
    strncpy(g_sol_status,  "STOP",   sizeof(g_sol_status) - 1);

  } else if (mode == "go") {
    solMode = FORCE_OFF;
    solManualOverride = true;
    strncpy(g_sol_last_kw, "manual", sizeof(g_sol_last_kw) - 1);
    strncpy(g_sol_status,  "GO",     sizeof(g_sol_status) - 1);

  } else if (mode == "normal") {
    solMode = NORMAL;
    solManualOverride = false;
    strncpy(g_sol_last_kw, "manual", sizeof(g_sol_last_kw) - 1);
    strncpy(g_sol_status,  "NORMAL", sizeof(g_sol_status) - 1);
    lastToggleMs = millis();  // avoid immediate flip

  } else {
    server.send(400, "text/plain", "invalid mode");
    return;
  }

  g_sol_last_kw[sizeof(g_sol_last_kw) - 1] = '\0';
  g_sol_status[sizeof(g_sol_status) - 1] = '\0';

  server.send(200, "text/plain", "ok");
}

void setupWiFiWeb() {
  WiFi.mode(WIFI_STA);
  WiFi.begin(WIFI_SSID, WIFI_PASS);
  Serial.print("WiFi connecting");
  while (WiFi.status() != WL_CONNECTED) {
    delay(250);
    Serial.print(".");
  }
  Serial.println();
  Serial.print("IP: ");
  Serial.println(WiFi.localIP());

  server.on("/", handleRoot);
  server.on("/events", HTTP_GET, handleEvents);
  server.on("/solenoid", HTTP_GET, handleSolenoidControl);
  server.begin();
  Serial.println("Web:  http:///");
}

void pumpWeb() {
  server.handleClient();

  // Push SSE updates ~5 Hz
  static uint32_t lastSend = 0;
  uint32_t now = millis();
  if (sseClient && sseClient.connected() && now - lastSend >= 200) {
    lastSend = now;

    char buf[1700];
    snprintf(buf, sizeof(buf),
      "data: {\"label\":\"%s\",\"score\":%.4f,"
      "\"r\":%u,\"g\":%u,\"b\":%u,"
      "\"solLast\":\"%s\",\"solStatus\":\"%s\",\"solMan\":%s,"
      "\"all\":%s}\n\n",
      g_label, (double)g_score,
      g_led_r, g_led_g, g_led_b,
      g_sol_last_kw, g_sol_status, (solManualOverride ? "true" : "false"),
      g_all_scores_json
    );
    sseClient.print(buf);
  }
}
// ==========================================================


// ---------- Per-label thresholds + colors ----------
static float threshold_for_label(const char* label) {
  const float DEFAULT = 0.80f;
  if (strcmp(label, "Stupid")     == 0) return 0.90f;
  if (strcmp(label, "Gryffindor") == 0) return 0.85f;
  if (strcmp(label, "Fantastic")  == 0) return 0.85f;
  if (strcmp(label, "Good_idea")  == 0) return 0.90f;
  return DEFAULT;
}

static void color_for_label(const char* label, uint8_t &r, uint8_t &g, uint8_t &b) {
  r = g = b = 0;
  if (strcmp(label, "Stupid") == 0)           { r = 200; g = 0;   b = 200; }
  else if (strcmp(label, "Gryffindor") == 0)  { r = 220; g = 0;   b = 0;   }
  else if (strcmp(label, "Fantastic") == 0)  { r = 0;   g = 0;   b = 200; }
  else if (strcmp(label, "Good_idea") == 0)  { r = 0;   g = 200; b = 0;   }
}
// --------------------------------------------------

// Build JSON array of all scores into g_all_scores_json
static void buildAllScoresJson(const ei_impulse_result_t &result) {
  size_t pos = 0;
  g_all_scores_json[0] = '[';
  pos = 1;

  for (size_t ix = 0; ix < EI_CLASSIFIER_LABEL_COUNT; ix++) {
    const char* lab = result.classification[ix].label;
    float v = result.classification[ix].value;

    if (ix > 0) {
      if (pos + 1 >= sizeof(g_all_scores_json)) break;
      g_all_scores_json[pos++] = ',';
    }

    char chunk[128];
    int n = snprintf(chunk, sizeof(chunk), "{\"l\":\"%s\",\"v\":%.6f}", lab, (double)v);
    if (n <= 0) break;
    if (pos + (size_t)n >= sizeof(g_all_scores_json) - 2) break;

    memcpy(&g_all_scores_json[pos], chunk, (size_t)n);
    pos += (size_t)n;
  }

  if (pos + 2 >= sizeof(g_all_scores_json)) pos = sizeof(g_all_scores_json) - 3;
  g_all_scores_json[pos++] = ']';
  g_all_scores_json[pos] = '\0';
}

static void applySolenoidMode() {
  uint32_t now = millis();
  if (solMode == FORCE_OFF) {
    solenoidWrite(false);
  } else if (solMode == FORCE_ON) {
    solenoidWrite(true);
  } else { // NORMAL
    if (now - lastToggleMs >= 1000) {
      lastToggleMs = now;
      solState = !solState;
    }
    solenoidWrite(solState);
  }
}

void setup() {
  Serial.begin(115200);
  delay(1500);

  // NeoPixel init + dim
  strip.begin();
  strip.setBrightness(20);
  setPixelRGB(0, 0, 0);

  // Solenoid pin
  pinMode(SOLENOID_PIN, OUTPUT);
  solenoidWrite(false); // start OFF

  // Init UI state
  setUIState("boot", 0.0f, 0, 0, 0, "none", "NORMAL");

  // WiFi + Web UI
  setupWiFiWeb();

  // I2S init via AudioTools
  auto cfg = i2s.defaultConfig(RX_MODE);
  cfg.sample_rate     = EI_CLASSIFIER_FREQUENCY;
  cfg.bits_per_sample = 32;
  cfg.channels        = 1;
  cfg.pin_ws          = I2S_WS_PIN;
  cfg.pin_bck         = I2S_SCK_PIN;
  cfg.pin_data        = I2S_SD_PIN;
  i2s.begin(cfg);

  Serial.println("I2S + Edge Impulse inference starting...");
  Serial.print("EI sample rate: ");
  Serial.println(EI_CLASSIFIER_FREQUENCY);
  Serial.print("EI window samples: ");
  Serial.println(EI_CLASSIFIER_DSP_INPUT_FRAME_SIZE);

  lastToggleMs = millis();
}

void loop() {
  pumpWeb();

  // Keep solenoid behavior active even while waiting
  applySolenoidMode();

  // Fill EI buffer (blocking until one full window)
  size_t filled = 0;
  while (filled < EI_CLASSIFIER_DSP_INPUT_FRAME_SIZE) {
    pumpWeb();
    applySolenoidMode();

    size_t bytesRead = i2s.readBytes((uint8_t*)raw_i2s, sizeof(raw_i2s));
    if (bytesRead == 0) continue;

    int samples = bytesRead / (int)sizeof(int32_t);
    for (int i = 0; i < samples && filled < EI_CLASSIFIER_DSP_INPUT_FRAME_SIZE; i++) {
      int32_t mic32 = raw_i2s[i];

      // Your existing scaling
      int32_t s = mic32 >> 12;
      float f   = (float)s / 8388608.0f;   // 2^23
      ei_audio[filled++] = f;
    }
  }

  ei::signal_t signal;
  int err = ei::numpy::signal_from_buffer(ei_audio, EI_CLASSIFIER_DSP_INPUT_FRAME_SIZE, &signal);
  if (err != 0) return;

  ei_impulse_result_t result = { 0 };
  err = run_classifier(&signal, &result, /*debug=*/false);
  if (err != EI_IMPULSE_OK) return;

  // Build "all scores" JSON for the HTML
  buildAllScoresJson(result);

  // Find top label
  size_t best_i = 0;
  float best_v = result.classification[0].value;
  for (size_t i = 1; i < EI_CLASSIFIER_LABEL_COUNT; i++) {
    if (result.classification[i].value > best_v) {
      best_v = result.classification[i].value;
      best_i = i;
    }
  }
  const char* best_label = result.classification[best_i].label;

  // ---- Per-label threshold + streak smoothing ----
  static uint8_t streak = 0;
  static char last_label[64] = {0};

  float THRESH = threshold_for_label(best_label);
  bool confident = (best_v >= THRESH);

  if (confident && last_label[0] != '\0' && strcmp(best_label, last_label) == 0) streak++;
  else streak = 1;

  strncpy(last_label, best_label, sizeof(last_label) - 1);
  last_label[sizeof(last_label) - 1] = '\0';

  bool fire = confident && (streak >= 1);

  // ---------------- LED LOGIC (LATCHED) ----------------
  if (fire) {
    uint8_t r, g, b;
    color_for_label(best_label, r, g, b);
    setPixelRGB(r, g, b);
    setUIState(best_label, best_v, r, g, b, g_sol_last_kw, g_sol_status);
  } else {
    // update label/score in UI without changing LED color
    setUIState(best_label, best_v, g_led_r, g_led_g, g_led_b, g_sol_last_kw, g_sol_status);
  }
  // -----------------------------------------------------

  // ---------------- Solenoid control from KEYWORDS ----------------
  // Only update solenoid mode from keywords if NOT manually overridden.
  // Requested mapping:
  //   Fantastic => STOP  (FORCE_ON)
  //   Stupid    => GO    (FORCE_OFF)
  if (fire && !solManualOverride) {
    if (strcmp(best_label, "Stupid") == 0) {
      solMode = FORCE_OFF;

      strncpy(g_sol_last_kw, "Stupid", sizeof(g_sol_last_kw) - 1);
      g_sol_last_kw[sizeof(g_sol_last_kw) - 1] = '\0';
      strncpy(g_sol_status, "GO", sizeof(g_sol_status) - 1);
      g_sol_status[sizeof(g_sol_status) - 1] = '\0';

    } else if (strcmp(best_label, "Gryffindor") == 0) {
      solMode = FORCE_OFF;

      strncpy(g_sol_last_kw, "Gryffindor", sizeof(g_sol_last_kw) - 1);
      g_sol_last_kw[sizeof(g_sol_last_kw) - 1] = '\0';
      strncpy(g_sol_status, "GO", sizeof(g_sol_status) - 1);
      g_sol_status[sizeof(g_sol_status) - 1] = '\0';

    }
     else if (strcmp(best_label, "Fantastic") == 0) {
      solMode = FORCE_ON;

      strncpy(g_sol_last_kw, "Fantastic", sizeof(g_sol_last_kw) - 1);
      g_sol_last_kw[sizeof(g_sol_last_kw) - 1] = '\0';
      strncpy(g_sol_status, "STOP", sizeof(g_sol_status) - 1);
      g_sol_status[sizeof(g_sol_status) - 1] = '\0';

    } else if (strcmp(best_label, "Good_idea") == 0) {
      solMode = FORCE_ON;

      strncpy(g_sol_last_kw, "Good_idea", sizeof(g_sol_last_kw) - 1);
      g_sol_last_kw[sizeof(g_sol_last_kw) - 1] = '\0';
      strncpy(g_sol_status, "STOP", sizeof(g_sol_status) - 1);
      g_sol_status[sizeof(g_sol_status) - 1] = '\0';

    }
     else {
      solMode = NORMAL;
      lastToggleMs = millis();
      solState = false;
      // keep last GO/STOP latched
    }
  }
  // Apply mode immediately
  applySolenoidMode();
  // --------------------------------------------------

  Serial.print("best="); Serial.print(best_label);
  Serial.print(" v="); Serial.println(best_v, 3);

  pumpWeb();
  //delay(400);
}

Before reaching the final version of the arduino sketch, there were several iterations of debugging and refinement. For example, the initial version did not include the manual override logic. As in the video below, though the keyword-based control was effective (you can see the sand flow changing based on the keyword), the spotting loop was running a bit too fast.

Other Notes

PCB files and arduino codes can be found here. ChatGPT usage check here
Another thing super useful is that for MIT community, we can request temporal password for MIT Wi-fi via MIT Wi-fi Portal.