feat(m5stack-cardputer): Better mic+speaker support and update example accordingly

components/m5stack-cardputer/README.md

@@ -40,10 +40,14 @@ singleton hardware abstraction for initializing and using the subsystems:
   (0x34), ES8311 (0x18), and BMI270 (0x68).
 
 > [!NOTE]
-> The speaker and the microphone share I2S pins (GPIO 43 word-select / PDM
-> clock on the original; the BCK/WS pair on the ADV), so they cannot be used
-> at the same time; initializing one while the other is active will fail.
-> This matches the boards' hardware design.
+> On the original Cardputer the speaker and microphone cannot be used at the
+> same time: GPIO 43 doubles as the speaker's I2S word-select and the PDM
+> microphone's (MHz-range) clock - two different signals on one physical pin
+> - so initializing one while the other is active will fail. On the
+> Cardputer ADV both go through the ES8311 codec in full duplex on a single
+> I2S bus (shared bit/word clocks, separate data pins), so the speaker and
+> microphone can be used simultaneously; they share the I2S sample rate,
+> which is set by whichever subsystem is initialized first.
 
 ## Example
 

components/m5stack-cardputer/example/README.md

@@ -27,11 +27,21 @@ component to initialize and use the components on the M5Stack Cardputer:
 | Fn + `` ` `` | Clear the text area |
 | Fn + 1 (F1) | Show / hide the controls help popup |
 | Fn + 2 (F2) | Show / hide the IMU overlay (ADV) |
+| Fn + 3 (F3) | Start / stop recording from the microphone (ADV) |
+| Fn + 4 (F4) | Play back / stop the recording (ADV) |
 | Fn + number row | Show F1-F12 in the status bar |
 | G0 (BOOT) button | Cycle the RGB LED color |
 
 The controls are also printed to the log at startup.
 
+On the ADV the ES8311 codec runs the speaker and microphone in full duplex,
+so recording works while the speaker is active (key clicks and all). The
+recording buffer prefers PSRAM when present; since neither Cardputer variant
+ships with PSRAM, it normally falls back to a few seconds of voice-rate
+(16 kHz mono) audio in internal RAM. The original Cardputer cannot record in
+this example: its PDM microphone clock shares GPIO 43 with the speaker
+word-select, and the example uses the speaker.
+
 The status bar at the bottom of the screen shows the most recent key /
 modifier activity and is updated with the battery voltage every few seconds.
 

components/m5stack-cardputer/example/main/gui.hpp

@@ -39,6 +39,8 @@ class Gui {
                                            "fn+bksp      delete forward\n"
                                            "fn+1 (F1)    show/hide help\n"
                                            "fn+2 (F2)    show/hide IMU\n"
+                                           "fn+3 (F3)    record / stop\n"
+                                           "fn+4 (F4)    play recording\n"
                                            "G0 button    cycle LED color\n"
                                            "\n"
                                            "While help is open:\n"

components/m5stack-cardputer/example/main/m5stack_cardputer_example.cpp

@@ -1,9 +1,14 @@
+#include <atomic>
 #include <chrono>
 #include <cmath>
+#include <cstring>
 #include <numbers>
 #include <thread>
 #include <vector>
 
+#include <esp_heap_caps.h>
+#include <esp_timer.h>
+
 #include "m5stack-cardputer.hpp"
 
 #include "gui.hpp"
@@ -12,6 +17,26 @@ using namespace std::chrono_literals;
 
 static constexpr auto TAG = "cardputer_example";
 
+// Voice-quality sample rate for the speaker + microphone. On the ADV the two
+// run full duplex and share this rate; keeping it low keeps the recording
+// buffer small (neither Cardputer variant has PSRAM, so the recording
+// usually lives in internal RAM).
+static constexpr uint32_t AUDIO_SAMPLE_RATE_HZ = 16000;
+
+// Audio recording state (written by the microphone callback, read/controlled
+// from the keyboard callback and main loop)
+static constexpr size_t MAX_RECORDING_SECONDS = 30;     // when PSRAM is available
+static constexpr size_t FALLBACK_RECORDING_SECONDS = 3; // internal RAM fallback
+static uint8_t *recording_buffer = nullptr;
+static size_t recording_capacity = 0;
+static std::atomic<bool> recording{false};
+static std::atomic<size_t> recording_len{0};
+static std::atomic<bool> playing{false};
+// wall-clock bounds of the capture, for reporting the measured effective
+// sample rate (ordering is provided by the `recording` atomic)
+static int64_t recording_start_us = 0;
+static int64_t recording_last_us = 0;
+
 // Synthesize a short fading sine beep and queue it on the speaker
 static void play_beep(espp::M5StackCardputer &cardputer, float frequency_hz) {
   static constexpr float duration_s = 0.05f;
@@ -49,13 +74,12 @@ extern "C" void app_main(void) {
     return;
   }
 
-  // initialize the RGB LED and the sound subsystem (the microphone shares a
-  // pin with the speaker, so this example only uses the speaker)
+  // initialize the RGB LED and the sound subsystem
   if (!cardputer.initialize_led()) {
     logger.error("Failed to initialize RGB LED!");
     return;
   }
-  if (!cardputer.initialize_sound()) {
+  if (!cardputer.initialize_sound(AUDIO_SAMPLE_RATE_HZ)) {
     logger.error("Failed to initialize sound!");
     return;
   }
@@ -72,9 +96,10 @@ extern "C" void app_main(void) {
   // print the controls (also available on-screen via the fn+1 help popup)
   logger.info("Controls:\n{}", Gui::HELP_TEXT);
 
-  // whether the board has a working IMU; set after keyboard / variant
-  // detection below, referenced by the keypress callback
+  // whether the board has a working IMU / microphone; set after keyboard /
+  // variant detection below, referenced by the keypress callback
   static bool have_imu = false;
+  static bool have_mic = false;
 
   // the keyboard scanner delivers one event per key state change; use it to
   // drive the text editor, play key-click sounds, and show what's happening
@@ -97,6 +122,38 @@ extern "C" void app_main(void) {
         gui.set_status_text("No IMU on this board");
       }
       play_beep(cardputer, 660.0f);
+    } else if (event.special == espp::M5StackCardputer::SpecialKey::F3) {
+      // start / stop recording from the microphone
+      if (!have_mic || recording_capacity == 0) {
+        gui.set_status_text("No mic recording on this board");
+      } else if (recording) {
+        recording = false;
+        gui.set_status_text(
+            fmt::format("Recorded {:.1f}s (fn+4 plays)",
+                        static_cast<float>(recording_len) /
+                            (cardputer.microphone_sample_rate() * sizeof(int16_t))));
+      } else {
+        playing = false;
+        recording_len = 0;
+        recording_start_us = esp_timer_get_time();
+        recording_last_us = recording_start_us;
+        recording = true;
+        gui.set_status_text("Recording... (fn+3 stops)");
+      }
+      play_beep(cardputer, 660.0f);
+    } else if (event.special == espp::M5StackCardputer::SpecialKey::F4) {
+      // play back the recording (the main loop streams it to the speaker)
+      if (recording_len == 0) {
+        gui.set_status_text("Nothing recorded yet (fn+3 records)");
+      } else if (playing) {
+        playing = false;
+        gui.set_status_text("Playback stopped");
+      } else {
+        recording = false;
+        playing = true;
+        gui.set_status_text("Playing... (fn+4 stops)");
+      }
+      play_beep(cardputer, 660.0f);
     } else if (event.special != espp::M5StackCardputer::SpecialKey::NONE) {
       gui.handle_special_key(event.special);
       gui.set_status_text(espp::M5StackCardputer::special_key_name(event.special));
@@ -140,6 +197,54 @@ extern "C" void app_main(void) {
     }
   }
 
+  // On the ADV the ES8311 codec runs full duplex, so the microphone can be
+  // used at the same time as the speaker (it shares the speaker's sample
+  // rate). The original cannot: its PDM microphone clock and the speaker
+  // word-select share GPIO 43, and this example uses the speaker.
+  if (cardputer.variant() == espp::M5StackCardputer::Variant::ADV) {
+    auto mic_callback = [](const uint8_t *data, size_t num_bytes) {
+      if (!recording) {
+        return;
+      }
+      size_t offset = recording_len;
+      size_t to_copy = std::min(num_bytes, recording_capacity - offset);
+      if (to_copy > 0) {
+        memcpy(recording_buffer + offset, data, to_copy);
+        recording_last_us = esp_timer_get_time();
+        recording_len = offset + to_copy;
+      }
+      if (recording_len >= recording_capacity) {
+        // buffer full; the main loop notices recording went false and
+        // updates the status bar
+        recording = false;
+      }
+    };
+    have_mic = cardputer.initialize_microphone(mic_callback);
+    if (have_mic) {
+      // allocate the recording buffer: prefer PSRAM (neither Cardputer
+      // variant ships with it, but boards / mods that have it get a much
+      // longer recording), fall back to a few seconds in internal RAM
+      size_t bytes_per_second = cardputer.microphone_sample_rate() * sizeof(int16_t);
+      recording_capacity = MAX_RECORDING_SECONDS * bytes_per_second;
+      recording_buffer = static_cast<uint8_t *>(
+          heap_caps_malloc(recording_capacity, MALLOC_CAP_SPIRAM | MALLOC_CAP_8BIT));
+      if (recording_buffer == nullptr) {
+        recording_capacity = FALLBACK_RECORDING_SECONDS * bytes_per_second;
+        recording_buffer =
+            static_cast<uint8_t *>(heap_caps_malloc(recording_capacity, MALLOC_CAP_8BIT));
+      }
+      if (recording_buffer == nullptr) {
+        logger.warn("Could not allocate a recording buffer; recording disabled");
+        recording_capacity = 0;
+      } else {
+        logger.info("Recording buffer: {} KB ({} s at {} Hz)", recording_capacity / 1024,
+                    recording_capacity / bytes_per_second, cardputer.microphone_sample_rate());
+      }
+    } else {
+      logger.warn("Could not initialize the microphone!");
+    }
+  }
+
   // the G0 (BOOT) button cycles the RGB LED color
   static std::atomic<int> led_hue{0};
   auto button_callback = [&](const espp::Interrupt::Event &event) {
@@ -157,29 +262,65 @@ extern "C" void app_main(void) {
   // set the initial LED color
   cardputer.led(espp::Hsv(static_cast<float>(led_hue), 1.0f, 0.2f));
 
-  // periodically update the status bar with the battery voltage / state of
-  // charge, and (on the ADV) the IMU overlay with the latest accelerometer
-  // and gyroscope readings
-  static constexpr auto imu_period = 100ms;
-  int loops_per_battery_update = std::chrono::seconds(5) / imu_period;
+  // Main loop: stream any active playback to the speaker, update the IMU
+  // popup, and periodically show the battery voltage / state of charge in
+  // the status bar
+  static constexpr auto loop_period = 50ms;
+  const int loops_per_imu_update = 2; // 100 ms
+  const int loops_per_battery_update = std::chrono::seconds(5) / loop_period;
+  size_t play_offset = 0;
+  bool was_recording = false;
   int loop_count = 0;
   while (true) {
-    if (have_imu && gui.imu_visible()) {
+    // feed the active playback in chunks, advancing by however much the
+    // speaker's stream buffer accepted
+    if (playing) {
+      size_t len = recording_len;
+      if (play_offset >= len) {
+        playing = false;
+        play_offset = 0;
+        gui.set_status_text("Playback done");
+      } else {
+        play_offset += cardputer.play_audio(recording_buffer + play_offset,
+                                            std::min<size_t>(len - play_offset, 4096));
+      }
+    } else {
+      play_offset = 0;
+    }
+    // notice when the recording stopped (buffer filled up, or fn+3 / fn+4)
+    bool now_recording = recording;
+    if (was_recording && !now_recording) {
+      // report the measured capture rate: samples recorded over the wall
+      // clock they took to arrive should match the nominal sample rate
+      size_t num_samples = recording_len / sizeof(int16_t);
+      float elapsed_s = static_cast<float>(recording_last_us - recording_start_us) / 1e6f;
+      float effective_hz = elapsed_s > 0.0f ? num_samples / elapsed_s : 0.0f;
+      logger.info("Recorded {} samples in {:.2f} s (~{:.0f} Hz effective, {} Hz nominal)",
+                  num_samples, elapsed_s, effective_hz, cardputer.microphone_sample_rate());
+      gui.set_status_text(fmt::format("Recorded {:.1f}s (fn+4 plays)",
+                                      static_cast<float>(recording_len) /
+                                          (cardputer.microphone_sample_rate() * sizeof(int16_t))));
+    }
+    was_recording = now_recording;
+
+    if (have_imu && gui.imu_visible() && (loop_count % loops_per_imu_update) == 0) {
       auto imu = cardputer.imu();
       std::error_code ec;
-      if (imu->update(std::chrono::duration<float>(imu_period).count(), ec)) {
+      if (imu->update(std::chrono::duration<float>(loop_period * loops_per_imu_update).count(),
+                      ec)) {
         auto accel = imu->get_accelerometer();
         auto gyro = imu->get_gyroscope();
         gui.set_imu_text(fmt::format("a {:+.1f} {:+.1f} {:+.1f}\ng {:+5.0f} {:+5.0f} {:+5.0f}",
                                      accel.x, accel.y, accel.z, gyro.x, gyro.y, gyro.z));
       }
     }
-    if ((loop_count % loops_per_battery_update) == 0) {
+    // don't overwrite the recording / playback status with the battery
+    if ((loop_count % loops_per_battery_update) == 0 && !recording && !playing) {
       gui.set_status_text(fmt::format("Battery: {:.2f} V ({:.0f}%)", cardputer.battery_voltage(),
                                       cardputer.battery_soc()));
     }
     loop_count++;
-    std::this_thread::sleep_for(imu_period);
+    std::this_thread::sleep_for(loop_period);
   }
   //! [m5stack-cardputer example]
 }

components/m5stack-cardputer/example/sdkconfig.defaults

@@ -52,3 +52,9 @@ CONFIG_LV_USE_THEME_DEFAULT=y
 CONFIG_LV_THEME_DEFAULT_DARK=y
 CONFIG_LV_THEME_DEFAULT_GROW=y
 CONFIG_LV_THEME_DEFAULT_TRANSITION_TIME=80
+
+# Neither Cardputer variant ships with PSRAM, but enable support for it (the
+# example prefers it for the audio recording buffer when present);
+# ignore-not-found keeps PSRAM-less boards booting normally.
+CONFIG_SPIRAM=y
+CONFIG_SPIRAM_IGNORE_NOTFOUND=y