diff --git a/components/m5stack-cardputer/README.md b/components/m5stack-cardputer/README.md index be2539561..04955f53f 100644 --- a/components/m5stack-cardputer/README.md +++ b/components/m5stack-cardputer/README.md @@ -40,10 +40,14 @@ singleton hardware abstraction for initializing and using the subsystems: (0x34), ES8311 (0x18), and BMI270 (0x68). > [!NOTE] -> The speaker and the microphone share I2S pins (GPIO 43 word-select / PDM -> clock on the original; the BCK/WS pair on the ADV), so they cannot be used -> at the same time; initializing one while the other is active will fail. -> This matches the boards' hardware design. +> On the original Cardputer the speaker and microphone cannot be used at the +> same time: GPIO 43 doubles as the speaker's I2S word-select and the PDM +> microphone's (MHz-range) clock - two different signals on one physical pin +> - so initializing one while the other is active will fail. On the +> Cardputer ADV both go through the ES8311 codec in full duplex on a single +> I2S bus (shared bit/word clocks, separate data pins), so the speaker and +> microphone can be used simultaneously; they share the I2S sample rate, +> which is set by whichever subsystem is initialized first. ## Example diff --git a/components/m5stack-cardputer/example/README.md b/components/m5stack-cardputer/example/README.md index 82550dd8f..b55ebb915 100644 --- a/components/m5stack-cardputer/example/README.md +++ b/components/m5stack-cardputer/example/README.md @@ -27,11 +27,21 @@ component to initialize and use the components on the M5Stack Cardputer: | Fn + `` ` `` | Clear the text area | | Fn + 1 (F1) | Show / hide the controls help popup | | Fn + 2 (F2) | Show / hide the IMU overlay (ADV) | +| Fn + 3 (F3) | Start / stop recording from the microphone (ADV) | +| Fn + 4 (F4) | Play back / stop the recording (ADV) | | Fn + number row | Show F1-F12 in the status bar | | G0 (BOOT) button | Cycle the RGB LED color | The controls are also printed to the log at startup. +On the ADV the ES8311 codec runs the speaker and microphone in full duplex, +so recording works while the speaker is active (key clicks and all). The +recording buffer prefers PSRAM when present; since neither Cardputer variant +ships with PSRAM, it normally falls back to a few seconds of voice-rate +(16 kHz mono) audio in internal RAM. The original Cardputer cannot record in +this example: its PDM microphone clock shares GPIO 43 with the speaker +word-select, and the example uses the speaker. + The status bar at the bottom of the screen shows the most recent key / modifier activity and is updated with the battery voltage every few seconds. diff --git a/components/m5stack-cardputer/example/main/gui.hpp b/components/m5stack-cardputer/example/main/gui.hpp index b57add876..d51d7d2c4 100644 --- a/components/m5stack-cardputer/example/main/gui.hpp +++ b/components/m5stack-cardputer/example/main/gui.hpp @@ -39,6 +39,8 @@ class Gui { "fn+bksp delete forward\n" "fn+1 (F1) show/hide help\n" "fn+2 (F2) show/hide IMU\n" + "fn+3 (F3) record / stop\n" + "fn+4 (F4) play recording\n" "G0 button cycle LED color\n" "\n" "While help is open:\n" diff --git a/components/m5stack-cardputer/example/main/m5stack_cardputer_example.cpp b/components/m5stack-cardputer/example/main/m5stack_cardputer_example.cpp index 5d775c978..24c619943 100644 --- a/components/m5stack-cardputer/example/main/m5stack_cardputer_example.cpp +++ b/components/m5stack-cardputer/example/main/m5stack_cardputer_example.cpp @@ -1,9 +1,14 @@ +#include #include #include +#include #include #include #include +#include +#include + #include "m5stack-cardputer.hpp" #include "gui.hpp" @@ -12,6 +17,26 @@ using namespace std::chrono_literals; static constexpr auto TAG = "cardputer_example"; +// Voice-quality sample rate for the speaker + microphone. On the ADV the two +// run full duplex and share this rate; keeping it low keeps the recording +// buffer small (neither Cardputer variant has PSRAM, so the recording +// usually lives in internal RAM). +static constexpr uint32_t AUDIO_SAMPLE_RATE_HZ = 16000; + +// Audio recording state (written by the microphone callback, read/controlled +// from the keyboard callback and main loop) +static constexpr size_t MAX_RECORDING_SECONDS = 30; // when PSRAM is available +static constexpr size_t FALLBACK_RECORDING_SECONDS = 3; // internal RAM fallback +static uint8_t *recording_buffer = nullptr; +static size_t recording_capacity = 0; +static std::atomic recording{false}; +static std::atomic recording_len{0}; +static std::atomic playing{false}; +// wall-clock bounds of the capture, for reporting the measured effective +// sample rate (ordering is provided by the `recording` atomic) +static int64_t recording_start_us = 0; +static int64_t recording_last_us = 0; + // Synthesize a short fading sine beep and queue it on the speaker static void play_beep(espp::M5StackCardputer &cardputer, float frequency_hz) { static constexpr float duration_s = 0.05f; @@ -49,13 +74,12 @@ extern "C" void app_main(void) { return; } - // initialize the RGB LED and the sound subsystem (the microphone shares a - // pin with the speaker, so this example only uses the speaker) + // initialize the RGB LED and the sound subsystem if (!cardputer.initialize_led()) { logger.error("Failed to initialize RGB LED!"); return; } - if (!cardputer.initialize_sound()) { + if (!cardputer.initialize_sound(AUDIO_SAMPLE_RATE_HZ)) { logger.error("Failed to initialize sound!"); return; } @@ -72,9 +96,10 @@ extern "C" void app_main(void) { // print the controls (also available on-screen via the fn+1 help popup) logger.info("Controls:\n{}", Gui::HELP_TEXT); - // whether the board has a working IMU; set after keyboard / variant - // detection below, referenced by the keypress callback + // whether the board has a working IMU / microphone; set after keyboard / + // variant detection below, referenced by the keypress callback static bool have_imu = false; + static bool have_mic = false; // the keyboard scanner delivers one event per key state change; use it to // drive the text editor, play key-click sounds, and show what's happening @@ -97,6 +122,38 @@ extern "C" void app_main(void) { gui.set_status_text("No IMU on this board"); } play_beep(cardputer, 660.0f); + } else if (event.special == espp::M5StackCardputer::SpecialKey::F3) { + // start / stop recording from the microphone + if (!have_mic || recording_capacity == 0) { + gui.set_status_text("No mic recording on this board"); + } else if (recording) { + recording = false; + gui.set_status_text( + fmt::format("Recorded {:.1f}s (fn+4 plays)", + static_cast(recording_len) / + (cardputer.microphone_sample_rate() * sizeof(int16_t)))); + } else { + playing = false; + recording_len = 0; + recording_start_us = esp_timer_get_time(); + recording_last_us = recording_start_us; + recording = true; + gui.set_status_text("Recording... (fn+3 stops)"); + } + play_beep(cardputer, 660.0f); + } else if (event.special == espp::M5StackCardputer::SpecialKey::F4) { + // play back the recording (the main loop streams it to the speaker) + if (recording_len == 0) { + gui.set_status_text("Nothing recorded yet (fn+3 records)"); + } else if (playing) { + playing = false; + gui.set_status_text("Playback stopped"); + } else { + recording = false; + playing = true; + gui.set_status_text("Playing... (fn+4 stops)"); + } + play_beep(cardputer, 660.0f); } else if (event.special != espp::M5StackCardputer::SpecialKey::NONE) { gui.handle_special_key(event.special); gui.set_status_text(espp::M5StackCardputer::special_key_name(event.special)); @@ -140,6 +197,54 @@ extern "C" void app_main(void) { } } + // On the ADV the ES8311 codec runs full duplex, so the microphone can be + // used at the same time as the speaker (it shares the speaker's sample + // rate). The original cannot: its PDM microphone clock and the speaker + // word-select share GPIO 43, and this example uses the speaker. + if (cardputer.variant() == espp::M5StackCardputer::Variant::ADV) { + auto mic_callback = [](const uint8_t *data, size_t num_bytes) { + if (!recording) { + return; + } + size_t offset = recording_len; + size_t to_copy = std::min(num_bytes, recording_capacity - offset); + if (to_copy > 0) { + memcpy(recording_buffer + offset, data, to_copy); + recording_last_us = esp_timer_get_time(); + recording_len = offset + to_copy; + } + if (recording_len >= recording_capacity) { + // buffer full; the main loop notices recording went false and + // updates the status bar + recording = false; + } + }; + have_mic = cardputer.initialize_microphone(mic_callback); + if (have_mic) { + // allocate the recording buffer: prefer PSRAM (neither Cardputer + // variant ships with it, but boards / mods that have it get a much + // longer recording), fall back to a few seconds in internal RAM + size_t bytes_per_second = cardputer.microphone_sample_rate() * sizeof(int16_t); + recording_capacity = MAX_RECORDING_SECONDS * bytes_per_second; + recording_buffer = static_cast( + heap_caps_malloc(recording_capacity, MALLOC_CAP_SPIRAM | MALLOC_CAP_8BIT)); + if (recording_buffer == nullptr) { + recording_capacity = FALLBACK_RECORDING_SECONDS * bytes_per_second; + recording_buffer = + static_cast(heap_caps_malloc(recording_capacity, MALLOC_CAP_8BIT)); + } + if (recording_buffer == nullptr) { + logger.warn("Could not allocate a recording buffer; recording disabled"); + recording_capacity = 0; + } else { + logger.info("Recording buffer: {} KB ({} s at {} Hz)", recording_capacity / 1024, + recording_capacity / bytes_per_second, cardputer.microphone_sample_rate()); + } + } else { + logger.warn("Could not initialize the microphone!"); + } + } + // the G0 (BOOT) button cycles the RGB LED color static std::atomic led_hue{0}; auto button_callback = [&](const espp::Interrupt::Event &event) { @@ -157,29 +262,65 @@ extern "C" void app_main(void) { // set the initial LED color cardputer.led(espp::Hsv(static_cast(led_hue), 1.0f, 0.2f)); - // periodically update the status bar with the battery voltage / state of - // charge, and (on the ADV) the IMU overlay with the latest accelerometer - // and gyroscope readings - static constexpr auto imu_period = 100ms; - int loops_per_battery_update = std::chrono::seconds(5) / imu_period; + // Main loop: stream any active playback to the speaker, update the IMU + // popup, and periodically show the battery voltage / state of charge in + // the status bar + static constexpr auto loop_period = 50ms; + const int loops_per_imu_update = 2; // 100 ms + const int loops_per_battery_update = std::chrono::seconds(5) / loop_period; + size_t play_offset = 0; + bool was_recording = false; int loop_count = 0; while (true) { - if (have_imu && gui.imu_visible()) { + // feed the active playback in chunks, advancing by however much the + // speaker's stream buffer accepted + if (playing) { + size_t len = recording_len; + if (play_offset >= len) { + playing = false; + play_offset = 0; + gui.set_status_text("Playback done"); + } else { + play_offset += cardputer.play_audio(recording_buffer + play_offset, + std::min(len - play_offset, 4096)); + } + } else { + play_offset = 0; + } + // notice when the recording stopped (buffer filled up, or fn+3 / fn+4) + bool now_recording = recording; + if (was_recording && !now_recording) { + // report the measured capture rate: samples recorded over the wall + // clock they took to arrive should match the nominal sample rate + size_t num_samples = recording_len / sizeof(int16_t); + float elapsed_s = static_cast(recording_last_us - recording_start_us) / 1e6f; + float effective_hz = elapsed_s > 0.0f ? num_samples / elapsed_s : 0.0f; + logger.info("Recorded {} samples in {:.2f} s (~{:.0f} Hz effective, {} Hz nominal)", + num_samples, elapsed_s, effective_hz, cardputer.microphone_sample_rate()); + gui.set_status_text(fmt::format("Recorded {:.1f}s (fn+4 plays)", + static_cast(recording_len) / + (cardputer.microphone_sample_rate() * sizeof(int16_t)))); + } + was_recording = now_recording; + + if (have_imu && gui.imu_visible() && (loop_count % loops_per_imu_update) == 0) { auto imu = cardputer.imu(); std::error_code ec; - if (imu->update(std::chrono::duration(imu_period).count(), ec)) { + if (imu->update(std::chrono::duration(loop_period * loops_per_imu_update).count(), + ec)) { auto accel = imu->get_accelerometer(); auto gyro = imu->get_gyroscope(); gui.set_imu_text(fmt::format("a {:+.1f} {:+.1f} {:+.1f}\ng {:+5.0f} {:+5.0f} {:+5.0f}", accel.x, accel.y, accel.z, gyro.x, gyro.y, gyro.z)); } } - if ((loop_count % loops_per_battery_update) == 0) { + // don't overwrite the recording / playback status with the battery + if ((loop_count % loops_per_battery_update) == 0 && !recording && !playing) { gui.set_status_text(fmt::format("Battery: {:.2f} V ({:.0f}%)", cardputer.battery_voltage(), cardputer.battery_soc())); } loop_count++; - std::this_thread::sleep_for(imu_period); + std::this_thread::sleep_for(loop_period); } //! [m5stack-cardputer example] } diff --git a/components/m5stack-cardputer/example/sdkconfig.defaults b/components/m5stack-cardputer/example/sdkconfig.defaults index 6237a09eb..1343ed684 100644 --- a/components/m5stack-cardputer/example/sdkconfig.defaults +++ b/components/m5stack-cardputer/example/sdkconfig.defaults @@ -52,3 +52,9 @@ CONFIG_LV_USE_THEME_DEFAULT=y CONFIG_LV_THEME_DEFAULT_DARK=y CONFIG_LV_THEME_DEFAULT_GROW=y CONFIG_LV_THEME_DEFAULT_TRANSITION_TIME=80 + +# Neither Cardputer variant ships with PSRAM, but enable support for it (the +# example prefers it for the audio recording buffer when present); +# ignore-not-found keeps PSRAM-less boards booting normally. +CONFIG_SPIRAM=y +CONFIG_SPIRAM_IGNORE_NOTFOUND=y diff --git a/components/m5stack-cardputer/include/m5stack-cardputer.hpp b/components/m5stack-cardputer/include/m5stack-cardputer.hpp index 936f8cfb0..45782b13c 100644 --- a/components/m5stack-cardputer/include/m5stack-cardputer.hpp +++ b/components/m5stack-cardputer/include/m5stack-cardputer.hpp @@ -62,10 +62,15 @@ namespace espp { /// /// The class is a singleton and can be accessed using the get() method. /// -/// \note The speaker and the microphone share I2S pins (GPIO 43 word-select / -/// PDM clock on the original; GPIO 41/43 bit-clock and word-select on -/// the ADV), so they cannot be used at the same time. Initializing one -/// while the other is active will fail. +/// \note On the original Cardputer the speaker and microphone cannot be used +/// at the same time: GPIO 43 doubles as the speaker's I2S word-select +/// and the PDM microphone's (MHz-range) clock - two different signals +/// on one physical pin - so initializing one while the other is active +/// will fail. On the Cardputer ADV both go through the ES8311 codec in +/// full duplex on a single I2S bus (shared bit/word clocks, separate +/// data pins), so the speaker and microphone can be used +/// simultaneously; they share the I2S sample rate, which is set by +/// whichever subsystem is initialized first. /// /// \section m5stack_cardputer_example Example /// \snippet m5stack_cardputer_example.cpp m5stack-cardputer example @@ -361,8 +366,11 @@ class M5StackCardputer : public BaseComponent { /// come from the M5STACK_CARDPUTER_AUDIO_TASK_* Kconfig options. /// \return true if the sound subsystem was successfully initialized, false /// otherwise - /// \note The speaker shares I2S pins with the microphone, so this will - /// fail if the microphone has been initialized. + /// \note On the original Cardputer this will fail if the microphone has + /// been initialized (see the class notes). On the ADV the speaker + /// and microphone run full duplex and share the I2S sample rate; if + /// the microphone was initialized first, its sample rate is kept and + /// \p default_audio_rate is ignored (with a warning). bool initialize_sound(uint32_t default_audio_rate = 44100, const espp::Task::BaseConfig &task_config = { .name = "audio", @@ -402,16 +410,22 @@ class M5StackCardputer : public BaseComponent { /// Play the audio data /// \param data The audio data to play (16-bit signed mono samples) + /// \return The number of bytes actually queued (may be less than the data + /// size if the internal stream buffer is full) /// \note This function is non-blocking and queues the data for the audio - /// task to play - void play_audio(const std::vector &data); + /// task to play; to stream data larger than the internal buffer, + /// call it repeatedly, advancing by the returned number of bytes + size_t play_audio(const std::vector &data); /// Play the audio data /// \param data The audio data to play (16-bit signed mono samples) /// \param num_bytes The number of bytes to play + /// \return The number of bytes actually queued (may be less than \p + /// num_bytes if the internal stream buffer is full) /// \note This function is non-blocking and queues the data for the audio - /// task to play - void play_audio(const uint8_t *data, uint32_t num_bytes); + /// task to play; to stream data larger than the internal buffer, + /// call it repeatedly, advancing by the returned number of bytes + size_t play_audio(const uint8_t *data, uint32_t num_bytes); ///////////////////////////////////////////////////////////////////////////// // Microphone @@ -431,8 +445,12 @@ class M5StackCardputer : public BaseComponent { /// \note The callback runs in the microphone task's context, so the task's /// stack must be large enough for whatever the callback does with /// the audio data. - /// \note The microphone shares I2S pins with the speaker, so this will fail - /// if the sound subsystem has been initialized. + /// \note On the original Cardputer this will fail if the sound subsystem + /// has been initialized (see the class notes). On the ADV the + /// speaker and microphone run full duplex and share the I2S sample + /// rate; if the sound subsystem was initialized first, its sample + /// rate is kept and \p sample_rate is ignored (with a warning) - + /// check microphone_sample_rate() for the actual rate. bool initialize_microphone(const microphone_callback_t &callback, uint32_t sample_rate = 16000, const espp::Task::BaseConfig &task_config = { @@ -552,6 +570,8 @@ class M5StackCardputer : public BaseComponent { M5StackCardputer(); void lcd_wait_lines(); bool initialize_i2s(uint32_t default_audio_rate); + bool ensure_adv_i2s(uint32_t sample_rate); + bool es8311_ensure_common(); bool audio_task_callback(std::mutex &m, std::condition_variable &cv, bool &task_notified); bool microphone_task_callback(std::mutex &m, std::condition_variable &cv, bool &task_notified); bool keyboard_task_callback(std::mutex &m, std::condition_variable &cv, bool &task_notified); @@ -725,7 +745,12 @@ class M5StackCardputer : public BaseComponent { static constexpr int UPDATE_FREQUENCY = 60; static constexpr int calc_audio_buffer_size(int sample_rate) { - return sample_rate * NUM_CHANNELS * NUM_BYTES_PER_CHANNEL / UPDATE_FREQUENCY; + // NOTE: divide the rate by the update frequency FIRST so the result is + // always a whole number of samples. Otherwise rates that are not a + // multiple of the update frequency (e.g. 16 kHz) yield an odd byte + // count, and reading/writing partial samples shifts the I2S sample + // framing on every transfer - heard as loud static. + return (sample_rate / UPDATE_FREQUENCY) * NUM_CHANNELS * NUM_BYTES_PER_CHANNEL; } // Microphone. Original: SPM1423 PDM (clk = GPIO 43, shared with the @@ -840,6 +865,9 @@ class M5StackCardputer : public BaseComponent { // sound std::atomic sound_initialized_{false}; + // whether the ES8311's common (reset / clocking / analog power) registers + // have been written (ADV only; shared by the speaker and microphone paths) + bool es8311_common_initialized_{false}; std::atomic mute_{false}; std::atomic volume_{50.0f}; std::unique_ptr audio_task_{nullptr}; @@ -855,6 +883,9 @@ class M5StackCardputer : public BaseComponent { std::unique_ptr microphone_task_{nullptr}; i2s_chan_handle_t audio_rx_handle{nullptr}; std::vector audio_rx_buffer; + // true when the RX channel captures both 16-bit slots per frame (ADV full + // duplex); the microphone task then keeps only the left slot + bool mic_stereo_capture_{false}; i2s_pdm_rx_config_t mic_pdm_cfg; }; // class M5StackCardputer } // namespace espp diff --git a/components/m5stack-cardputer/src/audio.cpp b/components/m5stack-cardputer/src/audio.cpp index fcb9856c5..8c59687d7 100644 --- a/components/m5stack-cardputer/src/audio.cpp +++ b/components/m5stack-cardputer/src/audio.cpp @@ -9,6 +9,8 @@ using namespace espp; //////////////////////// bool M5StackCardputer::initialize_i2s(uint32_t default_audio_rate) { + // original Cardputer only: transmit-only standard I2S into the NS4168 + // amplifier (the ADV uses the full-duplex path in ensure_adv_i2s()) logger_.info("initializing i2s driver"); i2s_chan_config_t chan_cfg = I2S_CHANNEL_DEFAULT_CONFIG(i2s_port, I2S_ROLE_MASTER); @@ -28,14 +30,59 @@ bool M5StackCardputer::initialize_i2s(uint32_t default_audio_rate) { ESP_ERROR_CHECK(i2s_channel_init_std_mode(audio_tx_handle, &audio_std_cfg)); - auto buffer_size = calc_audio_buffer_size(default_audio_rate); - audio_tx_buffer.resize(buffer_size); + ESP_ERROR_CHECK(i2s_channel_enable(audio_tx_handle)); - audio_tx_stream = xStreamBufferCreate(buffer_size * 4, 0); + return true; +} - xStreamBufferReset(audio_tx_stream); +bool M5StackCardputer::ensure_adv_i2s(uint32_t sample_rate) { + // ADV only: the ES8311 codec is a full-duplex device on a single I2S bus + // (shared bit/word clocks; DAC data on dout, ADC data on din), so both the + // transmit and receive channels are allocated together on one port. The + // channels are enabled separately by initialize_sound() / + // initialize_microphone(). + if (audio_tx_handle != nullptr) { + // already created by whichever of sound / microphone was initialized + // first; the clock is shared, so a different requested rate cannot be + // honored + if (sample_rate != audio_std_cfg.clk_cfg.sample_rate_hz) { + logger_.warn("I2S already configured at {} Hz; ignoring requested {} Hz (the ADV speaker " + "and microphone share the I2S clock)", + audio_std_cfg.clk_cfg.sample_rate_hz, sample_rate); + } + return true; + } - ESP_ERROR_CHECK(i2s_channel_enable(audio_tx_handle)); + logger_.info("initializing full-duplex i2s driver at {} Hz", sample_rate); + + i2s_chan_config_t chan_cfg = I2S_CHANNEL_DEFAULT_CONFIG(i2s_port, I2S_ROLE_MASTER); + chan_cfg.auto_clear = true; // Auto clear the legacy data in the DMA buffer + ESP_ERROR_CHECK(i2s_new_channel(&chan_cfg, &audio_tx_handle, &audio_rx_handle)); + + audio_std_cfg = { + .clk_cfg = I2S_STD_CLK_DEFAULT_CONFIG(sample_rate), + .slot_cfg = I2S_STD_PHILIPS_SLOT_DEFAULT_CONFIG(I2S_DATA_BIT_WIDTH_16BIT, I2S_SLOT_MODE_MONO), + .gpio_cfg = {.mclk = GPIO_NUM_NC, + .bclk = i2s_bck_io, + .ws = i2s_ws_io, + .dout = i2s_do_io, + .din = mic_data_io, + .invert_flags = {.mclk_inv = false, .bclk_inv = false, .ws_inv = false}}, + }; + + ESP_ERROR_CHECK(i2s_channel_init_std_mode(audio_tx_handle, &audio_std_cfg)); + + // Receive in stereo (both 16-bit slots of every frame) even though the + // codec's ADC is mono: a mono RX slot configuration in this full-duplex + // setup does not deliver one sample per frame (recordings came out with + // every sample doubled, i.e. at half speed and an octave low). Capturing + // both slots gives a deterministic L,R word layout; the microphone task + // keeps only the left slot, where the ES8311 drives its ADC data. The + // transmit side is unaffected. + i2s_std_config_t rx_cfg = audio_std_cfg; + rx_cfg.slot_cfg = + I2S_STD_PHILIPS_SLOT_DEFAULT_CONFIG(I2S_DATA_BIT_WIDTH_16BIT, I2S_SLOT_MODE_STEREO); + ESP_ERROR_CHECK(i2s_channel_init_std_mode(audio_rx_handle, &rx_cfg)); return true; } @@ -46,31 +93,74 @@ bool M5StackCardputer::initialize_sound(uint32_t default_audio_rate, logger_.warn("Sound already initialized"); return true; } - if (microphone_initialized_) { - logger_.error("Cannot initialize sound: the microphone is active and shares GPIO {} with the " - "speaker (WS / PDM clock)", - static_cast(i2s_ws_io)); - return false; + + bool is_adv = variant() == Variant::ADV; + + if (is_adv) { + // full duplex with the microphone; create (or reuse) the shared channels + if (!ensure_adv_i2s(default_audio_rate)) { + logger_.error("Could not initialize I2S driver"); + return false; + } + } else { + // the original cannot run the speaker and PDM microphone at the same + // time: GPIO 43 doubles as the speaker WS and the microphone clock + if (microphone_initialized_) { + logger_.error("Cannot initialize sound: the microphone is active and shares GPIO {} with " + "the speaker (WS / PDM clock)", + static_cast(i2s_ws_io)); + return false; + } + if (!initialize_i2s(default_audio_rate)) { + logger_.error("Could not initialize I2S driver"); + return false; + } } - if (!initialize_i2s(default_audio_rate)) { - logger_.error("Could not initialize I2S driver"); + + // size the transmit buffering from the actual (possibly shared) rate + size_t buffer_size = calc_audio_buffer_size(audio_sample_rate()); + audio_tx_buffer.resize(buffer_size); + audio_tx_stream = xStreamBufferCreate(buffer_size * 4, 0); + if (audio_tx_stream == nullptr) { + logger_.error("Could not allocate the audio stream buffer"); + audio_tx_buffer.clear(); + if (is_adv) { + if (!microphone_initialized_) { + // the channels are shared with the microphone; only tear them down + // if it is not using them + i2s_del_channel(audio_tx_handle); + i2s_del_channel(audio_rx_handle); + audio_tx_handle = nullptr; + audio_rx_handle = nullptr; + } + } else { + i2s_channel_disable(audio_tx_handle); + i2s_del_channel(audio_tx_handle); + audio_tx_handle = nullptr; + } return false; } + xStreamBufferReset(audio_tx_stream); - // On the ADV the I2S signals go through an ES8311 codec (then an NS4150B - // amplifier) instead of directly into an NS4168 amplifier, so the codec - // must be configured. Do this after the I2S channel is enabled since the - // codec derives its clock from BCLK. - if (variant() == Variant::ADV) { + if (is_adv) { + ESP_ERROR_CHECK(i2s_channel_enable(audio_tx_handle)); + // On the ADV the I2S signals go through the ES8311 codec (then an + // NS4150B amplifier), so the codec's DAC path must be configured. Do + // this after the I2S channel is enabled since the codec derives its + // clock from BCLK. if (!initialize_es8311_speaker()) { logger_.error("Could not initialize the ES8311 codec"); i2s_channel_disable(audio_tx_handle); - i2s_del_channel(audio_tx_handle); - audio_tx_handle = nullptr; - if (audio_tx_stream) { - vStreamBufferDelete(audio_tx_stream); - audio_tx_stream = nullptr; + if (!microphone_initialized_) { + // the channels are shared with the microphone; only tear them down + // if it is not using them + i2s_del_channel(audio_tx_handle); + i2s_del_channel(audio_rx_handle); + audio_tx_handle = nullptr; + audio_rx_handle = nullptr; } + vStreamBufferDelete(audio_tx_stream); + audio_tx_stream = nullptr; audio_tx_buffer.clear(); return false; } @@ -98,7 +188,7 @@ bool M5StackCardputer::audio_task_callback(std::mutex &m, std::condition_variabl if (available > 0) { xStreamBufferReceive(audio_tx_stream, buffer, available, 0); - // The NS4168 has no volume control, so scale the samples in software + // The amplifier has no volume control, so scale the samples in software // according to the current volume / mute state float scale = mute_ ? 0.0f : (volume_ / 100.0f); auto *samples = reinterpret_cast(buffer); @@ -131,25 +221,38 @@ void M5StackCardputer::audio_sample_rate(uint32_t sample_rate) { return; } logger_.info("Setting audio sample rate to {} Hz", sample_rate); - // stop the channel + bool reconfigure_rx = variant() == Variant::ADV && microphone_initialized_; + // stop the channel(s); on the ADV the microphone shares the clock, so its + // channel must be reconfigured as well i2s_channel_disable(audio_tx_handle); + if (reconfigure_rx) { + i2s_channel_disable(audio_rx_handle); + } // update the sample rate audio_std_cfg.clk_cfg.sample_rate_hz = sample_rate; i2s_channel_reconfig_std_clock(audio_tx_handle, &audio_std_cfg.clk_cfg); + if (reconfigure_rx) { + i2s_channel_reconfig_std_clock(audio_rx_handle, &audio_std_cfg.clk_cfg); + mic_sample_rate_ = sample_rate; + } // clear the buffer xStreamBufferReset(audio_tx_stream); - // restart the channel + // restart the channel(s) i2s_channel_enable(audio_tx_handle); + if (reconfigure_rx) { + i2s_channel_enable(audio_rx_handle); + } } -void M5StackCardputer::play_audio(const std::vector &data) { - play_audio(data.data(), data.size()); +size_t M5StackCardputer::play_audio(const std::vector &data) { + return play_audio(data.data(), data.size()); } -void M5StackCardputer::play_audio(const uint8_t *data, uint32_t num_bytes) { +size_t M5StackCardputer::play_audio(const uint8_t *data, uint32_t num_bytes) { if (!sound_initialized_) { - return; + return 0; } - // don't block here - xStreamBufferSendFromISR(audio_tx_stream, data, num_bytes, NULL); + // don't block here; report how much was actually queued so callers can + // stream data larger than the buffer + return xStreamBufferSendFromISR(audio_tx_stream, data, num_bytes, NULL); } diff --git a/components/m5stack-cardputer/src/m5stack-cardputer.cpp b/components/m5stack-cardputer/src/m5stack-cardputer.cpp index 9c9aa3e42..076573db4 100644 --- a/components/m5stack-cardputer/src/m5stack-cardputer.cpp +++ b/components/m5stack-cardputer/src/m5stack-cardputer.cpp @@ -1,3 +1,6 @@ +#include +#include + #include "m5stack-cardputer.hpp" using namespace espp; @@ -71,15 +74,41 @@ bool M5StackCardputer::es8311_write(uint8_t reg, uint8_t value) { return internal_i2c_->write(es8311_address, buffer, 2); } -bool M5StackCardputer::initialize_es8311_speaker() { - logger_.info("Initializing ES8311 codec (speaker path)"); - // Minimal DAC bring-up, clocked from BCLK (there is no MCLK pin); matches - // the M5Unified Cardputer ADV speaker-enable sequence. +bool M5StackCardputer::es8311_ensure_common() { + // The reset / clocking / analog-power registers are shared by the DAC + // (speaker) and ADC (microphone) paths, and resetting the codec while the + // other path is running would glitch it - so write them exactly once. The + // clock manager enables both the DAC and ADC clocks (0xBF is the union of + // M5Unified's speaker-only 0xB5 and microphone-only 0xBA values), which is + // harmless when only one path is used. + if (es8311_common_initialized_) { + return true; + } const uint8_t init[][2] = { {0x00, 0x80}, // RESET: power on, CSM enabled - {0x01, 0xB5}, // CLOCK_MANAGER: MCLK from BCLK + {0x01, 0xBF}, // CLOCK_MANAGER: MCLK from BCLK, DAC + ADC clocks on {0x02, 0x18}, // CLOCK_MANAGER: MULT_PRE = 3 {0x0D, 0x01}, // SYSTEM: power up analog circuits + }; + es8311_common_initialized_ = + std::all_of(std::begin(init), std::end(init), [this](const auto &entry) { + if (!es8311_write(entry[0], entry[1])) { + logger_.error("Failed to write ES8311 register {:#04x}", entry[0]); + return false; + } + return true; + }); + return es8311_common_initialized_; +} + +bool M5StackCardputer::initialize_es8311_speaker() { + logger_.info("Initializing ES8311 codec (speaker / DAC path)"); + // Minimal DAC bring-up, clocked from BCLK (there is no MCLK pin); derived + // from the M5Unified Cardputer ADV speaker-enable sequence. + if (!es8311_ensure_common()) { + return false; + } + const uint8_t init[][2] = { {0x12, 0x00}, // SYSTEM: power up DAC {0x13, 0x10}, // SYSTEM: enable output to HP drive {0x32, 0xBF}, // DAC: volume 0 dB @@ -95,17 +124,22 @@ bool M5StackCardputer::initialize_es8311_speaker() { } bool M5StackCardputer::initialize_es8311_microphone() { - logger_.info("Initializing ES8311 codec (microphone path)"); + logger_.info("Initializing ES8311 codec (microphone / ADC path)"); // Minimal ADC bring-up for the analog MEMS microphone on MIC1, clocked - // from BCLK; matches the M5Unified Cardputer ADV microphone-enable + // from BCLK; derived from the M5Unified Cardputer ADV microphone-enable // sequence. + if (!es8311_ensure_common()) { + return false; + } + // The gain values follow the es8311 driver in the espp codec component + // (hardware-proven with the esp-box microphone): M5Unified's minimal + // sequence uses minimum PGA gain (0x14 = 0x10) and no digital mic gain, + // which records at a near-mute level. const uint8_t init[][2] = { - {0x00, 0x80}, // RESET: power on, CSM enabled - {0x01, 0xBA}, // CLOCK_MANAGER: MCLK from BCLK, ADC clocks on - {0x02, 0x18}, // CLOCK_MANAGER: MULT_PRE = 3 - {0x0D, 0x01}, // SYSTEM: power up analog circuits {0x0E, 0x02}, // SYSTEM: enable analog PGA / ADC modulator - {0x14, 0x10}, // SYSTEM: select Mic1p-Mic1n, minimum PGA gain + {0x14, 0x1A}, // SYSTEM: select Mic1p-Mic1n, raised analog PGA gain + {0x15, 0x40}, // ADC: soft-ramp / ALC configuration + {0x16, 0x24}, // ADC: mic digital gain scale {0x17, 0xBF}, // ADC: volume 0 dB {0x1C, 0x6A}, // ADC: equalizer bypass }; diff --git a/components/m5stack-cardputer/src/microphone.cpp b/components/m5stack-cardputer/src/microphone.cpp index c34001dfb..d000927c6 100644 --- a/components/m5stack-cardputer/src/microphone.cpp +++ b/components/m5stack-cardputer/src/microphone.cpp @@ -14,40 +14,42 @@ bool M5StackCardputer::initialize_microphone(const microphone_callback_t &callba logger_.warn("Microphone already initialized, not initializing again!"); return false; } - if (sound_initialized_) { - logger_.error("Cannot initialize microphone: the sound subsystem is active and shares I2S " - "pins with the microphone"); - return false; - } if (!callback) { logger_.error("A callback is required to receive the recorded audio data"); return false; } - microphone_callback_ = callback; + bool is_adv = variant() == Variant::ADV; - i2s_chan_config_t chan_cfg = I2S_CHANNEL_DEFAULT_CONFIG(mic_i2s_port, I2S_ROLE_MASTER); - ESP_ERROR_CHECK(i2s_new_channel(&chan_cfg, nullptr, &audio_rx_handle)); + if (!is_adv && sound_initialized_) { + // the original cannot run the speaker and PDM microphone at the same + // time: GPIO 43 doubles as the speaker WS and the microphone clock + logger_.error("Cannot initialize microphone: the sound subsystem is active and shares GPIO " + "{} with the microphone (WS / PDM clock)", + static_cast(mic_clk_io)); + return false; + } - if (variant() == Variant::ADV) { - // The ADV's analog MEMS microphone goes through the ES8311 codec's ADC, - // which streams standard I2S on the same BCK/WS as the speaker with its - // data out (ASDOUT) on GPIO 46 - i2s_std_config_t std_cfg = { - .clk_cfg = I2S_STD_CLK_DEFAULT_CONFIG(sample_rate), - .slot_cfg = - I2S_STD_PHILIPS_SLOT_DEFAULT_CONFIG(I2S_DATA_BIT_WIDTH_16BIT, I2S_SLOT_MODE_MONO), - .gpio_cfg = {.mclk = GPIO_NUM_NC, - .bclk = i2s_bck_io, - .ws = i2s_ws_io, - .dout = GPIO_NUM_NC, - .din = mic_data_io, - .invert_flags = {.mclk_inv = false, .bclk_inv = false, .ws_inv = false}}, - }; - ESP_ERROR_CHECK(i2s_channel_init_std_mode(audio_rx_handle, &std_cfg)); + microphone_callback_ = callback; + + if (is_adv) { + // The ADV's analog MEMS microphone goes through the ES8311 codec's ADC: + // full duplex with the speaker on a single I2S bus (shared BCK/WS, ADC + // data out on GPIO 46). Create (or reuse) the shared channels; the + // sample rate is shared with the speaker. + if (!ensure_adv_i2s(sample_rate)) { + logger_.error("Could not initialize I2S driver"); + return false; + } + mic_sample_rate_ = audio_std_cfg.clk_cfg.sample_rate_hz; + // the shared-bus RX channel captures both 16-bit slots of each frame + // (see ensure_adv_i2s()); the task callback keeps only the left slot + mic_stereo_capture_ = true; } else { // The original's SPM1423 is a PDM microphone (clock on GPIO 43, data on - // GPIO 46) + // GPIO 46), on its own I2S port + i2s_chan_config_t chan_cfg = I2S_CHANNEL_DEFAULT_CONFIG(mic_i2s_port, I2S_ROLE_MASTER); + ESP_ERROR_CHECK(i2s_new_channel(&chan_cfg, nullptr, &audio_rx_handle)); mic_pdm_cfg = { .clk_cfg = I2S_PDM_RX_CLK_DEFAULT_CONFIG(sample_rate), .slot_cfg = I2S_PDM_RX_SLOT_DEFAULT_CONFIG(I2S_DATA_BIT_WIDTH_16BIT, I2S_SLOT_MODE_MONO), @@ -59,23 +61,34 @@ bool M5StackCardputer::initialize_microphone(const microphone_callback_t &callba }, }; ESP_ERROR_CHECK(i2s_channel_init_pdm_rx_mode(audio_rx_handle, &mic_pdm_cfg)); + mic_sample_rate_ = sample_rate; } - mic_sample_rate_ = sample_rate; - - // buffer roughly one update period's worth of samples - audio_rx_buffer.resize(sample_rate * NUM_BYTES_PER_CHANNEL / UPDATE_FREQUENCY); + // buffer roughly one update period's worth of samples (at the actual, + // possibly shared, rate); a stereo capture carries two words per sample + // period + size_t rx_buffer_size = calc_audio_buffer_size(mic_sample_rate_); + if (mic_stereo_capture_) { + rx_buffer_size *= 2; + } + audio_rx_buffer.resize(rx_buffer_size); ESP_ERROR_CHECK(i2s_channel_enable(audio_rx_handle)); // On the ADV, configure the codec's ADC path now that BCLK is running (the // codec derives its clock from it) - if (variant() == Variant::ADV) { + if (is_adv) { if (!initialize_es8311_microphone()) { logger_.error("Could not initialize the ES8311 codec"); i2s_channel_disable(audio_rx_handle); - i2s_del_channel(audio_rx_handle); - audio_rx_handle = nullptr; + if (!sound_initialized_) { + // the channels are shared with the speaker; only tear them down if + // it is not using them + i2s_del_channel(audio_tx_handle); + i2s_del_channel(audio_rx_handle); + audio_tx_handle = nullptr; + audio_rx_handle = nullptr; + } return false; } } @@ -99,6 +112,16 @@ bool M5StackCardputer::microphone_task_callback(std::mutex &m, std::condition_va auto err = i2s_channel_read(audio_rx_handle, audio_rx_buffer.data(), audio_rx_buffer.size(), &bytes_read, portMAX_DELAY); if (err == ESP_OK && bytes_read > 0 && microphone_callback_) { + if (mic_stereo_capture_) { + // compact the L,R word pairs down to mono in place, keeping the left + // slot (the ES8311's ADC data) + auto *samples = reinterpret_cast(audio_rx_buffer.data()); + size_t num_frames = bytes_read / (2 * sizeof(int16_t)); + for (size_t i = 0; i < num_frames; i++) { + samples[i] = samples[2 * i]; + } + bytes_read = num_frames * sizeof(int16_t); + } microphone_callback_(audio_rx_buffer.data(), bytes_read); } return false; // don't stop the task