From b589692f55afd88ef0b8065296b0e5da3e932c6a Mon Sep 17 00:00:00 2001 From: oliverbhull Date: Thu, 10 Jul 2025 18:31:57 -0400 Subject: [PATCH 1/2] Add frame-by-frame streaming API to react-native-opus - Add streaming methods to TurboModule spec: initializeStreamDecoder, decodeOpusFrame, resetOpusStreamDecoder - Implement C++ streaming decoder with persistent state and Float32Array output - Update JS wrapper to handle Uint8Array to base64 conversion - Add streaming test functionality to example app - Update documentation with streaming API examples - Maintain backward compatibility with existing batch API --- README.md | 64 +++++++++++++++-- cpp/NativeOpusTurboModule.cpp | 126 ++++++++++++++++++++++++++++++++++ cpp/NativeOpusTurboModule.h | 12 ++++ example/App.tsx | 68 +++++++++++++++++- src/NativeOpusTurboModule.ts | 17 +++++ src/index.tsx | 25 +++++++ 6 files changed, 306 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 25405f4..6a48aed 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # react-native-opus -A library for decoding Opus audio packets in React Native. +A library for decoding Opus audio packets in React Native with both batch and frame-by-frame streaming capabilities. ## Installation @@ -10,9 +10,23 @@ npm install react-native-opus ## Usage -The `react-native-opus` library provides a function to decode Opus audio packets from base64-encoded strings. +The `react-native-opus` library provides functions to decode Opus audio packets from base64-encoded strings with both batch and frame-by-frame streaming capabilities. -### Example: Decoding an Opus Packet +## API Methods + +### Batch Decoding + +- **`decodeMultipleOpusPackets(base64String: string, frameSize: number)`**: Decodes multiple Opus packets from a base64-encoded string. The `frameSize` parameter specifies the frame size in bytes. + +### Frame-by-Frame Streaming + +- **`initializeStreamDecoder(sampleRate: number, channels: number)`**: Initializes the stream decoder with specified sample rate and channel count. +- **`decodeOpusFrame(frameData: Uint8Array)`**: Decodes a single Opus frame and returns Float32Array PCM data. +- **`resetOpusStreamDecoder()`**: Resets the stream decoder state. + +## Examples + +### Example: Batch Decoding ```js import { decodeMultipleOpusPackets } from 'react-native-opus'; @@ -33,9 +47,49 @@ async function decodeAudio() { decodeAudio(); ``` -### Key Function +### Example: Frame-by-Frame Streaming + +```js +import { + initializeStreamDecoder, + decodeOpusFrame, + resetOpusStreamDecoder +} from 'react-native-opus'; + +async function streamDecodeAudio() { + try { + // Initialize stream decoder for 16kHz mono audio + const initResult = await initializeStreamDecoder(16000, 1); + if (!initResult.success) { + throw new Error(initResult.error); + } + + // Decode individual frames + const frameData = new Uint8Array([/* your opus frame bytes */]); + const decodeResult = await decodeOpusFrame(frameData); + + if (decodeResult.success && decodeResult.pcmData) { + console.log(`Decoded ${decodeResult.samplesDecoded} samples`); + console.log('PCM data:', decodeResult.pcmData); // Float32Array + } + + // Reset decoder when done + await resetOpusStreamDecoder(); + } catch (error) { + console.error("Error streaming decode:", error); + } +} + +streamDecodeAudio(); +``` + +## Key Features -- **`decodeMultipleOpusPackets(base64String: string, frameSize: number)`**: Decodes a base64-encoded Opus packet. The `frameSize` parameter specifies the frame size in milliseconds (e.g., 40ms). +- **Batch Processing**: Decode multiple packets at once for efficiency +- **Frame-by-Frame Streaming**: Real-time decoding with persistent decoder state +- **Float32Array Output**: Direct PCM data for audio processing +- **Cross-Platform**: Works on both iOS and Android +- **Memory Efficient**: Optimized for mobile devices ## Contributing diff --git a/cpp/NativeOpusTurboModule.cpp b/cpp/NativeOpusTurboModule.cpp index 409e93f..f670b37 100644 --- a/cpp/NativeOpusTurboModule.cpp +++ b/cpp/NativeOpusTurboModule.cpp @@ -24,6 +24,10 @@ NativeOpusTurboModule::~NativeOpusTurboModule() { opus_decoder_destroy(opusDecoder); opusDecoder = nullptr; } + if (streamDecoder) { + opus_decoder_destroy(streamDecoder); + streamDecoder = nullptr; + } } // Base64 encoding/decoding utility methods @@ -286,4 +290,126 @@ jsi::Value NativeOpusTurboModule::saveDecodedDataAsWav(jsi::Runtime &rt, std::st return result; } +// Frame-by-frame streaming methods +jsi::Value NativeOpusTurboModule::initializeStreamDecoder(jsi::Runtime &rt, double sampleRate, double channels) { + jsi::Object result = jsi::Object(rt); + + try { + // Clean up existing stream decoder if any + if (streamDecoder) { + opus_decoder_destroy(streamDecoder); + streamDecoder = nullptr; + } + + // Create new stream decoder with specified parameters + streamSampleRate = static_cast(sampleRate); + streamChannels = static_cast(channels); + + int error = 0; + streamDecoder = opus_decoder_create(streamSampleRate, streamChannels, &error); + + if (error != OPUS_OK || !streamDecoder) { + result.setProperty(rt, "success", false); + result.setProperty(rt, "error", jsi::String::createFromUtf8(rt, opus_strerror(error))); + streamInitialized = false; + return result; + } + + streamInitialized = true; + result.setProperty(rt, "success", true); + + } catch (const std::exception& e) { + result.setProperty(rt, "success", false); + result.setProperty(rt, "error", jsi::String::createFromUtf8(rt, e.what())); + streamInitialized = false; + } + + return result; +} + +jsi::Value NativeOpusTurboModule::decodeOpusFrame(jsi::Runtime &rt, std::string base64Frame) { + jsi::Object result = jsi::Object(rt); + + if (!streamInitialized || !streamDecoder) { + result.setProperty(rt, "success", false); + result.setProperty(rt, "error", jsi::String::createFromUtf8(rt, "Stream decoder not initialized")); + return result; + } + + try { + // Decode base64 frame data + std::vector frameData = base64_decode(base64Frame); + + if (frameData.empty() && !base64Frame.empty()) { + result.setProperty(rt, "success", false); + result.setProperty(rt, "error", jsi::String::createFromUtf8(rt, "Invalid base64 frame data")); + return result; + } + + // Prepare output buffer for decoded PCM data + float pcmBuffer[MAX_FRAME_SIZE * streamChannels]; + + // Decode the Opus frame + int samplesDecoded = opus_decode_float( + streamDecoder, + frameData.data(), + static_cast(frameData.size()), + pcmBuffer, + MAX_FRAME_SIZE, + 0 + ); + + if (samplesDecoded < 0) { + result.setProperty(rt, "success", false); + result.setProperty(rt, "error", jsi::String::createFromUtf8(rt, opus_strerror(samplesDecoded))); + return result; + } + + // Create Float32Array from the decoded PCM data + size_t totalSamples = samplesDecoded * streamChannels; + size_t bufferSize = totalSamples * sizeof(float); + + auto arrayBuffer = jsi::ArrayBuffer(rt, bufferSize); + std::memcpy(arrayBuffer.data(rt), pcmBuffer, bufferSize); + + jsi::Object typedArray = rt.global().getPropertyAsObject(rt, "Float32Array"); + jsi::Value args[] = { + jsi::Value(rt, arrayBuffer), + jsi::Value(0), + jsi::Value(static_cast(totalSamples)) + }; + jsi::Object float32Array = typedArray.asFunction(rt).callAsConstructor(rt, args, 3).asObject(rt); + + result.setProperty(rt, "success", true); + result.setProperty(rt, "pcmData", float32Array); + result.setProperty(rt, "samplesDecoded", samplesDecoded); + + } catch (const std::exception& e) { + result.setProperty(rt, "success", false); + result.setProperty(rt, "error", jsi::String::createFromUtf8(rt, e.what())); + } + + return result; +} + +jsi::Value NativeOpusTurboModule::resetOpusStreamDecoder(jsi::Runtime &rt) { + jsi::Object result = jsi::Object(rt); + + if (!streamInitialized || !streamDecoder) { + result.setProperty(rt, "success", false); + result.setProperty(rt, "error", jsi::String::createFromUtf8(rt, "Stream decoder not initialized")); + return result; + } + + int error = opus_decoder_ctl(streamDecoder, OPUS_RESET_STATE); + if (error == OPUS_OK) { + result.setProperty(rt, "success", true); + } else { + result.setProperty(rt, "success", false); + result.setProperty(rt, "error", jsi::String::createFromUtf8(rt, opus_strerror(error))); + } + + return result; +} + } // namespace facebook::react diff --git a/cpp/NativeOpusTurboModule.h b/cpp/NativeOpusTurboModule.h index 4a74819..0f6738b 100644 --- a/cpp/NativeOpusTurboModule.h +++ b/cpp/NativeOpusTurboModule.h @@ -33,6 +33,11 @@ class NativeOpusTurboModule: public NativeOpusTurboModuleCxxSpec& input); @@ -41,6 +46,13 @@ class NativeOpusTurboModule: public NativeOpusTurboModuleCxxSpec(null); const [isLoading, setIsLoading] = useState(false); + const [streamResult, setStreamResult] = useState(null); + const [isStreamLoading, setIsStreamLoading] = useState(false); const backgroundStyle = { backgroundColor: isDarkMode ? '#000000' : '#F5F5F5', // Black or light gray @@ -56,6 +63,45 @@ function App(): React.JSX.Element { } }; + const handleStreamDecodePress = async () => { + setIsStreamLoading(true); + setStreamResult(null); + console.log('Attempting to stream decode...'); + try { + // Initialize stream decoder + const initResult = await initializeStreamDecoder(16000, 1); + console.log('Stream decoder initialized:', initResult); + + if (!initResult.success) { + throw new Error(initResult.error || 'Failed to initialize stream decoder'); + } + + // Create a sample frame from the base64 string (first 40 bytes) + const buffer = Buffer.from(BASE64_OPUS_STRING, 'base64'); + const frameData = new Uint8Array(buffer.slice(0, 40)); + + // Decode the frame + const decodeResult = await decodeOpusFrame(frameData); + console.log('Frame decode result:', decodeResult); + + if (decodeResult.success && decodeResult.pcmData) { + setStreamResult(`Stream decode succeeded! Decoded ${decodeResult.samplesDecoded} samples. PCM data length: ${decodeResult.pcmData.length}`); + } else { + throw new Error(decodeResult.error || 'Failed to decode frame'); + } + + } catch (error) { + console.error('Stream decoding failed:', error); + setStreamResult(null); + Alert.alert( + 'Stream Decoding Error', + error instanceof Error ? error.message : 'An unknown error occurred' + ); + } finally { + setIsStreamLoading(false); + } + }; + return ( @@ -79,6 +125,26 @@ function App(): React.JSX.Element { {decodedResult} )} + + [ + styles.button, + isDarkMode ? styles.buttonDark : styles.buttonLight, + pressed && (isDarkMode ? styles.buttonDarkPressed : styles.buttonLightPressed), + isStreamLoading && styles.buttonDisabled, + ]} + onPress={handleStreamDecodePress} + disabled={isStreamLoading} + > + + {isStreamLoading ? 'Stream Decoding...' : 'Test Stream Decode'} + + + {streamResult && ( + + {streamResult} + + )} ); diff --git a/src/NativeOpusTurboModule.ts b/src/NativeOpusTurboModule.ts index 173c706..0986eac 100644 --- a/src/NativeOpusTurboModule.ts +++ b/src/NativeOpusTurboModule.ts @@ -27,6 +27,23 @@ export interface Spec extends TurboModule { filepath?: string; error?: string; }>; + + // Frame-by-frame streaming methods + initializeStreamDecoder( + sampleRate: number, + channels: number + ): Promise<{ success: boolean; error?: string }>; + + decodeOpusFrame( + base64Frame: string + ): Promise<{ + success: boolean; + pcmData?: Float32Array; + samplesDecoded?: number; + error?: string; + }>; + + resetOpusStreamDecoder(): Promise<{ success: boolean; error?: string }>; } export default TurboModuleRegistry.getEnforcing('OpusTurbo'); diff --git a/src/index.tsx b/src/index.tsx index bc1a995..e0bbc05 100644 --- a/src/index.tsx +++ b/src/index.tsx @@ -29,4 +29,29 @@ export function saveDecodedDataAsWav( error?: string; }> { return OpusTurboModule.saveDecodedDataAsWav(decodedDataBase64, filepath, sampleRate, channels); +} + +// Frame-by-frame streaming API +export function initializeStreamDecoder( + sampleRate: number, + channels: number +): Promise<{ success: boolean; error?: string }> { + return OpusTurboModule.initializeStreamDecoder(sampleRate, channels); +} + +export function decodeOpusFrame( + frameData: Uint8Array +): Promise<{ + success: boolean; + pcmData?: Float32Array; + samplesDecoded?: number; + error?: string; +}> { + // Convert Uint8Array to base64 string for native module + const base64Frame = Buffer.from(frameData).toString('base64'); + return OpusTurboModule.decodeOpusFrame(base64Frame); +} + +export function resetOpusStreamDecoder(): Promise<{ success: boolean; error?: string }> { + return OpusTurboModule.resetOpusStreamDecoder(); } \ No newline at end of file From 2b6debaf969c43e1c15cf96f3a8bc30e915663c0 Mon Sep 17 00:00:00 2001 From: oliverbhull Date: Thu, 10 Jul 2025 18:34:40 -0400 Subject: [PATCH 2/2] Update README to focus on rn-stream-opus streaming capabilities - Rebrand from react-native-opus to rn-stream-opus - Emphasize real-time streaming as primary use case - Add practical streaming examples and OpusStreamer class - Highlight key features for streaming applications - Add use cases section for voice/audio streaming apps - Move batch API to legacy section --- README.md | 117 ++++++++++++++++++++++++++++++++++++------------------ 1 file changed, 79 insertions(+), 38 deletions(-) diff --git a/README.md b/README.md index 6a48aed..fb55169 100644 --- a/README.md +++ b/README.md @@ -1,60 +1,39 @@ -# react-native-opus +# rn-stream-opus -A library for decoding Opus audio packets in React Native with both batch and frame-by-frame streaming capabilities. +A React Native library optimized for real-time Opus audio streaming with frame-by-frame decoding capabilities. Built specifically for streaming audio applications that require low-latency, persistent decoder state. ## Installation ```sh -npm install react-native-opus +npm install rn-stream-opus ``` ## Usage -The `react-native-opus` library provides functions to decode Opus audio packets from base64-encoded strings with both batch and frame-by-frame streaming capabilities. +The `rn-stream-opus` library provides efficient frame-by-frame Opus decoding with persistent decoder state, perfect for real-time audio streaming applications. ## API Methods -### Batch Decoding - -- **`decodeMultipleOpusPackets(base64String: string, frameSize: number)`**: Decodes multiple Opus packets from a base64-encoded string. The `frameSize` parameter specifies the frame size in bytes. - -### Frame-by-Frame Streaming +### Primary Streaming API - **`initializeStreamDecoder(sampleRate: number, channels: number)`**: Initializes the stream decoder with specified sample rate and channel count. - **`decodeOpusFrame(frameData: Uint8Array)`**: Decodes a single Opus frame and returns Float32Array PCM data. - **`resetOpusStreamDecoder()`**: Resets the stream decoder state. -## Examples - -### Example: Batch Decoding +### Legacy Batch API -```js -import { decodeMultipleOpusPackets } from 'react-native-opus'; - -// Hardcoded base64 string (100 characters) -const base64String = "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"; - -// Decode the Opus packet -async function decodeAudio() { - try { - const decodedPackets = await decodeMultipleOpusPackets(base64String, 40); - console.log("Decoded packets:", decodedPackets); - } catch (error) { - console.error("Error decoding audio:", error); - } -} +- **`decodeMultipleOpusPackets(base64String: string, frameSize: number)`**: Decodes multiple Opus packets from a base64-encoded string. The `frameSize` parameter specifies the frame size in bytes. -decodeAudio(); -``` +## Examples -### Example: Frame-by-Frame Streaming +### Real-Time Streaming (Primary Use Case) ```js import { initializeStreamDecoder, decodeOpusFrame, resetOpusStreamDecoder -} from 'react-native-opus'; +} from 'rn-stream-opus'; async function streamDecodeAudio() { try { @@ -64,16 +43,19 @@ async function streamDecodeAudio() { throw new Error(initResult.error); } - // Decode individual frames + // Decode individual frames as they arrive const frameData = new Uint8Array([/* your opus frame bytes */]); const decodeResult = await decodeOpusFrame(frameData); if (decodeResult.success && decodeResult.pcmData) { console.log(`Decoded ${decodeResult.samplesDecoded} samples`); console.log('PCM data:', decodeResult.pcmData); // Float32Array + + // Process PCM data for real-time playback + processAudioData(decodeResult.pcmData); } - // Reset decoder when done + // Reset decoder when stream ends await resetOpusStreamDecoder(); } catch (error) { console.error("Error streaming decode:", error); @@ -83,13 +65,72 @@ async function streamDecodeAudio() { streamDecodeAudio(); ``` +### Streaming with Audio Processing + +```js +import { + initializeStreamDecoder, + decodeOpusFrame +} from 'rn-stream-opus'; + +class OpusStreamer { + constructor() { + this.initialized = false; + } + + async initialize(sampleRate = 16000, channels = 1) { + const result = await initializeStreamDecoder(sampleRate, channels); + this.initialized = result.success; + return result; + } + + async processFrame(opusFrame) { + if (!this.initialized) { + throw new Error('Decoder not initialized'); + } + + const result = await decodeOpusFrame(opusFrame); + + if (result.success && result.pcmData) { + // Direct access to Float32Array PCM data + return { + pcm: result.pcmData, + samples: result.samplesDecoded, + timestamp: Date.now() + }; + } + + throw new Error(result.error || 'Decode failed'); + } +} + +// Usage +const streamer = new OpusStreamer(); +await streamer.initialize(16000, 1); + +// Process frames as they arrive +const audioData = await streamer.processFrame(frameBytes); +playAudio(audioData.pcm); +``` + ## Key Features -- **Batch Processing**: Decode multiple packets at once for efficiency -- **Frame-by-Frame Streaming**: Real-time decoding with persistent decoder state -- **Float32Array Output**: Direct PCM data for audio processing -- **Cross-Platform**: Works on both iOS and Android -- **Memory Efficient**: Optimized for mobile devices +- **🎯 Real-Time Streaming**: Optimized for frame-by-frame decoding with persistent decoder state +- **⚡ Low Latency**: Minimal processing overhead for real-time audio applications +- **🔄 Float32Array Output**: Direct PCM data ready for audio processing and playback +- **📱 Cross-Platform**: Native TurboModule implementation for iOS and Android +- **🧠 Memory Efficient**: Optimized buffer management for mobile devices +- **🔧 Easy Integration**: Simple API designed for streaming audio applications +- **🔄 Persistent State**: Decoder maintains state across frames for continuous streams + +## Use Cases + +- **Real-time voice communication apps** +- **Audio streaming applications** +- **Voice memo recording/playback** +- **Live audio processing** +- **Bluetooth audio streaming** +- **WebRTC audio pipelines** ## Contributing