diff --git a/audio/audio_receive_stream.cc b/audio/audio_receive_stream.cc index 31e93b3c7d..3afc398962 100644 --- a/audio/audio_receive_stream.cc +++ b/audio/audio_receive_stream.cc @@ -367,6 +367,15 @@ void AudioReceiveStreamImpl::SetSink(AudioSinkInterface* sink) { void AudioReceiveStreamImpl::SetGain(float gain) { RTC_DCHECK_RUN_ON(&worker_thread_checker_); channel_receive_->SetChannelOutputVolumeScaling(gain); + + const bool muted = (gain == 0.f); + if(muted) { + channel_receive_->StopPlayout(); + } else { + channel_receive_->StartPlayout(); + } + // trying turn off/on hardware + audio_state()->ReceivingStreamMuted(this, muted); } bool AudioReceiveStreamImpl::SetBaseMinimumPlayoutDelayMs(int delay_ms) { diff --git a/audio/audio_state.cc b/audio/audio_state.cc index a0861eef6b..0c76994853 100644 --- a/audio/audio_state.cc +++ b/audio/audio_state.cc @@ -51,15 +51,9 @@ AudioTransport* AudioState::audio_transport() { return &audio_transport_; } -void AudioState::AddReceivingStream( - webrtc::AudioReceiveStreamInterface* stream) { +void AudioState::StartPlayout() +{ RTC_DCHECK_RUN_ON(&thread_checker_); - RTC_DCHECK_EQ(0, receiving_streams_.count(stream)); - receiving_streams_.insert(stream); - if (!config_.audio_mixer->AddSource( - static_cast(stream))) { - RTC_DLOG(LS_ERROR) << "Failed to add source to mixer."; - } // Make sure playback is initialized; start playing if enabled. UpdateNullAudioPollerState(); @@ -75,6 +69,52 @@ void AudioState::AddReceivingStream( } } +void AudioState::StopPlayout() +{ + RTC_DCHECK_RUN_ON(&thread_checker_); + + config_.audio_device_module->StopPlayout(); + UpdateNullAudioPollerState(); +} + +void AudioState::AddReceivingStream( + webrtc::AudioReceiveStreamInterface* stream) { + RTC_DCHECK_RUN_ON(&thread_checker_); + RTC_DCHECK_EQ(0, receiving_streams_.count(stream)); + receiving_streams_.insert(std::make_pair(stream, ReceiveStreamProperties())); + if (!config_.audio_mixer->AddSource( + static_cast(stream))) { + RTC_DLOG(LS_ERROR) << "Failed to add source to mixer."; + } + + StartPlayout(); +} + +void AudioState::ReceivingStreamMuted( + webrtc::AudioReceiveStreamInterface* stream, + bool muted) +{ + auto it = receiving_streams_.find(stream); + if (it != receiving_streams_.end()) { + if (it->second.muted != muted) { + it->second.muted = muted; + } + } + + bool allMuted = true; + for (const auto& pair: receiving_streams_) { + if (!pair.second.muted) { + allMuted = false; + break; + } + } + + if (allMuted) + StopPlayout(); + else + StartPlayout(); +} + void AudioState::RemoveReceivingStream( webrtc::AudioReceiveStreamInterface* stream) { RTC_DCHECK_RUN_ON(&thread_checker_); @@ -82,9 +122,11 @@ void AudioState::RemoveReceivingStream( RTC_DCHECK_EQ(1, count); config_.audio_mixer->RemoveSource( static_cast(stream)); - UpdateNullAudioPollerState(); + if (receiving_streams_.empty()) { - config_.audio_device_module->StopPlayout(); + StopPlayout(); + } else { + UpdateNullAudioPollerState(); } } @@ -131,8 +173,7 @@ void AudioState::SetPlayout(bool enabled) { config_.audio_device_module->StartPlayout(); } } else { - config_.audio_device_module->StopPlayout(); - UpdateNullAudioPollerState(); + StopPlayout(); } } } diff --git a/audio/audio_state.h b/audio/audio_state.h index 88aaaa3697..86e5bca6b3 100644 --- a/audio/audio_state.h +++ b/audio/audio_state.h @@ -53,6 +53,7 @@ class AudioState : public webrtc::AudioState { } void AddReceivingStream(webrtc::AudioReceiveStreamInterface* stream); + void ReceivingStreamMuted(webrtc::AudioReceiveStreamInterface* stream, bool); void RemoveReceivingStream(webrtc::AudioReceiveStreamInterface* stream); void AddSendingStream(webrtc::AudioSendStream* stream, @@ -64,6 +65,9 @@ class AudioState : public webrtc::AudioState { void UpdateAudioTransportWithSendingStreams(); void UpdateNullAudioPollerState() RTC_RUN_ON(&thread_checker_); + void StartPlayout(); + void StopPlayout(); + SequenceChecker thread_checker_; SequenceChecker process_thread_checker_{SequenceChecker::kDetached}; const webrtc::AudioState::Config config_; @@ -79,7 +83,10 @@ class AudioState : public webrtc::AudioState { // stats are still updated. RepeatingTaskHandle null_audio_poller_ RTC_GUARDED_BY(&thread_checker_); - webrtc::flat_set receiving_streams_; + struct ReceiveStreamProperties { + bool muted = false; + }; + std::map receiving_streams_; struct StreamProperties { int sample_rate_hz = 0; size_t num_channels = 0; diff --git a/audio/channel_receive.cc b/audio/channel_receive.cc index b2fa0bf3cc..ba77c7960c 100644 --- a/audio/channel_receive.cc +++ b/audio/channel_receive.cc @@ -603,6 +603,7 @@ void ChannelReceive::SetSink(AudioSinkInterface* sink) { void ChannelReceive::StartPlayout() { RTC_DCHECK_RUN_ON(&worker_thread_checker_); + acm_receiver_.FlushBuffers(); playing_ = true; } diff --git a/call/rtp_transport_controller_send.cc b/call/rtp_transport_controller_send.cc index f667417239..fdf4c2d64d 100644 --- a/call/rtp_transport_controller_send.cc +++ b/call/rtp_transport_controller_send.cc @@ -339,10 +339,10 @@ void RtpTransportControllerSend::OnNetworkRouteChanged( auto kv = result.first; bool inserted = result.second; if (inserted || !(kv->second == network_route)) { - RTC_LOG(LS_INFO) << "Network route changed on transport " << transport_name + RTC_LOG(LS_VERBOSE) << "Network route changed on transport " << transport_name << ": new_route = " << network_route.DebugString(); if (!inserted) { - RTC_LOG(LS_INFO) << "old_route = " << kv->second.DebugString(); + RTC_LOG(LS_VERBOSE) << "old_route = " << kv->second.DebugString(); } } diff --git a/modules/congestion_controller/goog_cc/probe_controller.cc b/modules/congestion_controller/goog_cc/probe_controller.cc index 44cb0a79f0..4877c05b83 100644 --- a/modules/congestion_controller/goog_cc/probe_controller.cc +++ b/modules/congestion_controller/goog_cc/probe_controller.cc @@ -352,7 +352,7 @@ std::vector ProbeController::SetEstimatedBitrate( ? network_estimate_->link_capacity_upper * config_.further_probe_threshold : DataRate::PlusInfinity(); - RTC_LOG(LS_INFO) << "Measured bitrate: " << bitrate + RTC_LOG(LS_VERBOSE) << "Measured bitrate: " << bitrate << " Minimum to probe further: " << min_bitrate_to_probe_further_ << " upper limit: " << network_state_estimate_probe_further_limit; @@ -409,7 +409,7 @@ std::vector ProbeController::RequestProbe( if (min_expected_probe_result > estimated_bitrate_ && time_since_drop < kBitrateDropTimeout && time_since_probe > kMinTimeBetweenAlrProbes) { - RTC_LOG(LS_INFO) << "Detected big bandwidth drop, start probing."; + RTC_LOG(LS_VERBOSE) << "Detected big bandwidth drop, start probing."; // Track how often we probe in response to bandwidth drop in ALR. RTC_HISTOGRAM_COUNTS_10000( "WebRTC.BWE.BweDropProbingIntervalInS", @@ -501,7 +501,7 @@ std::vector ProbeController::Process(Timestamp at_time) { if (at_time - time_last_probing_initiated_ > kMaxWaitingTimeForProbingResult) { if (state_ == State::kWaitingForProbingResult) { - RTC_LOG(LS_INFO) << "kWaitingForProbingResult: timeout"; + RTC_LOG(LS_VERBOSE) << "kWaitingForProbingResult: timeout"; UpdateState(State::kProbingComplete); } } diff --git a/modules/rtp_rtcp/source/rtp_rtcp_impl2.cc b/modules/rtp_rtcp/source/rtp_rtcp_impl2.cc index 03dab0220d..5bb4bd5535 100644 --- a/modules/rtp_rtcp/source/rtp_rtcp_impl2.cc +++ b/modules/rtp_rtcp/source/rtp_rtcp_impl2.cc @@ -775,7 +775,7 @@ void ModuleRtpRtcpImpl2::MaybeSendRtcpAtOrAfterTimestamp( TimeDelta delta = execution_time - now; // TaskQueue may run task 1ms earlier, so don't print warning if in this case. if (delta > TimeDelta::Millis(1)) { - RTC_DLOG(LS_WARNING) << "BUGBUG: Task queue scheduled delayed call " + RTC_DLOG(LS_VERBOSE) << "BUGBUG: Task queue scheduled delayed call " << delta << " too early."; } diff --git a/p2p/base/connection.cc b/p2p/base/connection.cc index 762b69259e..02674812f8 100644 --- a/p2p/base/connection.cc +++ b/p2p/base/connection.cc @@ -819,7 +819,7 @@ bool Connection::pruned() const { void Connection::Prune() { RTC_DCHECK_RUN_ON(network_thread_); if (!pruned_ || active()) { - RTC_LOG(LS_INFO) << ToString() << ": Connection pruned"; + RTC_LOG(LS_VERBOSE) << ToString() << ": Connection pruned"; pruned_ = true; requests_.Clear(); set_write_state(STATE_WRITE_TIMEOUT); @@ -1406,7 +1406,7 @@ void Connection::OnConnectionRequestResponse(StunRequest* request, RTC_DCHECK_RUN_ON(network_thread_); // Log at LS_INFO if we receive a ping response on an unwritable // connection. - rtc::LoggingSeverity sev = !writable() ? rtc::LS_INFO : rtc::LS_VERBOSE; + rtc::LoggingSeverity sev = !writable() ? rtc::LS_VERBOSE : rtc::LS_VERBOSE; int rtt = request->Elapsed(); @@ -1530,7 +1530,7 @@ void Connection::OnConnectionRequestTimeout(ConnectionRequest* request) { void Connection::OnConnectionRequestSent(ConnectionRequest* request) { RTC_DCHECK_RUN_ON(network_thread_); // Log at LS_INFO if we send a ping on an unwritable connection. - rtc::LoggingSeverity sev = !writable() ? rtc::LS_INFO : rtc::LS_VERBOSE; + rtc::LoggingSeverity sev = !writable() ? rtc::LS_VERBOSE : rtc::LS_VERBOSE; RTC_LOG_V(sev) << ToString() << ": Sent " << StunMethodToString(request->msg()->type()) << ", id=" << rtc::hex_encode(request->id()) diff --git a/p2p/base/port.cc b/p2p/base/port.cc index 25105dcdb0..ff15e5f529 100644 --- a/p2p/base/port.cc +++ b/p2p/base/port.cc @@ -135,7 +135,7 @@ Port::Port(const PortParametersRef& args, network_->SignalTypeChanged.connect(this, &Port::OnNetworkTypeChanged); PostDestroyIfDead(/*delayed=*/true); - RTC_LOG(LS_INFO) << ToString() << ": Port created with network cost " + RTC_LOG(LS_VERBOSE) << ToString() << ": Port created with network cost " << network_cost_; } @@ -338,7 +338,7 @@ void Port::OnReadPacket(const rtc::ReceivedPacket& packet, ProtocolType proto) { } else if (!msg) { // STUN message handled already } else if (msg->type() == STUN_BINDING_REQUEST) { - RTC_LOG(LS_INFO) << "Received " << StunMethodToString(msg->type()) + RTC_LOG(LS_VERBOSE) << "Received " << StunMethodToString(msg->type()) << " id=" << rtc::hex_encode(msg->transaction_id()) << " from unknown address " << addr.ToSensitiveString(); // We need to signal an unknown address before we handle any role conflict @@ -347,7 +347,7 @@ void Port::OnReadPacket(const rtc::ReceivedPacket& packet, ProtocolType proto) { SignalUnknownAddress(this, addr, proto, msg.get(), remote_username, false); // Check for role conflicts. if (!MaybeIceRoleConflict(addr, msg.get(), remote_username)) { - RTC_LOG(LS_INFO) << "Received conflicting role from the peer."; + RTC_LOG(LS_VERBOSE) << "Received conflicting role from the peer."; return; } } else if (msg->type() == GOOG_PING_REQUEST) { @@ -737,7 +737,7 @@ void Port::SendBindingErrorResponse(StunMessage* message, options.info_signaled_after_sent.packet_type = rtc::PacketType::kIceConnectivityCheckResponse; SendTo(buf.Data(), buf.Length(), addr, options, false); - RTC_LOG(LS_INFO) << ToString() << ": Sending STUN " + RTC_LOG(LS_VERBOSE) << ToString() << ": Sending STUN " << StunMethodToString(response.type()) << ": reason=" << reason << " to " << addr.ToSensitiveString(); @@ -855,7 +855,7 @@ void Port::UpdateNetworkCost() { if (network_cost_ == new_cost) { return; } - RTC_LOG(LS_INFO) << "Network cost changed from " << network_cost_ << " to " + RTC_LOG(LS_VERBOSE) << "Network cost changed from " << network_cost_ << " to " << new_cost << ". Number of candidates created: " << candidates_.size() << ". Number of connections created: " @@ -917,7 +917,7 @@ void Port::DestroyConnectionInternal(Connection* conn, bool async) { void Port::Destroy() { RTC_DCHECK(connections_.empty()); - RTC_LOG(LS_INFO) << ToString() << ": Port deleted"; + RTC_LOG(LS_VERBOSE) << ToString() << ": Port deleted"; SendPortDestroyed(this); delete this; } diff --git a/p2p/base/stun_port.cc b/p2p/base/stun_port.cc index fd1eb72a92..b161c5c699 100644 --- a/p2p/base/stun_port.cc +++ b/p2p/base/stun_port.cc @@ -98,7 +98,7 @@ class StunBindingRequest : public StunRequest { } } void OnTimeout() override { - RTC_LOG(LS_ERROR) << "Binding request timed out from " + RTC_LOG(LS_WARNING) << "Binding request timed out from " << port_->GetLocalAddress().ToSensitiveString() << " (" << port_->Network()->name() << ")"; port_->OnStunBindingOrResolveRequestFailed( @@ -428,7 +428,7 @@ void UDPPort::ResolveStunAddress(const rtc::SocketAddress& stun_addr) { })); } - RTC_LOG(LS_INFO) << ToString() << ": Starting STUN host lookup for " + RTC_LOG(LS_VERBOSE) << ToString() << ": Starting STUN host lookup for " << stun_addr.ToSensitiveString(); resolver_->Resolve(stun_addr, Network()->family(), field_trials()); } @@ -586,7 +586,7 @@ void UDPPort::OnSendPacket(const void* data, size_t size, StunRequest* req) { options.info_signaled_after_sent.packet_type = rtc::PacketType::kStunMessage; CopyPortInformationToPacketInfo(&options.info_signaled_after_sent); if (socket_->SendTo(data, size, sreq->server_addr(), options) < 0) { - RTC_LOG_ERR_EX(LS_ERROR, socket_->GetError()) + RTC_LOG_ERR_EX(LS_WARNING, socket_->GetError()) << "UDP send of " << size << " bytes to host " << sreq->server_addr().ToSensitiveNameAndAddressString() << " failed with error " << error_; diff --git a/p2p/base/turn_port.cc b/p2p/base/turn_port.cc index 71deb693c1..3edcbb3fb7 100644 --- a/p2p/base/turn_port.cc +++ b/p2p/base/turn_port.cc @@ -428,7 +428,7 @@ void TurnPort::PrepareAddress() { // Insert the current address to prevent redirection pingpong. attempted_server_addresses_.insert(server_address_.address); - RTC_LOG(LS_INFO) + RTC_LOG(LS_VERBOSE) << ToString() << ": Trying to connect to TURN server via " << ProtoToString(server_address_.proto) << " @ " << server_address_.address.ToSensitiveNameAndAddressString(); @@ -570,7 +570,7 @@ void TurnPort::OnSocketConnect(rtc::AsyncPacketSocket* socket) { server_address_.address = socket_->GetRemoteAddress(); } - RTC_LOG(LS_INFO) << "TurnPort connected to " + RTC_LOG(LS_VERBOSE) << "TurnPort connected to " << socket->GetRemoteAddress().ToSensitiveString() << " using tcp."; SendRequest(new TurnAllocateRequest(this), 0); @@ -594,7 +594,7 @@ void TurnPort::OnAllocateMismatch() { return; } - RTC_LOG(LS_INFO) << ToString() + RTC_LOG(LS_VERBOSE) << ToString() << ": Allocating a new socket after " "STUN_ERROR_ALLOCATION_MISMATCH, retry: " << allocate_mismatch_retries_ + 1; @@ -848,7 +848,7 @@ bool TurnPort::SetAlternateServer(const rtc::SocketAddress& address) { return false; } - RTC_LOG(LS_INFO) << ToString() << ": Redirecting from TURN server [" + RTC_LOG(LS_VERBOSE) << ToString() << ": Redirecting from TURN server [" << server_address_.address.ToSensitiveNameAndAddressString() << "] to TURN server [" << address.ToSensitiveNameAndAddressString() << "]"; @@ -863,7 +863,7 @@ void TurnPort::ResolveTurnAddress(const rtc::SocketAddress& address) { if (resolver_) return; - RTC_LOG(LS_INFO) << ToString() << ": Starting TURN host lookup for " + RTC_LOG(LS_VERBOSE) << ToString() << ": Starting TURN host lookup for " << address.ToSensitiveString(); resolver_ = socket_factory()->CreateAsyncDnsResolver(); auto callback = [this] { @@ -909,7 +909,7 @@ void TurnPort::OnSendStunPacket(const void* data, options.info_signaled_after_sent.packet_type = rtc::PacketType::kTurnMessage; CopyPortInformationToPacketInfo(&options.info_signaled_after_sent); if (Send(data, size, options) < 0) { - RTC_LOG(LS_ERROR) << ToString() << ": Failed to send TURN message, error: " + RTC_LOG(LS_WARNING) << ToString() << ": Failed to send TURN message, error: " << socket_->GetError(); } } @@ -1170,7 +1170,7 @@ bool TurnPort::ScheduleRefresh(uint32_t lifetime) { } SendRequest(new TurnRefreshRequest(this), delay); - RTC_LOG(LS_INFO) << ToString() << ": Scheduled refresh in " << delay << "ms."; + RTC_LOG(LS_VERBOSE) << ToString() << ": Scheduled refresh in " << delay << "ms."; return true; } @@ -1380,13 +1380,13 @@ TurnAllocateRequest::TurnAllocateRequest(TurnPort* port) } void TurnAllocateRequest::OnSent() { - RTC_LOG(LS_INFO) << port_->ToString() << ": TURN allocate request sent, id=" + RTC_LOG(LS_VERBOSE) << port_->ToString() << ": TURN allocate request sent, id=" << rtc::hex_encode(id()); StunRequest::OnSent(); } void TurnAllocateRequest::OnResponse(StunMessage* response) { - RTC_LOG(LS_INFO) << port_->ToString() + RTC_LOG(LS_VERBOSE) << port_->ToString() << ": TURN allocate requested successfully, id=" << rtc::hex_encode(id()) << ", code=0" // Makes logging easier to parse. @@ -1432,7 +1432,7 @@ void TurnAllocateRequest::OnErrorResponse(StunMessage* response) { // Process error response according to RFC5766, Section 6.4. int error_code = response->GetErrorCodeValue(); - RTC_LOG(LS_INFO) << port_->ToString() + RTC_LOG(LS_VERBOSE) << port_->ToString() << ": Received TURN allocate error response, id=" << rtc::hex_encode(id()) << ", code=" << error_code << ", rtt=" << Elapsed(); @@ -1462,7 +1462,7 @@ void TurnAllocateRequest::OnErrorResponse(StunMessage* response) { } void TurnAllocateRequest::OnTimeout() { - RTC_LOG(LS_WARNING) << port_->ToString() << ": TURN allocate request " + RTC_LOG(LS_VERBOSE) << port_->ToString() << ": TURN allocate request " << rtc::hex_encode(id()) << " timeout"; port_->OnAllocateRequestTimeout(); } @@ -1529,7 +1529,7 @@ void TurnAllocateRequest::OnTryAlternate(StunMessage* response, int code) { const StunByteStringAttribute* realm_attr = response->GetByteString(STUN_ATTR_REALM); if (realm_attr) { - RTC_LOG(LS_INFO) << port_->ToString() + RTC_LOG(LS_VERBOSE) << port_->ToString() << ": Applying STUN_ATTR_REALM attribute in " "try alternate error response."; port_->set_realm(realm_attr->string_view()); @@ -1538,7 +1538,7 @@ void TurnAllocateRequest::OnTryAlternate(StunMessage* response, int code) { const StunByteStringAttribute* nonce_attr = response->GetByteString(STUN_ATTR_NONCE); if (nonce_attr) { - RTC_LOG(LS_INFO) << port_->ToString() + RTC_LOG(LS_VERBOSE) << port_->ToString() << ": Applying STUN_ATTR_NONCE attribute in " "try alternate error response."; port_->set_nonce(nonce_attr->string_view()); @@ -1570,13 +1570,13 @@ TurnRefreshRequest::TurnRefreshRequest(TurnPort* port, int lifetime /*= -1*/) } void TurnRefreshRequest::OnSent() { - RTC_LOG(LS_INFO) << port_->ToString() << ": TURN refresh request sent, id=" + RTC_LOG(LS_VERBOSE) << port_->ToString() << ": TURN refresh request sent, id=" << rtc::hex_encode(id()); StunRequest::OnSent(); } void TurnRefreshRequest::OnResponse(StunMessage* response) { - RTC_LOG(LS_INFO) << port_->ToString() + RTC_LOG(LS_VERBOSE) << port_->ToString() << ": TURN refresh requested successfully, id=" << rtc::hex_encode(id()) << ", code=0" // Makes logging easier to parse. @@ -1666,14 +1666,14 @@ TurnCreatePermissionRequest::~TurnCreatePermissionRequest() { } void TurnCreatePermissionRequest::OnSent() { - RTC_LOG(LS_INFO) << port_->ToString() + RTC_LOG(LS_VERBOSE) << port_->ToString() << ": TURN create permission request sent, id=" << rtc::hex_encode(id()); StunRequest::OnSent(); } void TurnCreatePermissionRequest::OnResponse(StunMessage* response) { - RTC_LOG(LS_INFO) << port_->ToString() + RTC_LOG(LS_VERBOSE) << port_->ToString() << ": TURN permission requested successfully, id=" << rtc::hex_encode(id()) << ", code=0" // Makes logging easier to parse. @@ -1739,14 +1739,14 @@ TurnChannelBindRequest::~TurnChannelBindRequest() { } void TurnChannelBindRequest::OnSent() { - RTC_LOG(LS_INFO) << port_->ToString() + RTC_LOG(LS_VERBOSE) << port_->ToString() << ": TURN channel bind request sent, id=" << rtc::hex_encode(id()); StunRequest::OnSent(); } void TurnChannelBindRequest::OnResponse(StunMessage* response) { - RTC_LOG(LS_INFO) << port_->ToString() + RTC_LOG(LS_VERBOSE) << port_->ToString() << ": TURN channel bind requested successfully, id=" << rtc::hex_encode(id()) << ", code=0" // Makes logging easier to parse. @@ -1761,7 +1761,7 @@ void TurnChannelBindRequest::OnResponse(StunMessage* response) { // permission from expiring. TimeDelta delay = kTurnPermissionTimeout - TimeDelta::Minutes(1); entry_->SendChannelBindRequest(delay.ms()); - RTC_LOG(LS_INFO) << port_->ToString() << ": Scheduled channel bind in " + RTC_LOG(LS_VERBOSE) << port_->ToString() << ": Scheduled channel bind in " << delay.ms() << "ms."; } } @@ -1861,7 +1861,7 @@ int TurnEntry::Send(const void* data, } void TurnEntry::OnCreatePermissionSuccess() { - RTC_LOG(LS_INFO) << port_->ToString() << ": Create permission for " + RTC_LOG(LS_VERBOSE) << port_->ToString() << ": Create permission for " << ext_addr_.ToSensitiveString() << " succeeded"; if (port_->callbacks_for_test_) { port_->callbacks_for_test_->OnTurnCreatePermissionResult( @@ -1875,7 +1875,7 @@ void TurnEntry::OnCreatePermissionSuccess() { // times out. TimeDelta delay = kTurnPermissionTimeout - TimeDelta::Minutes(1); SendCreatePermissionRequest(delay.ms()); - RTC_LOG(LS_INFO) << port_->ToString() + RTC_LOG(LS_VERBOSE) << port_->ToString() << ": Scheduled create-permission-request in " << delay.ms() << "ms."; } @@ -1904,7 +1904,7 @@ void TurnEntry::OnCreatePermissionTimeout() { } void TurnEntry::OnChannelBindSuccess() { - RTC_LOG(LS_INFO) << port_->ToString() << ": Successful channel bind for " + RTC_LOG(LS_VERBOSE) << port_->ToString() << ": Successful channel bind for " << ext_addr_.ToSensitiveString(); RTC_DCHECK(state_ == STATE_BINDING || state_ == STATE_BOUND); state_ = STATE_BOUND; diff --git a/p2p/client/basic_port_allocator.cc b/p2p/client/basic_port_allocator.cc index c8b293a290..23bdfaacad 100644 --- a/p2p/client/basic_port_allocator.cc +++ b/p2p/client/basic_port_allocator.cc @@ -111,9 +111,9 @@ void FilterNetworks(std::vector* networks, if (start_to_remove == networks->end()) { return; } - RTC_LOG(LS_INFO) << "Filtered out " << filter.description << " networks:"; + RTC_LOG(LS_VERBOSE) << "Filtered out " << filter.description << " networks:"; for (auto it = start_to_remove; it != networks->end(); ++it) { - RTC_LOG(LS_INFO) << (*it)->ToString(); + RTC_LOG(LS_VERBOSE) << (*it)->ToString(); } networks->erase(start_to_remove, networks->end()); } @@ -376,7 +376,7 @@ void BasicPortAllocatorSession::StartGettingPorts() { network_thread_->PostTask( SafeTask(network_safety_.flag(), [this] { GetPortConfigurations(); })); - RTC_LOG(LS_INFO) << "Start getting ports with turn_port_prune_policy " + RTC_LOG(LS_VERBOSE) << "Start getting ports with turn_port_prune_policy " << turn_port_prune_policy_; } @@ -451,7 +451,7 @@ void BasicPortAllocatorSession::RegatherOnFailedNetworks() { return; } - RTC_LOG(LS_INFO) << "Regather candidates on failed networks"; + RTC_LOG(LS_VERBOSE) << "Regather candidates on failed networks"; // Mark a sequence as "network failed" if its network is in the list of failed // networks, so that it won't be considered as equivalent when the session @@ -477,7 +477,7 @@ void BasicPortAllocatorSession::Regather( // the candidates on the remote side. std::vector ports_to_prune = GetUnprunedPorts(networks); if (!ports_to_prune.empty()) { - RTC_LOG(LS_INFO) << "Prune " << ports_to_prune.size() << " ports"; + RTC_LOG(LS_VERBOSE) << "Prune " << ports_to_prune.size() << " ports"; PrunePortsAndRemoveCandidates(ports_to_prune); } @@ -820,7 +820,7 @@ void BasicPortAllocatorSession::DoAllocate(bool disable_equivalent) { << "Machine has no networks; no ports will be allocated"; done_signal_needed = true; } else { - RTC_LOG(LS_INFO) << "Allocate ports on " << NetworksToString(networks); + RTC_LOG(LS_VERBOSE) << "Allocate ports on " << NetworksToString(networks); PortConfiguration* config = configs_.empty() ? nullptr : configs_.back().get(); for (uint32_t i = 0; i < networks.size(); ++i) { @@ -895,7 +895,7 @@ void BasicPortAllocatorSession::OnNetworksChanged() { } std::vector ports_to_prune = GetUnprunedPorts(failed_networks); if (!ports_to_prune.empty()) { - RTC_LOG(LS_INFO) << "Prune " << ports_to_prune.size() + RTC_LOG(LS_VERBOSE) << "Prune " << ports_to_prune.size() << " ports because their networks were gone"; PrunePortsAndRemoveCandidates(ports_to_prune); } @@ -910,7 +910,7 @@ void BasicPortAllocatorSession::OnNetworksChanged() { } if (!network_manager_started_) { - RTC_LOG(LS_INFO) << "Network manager has started"; + RTC_LOG(LS_VERBOSE) << "Network manager has started"; network_manager_started_ = true; } } @@ -933,7 +933,7 @@ void BasicPortAllocatorSession::AddAllocatedPort(Port* port, if (!port) return; - RTC_LOG(LS_INFO) << "Adding allocated port for " << content_name(); + RTC_LOG(LS_VERBOSE) << "Adding allocated port for " << content_name(); port->set_content_name(content_name()); port->set_component(component()); port->set_generation(generation()); @@ -953,7 +953,7 @@ void BasicPortAllocatorSession::AddAllocatedPort(Port* port, [this](PortInterface* port) { OnPortDestroyed(port); }); port->SignalPortError.connect(this, &BasicPortAllocatorSession::OnPortError); - RTC_LOG(LS_INFO) << port->ToString() << ": Added port to allocator"; + RTC_LOG(LS_VERBOSE) << port->ToString() << ": Added port to allocator"; port->PrepareAddress(); } @@ -970,7 +970,7 @@ void BasicPortAllocatorSession::OnCandidateReady(Port* port, RTC_DCHECK_RUN_ON(network_thread_); PortData* data = FindPort(port); RTC_DCHECK(data != NULL); - RTC_LOG(LS_INFO) << port->ToString() + RTC_LOG(LS_VERBOSE) << port->ToString() << ": Gathered candidate: " << c.ToSensitiveString(); // Discarding any candidate signal if port allocation status is // already done with gathering. @@ -1003,7 +1003,7 @@ void BasicPortAllocatorSession::OnCandidateReady(Port* port, // If the current port is not pruned yet, SignalPortReady. if (!data->pruned()) { - RTC_LOG(LS_INFO) << port->ToString() << ": Port ready."; + RTC_LOG(LS_VERBOSE) << port->ToString() << ": Port ready."; SignalPortReady(this, port); port->KeepAliveUntilPruned(); } @@ -1014,7 +1014,7 @@ void BasicPortAllocatorSession::OnCandidateReady(Port* port, candidates.push_back(allocator_->SanitizeCandidate(c)); SignalCandidatesReady(this, candidates); } else { - RTC_LOG(LS_INFO) << "Discarding candidate because it doesn't match filter."; + RTC_LOG(LS_VERBOSE) << "Discarding candidate because it doesn't match filter."; } // If we have pruned any port, maybe need to signal port allocation done. @@ -1063,7 +1063,7 @@ bool BasicPortAllocatorSession::PruneNewlyPairableTurnPort( if (data.port()->Network()->name() == network_name && data.port()->Type() == IceCandidateType::kRelay && data.ready() && &data != newly_pairable_port_data) { - RTC_LOG(LS_INFO) << "Port pruned: " + RTC_LOG(LS_VERBOSE) << "Port pruned: " << newly_pairable_port_data->port()->ToString(); newly_pairable_port_data->Prune(); return true; @@ -1099,7 +1099,7 @@ bool BasicPortAllocatorSession::PruneTurnPorts(Port* newly_pairable_turn_port) { } if (!ports_to_prune.empty()) { - RTC_LOG(LS_INFO) << "Prune " << ports_to_prune.size() + RTC_LOG(LS_VERBOSE) << "Prune " << ports_to_prune.size() << " low-priority TURN ports"; PrunePortsAndRemoveCandidates(ports_to_prune); } @@ -1115,7 +1115,7 @@ void BasicPortAllocatorSession::PruneAllPorts() { void BasicPortAllocatorSession::OnPortComplete(Port* port) { RTC_DCHECK_RUN_ON(network_thread_); - RTC_LOG(LS_INFO) << port->ToString() + RTC_LOG(LS_VERBOSE) << port->ToString() << ": Port completed gathering candidates."; PortData* data = FindPort(port); RTC_DCHECK(data != NULL); @@ -1133,7 +1133,7 @@ void BasicPortAllocatorSession::OnPortComplete(Port* port) { void BasicPortAllocatorSession::OnPortError(Port* port) { RTC_DCHECK_RUN_ON(network_thread_); - RTC_LOG(LS_INFO) << port->ToString() + RTC_LOG(LS_VERBOSE) << port->ToString() << ": Port encountered error while gathering candidates."; PortData* data = FindPort(port); RTC_DCHECK(data != NULL); @@ -1187,9 +1187,9 @@ void BasicPortAllocatorSession::MaybeSignalCandidatesAllocationDone() { RTC_DCHECK_RUN_ON(network_thread_); if (CandidatesAllocationDone()) { if (pooled()) { - RTC_LOG(LS_INFO) << "All candidates gathered for pooled session."; + RTC_LOG(LS_VERBOSE) << "All candidates gathered for pooled session."; } else { - RTC_LOG(LS_INFO) << "All candidates gathered for " << content_name() + RTC_LOG(LS_VERBOSE) << "All candidates gathered for " << content_name() << ":" << component() << ":" << generation(); } for (const auto& event : candidate_error_events_) { @@ -1206,7 +1206,7 @@ void BasicPortAllocatorSession::OnPortDestroyed(PortInterface* port) { iter != ports_.end(); ++iter) { if (port == iter->port()) { ports_.erase(iter); - RTC_LOG(LS_INFO) << port->ToString() << ": Removed port from allocator (" + RTC_LOG(LS_VERBOSE) << port->ToString() << ": Removed port from allocator (" << static_cast(ports_.size()) << " remaining)"; return; } @@ -1260,7 +1260,7 @@ void BasicPortAllocatorSession::PrunePortsAndRemoveCandidates( SignalPortsPruned(this, pruned_ports); } if (!removed_candidates.empty()) { - RTC_LOG(LS_INFO) << "Removed " << removed_candidates.size() + RTC_LOG(LS_VERBOSE) << "Removed " << removed_candidates.size() << " candidates"; SignalCandidatesRemoved(this, removed_candidates); } @@ -1421,7 +1421,7 @@ void AllocationSequence::Process(int epoch) { return; // Perform all of the phases in the current step. - RTC_LOG(LS_INFO) << network_->ToString() + RTC_LOG(LS_VERBOSE) << network_->ToString() << ": Allocation Phase=" << PHASE_NAMES[phase_]; switch (phase_) { @@ -1503,7 +1503,7 @@ void AllocationSequence::CreateUDPPorts() { // If STUN is not disabled, setting stun server address to port. if (!IsFlagSet(PORTALLOCATOR_DISABLE_STUN)) { if (config_ && !config_->StunServers().empty()) { - RTC_LOG(LS_INFO) + RTC_LOG(LS_VERBOSE) << "AllocationSequence: UDPPort will be handling the " "STUN candidate generation."; port->set_server_addresses(config_->StunServers()); @@ -1615,7 +1615,7 @@ void AllocationSequence::CreateTurnPort(const RelayServerConfig& config, int server_ip_family = relay_port->address.ipaddr().family(); int local_ip_family = network_->GetBestIP().family(); if (server_ip_family != AF_UNSPEC && server_ip_family != local_ip_family) { - RTC_LOG(LS_INFO) + RTC_LOG(LS_VERBOSE) << "Server and local address families are not compatible. " "Server address: " << relay_port->address.ipaddr().ToSensitiveString() diff --git a/pc/sdp_offer_answer.cc b/pc/sdp_offer_answer.cc index 199d7ecfbe..cecefebc97 100644 --- a/pc/sdp_offer_answer.cc +++ b/pc/sdp_offer_answer.cc @@ -192,7 +192,7 @@ bool CheckForRemoteIceRestart(const SessionDescriptionInterface* old_desc, if (cricket::IceCredentialsChanged( old_transport_desc->ice_ufrag, old_transport_desc->ice_pwd, new_transport_desc->ice_ufrag, new_transport_desc->ice_pwd)) { - RTC_LOG(LS_INFO) << "Remote peer requests ICE restart for " << content_name + RTC_LOG(LS_VERBOSE) << "Remote peer requests ICE restart for " << content_name << "."; return true; } @@ -2173,7 +2173,7 @@ void SdpOfferAnswerHandler::ApplyRemoteDescriptionUpdateTransceiverState( stream_ids = media_desc->streams()[0].stream_ids(); } - RTC_LOG(LS_INFO) << "Processing the MSIDs for MID=" << content->name + RTC_LOG(LS_VERBOSE) << "Processing the MSIDs for MID=" << content->name << " (" << GetStreamIdsString(stream_ids) << ")."; SetAssociatedRemoteStreams(transceiver->receiver_internal(), stream_ids, &added_streams, &removed_streams); @@ -2183,7 +2183,7 @@ void SdpOfferAnswerHandler::ApplyRemoteDescriptionUpdateTransceiverState( // process the addition of a remote track for the media description. if (!transceiver->fired_direction() || !RtpTransceiverDirectionHasRecv(*transceiver->fired_direction())) { - RTC_LOG(LS_INFO) << "Processing the addition of a remote track for MID=" + RTC_LOG(LS_VERBOSE) << "Processing the addition of a remote track for MID=" << content->name << "."; // Since the transceiver is passed to the user in an // OnTrack event, we must use the proxied transceiver. @@ -2228,7 +2228,7 @@ void SdpOfferAnswerHandler::ApplyRemoteDescriptionUpdateTransceiverState( // 2.2.8.1.12: If the media description is rejected, and transceiver is // not already stopped, stop the RTCRtpTransceiver transceiver. if (content->rejected && !transceiver->stopped()) { - RTC_LOG(LS_INFO) << "Stopping transceiver for MID=" << content->name + RTC_LOG(LS_VERBOSE) << "Stopping transceiver for MID=" << content->name << " since the media section was rejected."; transceiver->StopTransceiverProcedure(); } @@ -2708,7 +2708,7 @@ AddIceCandidateResult SdpOfferAnswerHandler::AddIceCandidateInternal( } if (!ready) { - RTC_LOG(LS_INFO) << "AddIceCandidate: Not ready to use candidate."; + RTC_LOG(LS_VERBOSE) << "AddIceCandidate: Not ready to use candidate."; return kAddIceCandidateFailNotReady; } @@ -2885,7 +2885,7 @@ void SdpOfferAnswerHandler::ChangeSignalingState( if (signaling_state_ == signaling_state) { return; } - RTC_LOG(LS_INFO) << "Session: " << pc_->session_id() << " Old state: " + RTC_LOG(LS_VERBOSE) << "Session: " << pc_->session_id() << " Old state: " << PeerConnectionInterface::AsString(signaling_state_) << " New state: " << PeerConnectionInterface::AsString(signaling_state); @@ -3798,7 +3798,7 @@ RTCError SdpOfferAnswerHandler::UpdateTransceiversAndDataChannels( const auto data_mid = pc_->sctp_mid(); if (data_mid && new_content.name != data_mid.value()) { // Ignore all but the first data section. - RTC_LOG(LS_INFO) << "Ignoring data media section with MID=" + RTC_LOG(LS_VERBOSE) << "Ignoring data media section with MID=" << new_content.name; continue; } @@ -3808,7 +3808,7 @@ RTCError SdpOfferAnswerHandler::UpdateTransceiversAndDataChannels( return error; } } else if (media_type == cricket::MEDIA_TYPE_UNSUPPORTED) { - RTC_LOG(LS_INFO) << "Ignoring unsupported media type"; + RTC_LOG(LS_VERBOSE) << "Ignoring unsupported media type"; } else { LOG_AND_RETURN_ERROR(RTCErrorType::INTERNAL_ERROR, "Unknown section type."); @@ -3873,7 +3873,7 @@ SdpOfferAnswerHandler::AssociateTransceiver( // If no RtpTransceiver was found in the previous step, create one with a // recvonly direction. if (!transceiver) { - RTC_LOG(LS_INFO) << "Adding " + RTC_LOG(LS_VERBOSE) << "Adding " << cricket::MediaTypeToString(media_desc->type()) << " transceiver for MID=" << content.name << " at i=" << mline_index @@ -3990,7 +3990,7 @@ RTCError SdpOfferAnswerHandler::UpdateDataChannelTransport( const cricket::ContentInfo& content, const cricket::ContentGroup* bundle_group) { if (content.rejected) { - RTC_LOG(LS_INFO) << "Rejected data channel transport with mid=" + RTC_LOG(LS_VERBOSE) << "Rejected data channel transport with mid=" << content.mid(); rtc::StringBuilder sb; @@ -4066,7 +4066,7 @@ void SdpOfferAnswerHandler::FillInMissingRemoteMids( RTC_DCHECK(!new_mid.empty()); content.name = new_mid; new_remote_description->transport_infos()[i].content_name = new_mid; - RTC_LOG(LS_INFO) << "SetRemoteDescription: Remote media section at i=" << i + RTC_LOG(LS_VERBOSE) << "SetRemoteDescription: Remote media section at i=" << i << " is missing an a=mid line. Filling in the value '" << new_mid << "' " << source_explanation << "."; } @@ -4595,7 +4595,7 @@ void SdpOfferAnswerHandler::RemoveRecvDirectionFromReceivingTransceiversOfType( RtpTransceiverDirection new_direction = RtpTransceiverDirectionWithRecvSet(transceiver->direction(), false); if (new_direction != transceiver->direction()) { - RTC_LOG(LS_INFO) << "Changing " << cricket::MediaTypeToString(media_type) + RTC_LOG(LS_VERBOSE) << "Changing " << cricket::MediaTypeToString(media_type) << " transceiver (MID=" << transceiver->mid().value_or("") << ") from " << RtpTransceiverDirectionToString( @@ -4612,7 +4612,7 @@ void SdpOfferAnswerHandler::AddUpToOneReceivingTransceiverOfType( cricket::MediaType media_type) { RTC_DCHECK_RUN_ON(signaling_thread()); if (GetReceivingTransceiversOfType(media_type).empty()) { - RTC_LOG(LS_INFO) + RTC_LOG(LS_VERBOSE) << "Adding one recvonly " << cricket::MediaTypeToString(media_type) << " transceiver since CreateOffer specified offer_to_receive=1"; RtpTransceiverInit init; @@ -4643,7 +4643,7 @@ void SdpOfferAnswerHandler::ProcessRemovalOfRemoteTrack( std::vector>* remove_list, std::vector>* removed_streams) { RTC_DCHECK(transceiver->mid()); - RTC_LOG(LS_INFO) << "Processing the removal of a track for MID=" + RTC_LOG(LS_VERBOSE) << "Processing the removal of a track for MID=" << *transceiver->mid(); std::vector> previous_streams = transceiver->internal()->receiver_internal()->streams(); @@ -4977,14 +4977,14 @@ void SdpOfferAnswerHandler::RemoveStoppedTransceivers() { transceiver->internal(), remote_description()); if ((local_content && local_content->rejected) || (remote_content && remote_content->rejected)) { - RTC_LOG(LS_INFO) << "Dissociating transceiver" + RTC_LOG(LS_VERBOSE) << "Dissociating transceiver" " since the media section is being recycled."; transceiver->internal()->set_mid(absl::nullopt); transceiver->internal()->set_mline_index(absl::nullopt); } else if (!local_content && !remote_content) { // TODO(bugs.webrtc.org/11973): Consider if this should be removed already // See https://github.com/w3c/webrtc-pc/issues/2576 - RTC_LOG(LS_INFO) + RTC_LOG(LS_VERBOSE) << "Dropping stopped transceiver that was never associated"; } transceivers()->Remove(transceiver); @@ -5057,7 +5057,7 @@ bool SdpOfferAnswerHandler::UseCandidatesInRemoteDescription() { bool valid = false; if (!ReadyToUseRemoteCandidate(candidate, remote_desc, &valid)) { if (valid) { - RTC_LOG(LS_INFO) + RTC_LOG(LS_VERBOSE) << "UseCandidatesInRemoteDescription: Not ready to use " "candidate."; } diff --git a/pc/usage_pattern.cc b/pc/usage_pattern.cc index 848472148f..c509496cb8 100644 --- a/pc/usage_pattern.cc +++ b/pc/usage_pattern.cc @@ -21,7 +21,7 @@ void UsagePattern::NoteUsageEvent(UsageEvent event) { } void UsagePattern::ReportUsagePattern(PeerConnectionObserver* observer) const { - RTC_DLOG(LS_INFO) << "Usage signature is " << usage_event_accumulator_; + RTC_DLOG(LS_VERBOSE) << "Usage signature is " << usage_event_accumulator_; RTC_HISTOGRAM_ENUMERATION_SPARSE("WebRTC.PeerConnection.UsagePattern", usage_event_accumulator_, static_cast(UsageEvent::MAX_VALUE)); @@ -39,7 +39,7 @@ void UsagePattern::ReportUsagePattern(PeerConnectionObserver* observer) const { if (observer) { observer->OnInterestingUsage(usage_event_accumulator_); } else { - RTC_LOG(LS_INFO) << "Interesting usage signature " + RTC_LOG(LS_VERBOSE) << "Interesting usage signature " << usage_event_accumulator_ << " observed after observer shutdown"; } diff --git a/sdk/BUILD.gn b/sdk/BUILD.gn index e7ec3e3614..b2c5d9d5be 100644 --- a/sdk/BUILD.gn +++ b/sdk/BUILD.gn @@ -917,6 +917,9 @@ if (is_ios || is_mac) { ] configs += [ "..:no_global_constructors" ] sources = [ + "objc/api/RTCAudioRendererAdapter+Private.h", + "objc/api/RTCAudioRendererAdapter.h", + "objc/api/RTCAudioRendererAdapter.mm", "objc/api/peerconnection/RTCAudioSource+Private.h", "objc/api/peerconnection/RTCAudioSource.h", "objc/api/peerconnection/RTCAudioSource.mm", @@ -1117,6 +1120,7 @@ if (is_ios || is_mac) { "objc/unittests/RTCAudioDeviceModule_xctest.mm", "objc/unittests/RTCAudioDevice_xctest.mm", "objc/unittests/RTCAudioSessionTest.mm", + "objc/unittests/RTCAudioTrack_xctest.mm", "objc/unittests/RTCCVPixelBuffer_xctest.mm", "objc/unittests/RTCCallbackLogger_xctest.m", "objc/unittests/RTCCameraVideoCapturerTests.mm", diff --git a/sdk/objc/api/RTCAudioRendererAdapter+Private.h b/sdk/objc/api/RTCAudioRendererAdapter+Private.h new file mode 100644 index 0000000000..f8e1b39ab7 --- /dev/null +++ b/sdk/objc/api/RTCAudioRendererAdapter+Private.h @@ -0,0 +1,40 @@ +/* + * Copyright 2026 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "RTCAudioRendererAdapter.h" + +#import "api/peerconnection/RTCAudioTrack.h" + +#include "api/media_stream_interface.h" + +NS_ASSUME_NONNULL_BEGIN + +@interface RTCAudioRendererAdapter () + +/** + * The Objective-C audio renderer passed to this adapter during construction. + * Calls made to the native AudioTrackSinkInterface will be adapted and passed + * to this renderer. + */ +@property(nonatomic, readonly) id audioRenderer; + +/** + * The native AudioTrackSinkInterface surface exposed by this adapter. This + * pointer is unsafe and owned by this class. + */ +@property(nonatomic, readonly) webrtc::AudioTrackSinkInterface *nativeAudioRenderer; + +/** Initialize an RTCAudioRendererAdapter with an RTCAudioRenderer. */ +- (instancetype)initWithNativeRenderer:(id)audioRenderer + NS_DESIGNATED_INITIALIZER; + +@end + +NS_ASSUME_NONNULL_END diff --git a/sdk/objc/api/RTCAudioRendererAdapter.h b/sdk/objc/api/RTCAudioRendererAdapter.h new file mode 100644 index 0000000000..cff8d8cfd7 --- /dev/null +++ b/sdk/objc/api/RTCAudioRendererAdapter.h @@ -0,0 +1,26 @@ +/* + * Copyright 2026 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import + +NS_ASSUME_NONNULL_BEGIN + +/* + * Creates a webrtc::AudioTrackSinkInterface surface for an RTCAudioRenderer. + * The sink is used by WebRTC audio rendering code - this adapter forwards + * callbacks to the RTCAudioRenderer supplied during construction. + */ +@interface RTCAudioRendererAdapter : NSObject + +- (instancetype)init NS_UNAVAILABLE; + +@end + +NS_ASSUME_NONNULL_END diff --git a/sdk/objc/api/RTCAudioRendererAdapter.mm b/sdk/objc/api/RTCAudioRendererAdapter.mm new file mode 100644 index 0000000000..1bca08a852 --- /dev/null +++ b/sdk/objc/api/RTCAudioRendererAdapter.mm @@ -0,0 +1,76 @@ +/* + * Copyright 2026 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "RTCAudioRendererAdapter+Private.h" + +#include + +namespace webrtc { + +class AudioRendererAdapter : public webrtc::AudioTrackSinkInterface { + public: + explicit AudioRendererAdapter(::RTCAudioRendererAdapter* adapter) + : adapter_(adapter) {} + + void OnData(const void* audio_data, + int bits_per_sample, + int sample_rate, + size_t number_of_channels, + size_t number_of_frames) override { + [adapter_.audioRenderer renderPCMData:audio_data + bitsPerSample:bits_per_sample + sampleRate:sample_rate + numberOfChannels:number_of_channels + numberOfFrames:number_of_frames + absoluteCaptureTimestampMs:0 + timestampIsValid:NO]; + } + + void OnData(const void* audio_data, + int bits_per_sample, + int sample_rate, + size_t number_of_channels, + size_t number_of_frames, + absl::optional absolute_capture_timestamp_ms) override { + [adapter_.audioRenderer renderPCMData:audio_data + bitsPerSample:bits_per_sample + sampleRate:sample_rate + numberOfChannels:number_of_channels + numberOfFrames:number_of_frames + absoluteCaptureTimestampMs:absolute_capture_timestamp_ms.value_or(0) + timestampIsValid:absolute_capture_timestamp_ms.has_value()]; + } + + private: + __weak ::RTCAudioRendererAdapter* adapter_; +}; + +} // namespace webrtc + +@implementation RTCAudioRendererAdapter { + std::unique_ptr _adapter; +} + +@synthesize audioRenderer = _audioRenderer; + +- (instancetype)initWithNativeRenderer:(id)audioRenderer { + NSParameterAssert(audioRenderer); + if (self = [super init]) { + _audioRenderer = audioRenderer; + _adapter = std::make_unique(self); + } + return self; +} + +- (webrtc::AudioTrackSinkInterface*)nativeAudioRenderer { + return _adapter.get(); +} + +@end diff --git a/sdk/objc/api/peerconnection/RTCAudioTrack.h b/sdk/objc/api/peerconnection/RTCAudioTrack.h index 95eb5d3d48..43ef87b9ce 100644 --- a/sdk/objc/api/peerconnection/RTCAudioTrack.h +++ b/sdk/objc/api/peerconnection/RTCAudioTrack.h @@ -8,6 +8,10 @@ * be found in the AUTHORS file in the root of the source tree. */ +#import +#include +#include + #import "RTCMacros.h" #import "RTCMediaStreamTrack.h" @@ -15,13 +19,37 @@ NS_ASSUME_NONNULL_BEGIN @class RTC_OBJC_TYPE(RTCAudioSource); +/** + * Receives decoded PCM samples from an RTCAudioTrack. + * + * The audio_data pointer is only valid for the lifetime of the callback. + * Callbacks may be delivered on a WebRTC audio thread. + */ +@protocol RTC_OBJC_TYPE(RTCAudioRenderer) + +- (void)renderPCMData:(const void *)audio_data + bitsPerSample:(int)bits_per_sample + sampleRate:(int)sample_rate + numberOfChannels:(size_t)number_of_channels + numberOfFrames:(size_t)number_of_frames +absoluteCaptureTimestampMs:(int64_t)absolute_capture_timestamp_ms + timestampIsValid:(BOOL)timestamp_is_valid; + +@end + RTC_OBJC_EXPORT @interface RTC_OBJC_TYPE (RTCAudioTrack) : RTC_OBJC_TYPE(RTCMediaStreamTrack) - (instancetype)init NS_UNAVAILABLE; /** The audio source for this audio track. */ -@property(nonatomic, readonly) RTC_OBJC_TYPE(RTCAudioSource) * source; +@property(nonatomic, readonly) RTC_OBJC_TYPE(RTCAudioSource) *source; + +/** Register a renderer that will receive decoded PCM frames on this track. */ +- (void)addRenderer:(id)renderer; + +/** Deregister a previously registered renderer. */ +- (void)removeRenderer:(id)renderer; @end diff --git a/sdk/objc/api/peerconnection/RTCAudioTrack.mm b/sdk/objc/api/peerconnection/RTCAudioTrack.mm index 5c1736f436..6192c45e75 100644 --- a/sdk/objc/api/peerconnection/RTCAudioTrack.mm +++ b/sdk/objc/api/peerconnection/RTCAudioTrack.mm @@ -10,14 +10,19 @@ #import "RTCAudioTrack+Private.h" +#import "api/RTCAudioRendererAdapter+Private.h" #import "RTCAudioSource+Private.h" #import "RTCMediaStreamTrack+Private.h" #import "RTCPeerConnectionFactory+Private.h" #import "helpers/NSString+StdString.h" #include "rtc_base/checks.h" +#include "rtc_base/logging.h" -@implementation RTC_OBJC_TYPE (RTCAudioTrack) +@implementation RTC_OBJC_TYPE (RTCAudioTrack) { + rtc::Thread *_workerThread; + NSMutableArray *_adapters /* accessed on _workerThread */; +} @synthesize source = _source; @@ -43,7 +48,17 @@ - (instancetype)initWithFactory:(RTC_OBJC_TYPE(RTCPeerConnectionFactory) *)facto NSParameterAssert(factory); NSParameterAssert(nativeTrack); NSParameterAssert(type == RTCMediaStreamTrackTypeAudio); - return [super initWithFactory:factory nativeTrack:nativeTrack type:type]; + if (self = [super initWithFactory:factory nativeTrack:nativeTrack type:type]) { + _adapters = [NSMutableArray array]; + _workerThread = factory.workerThread; + } + return self; +} + +- (void)dealloc { + for (RTCAudioRendererAdapter *adapter in _adapters) { + self.nativeAudioTrack->RemoveSink(adapter.nativeAudioRenderer); + } } - (RTC_OBJC_TYPE(RTCAudioSource) *)source { @@ -57,6 +72,50 @@ - (instancetype)initWithFactory:(RTC_OBJC_TYPE(RTCPeerConnectionFactory) *)facto return _source; } +- (void)addRenderer:(id)renderer { + if (!_workerThread->IsCurrent()) { + _workerThread->BlockingCall([renderer, self] { [self addRenderer:renderer]; }); + return; + } + + for (RTCAudioRendererAdapter *adapter in _adapters) { + if (adapter.audioRenderer == renderer) { + RTC_LOG(LS_INFO) << "|renderer| is already attached to this track"; + return; + } + } + + RTCAudioRendererAdapter *adapter = + [[RTCAudioRendererAdapter alloc] initWithNativeRenderer:renderer]; + [_adapters addObject:adapter]; + self.nativeAudioTrack->AddSink(adapter.nativeAudioRenderer); +} + +- (void)removeRenderer:(id)renderer { + if (!_workerThread->IsCurrent()) { + _workerThread->BlockingCall([renderer, self] { [self removeRenderer:renderer]; }); + return; + } + + __block NSUInteger indexToRemove = NSNotFound; + [_adapters enumerateObjectsUsingBlock:^(RTCAudioRendererAdapter *adapter, + NSUInteger idx, + BOOL *stop) { + if (adapter.audioRenderer == renderer) { + indexToRemove = idx; + *stop = YES; + } + }]; + if (indexToRemove == NSNotFound) { + RTC_LOG(LS_INFO) << "removeRenderer called with a renderer that has not been previously added"; + return; + } + + RTCAudioRendererAdapter *adapterToRemove = [_adapters objectAtIndex:indexToRemove]; + self.nativeAudioTrack->RemoveSink(adapterToRemove.nativeAudioRenderer); + [_adapters removeObjectAtIndex:indexToRemove]; +} + #pragma mark - Private - (rtc::scoped_refptr)nativeAudioTrack { diff --git a/sdk/objc/components/audio/RTCAudioSession+Private.h b/sdk/objc/components/audio/RTCAudioSession+Private.h index 2be1b9fb3d..e7e1938898 100644 --- a/sdk/objc/components/audio/RTCAudioSession+Private.h +++ b/sdk/objc/components/audio/RTCAudioSession+Private.h @@ -17,11 +17,6 @@ NS_ASSUME_NONNULL_BEGIN @interface RTC_OBJC_TYPE (RTCAudioSession) () - /** Number of times setActive:YES has succeeded without a balanced call to - * setActive:NO. - */ - @property(nonatomic, readonly) int activationCount; - /** The number of times `beginWebRTCSession` was called without a balanced call * to `endWebRTCSession`. */ diff --git a/sdk/objc/components/audio/RTCAudioSession.h b/sdk/objc/components/audio/RTCAudioSession.h index 2730664858..2711558d1b 100644 --- a/sdk/objc/components/audio/RTCAudioSession.h +++ b/sdk/objc/components/audio/RTCAudioSession.h @@ -141,6 +141,11 @@ RTC_OBJC_EXPORT */ @property(nonatomic, readonly) BOOL isActive; +/** Number of times setActive:YES has succeeded without a balanced call to +* setActive:NO. +*/ +@property(nonatomic, readonly) int activationCount; + /** If YES, WebRTC will not initialize the audio unit automatically when an * audio track is ready for playout or recording. Instead, applications should * call setIsAudioEnabled. If NO, WebRTC will initialize the audio unit diff --git a/sdk/objc/native/src/audio/audio_device_ios.mm b/sdk/objc/native/src/audio/audio_device_ios.mm index 78420ec232..d744f94921 100644 --- a/sdk/objc/native/src/audio/audio_device_ios.mm +++ b/sdk/objc/native/src/audio/audio_device_ios.mm @@ -224,19 +224,19 @@ static void LogDeviceInfo() { RTC_DCHECK(audio_is_initialized_); RTC_DCHECK(!playing_.load()); RTC_DCHECK(audio_unit_); + if (fine_audio_buffer_) { fine_audio_buffer_->ResetPlayout(); } - if (!recording_.load() && audio_unit_->GetState() == VoiceProcessingAudioUnit::kInitialized) { - OSStatus result = audio_unit_->Start(); - if (result != noErr) { - RTC_OBJC_TYPE(RTCAudioSession)* session = [RTC_OBJC_TYPE(RTCAudioSession) sharedInstance]; - [session notifyAudioUnitStartFailedWithError:result]; - RTCLogError(@"StartPlayout failed to start audio unit, reason %d", result); - return -1; - } - RTC_LOG(LS_INFO) << "Voice-Processing I/O audio unit is now started"; + + // StartPlayout() can lead to audio unit recreate and io thread may be different after + io_thread_checker_.Detach(); + + if(!audio_unit_->StartPlayout()) { + RTCLogError(@"StartPlayout failed to enable playout in audio unit."); + return -1; } + playing_.store(1, std::memory_order_release); num_playout_callbacks_ = 0; num_detected_playout_glitches_ = 0; @@ -252,6 +252,11 @@ static void LogDeviceInfo() { if (!recording_.load()) { ShutdownPlayOrRecord(); audio_is_initialized_ = false; + } else { + if(!audio_unit_->StopPlayout()) { + RTCLogError(@"StopPlayout failed to disable playout in audio unit."); + return -1; + } } playing_.store(0, std::memory_order_release); @@ -280,19 +285,24 @@ static void LogDeviceInfo() { RTC_DCHECK(audio_is_initialized_); RTC_DCHECK(!recording_.load()); RTC_DCHECK(audio_unit_); + + if(!audio_unit_->CanRecord()) { + RTCLogError(@"VoiceProcessingAudioUnit is not ready to start recording"); + return -1; + } + if (fine_audio_buffer_) { fine_audio_buffer_->ResetRecord(); } - if (!playing_.load() && audio_unit_->GetState() == VoiceProcessingAudioUnit::kInitialized) { - OSStatus result = audio_unit_->Start(); - if (result != noErr) { - RTC_OBJC_TYPE(RTCAudioSession)* session = [RTC_OBJC_TYPE(RTCAudioSession) sharedInstance]; - [session notifyAudioUnitStartFailedWithError:result]; - RTCLogError(@"StartRecording failed to start audio unit, reason %d", result); - return -1; - } - RTC_LOG(LS_INFO) << "Voice-Processing I/O audio unit is now started"; + + // StartRecording() can lead to audio unit recreate and io thread may be different after + io_thread_checker_.Detach(); + + if(!audio_unit_->StartRecording()) { + RTCLogError(@"StartRecording failed to enable recording in audio unit."); + return -1; } + recording_.store(1, std::memory_order_release); return 0; } @@ -306,6 +316,11 @@ static void LogDeviceInfo() { if (!playing_.load()) { ShutdownPlayOrRecord(); audio_is_initialized_ = false; + } else { + if(!audio_unit_->StopRecording()) { + RTCLogError(@"StopRecording failed to disable recording in audio unit."); + return -1; + } } recording_.store(0, std::memory_order_release); return 0; @@ -589,7 +604,7 @@ static void LogDeviceInfo() { SetupAudioBuffersForActiveAudioSession(); // Initialize the audio unit again with the new sample rate. - if (!audio_unit_->Initialize(playout_parameters_.sample_rate())) { + if (!audio_unit_->Initialize(playout_parameters_.sample_rate(), playing_, recording_)) { RTCLogError(@"Failed to initialize the audio unit with sample rate: %d", playout_parameters_.sample_rate()); return; @@ -715,10 +730,6 @@ static void LogDeviceInfo() { RTC_DCHECK(!audio_unit_); audio_unit_.reset(new VoiceProcessingAudioUnit(bypass_voice_processing_, this)); - if (!audio_unit_->Init()) { - audio_unit_.reset(); - return false; - } return true; } @@ -747,13 +758,9 @@ static void LogDeviceInfo() { bool should_stop_audio_unit = false; switch (audio_unit_->GetState()) { - case VoiceProcessingAudioUnit::kInitRequired: - RTCLog(@"VPAU state: InitRequired"); - RTC_DCHECK_NOTREACHED(); - break; case VoiceProcessingAudioUnit::kUninitialized: RTCLog(@"VPAU state: Uninitialized"); - should_initialize_audio_unit = can_play_or_record; + should_initialize_audio_unit = can_play_or_record && (playing_ || recording_); should_start_audio_unit = should_initialize_audio_unit && (playing_.load() || recording_.load()); break; @@ -774,7 +781,7 @@ static void LogDeviceInfo() { RTCLog(@"Initializing audio unit for UpdateAudioUnit"); ConfigureAudioSession(); SetupAudioBuffersForActiveAudioSession(); - if (!audio_unit_->Initialize(playout_parameters_.sample_rate())) { + if (!audio_unit_->Initialize(playout_parameters_.sample_rate(), playing_, recording_)) { RTCLogError(@"Failed to initialize audio unit."); return; } @@ -900,7 +907,7 @@ static void LogDeviceInfo() { return false; } SetupAudioBuffersForActiveAudioSession(); - audio_unit_->Initialize(playout_parameters_.sample_rate()); + audio_unit_->InitializeSampleRate(playout_parameters_.sample_rate()); } // Release the lock. diff --git a/sdk/objc/native/src/audio/voice_processing_audio_unit.h b/sdk/objc/native/src/audio/voice_processing_audio_unit.h index ed9dd98568..fb001a2510 100644 --- a/sdk/objc/native/src/audio/voice_processing_audio_unit.h +++ b/sdk/objc/native/src/audio/voice_processing_audio_unit.h @@ -11,6 +11,8 @@ #ifndef SDK_OBJC_NATIVE_SRC_AUDIO_VOICE_PROCESSING_AUDIO_UNIT_H_ #define SDK_OBJC_NATIVE_SRC_AUDIO_VOICE_PROCESSING_AUDIO_UNIT_H_ +#include + #include namespace webrtc { @@ -52,9 +54,7 @@ class VoiceProcessingAudioUnit { // TODO(tkchin): enum for state and state checking. enum State : int32_t { - // Init() should be called. - kInitRequired, - // Audio unit created but not initialized. + // Audio unit not initialized. kUninitialized, // Initialized but not started. Equivalent to stopped. kInitialized, @@ -65,17 +65,29 @@ class VoiceProcessingAudioUnit { // Number of bytes per audio sample for 16-bit signed integer representation. static const UInt32 kBytesPerSample; + private: // Initializes this class by creating the underlying audio unit instance. // Creates a Voice-Processing I/O unit and configures it for full-duplex // audio. The selected stream format is selected to avoid internal resampling // and to match the 10ms callback rate for WebRTC as well as possible. // Does not intialize the audio unit. - bool Init(); + bool CreateVoiceProcessingAU(); + bool CreatePlaybackAU(); +public: VoiceProcessingAudioUnit::State GetState() const; + bool CanRecord() const; + + void InitializeSampleRate(Float64 sample_rate); // Initializes the underlying audio unit with the given sample rate. - bool Initialize(Float64 sample_rate); + bool Initialize(Float64 sample_rate, bool enable_playout, bool enable_recording); + + + bool StartRecording(); + bool StopRecording(); + bool StartPlayout(); + bool StopPlayout(); // Starts the underlying audio unit. OSStatus Start(); @@ -83,8 +95,8 @@ class VoiceProcessingAudioUnit { // Stops the underlying audio unit. bool Stop(); - // Uninitializes the underlying audio unit. - bool Uninitialize(); + // Uninitializes and disposes the underlying audio unit. + void Uninitialize(); // Calls render on the underlying audio unit. OSStatus Render(AudioUnitRenderActionFlags* flags, @@ -127,13 +139,22 @@ class VoiceProcessingAudioUnit { // implementation file for details on format. AudioStreamBasicDescription GetFormat(Float64 sample_rate) const; + void UninitializeAudioUnit(); // Deletes the underlying audio unit. void DisposeAudioUnit(); + static std::atomic_bool mic_owning_; + const bool bypass_voice_processing_; VoiceProcessingAudioUnitObserver* observer_; AudioUnit vpio_unit_; VoiceProcessingAudioUnit::State state_; + + // Following keep configuration requested externally. + // And exact audio unit configuration can be different. + Float64 sample_rate_; + bool playout_enabled_; + bool recording_enabled_; }; } // namespace ios_adm } // namespace webrtc diff --git a/sdk/objc/native/src/audio/voice_processing_audio_unit.mm b/sdk/objc/native/src/audio/voice_processing_audio_unit.mm index 3905b6857a..30cca24579 100644 --- a/sdk/objc/native/src/audio/voice_processing_audio_unit.mm +++ b/sdk/objc/native/src/audio/voice_processing_audio_unit.mm @@ -14,8 +14,10 @@ #include "system_wrappers/include/metrics.h" #import "base/RTCLogging.h" +#import "sdk/objc/components/audio/RTCAudioSession.h" #import "sdk/objc/components/audio/RTCAudioSessionConfiguration.h" + #if !defined(NDEBUG) static void LogStreamDescription(AudioStreamBasicDescription description) { char formatIdString[5]; @@ -71,12 +73,17 @@ static OSStatus GetAGCState(AudioUnit audio_unit, UInt32* enabled) { return result; } +std::atomic_bool VoiceProcessingAudioUnit::mic_owning_(false); + VoiceProcessingAudioUnit::VoiceProcessingAudioUnit(bool bypass_voice_processing, VoiceProcessingAudioUnitObserver* observer) : bypass_voice_processing_(bypass_voice_processing), observer_(observer), vpio_unit_(nullptr), - state_(kInitRequired) { + state_(kUninitialized), + sample_rate_(), + playout_enabled_(false), + recording_enabled_(false){ RTC_DCHECK(observer); } @@ -86,46 +93,86 @@ static OSStatus GetAGCState(AudioUnit audio_unit, UInt32* enabled) { const UInt32 VoiceProcessingAudioUnit::kBytesPerSample = 2; -bool VoiceProcessingAudioUnit::Init() { - RTC_DCHECK_EQ(state_, kInitRequired); +bool VoiceProcessingAudioUnit::CanRecord() const { + RTC_OBJC_TYPE(RTCAudioSession) *session = [RTC_OBJC_TYPE(RTCAudioSession) sharedInstance]; + return + [session.category isEqualToString: AVAudioSessionCategoryPlayAndRecord] && + !mic_owning_.load(std::memory_order_relaxed); +} - // Create an audio component description to identify the Voice Processing - // I/O audio unit. - AudioComponentDescription vpio_unit_description; - vpio_unit_description.componentType = kAudioUnitType_Output; - vpio_unit_description.componentSubType = kAudioUnitSubType_VoiceProcessingIO; - vpio_unit_description.componentManufacturer = kAudioUnitManufacturer_Apple; - vpio_unit_description.componentFlags = 0; - vpio_unit_description.componentFlagsMask = 0; +void VoiceProcessingAudioUnit::InitializeSampleRate(Float64 sample_rate) +{ + RTC_DCHECK_GE(state_, kUninitialized); + RTCLog(@"Initializing audio unit sample rate: %f", sample_rate); - // Obtain an audio unit instance given the description. - AudioComponent found_vpio_unit_ref = - AudioComponentFindNext(nullptr, &vpio_unit_description); + sample_rate_ = sample_rate; +} - // Create a Voice Processing IO audio unit. - OSStatus result = noErr; - result = AudioComponentInstanceNew(found_vpio_unit_ref, &vpio_unit_); - if (result != noErr) { - vpio_unit_ = nullptr; - RTCLogError(@"AudioComponentInstanceNew failed. Error=%ld.", (long)result); +bool VoiceProcessingAudioUnit::Initialize(Float64 sample_rate, bool enable_playout, bool enable_recording) +{ + RTC_DCHECK_GE(state_, kUninitialized); + RTC_DCHECK(!vpio_unit_); + RTC_DCHECK(enable_playout || enable_recording); + RTCLog(@"Initializing audio unit with sample rate: %f", sample_rate_); + + if (!enable_recording && !enable_playout) { + RTCLogError(@"It's not possible to create not playing nor recording audio unit"); return false; } - // Enable input on the input scope of the input element. - UInt32 enable_input = 1; - result = AudioUnitSetProperty(vpio_unit_, kAudioOutputUnitProperty_EnableIO, - kAudioUnitScope_Input, kInputBus, &enable_input, - sizeof(enable_input)); - if (result != noErr) { - DisposeAudioUnit(); - RTCLogError(@"Failed to enable input on input scope of input element. " - "Error=%ld.", - (long)result); + sample_rate_ = sample_rate; + playout_enabled_ = enable_playout; + recording_enabled_ = enable_recording; + + RTC_OBJC_TYPE(RTCAudioSession) *session = [RTC_OBJC_TYPE(RTCAudioSession) sharedInstance]; + if (enable_recording && ![session.category isEqualToString: AVAudioSessionCategoryPlayAndRecord]) { + RTCLogError(@"Trying to record without PlayAndRecord AVAudioSession category enabled"); + return false; + } + bool expected_mic_owning = false; + if (enable_recording && !mic_owning_.compare_exchange_strong(expected_mic_owning, true, std::memory_order_relaxed)) { + RTCLogError(@"Trying to acquire mic second time"); + return false; + } + + // Playout should be always enabled. + // * For recording: to avoid glitches in mic input on playback enable/disable + // * For playback: you creating it for playback, or why you creating it at all? + enable_playout = true; // playout always enabled + + bool auCreated; + + if(enable_recording) { + auCreated = CreateVoiceProcessingAU(); + } else { + auCreated = CreatePlaybackAU(); + } + + if(!auCreated) { + if(enable_recording) + mic_owning_.store(false, std::memory_order_relaxed); return false; } + OSStatus result = noErr; + + if(enable_recording) { + // Enable input on the input scope of the input element. + UInt32 enable_input = enable_recording ? 1 : 0; + result = AudioUnitSetProperty(vpio_unit_, kAudioOutputUnitProperty_EnableIO, + kAudioUnitScope_Input, kInputBus, &enable_input, + sizeof(enable_input)); + if (result != noErr) { + DisposeAudioUnit(); + RTCLogError(@"Failed to enable input on input scope of input element. " + "Error=%ld.", + (long)result); + return false; + } + } + // Enable output on the output scope of the output element. - UInt32 enable_output = 1; + UInt32 enable_output = enable_playout ? 1 : 0; result = AudioUnitSetProperty(vpio_unit_, kAudioOutputUnitProperty_EnableIO, kAudioUnitScope_Output, kOutputBus, &enable_output, sizeof(enable_output)); @@ -153,66 +200,58 @@ static OSStatus GetAGCState(AudioUnit audio_unit, UInt32* enabled) { return false; } - // Disable AU buffer allocation for the recorder, we allocate our own. - // TODO(henrika): not sure that it actually saves resource to make this call. - UInt32 flag = 0; - result = AudioUnitSetProperty( - vpio_unit_, kAudioUnitProperty_ShouldAllocateBuffer, - kAudioUnitScope_Output, kInputBus, &flag, sizeof(flag)); - if (result != noErr) { - DisposeAudioUnit(); - RTCLogError(@"Failed to disable buffer allocation on the input bus. " - "Error=%ld.", - (long)result); - return false; - } + if(enable_recording) { + // Disable AU buffer allocation for the recorder, we allocate our own. + // TODO(henrika): not sure that it actually saves resource to make this call. + UInt32 flag = 0; + result = AudioUnitSetProperty( + vpio_unit_, kAudioUnitProperty_ShouldAllocateBuffer, + kAudioUnitScope_Output, kInputBus, &flag, sizeof(flag)); + if (result != noErr) { + DisposeAudioUnit(); + RTCLogError(@"Failed to disable buffer allocation on the input bus. " + "Error=%ld.", + (long)result); + return false; + } - // Specify the callback to be called by the I/O thread to us when input audio - // is available. The recorded samples can then be obtained by calling the - // AudioUnitRender() method. - AURenderCallbackStruct input_callback; - input_callback.inputProc = OnDeliverRecordedData; - input_callback.inputProcRefCon = this; - result = AudioUnitSetProperty(vpio_unit_, - kAudioOutputUnitProperty_SetInputCallback, - kAudioUnitScope_Global, kInputBus, - &input_callback, sizeof(input_callback)); - if (result != noErr) { - DisposeAudioUnit(); - RTCLogError(@"Failed to specify the input callback on the input bus. " - "Error=%ld.", - (long)result); - return false; + // Specify the callback to be called by the I/O thread to us when input audio + // is available. The recorded samples can then be obtained by calling the + // AudioUnitRender() method. + AURenderCallbackStruct input_callback; + input_callback.inputProc = OnDeliverRecordedData; + input_callback.inputProcRefCon = this; + result = AudioUnitSetProperty(vpio_unit_, + kAudioOutputUnitProperty_SetInputCallback, + kAudioUnitScope_Global, kInputBus, + &input_callback, sizeof(input_callback)); + if (result != noErr) { + DisposeAudioUnit(); + RTCLogError(@"Failed to specify the input callback on the input bus. " + "Error=%ld.", + (long)result); + return false; + } } - state_ = kUninitialized; - return true; -} - -VoiceProcessingAudioUnit::State VoiceProcessingAudioUnit::GetState() const { - return state_; -} - -bool VoiceProcessingAudioUnit::Initialize(Float64 sample_rate) { - RTC_DCHECK_GE(state_, kUninitialized); - RTCLog(@"Initializing audio unit with sample rate: %f", sample_rate); - - OSStatus result = noErr; - AudioStreamBasicDescription format = GetFormat(sample_rate); + AudioStreamBasicDescription format = GetFormat(sample_rate_); UInt32 size = sizeof(format); #if !defined(NDEBUG) LogStreamDescription(format); #endif - // Set the format on the output scope of the input element/bus. - result = - AudioUnitSetProperty(vpio_unit_, kAudioUnitProperty_StreamFormat, - kAudioUnitScope_Output, kInputBus, &format, size); - if (result != noErr) { - RTCLogError(@"Failed to set format on output scope of input bus. " - "Error=%ld.", - (long)result); - return false; + if(enable_recording) { + // Set the format on the output scope of the input element/bus. + result = + AudioUnitSetProperty(vpio_unit_, kAudioUnitProperty_StreamFormat, + kAudioUnitScope_Output, kInputBus, &format, size); + if (result != noErr) { + DisposeAudioUnit(); + RTCLogError(@"Failed to set format on output scope of input bus. " + "Error=%ld.", + (long)result); + return false; + } } // Set the format on the input scope of the output element/bus. @@ -220,13 +259,14 @@ static OSStatus GetAGCState(AudioUnit audio_unit, UInt32* enabled) { AudioUnitSetProperty(vpio_unit_, kAudioUnitProperty_StreamFormat, kAudioUnitScope_Input, kOutputBus, &format, size); if (result != noErr) { + DisposeAudioUnit(); RTCLogError(@"Failed to set format on input scope of output bus. " "Error=%ld.", (long)result); return false; } - // Initialize the Voice Processing I/O unit instance. + // Initialize the audio unit instance. // Calls to AudioUnitInitialize() can fail if called back-to-back on // different ADM instances. The error message in this case is -66635 which is // undocumented. Tests have shown that calling AudioUnitInitialize a second @@ -235,11 +275,12 @@ static OSStatus GetAGCState(AudioUnit audio_unit, UInt32* enabled) { int failed_initalize_attempts = 0; result = AudioUnitInitialize(vpio_unit_); while (result != noErr) { - RTCLogError(@"Failed to initialize the Voice Processing I/O unit. " + RTCLogError(@"Failed to initialize the audio unit. " "Error=%ld.", (long)result); ++failed_initalize_attempts; if (failed_initalize_attempts == kMaxNumberOfAudioUnitInitializeAttempts) { + DisposeAudioUnit(); // Max number of initialization attempts exceeded, hence abort. RTCLogError(@"Too many initialization attempts."); return false; @@ -249,10 +290,10 @@ static OSStatus GetAGCState(AudioUnit audio_unit, UInt32* enabled) { result = AudioUnitInitialize(vpio_unit_); } if (result == noErr) { - RTCLog(@"Voice Processing I/O unit is now initialized."); + RTCLog(@"Audio unit is now initialized."); } - if (bypass_voice_processing_) { + if (bypass_voice_processing_ && enable_recording) { // Attempt to disable builtin voice processing. UInt32 toggle = 1; result = AudioUnitSetProperty(vpio_unit_, @@ -270,6 +311,12 @@ static OSStatus GetAGCState(AudioUnit audio_unit, UInt32* enabled) { return true; } + state_ = kInitialized; + + if(!enable_recording) { + return true; + } + // AGC should be enabled by default for Voice Processing I/O units but it is // checked below and enabled explicitly if needed. This scheme is used // to be absolutely sure that the AGC is enabled since we have seen cases @@ -327,7 +374,181 @@ static OSStatus GetAGCState(AudioUnit audio_unit, UInt32* enabled) { RTCLog(@"WebRTC.Audio.BuiltInAGCIsEnabled: %u", static_cast(agc_is_enabled)); - state_ = kInitialized; + return true; +} + +bool VoiceProcessingAudioUnit::CreatePlaybackAU() { + RTC_DCHECK_EQ(state_, kUninitialized); + RTC_DCHECK(!vpio_unit_); + + RTCLog(@"Creating Playback audio unit..."); + + // Create an audio component description to identify the Playback + // I/O audio unit. + AudioComponentDescription vpio_unit_description; + vpio_unit_description.componentType = kAudioUnitType_Output; + vpio_unit_description.componentSubType = kAudioUnitSubType_RemoteIO; + vpio_unit_description.componentManufacturer = kAudioUnitManufacturer_Apple; + vpio_unit_description.componentFlags = 0; + vpio_unit_description.componentFlagsMask = 0; + + // Obtain an audio unit instance given the description. + AudioComponent found_vpio_unit_ref = + AudioComponentFindNext(nullptr, &vpio_unit_description); + + // Create a Playback audio unit. + OSStatus result = noErr; + result = AudioComponentInstanceNew(found_vpio_unit_ref, &vpio_unit_); + if (result != noErr) { + vpio_unit_ = nullptr; + RTCLogError(@"AudioComponentInstanceNew failed. Error=%ld.", (long)result); + return false; + } + + return true; +} + +bool VoiceProcessingAudioUnit::CreateVoiceProcessingAU() { + RTC_DCHECK_EQ(state_, kUninitialized); + RTC_DCHECK(!vpio_unit_); + + RTCLog(@"Creating Voice Processing audio unit..."); + + // Create an audio component description to identify the Voice Processing + // I/O audio unit. + AudioComponentDescription vpio_unit_description; + vpio_unit_description.componentType = kAudioUnitType_Output; + vpio_unit_description.componentSubType = kAudioUnitSubType_VoiceProcessingIO; + vpio_unit_description.componentManufacturer = kAudioUnitManufacturer_Apple; + vpio_unit_description.componentFlags = 0; + vpio_unit_description.componentFlagsMask = 0; + + // Obtain an audio unit instance given the description. + AudioComponent found_vpio_unit_ref = + AudioComponentFindNext(nullptr, &vpio_unit_description); + + // Create a Voice Processing IO audio unit. + OSStatus result = noErr; + result = AudioComponentInstanceNew(found_vpio_unit_ref, &vpio_unit_); + if (result != noErr) { + vpio_unit_ = nullptr; + RTCLogError(@"AudioComponentInstanceNew failed. Error=%ld.", (long)result); + return false; + } + + return true; +} + +VoiceProcessingAudioUnit::State VoiceProcessingAudioUnit::GetState() const { + return state_; +} + +bool VoiceProcessingAudioUnit::StartPlayout() { + switch (state_) { + case kUninitialized: + break; + case kInitialized: + if(playout_enabled_ || recording_enabled_) { // already configured (recording always enables playout), just need to start + playout_enabled_ = true; + return Start() == noErr; + } + break; + case kStarted: + if(playout_enabled_ || recording_enabled_) { // already playing (recording always enables playout), nothing to do + playout_enabled_ = true; + return true; + } + break; + } + + RTC_DCHECK(!vpio_unit_ && state_ == kUninitialized); // other state is just not possible here... + DisposeAudioUnit(); // useless here, but just to be sure... + + if(!Initialize(sample_rate_, true, false)) { + return false; + } + + return Start() == noErr; +} + +bool VoiceProcessingAudioUnit::StopPlayout() { + if(recording_enabled_) { // playout always enabled if recording is enabled + playout_enabled_ = false; + return true; + } + + switch (state_) { + case kUninitialized: + break; + case kInitialized: + [[fallthrough]]; + case kStarted: + if(!playout_enabled_) // not playig, nothing to stop + return true; + break; + } + + DisposeAudioUnit(); + + return true; +} + +bool VoiceProcessingAudioUnit::StartRecording() { + if(!CanRecord()) { + RTCLogError(@"VoiceProcessingAudioUnit is not ready to start recording"); + return false; + } + + const bool playout_was_enabled = playout_enabled_; + + switch(state_) { + case kUninitialized: + break; + case kInitialized: + if(recording_enabled_) // already configured, just need to start + return Start() == noErr; + break; + case kStarted: + if(recording_enabled_) // already recording, nothing to do + return true; + break; + } + + DisposeAudioUnit(); + + if(!Initialize(sample_rate_, playout_was_enabled, true)) { + return false; + } + + return Start() == noErr; +} + +bool VoiceProcessingAudioUnit::StopRecording() { + const bool playout_was_enabled = playout_enabled_; + const bool was_started = (state_ == kStarted); + + switch(state_) { + case kUninitialized: + return true; // nothing to stop + case kInitialized: + [[fallthrough]]; + case kStarted: + if(!recording_enabled_) // not recording, nothing to stop + return true; + break; + } + + DisposeAudioUnit(); + + if(playout_was_enabled) { + // if playout was active it's required to recreate audio unit to restore playout + if(!Initialize(sample_rate_, true, false)) + return false; + + if(was_started) + return Start() == noErr; + } + return true; } @@ -362,20 +583,22 @@ static OSStatus GetAGCState(AudioUnit audio_unit, UInt32* enabled) { return true; } -bool VoiceProcessingAudioUnit::Uninitialize() { +void VoiceProcessingAudioUnit::UninitializeAudioUnit() { RTC_DCHECK_GE(state_, kUninitialized); RTCLog(@"Unintializing audio unit."); OSStatus result = AudioUnitUninitialize(vpio_unit_); if (result != noErr) { RTCLogError(@"Failed to uninitialize audio unit. Error=%ld", (long)result); - return false; } else { RTCLog(@"Uninitialized audio unit."); } state_ = kUninitialized; - return true; +} + +void VoiceProcessingAudioUnit::Uninitialize() { + DisposeAudioUnit(); } OSStatus VoiceProcessingAudioUnit::Render(AudioUnitRenderActionFlags* flags, @@ -462,25 +685,36 @@ static OSStatus GetAGCState(AudioUnit audio_unit, UInt32* enabled) { void VoiceProcessingAudioUnit::DisposeAudioUnit() { if (vpio_unit_) { + const bool recording_was_enabled = recording_enabled_; + switch (state_) { case kStarted: Stop(); [[fallthrough]]; case kInitialized: - Uninitialize(); + UninitializeAudioUnit(); break; case kUninitialized: - case kInitRequired: break; } - RTCLog(@"Disposing audio unit."); + if(recording_was_enabled) + RTCLog(@"Disposing Voice Processing audio unit."); + else + RTCLog(@"Disposing Playback audio unit."); + OSStatus result = AudioComponentInstanceDispose(vpio_unit_); if (result != noErr) { RTCLogError(@"AudioComponentInstanceDispose failed. Error=%ld.", (long)result); } vpio_unit_ = nullptr; + + playout_enabled_ = false; + recording_enabled_ = false; + + if(recording_was_enabled) + mic_owning_.store(false, std::memory_order_relaxed); } } diff --git a/sdk/objc/unittests/RTCAudioTrack_xctest.mm b/sdk/objc/unittests/RTCAudioTrack_xctest.mm new file mode 100644 index 0000000000..52c4766452 --- /dev/null +++ b/sdk/objc/unittests/RTCAudioTrack_xctest.mm @@ -0,0 +1,258 @@ +/* + * Copyright 2026 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import +#import + +#include +#include +#include + +#include "absl/types/optional.h" +#include "api/media_stream_interface.h" +#include "api/scoped_refptr.h" +#include "rtc_base/gunit.h" +#include "rtc_base/ref_counted_object.h" + +#import "api/peerconnection/RTCAudioTrack.h" +#import "api/peerconnection/RTCMediaStreamTrack+Private.h" +#import "api/peerconnection/RTCPeerConnectionFactory.h" + +namespace { + +class FakeAudioTrack : public webrtc::AudioTrackInterface { + public: + explicit FakeAudioTrack(std::string track_id) + : track_id_(std::move(track_id)), enabled_(true), state_(kLive) {} + + std::string kind() const override { return kAudioKind; } + std::string id() const override { return track_id_; } + + bool enabled() const override { return enabled_; } + bool set_enabled(bool enable) override { + enabled_ = enable; + return true; + } + + TrackState state() const override { return state_; } + + webrtc::AudioSourceInterface* GetSource() const override { return nullptr; } + + void AddSink(webrtc::AudioTrackSinkInterface* sink) override { + ++add_sink_call_count_; + sinks_.push_back(sink); + } + + void RemoveSink(webrtc::AudioTrackSinkInterface* sink) override { + ++remove_sink_call_count_; + sinks_.erase(std::remove(sinks_.begin(), sinks_.end(), sink), sinks_.end()); + } + + bool GetSignalLevel(int* level) override { + if (level) { + *level = 0; + } + return true; + } + + rtc::scoped_refptr GetAudioProcessor() override { + return nullptr; + } + + void RegisterObserver(webrtc::ObserverInterface* observer) override {} + void UnregisterObserver(webrtc::ObserverInterface* observer) override {} + + size_t sink_count() const { return sinks_.size(); } + size_t add_sink_call_count() const { return add_sink_call_count_; } + size_t remove_sink_call_count() const { return remove_sink_call_count_; } + + void DeliverData(const void* audio_data, + int bits_per_sample, + int sample_rate, + size_t number_of_channels, + size_t number_of_frames, + absl::optional absolute_capture_timestamp_ms) { + auto sinks_snapshot = sinks_; + for (webrtc::AudioTrackSinkInterface* sink : sinks_snapshot) { + if (absolute_capture_timestamp_ms.has_value()) { + sink->OnData(audio_data, bits_per_sample, sample_rate, number_of_channels, + number_of_frames, absolute_capture_timestamp_ms); + } else { + sink->OnData(audio_data, bits_per_sample, sample_rate, number_of_channels, + number_of_frames); + } + } + } + + protected: + ~FakeAudioTrack() override = default; + + private: + std::string track_id_; + bool enabled_; + TrackState state_; + size_t add_sink_call_count_ = 0; + size_t remove_sink_call_count_ = 0; + std::vector sinks_; +}; + +} // namespace + +@interface RTCAudioTrackTestRenderer : NSObject +@property(nonatomic, assign) NSInteger callbackCount; +@property(nonatomic, assign) const void *lastAudioData; +@property(nonatomic, assign) int lastBitsPerSample; +@property(nonatomic, assign) int lastSampleRate; +@property(nonatomic, assign) size_t lastNumberOfChannels; +@property(nonatomic, assign) size_t lastNumberOfFrames; +@property(nonatomic, assign) int64_t lastAbsoluteCaptureTimestampMs; +@property(nonatomic, assign) BOOL lastTimestampIsValid; +@end + +@implementation RTCAudioTrackTestRenderer + +- (void)renderPCMData:(const void *)audio_data + bitsPerSample:(int)bits_per_sample + sampleRate:(int)sample_rate + numberOfChannels:(size_t)number_of_channels + numberOfFrames:(size_t)number_of_frames +absoluteCaptureTimestampMs:(int64_t)absolute_capture_timestamp_ms + timestampIsValid:(BOOL)timestamp_is_valid { + self.callbackCount += 1; + self.lastAudioData = audio_data; + self.lastBitsPerSample = bits_per_sample; + self.lastSampleRate = sample_rate; + self.lastNumberOfChannels = number_of_channels; + self.lastNumberOfFrames = number_of_frames; + self.lastAbsoluteCaptureTimestampMs = absolute_capture_timestamp_ms; + self.lastTimestampIsValid = timestamp_is_valid; +} + +@end + +@interface RTCAudioTrackTests : XCTestCase +@end + +@implementation RTCAudioTrackTests + +- (RTC_OBJC_TYPE(RTCAudioTrack) *)audioTrackWithFactory:(RTC_OBJC_TYPE(RTCPeerConnectionFactory) *)factory + nativeTrack:(rtc::scoped_refptr)nativeTrack { + rtc::scoped_refptr mediaTrack = nativeTrack; + RTC_OBJC_TYPE(RTCMediaStreamTrack) *track = [RTC_OBJC_TYPE(RTCMediaStreamTrack) + mediaTrackForNativeTrack:mediaTrack + factory:factory]; + EXPECT_TRUE([track isKindOfClass:[RTC_OBJC_TYPE(RTCAudioTrack) class]]); + return (RTC_OBJC_TYPE(RTCAudioTrack) *)track; +} + +- (void)testAddAndRemoveRendererAreIdempotent { + RTC_OBJC_TYPE(RTCPeerConnectionFactory) *factory = + [[RTC_OBJC_TYPE(RTCPeerConnectionFactory) alloc] init]; + rtc::scoped_refptr nativeTrack = + rtc::make_ref_counted("audio-track-idempotent"); + RTC_OBJC_TYPE(RTCAudioTrack) *track = [self audioTrackWithFactory:factory nativeTrack:nativeTrack]; + RTCAudioTrackTestRenderer *renderer = [[RTCAudioTrackTestRenderer alloc] init]; + + [track addRenderer:renderer]; + EXPECT_EQ(nativeTrack->add_sink_call_count(), 1u); + EXPECT_EQ(nativeTrack->sink_count(), 1u); + + [track addRenderer:renderer]; + EXPECT_EQ(nativeTrack->add_sink_call_count(), 1u); + EXPECT_EQ(nativeTrack->sink_count(), 1u); + + [track removeRenderer:renderer]; + EXPECT_EQ(nativeTrack->remove_sink_call_count(), 1u); + EXPECT_EQ(nativeTrack->sink_count(), 0u); + + [track removeRenderer:renderer]; + EXPECT_EQ(nativeTrack->remove_sink_call_count(), 1u); + EXPECT_EQ(nativeTrack->sink_count(), 0u); +} + +- (void)testCallbackDeliveryForwardsAllMetadata { + RTC_OBJC_TYPE(RTCPeerConnectionFactory) *factory = + [[RTC_OBJC_TYPE(RTCPeerConnectionFactory) alloc] init]; + rtc::scoped_refptr nativeTrack = + rtc::make_ref_counted("audio-track-callback"); + RTC_OBJC_TYPE(RTCAudioTrack) *track = [self audioTrackWithFactory:factory nativeTrack:nativeTrack]; + RTCAudioTrackTestRenderer *renderer = [[RTCAudioTrackTestRenderer alloc] init]; + + [track addRenderer:renderer]; + + int16_t samples[4] = {1, 2, 3, 4}; + nativeTrack->DeliverData(samples, 16, 48000, 2, 2, 1234); + + EXPECT_EQ(renderer.callbackCount, 1); + EXPECT_EQ(renderer.lastAudioData, samples); + EXPECT_EQ(renderer.lastBitsPerSample, 16); + EXPECT_EQ(renderer.lastSampleRate, 48000); + EXPECT_EQ(renderer.lastNumberOfChannels, 2u); + EXPECT_EQ(renderer.lastNumberOfFrames, 2u); + EXPECT_TRUE(renderer.lastTimestampIsValid); + EXPECT_EQ(renderer.lastAbsoluteCaptureTimestampMs, 1234); + + nativeTrack->DeliverData(samples, 16, 48000, 2, 2, absl::nullopt); + + EXPECT_EQ(renderer.callbackCount, 2); + EXPECT_FALSE(renderer.lastTimestampIsValid); +} + +- (void)testMultipleRenderersReceiveCallbacks { + RTC_OBJC_TYPE(RTCPeerConnectionFactory) *factory = + [[RTC_OBJC_TYPE(RTCPeerConnectionFactory) alloc] init]; + rtc::scoped_refptr nativeTrack = + rtc::make_ref_counted("audio-track-multi-renderer"); + RTC_OBJC_TYPE(RTCAudioTrack) *track = [self audioTrackWithFactory:factory nativeTrack:nativeTrack]; + RTCAudioTrackTestRenderer *rendererOne = [[RTCAudioTrackTestRenderer alloc] init]; + RTCAudioTrackTestRenderer *rendererTwo = [[RTCAudioTrackTestRenderer alloc] init]; + + [track addRenderer:rendererOne]; + [track addRenderer:rendererTwo]; + + EXPECT_EQ(nativeTrack->sink_count(), 2u); + + int16_t samples[4] = {1, 2, 3, 4}; + nativeTrack->DeliverData(samples, 16, 48000, 2, 2, 2000); + + EXPECT_EQ(rendererOne.callbackCount, 1); + EXPECT_EQ(rendererTwo.callbackCount, 1); + + [track removeRenderer:rendererOne]; + EXPECT_EQ(nativeTrack->sink_count(), 1u); + + nativeTrack->DeliverData(samples, 16, 48000, 2, 2, 2500); + + EXPECT_EQ(rendererOne.callbackCount, 1); + EXPECT_EQ(rendererTwo.callbackCount, 2); +} + +- (void)testTrackDeallocRemovesAllSinks { + rtc::scoped_refptr nativeTrack = + rtc::make_ref_counted("audio-track-dealloc"); + + @autoreleasepool { + RTC_OBJC_TYPE(RTCPeerConnectionFactory) *factory = + [[RTC_OBJC_TYPE(RTCPeerConnectionFactory) alloc] init]; + RTC_OBJC_TYPE(RTCAudioTrack) *track = [self audioTrackWithFactory:factory nativeTrack:nativeTrack]; + RTCAudioTrackTestRenderer *rendererOne = [[RTCAudioTrackTestRenderer alloc] init]; + RTCAudioTrackTestRenderer *rendererTwo = [[RTCAudioTrackTestRenderer alloc] init]; + + [track addRenderer:rendererOne]; + [track addRenderer:rendererTwo]; + + EXPECT_EQ(nativeTrack->sink_count(), 2u); + } + + EXPECT_EQ(nativeTrack->remove_sink_call_count(), 2u); + EXPECT_EQ(nativeTrack->sink_count(), 0u); +} + +@end