/* * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. * * Use of this source code is governed by a BSD-style license * that can be found in the LICENSE file in the root of the source * tree. An additional intellectual property rights grant can be found * in the file PATENTS. All contributing project authors may * be found in the AUTHORS file in the root of the source tree. */ #include "audio/audio_send_stream.h" #include #include #include #include #include "api/audio_codecs/audio_encoder.h" #include "api/audio_codecs/audio_encoder_factory.h" #include "api/audio_codecs/audio_format.h" #include "api/call/transport.h" #include "api/crypto/frame_encryptor_interface.h" #include "api/function_view.h" #include "api/rtc_event_log/rtc_event_log.h" #include "audio/audio_state.h" #include "audio/channel_send.h" #include "audio/conversion.h" #include "call/rtp_config.h" #include "call/rtp_transport_controller_send_interface.h" #include "common_audio/vad/include/vad.h" #include "logging/rtc_event_log/events/rtc_event_audio_send_stream_config.h" #include "logging/rtc_event_log/rtc_stream_config.h" #include "modules/audio_coding/codecs/cng/audio_encoder_cng.h" #include "modules/audio_coding/codecs/red/audio_encoder_copy_red.h" #include "modules/audio_processing/include/audio_processing.h" #include "modules/rtp_rtcp/source/rtp_header_extensions.h" #include "rtc_base/checks.h" #include "rtc_base/event.h" #include "rtc_base/logging.h" #include "rtc_base/strings/audio_format_to_string.h" #include "rtc_base/task_queue.h" #include "system_wrappers/include/field_trial.h" namespace webrtc { namespace { void UpdateEventLogStreamConfig(RtcEventLog* event_log, const AudioSendStream::Config& config, const AudioSendStream::Config* old_config) { using SendCodecSpec = AudioSendStream::Config::SendCodecSpec; // Only update if any of the things we log have changed. auto payload_types_equal = [](const absl::optional& a, const absl::optional& b) { if (a.has_value() && b.has_value()) { return a->format.name == b->format.name && a->payload_type == b->payload_type; } return !a.has_value() && !b.has_value(); }; if (old_config && config.rtp.ssrc == old_config->rtp.ssrc && config.rtp.extensions == old_config->rtp.extensions && payload_types_equal(config.send_codec_spec, old_config->send_codec_spec)) { return; } auto rtclog_config = std::make_unique(); rtclog_config->local_ssrc = config.rtp.ssrc; rtclog_config->rtp_extensions = config.rtp.extensions; if (config.send_codec_spec) { rtclog_config->codecs.emplace_back(config.send_codec_spec->format.name, config.send_codec_spec->payload_type, 0); } event_log->Log(std::make_unique( std::move(rtclog_config))); } } // namespace constexpr char AudioAllocationConfig::kKey[]; std::unique_ptr AudioAllocationConfig::Parser() { return StructParametersParser::Create( // "min", &min_bitrate, // "max", &max_bitrate, // "prio_rate", &priority_bitrate, // "prio_rate_raw", &priority_bitrate_raw, // "rate_prio", &bitrate_priority); } AudioAllocationConfig::AudioAllocationConfig() { Parser()->Parse(field_trial::FindFullName(kKey)); if (priority_bitrate_raw && !priority_bitrate.IsZero()) { RTC_LOG(LS_WARNING) << "'priority_bitrate' and '_raw' are mutually " "exclusive but both were configured."; } } namespace internal { AudioSendStream::AudioSendStream( Clock* clock, const webrtc::AudioSendStream::Config& config, const rtc::scoped_refptr& audio_state, TaskQueueFactory* task_queue_factory, ProcessThread* module_process_thread, RtpTransportControllerSendInterface* rtp_transport, BitrateAllocatorInterface* bitrate_allocator, RtcEventLog* event_log, RtcpRttStats* rtcp_rtt_stats, const absl::optional& suspended_rtp_state) : AudioSendStream(clock, config, audio_state, task_queue_factory, rtp_transport, bitrate_allocator, event_log, suspended_rtp_state, voe::CreateChannelSend( clock, task_queue_factory, module_process_thread, config.send_transport, rtcp_rtt_stats, event_log, config.frame_encryptor, config.crypto_options, config.rtp.extmap_allow_mixed, config.rtcp_report_interval_ms, config.rtp.ssrc, config.frame_transformer, rtp_transport->transport_feedback_observer())) {} AudioSendStream::AudioSendStream( Clock* clock, const webrtc::AudioSendStream::Config& config, const rtc::scoped_refptr& audio_state, TaskQueueFactory* task_queue_factory, RtpTransportControllerSendInterface* rtp_transport, BitrateAllocatorInterface* bitrate_allocator, RtcEventLog* event_log, const absl::optional& suspended_rtp_state, std::unique_ptr channel_send) : clock_(clock), worker_queue_(rtp_transport->GetWorkerQueue()), allocate_audio_without_feedback_( field_trial::IsEnabled("WebRTC-Audio-ABWENoTWCC")), enable_audio_alr_probing_( !field_trial::IsDisabled("WebRTC-Audio-AlrProbing")), send_side_bwe_with_overhead_( !field_trial::IsDisabled("WebRTC-SendSideBwe-WithOverhead")), config_(Config(/*send_transport=*/nullptr)), audio_state_(audio_state), channel_send_(std::move(channel_send)), event_log_(event_log), use_legacy_overhead_calculation_( field_trial::IsEnabled("WebRTC-Audio-LegacyOverhead")), bitrate_allocator_(bitrate_allocator), rtp_transport_(rtp_transport), rtp_rtcp_module_(channel_send_->GetRtpRtcp()), suspended_rtp_state_(suspended_rtp_state) { RTC_LOG(LS_INFO) << "AudioSendStream: " << config.rtp.ssrc; RTC_DCHECK(worker_queue_); RTC_DCHECK(audio_state_); RTC_DCHECK(channel_send_); RTC_DCHECK(bitrate_allocator_); RTC_DCHECK(rtp_transport); RTC_DCHECK(rtp_rtcp_module_); RTC_DCHECK_RUN_ON(&worker_thread_checker_); ConfigureStream(config, true); UpdateCachedTargetAudioBitrateConstraints(); pacer_thread_checker_.Detach(); } AudioSendStream::~AudioSendStream() { RTC_DCHECK_RUN_ON(&worker_thread_checker_); RTC_LOG(LS_INFO) << "~AudioSendStream: " << config_.rtp.ssrc; RTC_DCHECK(!sending_); channel_send_->ResetSenderCongestionControlObjects(); // Blocking call to synchronize state with worker queue to ensure that there // are no pending tasks left that keeps references to audio. rtc::Event thread_sync_event; worker_queue_->PostTask([&] { thread_sync_event.Set(); }); thread_sync_event.Wait(rtc::Event::kForever); } const webrtc::AudioSendStream::Config& AudioSendStream::GetConfig() const { RTC_DCHECK_RUN_ON(&worker_thread_checker_); return config_; } void AudioSendStream::Reconfigure( const webrtc::AudioSendStream::Config& new_config) { RTC_DCHECK_RUN_ON(&worker_thread_checker_); ConfigureStream(new_config, false); } AudioSendStream::ExtensionIds AudioSendStream::FindExtensionIds( const std::vector& extensions) { ExtensionIds ids; for (const auto& extension : extensions) { if (extension.uri == RtpExtension::kAudioLevelUri) { ids.audio_level = extension.id; } else if (extension.uri == RtpExtension::kAbsSendTimeUri) { ids.abs_send_time = extension.id; } else if (extension.uri == RtpExtension::kTransportSequenceNumberUri) { ids.transport_sequence_number = extension.id; } else if (extension.uri == RtpExtension::kMidUri) { ids.mid = extension.id; } else if (extension.uri == RtpExtension::kRidUri) { ids.rid = extension.id; } else if (extension.uri == RtpExtension::kRepairedRidUri) { ids.repaired_rid = extension.id; } else if (extension.uri == RtpExtension::kAbsoluteCaptureTimeUri) { ids.abs_capture_time = extension.id; } } return ids; } int AudioSendStream::TransportSeqNumId(const AudioSendStream::Config& config) { return FindExtensionIds(config.rtp.extensions).transport_sequence_number; } void AudioSendStream::ConfigureStream( const webrtc::AudioSendStream::Config& new_config, bool first_time) { RTC_LOG(LS_INFO) << "AudioSendStream::ConfigureStream: " << new_config.ToString(); UpdateEventLogStreamConfig(event_log_, new_config, first_time ? nullptr : &config_); const auto& old_config = config_; // Configuration parameters which cannot be changed. RTC_DCHECK(first_time || old_config.send_transport == new_config.send_transport); RTC_DCHECK(first_time || old_config.rtp.ssrc == new_config.rtp.ssrc); if (suspended_rtp_state_ && first_time) { rtp_rtcp_module_->SetRtpState(*suspended_rtp_state_); } if (first_time || old_config.rtp.c_name != new_config.rtp.c_name) { channel_send_->SetRTCP_CNAME(new_config.rtp.c_name); } // Enable the frame encryptor if a new frame encryptor has been provided. if (first_time || new_config.frame_encryptor != old_config.frame_encryptor) { channel_send_->SetFrameEncryptor(new_config.frame_encryptor); } if (first_time || new_config.frame_transformer != old_config.frame_transformer) { channel_send_->SetEncoderToPacketizerFrameTransformer( new_config.frame_transformer); } if (first_time || new_config.rtp.extmap_allow_mixed != old_config.rtp.extmap_allow_mixed) { rtp_rtcp_module_->SetExtmapAllowMixed(new_config.rtp.extmap_allow_mixed); } const ExtensionIds old_ids = FindExtensionIds(old_config.rtp.extensions); const ExtensionIds new_ids = FindExtensionIds(new_config.rtp.extensions); // Audio level indication if (first_time || new_ids.audio_level != old_ids.audio_level) { channel_send_->SetSendAudioLevelIndicationStatus(new_ids.audio_level != 0, new_ids.audio_level); } if (first_time || new_ids.abs_send_time != old_ids.abs_send_time) { rtp_rtcp_module_->DeregisterSendRtpHeaderExtension( kRtpExtensionAbsoluteSendTime); if (new_ids.abs_send_time) { rtp_rtcp_module_->RegisterRtpHeaderExtension(AbsoluteSendTime::kUri, new_ids.abs_send_time); } } bool transport_seq_num_id_changed = new_ids.transport_sequence_number != old_ids.transport_sequence_number; if (first_time || (transport_seq_num_id_changed && !allocate_audio_without_feedback_)) { if (!first_time) { channel_send_->ResetSenderCongestionControlObjects(); } RtcpBandwidthObserver* bandwidth_observer = nullptr; if (!allocate_audio_without_feedback_ && new_ids.transport_sequence_number != 0) { rtp_rtcp_module_->RegisterRtpHeaderExtension( TransportSequenceNumber::kUri, new_ids.transport_sequence_number); // Probing in application limited region is only used in combination with // send side congestion control, wich depends on feedback packets which // requires transport sequence numbers to be enabled. // Optionally request ALR probing but do not override any existing // request from other streams. if (enable_audio_alr_probing_) { rtp_transport_->EnablePeriodicAlrProbing(true); } bandwidth_observer = rtp_transport_->GetBandwidthObserver(); } channel_send_->RegisterSenderCongestionControlObjects(rtp_transport_, bandwidth_observer); } // MID RTP header extension. if ((first_time || new_ids.mid != old_ids.mid || new_config.rtp.mid != old_config.rtp.mid) && new_ids.mid != 0 && !new_config.rtp.mid.empty()) { rtp_rtcp_module_->RegisterRtpHeaderExtension(RtpMid::kUri, new_ids.mid); rtp_rtcp_module_->SetMid(new_config.rtp.mid); } // RID RTP header extension if ((first_time || new_ids.rid != old_ids.rid || new_ids.repaired_rid != old_ids.repaired_rid || new_config.rtp.rid != old_config.rtp.rid)) { if (new_ids.rid != 0 || new_ids.repaired_rid != 0) { if (new_config.rtp.rid.empty()) { rtp_rtcp_module_->DeregisterSendRtpHeaderExtension(RtpStreamId::kUri); } else if (new_ids.repaired_rid != 0) { rtp_rtcp_module_->RegisterRtpHeaderExtension(RtpStreamId::kUri, new_ids.repaired_rid); } else { rtp_rtcp_module_->RegisterRtpHeaderExtension(RtpStreamId::kUri, new_ids.rid); } } rtp_rtcp_module_->SetRid(new_config.rtp.rid); } if (first_time || new_ids.abs_capture_time != old_ids.abs_capture_time) { rtp_rtcp_module_->DeregisterSendRtpHeaderExtension( kRtpExtensionAbsoluteCaptureTime); if (new_ids.abs_capture_time) { rtp_rtcp_module_->RegisterRtpHeaderExtension( AbsoluteCaptureTimeExtension::kUri, new_ids.abs_capture_time); } } if (!ReconfigureSendCodec(new_config)) { RTC_LOG(LS_ERROR) << "Failed to set up send codec state."; } // Set currently known overhead (used in ANA, opus only). { MutexLock lock(&overhead_per_packet_lock_); UpdateOverheadForEncoder(); } channel_send_->CallEncoder([this](AudioEncoder* encoder) { RTC_DCHECK_RUN_ON(&worker_thread_checker_); if (!encoder) { return; } frame_length_range_ = encoder->GetFrameLengthRange(); UpdateCachedTargetAudioBitrateConstraints(); }); if (sending_) { ReconfigureBitrateObserver(new_config); } config_ = new_config; if (!first_time) { UpdateCachedTargetAudioBitrateConstraints(); } } void AudioSendStream::Start() { RTC_DCHECK_RUN_ON(&worker_thread_checker_); if (sending_) { return; } if (!config_.has_dscp && config_.min_bitrate_bps != -1 && config_.max_bitrate_bps != -1 && (allocate_audio_without_feedback_ || TransportSeqNumId(config_) != 0)) { rtp_transport_->AccountForAudioPacketsInPacedSender(true); if (send_side_bwe_with_overhead_) rtp_transport_->IncludeOverheadInPacedSender(); rtp_rtcp_module_->SetAsPartOfAllocation(true); ConfigureBitrateObserver(); } else { rtp_rtcp_module_->SetAsPartOfAllocation(false); } channel_send_->StartSend(); sending_ = true; audio_state()->AddSendingStream(this, encoder_sample_rate_hz_, encoder_num_channels_); } void AudioSendStream::Stop() { RTC_DCHECK_RUN_ON(&worker_thread_checker_); if (!sending_) { return; } RemoveBitrateObserver(); channel_send_->StopSend(); sending_ = false; audio_state()->RemoveSendingStream(this); } void AudioSendStream::SendAudioData(std::unique_ptr audio_frame) { RTC_CHECK_RUNS_SERIALIZED(&audio_capture_race_checker_); RTC_DCHECK_GT(audio_frame->sample_rate_hz_, 0); double duration = static_cast(audio_frame->samples_per_channel_) / audio_frame->sample_rate_hz_; { // Note: SendAudioData() passes the frame further down the pipeline and it // may eventually get sent. But this method is invoked even if we are not // connected, as long as we have an AudioSendStream (created as a result of // an O/A exchange). This means that we are calculating audio levels whether // or not we are sending samples. // TODO(https://crbug.com/webrtc/10771): All "media-source" related stats // should move from send-streams to the local audio sources or tracks; a // send-stream should not be required to read the microphone audio levels. MutexLock lock(&audio_level_lock_); audio_level_.ComputeLevel(*audio_frame, duration); } channel_send_->ProcessAndEncodeAudio(std::move(audio_frame)); } bool AudioSendStream::SendTelephoneEvent(int payload_type, int payload_frequency, int event, int duration_ms) { RTC_DCHECK_RUN_ON(&worker_thread_checker_); channel_send_->SetSendTelephoneEventPayloadType(payload_type, payload_frequency); return channel_send_->SendTelephoneEventOutband(event, duration_ms); } void AudioSendStream::SetMuted(bool muted) { RTC_DCHECK_RUN_ON(&worker_thread_checker_); channel_send_->SetInputMute(muted); } webrtc::AudioSendStream::Stats AudioSendStream::GetStats() const { return GetStats(true); } webrtc::AudioSendStream::Stats AudioSendStream::GetStats( bool has_remote_tracks) const { RTC_DCHECK_RUN_ON(&worker_thread_checker_); webrtc::AudioSendStream::Stats stats; stats.local_ssrc = config_.rtp.ssrc; stats.target_bitrate_bps = channel_send_->GetBitrate(); webrtc::CallSendStatistics call_stats = channel_send_->GetRTCPStatistics(); stats.payload_bytes_sent = call_stats.payload_bytes_sent; stats.header_and_padding_bytes_sent = call_stats.header_and_padding_bytes_sent; stats.retransmitted_bytes_sent = call_stats.retransmitted_bytes_sent; stats.packets_sent = call_stats.packetsSent; stats.retransmitted_packets_sent = call_stats.retransmitted_packets_sent; // RTT isn't known until a RTCP report is received. Until then, VoiceEngine // returns 0 to indicate an error value. if (call_stats.rttMs > 0) { stats.rtt_ms = call_stats.rttMs; } if (config_.send_codec_spec) { const auto& spec = *config_.send_codec_spec; stats.codec_name = spec.format.name; stats.codec_payload_type = spec.payload_type; // Get data from the last remote RTCP report. for (const auto& block : channel_send_->GetRemoteRTCPReportBlocks()) { // Lookup report for send ssrc only. if (block.source_SSRC == stats.local_ssrc) { stats.packets_lost = block.cumulative_num_packets_lost; stats.fraction_lost = Q8ToFloat(block.fraction_lost); // Convert timestamps to milliseconds. if (spec.format.clockrate_hz / 1000 > 0) { stats.jitter_ms = block.interarrival_jitter / (spec.format.clockrate_hz / 1000); } break; } } } { MutexLock lock(&audio_level_lock_); stats.audio_level = audio_level_.LevelFullRange(); stats.total_input_energy = audio_level_.TotalEnergy(); stats.total_input_duration = audio_level_.TotalDuration(); } stats.typing_noise_detected = audio_state()->typing_noise_detected(); stats.ana_statistics = channel_send_->GetANAStatistics(); AudioProcessing* ap = audio_state_->audio_processing(); if (ap) { stats.apm_statistics = ap->GetStatistics(has_remote_tracks); } stats.report_block_datas = std::move(call_stats.report_block_datas); return stats; } void AudioSendStream::DeliverRtcp(const uint8_t* packet, size_t length) { RTC_DCHECK_RUN_ON(&worker_thread_checker_); channel_send_->ReceivedRTCPPacket(packet, length); { // Poll if overhead has changed, which it can do if ack triggers us to stop // sending mid/rid. MutexLock lock(&overhead_per_packet_lock_); UpdateOverheadForEncoder(); } UpdateCachedTargetAudioBitrateConstraints(); } uint32_t AudioSendStream::OnBitrateUpdated(BitrateAllocationUpdate update) { RTC_DCHECK_RUN_ON(worker_queue_); // Pick a target bitrate between the constraints. Overrules the allocator if // it 1) allocated a bitrate of zero to disable the stream or 2) allocated a // higher than max to allow for e.g. extra FEC. RTC_DCHECK(cached_constraints_.has_value()); update.target_bitrate.Clamp(cached_constraints_->min, cached_constraints_->max); update.stable_target_bitrate.Clamp(cached_constraints_->min, cached_constraints_->max); channel_send_->OnBitrateAllocation(update); // The amount of audio protection is not exposed by the encoder, hence // always returning 0. return 0; } void AudioSendStream::SetTransportOverhead( int transport_overhead_per_packet_bytes) { RTC_DCHECK_RUN_ON(&worker_thread_checker_); { MutexLock lock(&overhead_per_packet_lock_); transport_overhead_per_packet_bytes_ = transport_overhead_per_packet_bytes; UpdateOverheadForEncoder(); } UpdateCachedTargetAudioBitrateConstraints(); } void AudioSendStream::UpdateOverheadForEncoder() { RTC_DCHECK_RUN_ON(&worker_thread_checker_); size_t overhead_per_packet_bytes = GetPerPacketOverheadBytes(); if (overhead_per_packet_ == overhead_per_packet_bytes) { return; } overhead_per_packet_ = overhead_per_packet_bytes; channel_send_->CallEncoder([&](AudioEncoder* encoder) { encoder->OnReceivedOverhead(overhead_per_packet_bytes); }); if (total_packet_overhead_bytes_ != overhead_per_packet_bytes) { total_packet_overhead_bytes_ = overhead_per_packet_bytes; if (registered_with_allocator_) { ConfigureBitrateObserver(); } } } size_t AudioSendStream::TestOnlyGetPerPacketOverheadBytes() const { MutexLock lock(&overhead_per_packet_lock_); return GetPerPacketOverheadBytes(); } size_t AudioSendStream::GetPerPacketOverheadBytes() const { return transport_overhead_per_packet_bytes_ + rtp_rtcp_module_->ExpectedPerPacketOverhead(); } RtpState AudioSendStream::GetRtpState() const { return rtp_rtcp_module_->GetRtpState(); } const voe::ChannelSendInterface* AudioSendStream::GetChannel() const { return channel_send_.get(); } internal::AudioState* AudioSendStream::audio_state() { internal::AudioState* audio_state = static_cast(audio_state_.get()); RTC_DCHECK(audio_state); return audio_state; } const internal::AudioState* AudioSendStream::audio_state() const { internal::AudioState* audio_state = static_cast(audio_state_.get()); RTC_DCHECK(audio_state); return audio_state; } void AudioSendStream::StoreEncoderProperties(int sample_rate_hz, size_t num_channels) { encoder_sample_rate_hz_ = sample_rate_hz; encoder_num_channels_ = num_channels; if (sending_) { // Update AudioState's information about the stream. audio_state()->AddSendingStream(this, sample_rate_hz, num_channels); } } // Apply current codec settings to a single voe::Channel used for sending. bool AudioSendStream::SetupSendCodec(const Config& new_config) { RTC_DCHECK(new_config.send_codec_spec); const auto& spec = *new_config.send_codec_spec; RTC_DCHECK(new_config.encoder_factory); std::unique_ptr encoder = new_config.encoder_factory->MakeAudioEncoder( spec.payload_type, spec.format, new_config.codec_pair_id); if (!encoder) { RTC_DLOG(LS_ERROR) << "Unable to create encoder for " << rtc::ToString(spec.format); return false; } // If a bitrate has been specified for the codec, use it over the // codec's default. if (spec.target_bitrate_bps) { encoder->OnReceivedTargetAudioBitrate(*spec.target_bitrate_bps); } // Enable ANA if configured (currently only used by Opus). if (new_config.audio_network_adaptor_config) { if (encoder->EnableAudioNetworkAdaptor( *new_config.audio_network_adaptor_config, event_log_)) { RTC_LOG(LS_INFO) << "Audio network adaptor enabled on SSRC " << new_config.rtp.ssrc; } else { RTC_LOG(LS_INFO) << "Failed to enable Audio network adaptor on SSRC " << new_config.rtp.ssrc; } } // Wrap the encoder in an AudioEncoderCNG, if VAD is enabled. if (spec.cng_payload_type) { AudioEncoderCngConfig cng_config; cng_config.num_channels = encoder->NumChannels(); cng_config.payload_type = *spec.cng_payload_type; cng_config.speech_encoder = std::move(encoder); cng_config.vad_mode = Vad::kVadNormal; encoder = CreateComfortNoiseEncoder(std::move(cng_config)); RegisterCngPayloadType(*spec.cng_payload_type, new_config.send_codec_spec->format.clockrate_hz); } // Wrap the encoder in a RED encoder, if RED is enabled. if (spec.red_payload_type) { AudioEncoderCopyRed::Config red_config; red_config.payload_type = *spec.red_payload_type; red_config.speech_encoder = std::move(encoder); encoder = std::make_unique(std::move(red_config)); } // Set currently known overhead (used in ANA, opus only). // If overhead changes later, it will be updated in UpdateOverheadForEncoder. { MutexLock lock(&overhead_per_packet_lock_); size_t overhead = GetPerPacketOverheadBytes(); if (overhead > 0) { encoder->OnReceivedOverhead(overhead); } } StoreEncoderProperties(encoder->SampleRateHz(), encoder->NumChannels()); channel_send_->SetEncoder(new_config.send_codec_spec->payload_type, std::move(encoder)); return true; } bool AudioSendStream::ReconfigureSendCodec(const Config& new_config) { const auto& old_config = config_; if (!new_config.send_codec_spec) { // We cannot de-configure a send codec. So we will do nothing. // By design, the send codec should have not been configured. RTC_DCHECK(!old_config.send_codec_spec); return true; } if (new_config.send_codec_spec == old_config.send_codec_spec && new_config.audio_network_adaptor_config == old_config.audio_network_adaptor_config) { return true; } // If we have no encoder, or the format or payload type's changed, create a // new encoder. if (!old_config.send_codec_spec || new_config.send_codec_spec->format != old_config.send_codec_spec->format || new_config.send_codec_spec->payload_type != old_config.send_codec_spec->payload_type) { return SetupSendCodec(new_config); } const absl::optional& new_target_bitrate_bps = new_config.send_codec_spec->target_bitrate_bps; // If a bitrate has been specified for the codec, use it over the // codec's default. if (new_target_bitrate_bps && new_target_bitrate_bps != old_config.send_codec_spec->target_bitrate_bps) { channel_send_->CallEncoder([&](AudioEncoder* encoder) { encoder->OnReceivedTargetAudioBitrate(*new_target_bitrate_bps); }); } ReconfigureANA(new_config); ReconfigureCNG(new_config); return true; } void AudioSendStream::ReconfigureANA(const Config& new_config) { if (new_config.audio_network_adaptor_config == config_.audio_network_adaptor_config) { return; } if (new_config.audio_network_adaptor_config) { // This lock needs to be acquired before CallEncoder, since it aquires // another lock and we need to maintain the same order at all call sites to // avoid deadlock. MutexLock lock(&overhead_per_packet_lock_); size_t overhead = GetPerPacketOverheadBytes(); channel_send_->CallEncoder([&](AudioEncoder* encoder) { if (encoder->EnableAudioNetworkAdaptor( *new_config.audio_network_adaptor_config, event_log_)) { RTC_LOG(LS_INFO) << "Audio network adaptor enabled on SSRC " << new_config.rtp.ssrc; if (overhead > 0) { encoder->OnReceivedOverhead(overhead); } } else { RTC_LOG(LS_INFO) << "Failed to enable Audio network adaptor on SSRC " << new_config.rtp.ssrc; } }); } else { channel_send_->CallEncoder( [&](AudioEncoder* encoder) { encoder->DisableAudioNetworkAdaptor(); }); RTC_LOG(LS_INFO) << "Audio network adaptor disabled on SSRC " << new_config.rtp.ssrc; } } void AudioSendStream::ReconfigureCNG(const Config& new_config) { if (new_config.send_codec_spec->cng_payload_type == config_.send_codec_spec->cng_payload_type) { return; } // Register the CNG payload type if it's been added, don't do anything if CNG // is removed. Payload types must not be redefined. if (new_config.send_codec_spec->cng_payload_type) { RegisterCngPayloadType(*new_config.send_codec_spec->cng_payload_type, new_config.send_codec_spec->format.clockrate_hz); } // Wrap or unwrap the encoder in an AudioEncoderCNG. channel_send_->ModifyEncoder([&](std::unique_ptr* encoder_ptr) { std::unique_ptr old_encoder(std::move(*encoder_ptr)); auto sub_encoders = old_encoder->ReclaimContainedEncoders(); if (!sub_encoders.empty()) { // Replace enc with its sub encoder. We need to put the sub // encoder in a temporary first, since otherwise the old value // of enc would be destroyed before the new value got assigned, // which would be bad since the new value is a part of the old // value. auto tmp = std::move(sub_encoders[0]); old_encoder = std::move(tmp); } if (new_config.send_codec_spec->cng_payload_type) { AudioEncoderCngConfig config; config.speech_encoder = std::move(old_encoder); config.num_channels = config.speech_encoder->NumChannels(); config.payload_type = *new_config.send_codec_spec->cng_payload_type; config.vad_mode = Vad::kVadNormal; *encoder_ptr = CreateComfortNoiseEncoder(std::move(config)); } else { *encoder_ptr = std::move(old_encoder); } }); } void AudioSendStream::ReconfigureBitrateObserver( const webrtc::AudioSendStream::Config& new_config) { // Since the Config's default is for both of these to be -1, this test will // allow us to configure the bitrate observer if the new config has bitrate // limits set, but would only have us call RemoveBitrateObserver if we were // previously configured with bitrate limits. if (config_.min_bitrate_bps == new_config.min_bitrate_bps && config_.max_bitrate_bps == new_config.max_bitrate_bps && config_.bitrate_priority == new_config.bitrate_priority && TransportSeqNumId(config_) == TransportSeqNumId(new_config) && config_.audio_network_adaptor_config == new_config.audio_network_adaptor_config) { return; } if (!new_config.has_dscp && new_config.min_bitrate_bps != -1 && new_config.max_bitrate_bps != -1 && TransportSeqNumId(new_config) != 0) { rtp_transport_->AccountForAudioPacketsInPacedSender(true); if (send_side_bwe_with_overhead_) rtp_transport_->IncludeOverheadInPacedSender(); // We may get a callback immediately as the observer is registered, so // make sure the bitrate limits in config_ are up-to-date. config_.min_bitrate_bps = new_config.min_bitrate_bps; config_.max_bitrate_bps = new_config.max_bitrate_bps; config_.bitrate_priority = new_config.bitrate_priority; ConfigureBitrateObserver(); rtp_rtcp_module_->SetAsPartOfAllocation(true); } else { rtp_transport_->AccountForAudioPacketsInPacedSender(false); RemoveBitrateObserver(); rtp_rtcp_module_->SetAsPartOfAllocation(false); } } void AudioSendStream::ConfigureBitrateObserver() { // This either updates the current observer or adds a new observer. // TODO(srte): Add overhead compensation here. auto constraints = GetMinMaxBitrateConstraints(); RTC_DCHECK(constraints.has_value()); DataRate priority_bitrate = allocation_settings_.priority_bitrate; if (send_side_bwe_with_overhead_) { if (use_legacy_overhead_calculation_) { // OverheadPerPacket = Ipv4(20B) + UDP(8B) + SRTP(10B) + RTP(12) constexpr int kOverheadPerPacket = 20 + 8 + 10 + 12; const TimeDelta kMinPacketDuration = TimeDelta::Millis(20); DataRate max_overhead = DataSize::Bytes(kOverheadPerPacket) / kMinPacketDuration; priority_bitrate += max_overhead; } else { RTC_DCHECK(frame_length_range_); const DataSize overhead_per_packet = DataSize::Bytes(total_packet_overhead_bytes_); DataRate min_overhead = overhead_per_packet / frame_length_range_->second; priority_bitrate += min_overhead; } } if (allocation_settings_.priority_bitrate_raw) priority_bitrate = *allocation_settings_.priority_bitrate_raw; worker_queue_->PostTask([this, constraints, priority_bitrate, config_bitrate_priority = config_.bitrate_priority] { RTC_DCHECK_RUN_ON(worker_queue_); bitrate_allocator_->AddObserver( this, MediaStreamAllocationConfig{ constraints->min.bps(), constraints->max.bps(), 0, priority_bitrate.bps(), true, allocation_settings_.bitrate_priority.value_or( config_bitrate_priority)}); }); registered_with_allocator_ = true; } void AudioSendStream::RemoveBitrateObserver() { registered_with_allocator_ = false; rtc::Event thread_sync_event; worker_queue_->PostTask([this, &thread_sync_event] { RTC_DCHECK_RUN_ON(worker_queue_); bitrate_allocator_->RemoveObserver(this); thread_sync_event.Set(); }); thread_sync_event.Wait(rtc::Event::kForever); } absl::optional AudioSendStream::GetMinMaxBitrateConstraints() const { if (config_.min_bitrate_bps < 0 || config_.max_bitrate_bps < 0) { RTC_LOG(LS_WARNING) << "Config is invalid: min_bitrate_bps=" << config_.min_bitrate_bps << "; max_bitrate_bps=" << config_.max_bitrate_bps << "; both expected greater or equal to 0"; return absl::nullopt; } TargetAudioBitrateConstraints constraints{ DataRate::BitsPerSec(config_.min_bitrate_bps), DataRate::BitsPerSec(config_.max_bitrate_bps)}; // If bitrates were explicitly overriden via field trial, use those values. if (allocation_settings_.min_bitrate) constraints.min = *allocation_settings_.min_bitrate; if (allocation_settings_.max_bitrate) constraints.max = *allocation_settings_.max_bitrate; RTC_DCHECK_GE(constraints.min, DataRate::Zero()); RTC_DCHECK_GE(constraints.max, DataRate::Zero()); if (constraints.max < constraints.min) { RTC_LOG(LS_WARNING) << "TargetAudioBitrateConstraints::max is less than " << "TargetAudioBitrateConstraints::min"; return absl::nullopt; } if (send_side_bwe_with_overhead_) { if (use_legacy_overhead_calculation_) { // OverheadPerPacket = Ipv4(20B) + UDP(8B) + SRTP(10B) + RTP(12) const DataSize kOverheadPerPacket = DataSize::Bytes(20 + 8 + 10 + 12); const TimeDelta kMaxFrameLength = TimeDelta::Millis(60); // Based on Opus spec const DataRate kMinOverhead = kOverheadPerPacket / kMaxFrameLength; constraints.min += kMinOverhead; constraints.max += kMinOverhead; } else { if (!frame_length_range_.has_value()) { RTC_LOG(LS_WARNING) << "frame_length_range_ is not set"; return absl::nullopt; } const DataSize kOverheadPerPacket = DataSize::Bytes(total_packet_overhead_bytes_); constraints.min += kOverheadPerPacket / frame_length_range_->second; constraints.max += kOverheadPerPacket / frame_length_range_->first; } } return constraints; } void AudioSendStream::RegisterCngPayloadType(int payload_type, int clockrate_hz) { channel_send_->RegisterCngPayloadType(payload_type, clockrate_hz); } void AudioSendStream::UpdateCachedTargetAudioBitrateConstraints() { absl::optional new_constraints = GetMinMaxBitrateConstraints(); if (!new_constraints.has_value()) { return; } worker_queue_->PostTask([this, new_constraints]() { RTC_DCHECK_RUN_ON(worker_queue_); cached_constraints_ = new_constraints; }); } } // namespace internal } // namespace webrtc