Note: We no longer publish the latest version of our code here. We primarily use a kumc-bmi github organization. The heron ETL repository, in particular, is not public. Peers in the informatics community should see MultiSiteDev for details on requesting access.

source: webrtc/webrtc/modules/audio_coding/neteq4/neteq_impl.cc @ 0:4bda6873e34c

pub_scrub_3792 tip
Last change on this file since 0:4bda6873e34c was 0:4bda6873e34c, checked in by Michael Prittie <mprittie@…>, 6 years ago

Scrubbed password for publication.

File size: 71.8 KB
Line 
1/*
2 *  Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
3 *
4 *  Use of this source code is governed by a BSD-style license
5 *  that can be found in the LICENSE file in the root of the source
6 *  tree. An additional intellectual property rights grant can be found
7 *  in the file PATENTS.  All contributing project authors may
8 *  be found in the AUTHORS file in the root of the source tree.
9 */
10
11#include "webrtc/modules/audio_coding/neteq4/neteq_impl.h"
12
13#include <assert.h>
14#include <memory.h>  // memset
15
16#include <algorithm>
17
18#include "webrtc/common_audio/signal_processing/include/signal_processing_library.h"
19#include "webrtc/modules/audio_coding/neteq4/accelerate.h"
20#include "webrtc/modules/audio_coding/neteq4/background_noise.h"
21#include "webrtc/modules/audio_coding/neteq4/buffer_level_filter.h"
22#include "webrtc/modules/audio_coding/neteq4/comfort_noise.h"
23#include "webrtc/modules/audio_coding/neteq4/decision_logic.h"
24#include "webrtc/modules/audio_coding/neteq4/decoder_database.h"
25#include "webrtc/modules/audio_coding/neteq4/defines.h"
26#include "webrtc/modules/audio_coding/neteq4/delay_manager.h"
27#include "webrtc/modules/audio_coding/neteq4/delay_peak_detector.h"
28#include "webrtc/modules/audio_coding/neteq4/dtmf_buffer.h"
29#include "webrtc/modules/audio_coding/neteq4/dtmf_tone_generator.h"
30#include "webrtc/modules/audio_coding/neteq4/expand.h"
31#include "webrtc/modules/audio_coding/neteq4/interface/audio_decoder.h"
32#include "webrtc/modules/audio_coding/neteq4/merge.h"
33#include "webrtc/modules/audio_coding/neteq4/normal.h"
34#include "webrtc/modules/audio_coding/neteq4/packet_buffer.h"
35#include "webrtc/modules/audio_coding/neteq4/packet.h"
36#include "webrtc/modules/audio_coding/neteq4/payload_splitter.h"
37#include "webrtc/modules/audio_coding/neteq4/post_decode_vad.h"
38#include "webrtc/modules/audio_coding/neteq4/preemptive_expand.h"
39#include "webrtc/modules/audio_coding/neteq4/sync_buffer.h"
40#include "webrtc/modules/audio_coding/neteq4/timestamp_scaler.h"
41#include "webrtc/modules/interface/module_common_types.h"
42#include "webrtc/system_wrappers/interface/critical_section_wrapper.h"
43#include "webrtc/system_wrappers/interface/logging.h"
44
45// Modify the code to obtain backwards bit-exactness. Once bit-exactness is no
46// longer required, this #define should be removed (and the code that it
47// enables).
48#define LEGACY_BITEXACT
49
50namespace webrtc {
51
52NetEqImpl::NetEqImpl(int fs,
53                     BufferLevelFilter* buffer_level_filter,
54                     DecoderDatabase* decoder_database,
55                     DelayManager* delay_manager,
56                     DelayPeakDetector* delay_peak_detector,
57                     DtmfBuffer* dtmf_buffer,
58                     DtmfToneGenerator* dtmf_tone_generator,
59                     PacketBuffer* packet_buffer,
60                     PayloadSplitter* payload_splitter,
61                     TimestampScaler* timestamp_scaler)
62    : buffer_level_filter_(buffer_level_filter),
63      decoder_database_(decoder_database),
64      delay_manager_(delay_manager),
65      delay_peak_detector_(delay_peak_detector),
66      dtmf_buffer_(dtmf_buffer),
67      dtmf_tone_generator_(dtmf_tone_generator),
68      packet_buffer_(packet_buffer),
69      payload_splitter_(payload_splitter),
70      timestamp_scaler_(timestamp_scaler),
71      vad_(new PostDecodeVad()),
72      last_mode_(kModeNormal),
73      mute_factor_array_(NULL),
74      decoded_buffer_length_(kMaxFrameSize),
75      decoded_buffer_(new int16_t[decoded_buffer_length_]),
76      playout_timestamp_(0),
77      new_codec_(false),
78      timestamp_(0),
79      reset_decoder_(false),
80      current_rtp_payload_type_(0xFF),  // Invalid RTP payload type.
81      current_cng_rtp_payload_type_(0xFF),  // Invalid RTP payload type.
82      ssrc_(0),
83      first_packet_(true),
84      error_code_(0),
85      decoder_error_code_(0),
86      crit_sect_(CriticalSectionWrapper::CreateCriticalSection()),
87      decoded_packet_sequence_number_(-1),
88      decoded_packet_timestamp_(0) {
89  if (fs != 8000 && fs != 16000 && fs != 32000 && fs != 48000) {
90    LOG(LS_ERROR) << "Sample rate " << fs << " Hz not supported. " <<
91        "Changing to 8000 Hz.";
92    fs = 8000;
93  }
94  LOG(LS_INFO) << "Create NetEqImpl object with fs = " << fs << ".";
95  fs_hz_ = fs;
96  fs_mult_ = fs / 8000;
97  output_size_samples_ = kOutputSizeMs * 8 * fs_mult_;
98  decoder_frame_length_ = 3 * output_size_samples_;
99  WebRtcSpl_Init();
100  decision_logic_.reset(DecisionLogic::Create(fs_hz_, output_size_samples_,
101                                              kPlayoutOn,
102                                              decoder_database_.get(),
103                                              *packet_buffer_.get(),
104                                              delay_manager_.get(),
105                                              buffer_level_filter_.get()));
106  SetSampleRateAndChannels(fs, 1);  // Default is 1 channel.
107}
108
109NetEqImpl::~NetEqImpl() {
110  LOG(LS_INFO) << "Deleting NetEqImpl object.";
111}
112
113int NetEqImpl::InsertPacket(const WebRtcRTPHeader& rtp_header,
114                            const uint8_t* payload,
115                            int length_bytes,
116                            uint32_t receive_timestamp) {
117  CriticalSectionScoped lock(crit_sect_.get());
118  LOG(LS_VERBOSE) << "InsertPacket: ts=" << rtp_header.header.timestamp <<
119      ", sn=" << rtp_header.header.sequenceNumber <<
120      ", pt=" << static_cast<int>(rtp_header.header.payloadType) <<
121      ", ssrc=" << rtp_header.header.ssrc <<
122      ", len=" << length_bytes;
123  int error = InsertPacketInternal(rtp_header, payload, length_bytes,
124                                   receive_timestamp, false);
125  if (error != 0) {
126    LOG_FERR1(LS_WARNING, InsertPacketInternal, error);
127    error_code_ = error;
128    return kFail;
129  }
130  return kOK;
131}
132
133int NetEqImpl::InsertSyncPacket(const WebRtcRTPHeader& rtp_header,
134                                uint32_t receive_timestamp) {
135  CriticalSectionScoped lock(crit_sect_.get());
136  LOG(LS_VERBOSE) << "InsertPacket-Sync: ts="
137      << rtp_header.header.timestamp <<
138      ", sn=" << rtp_header.header.sequenceNumber <<
139      ", pt=" << static_cast<int>(rtp_header.header.payloadType) <<
140      ", ssrc=" << rtp_header.header.ssrc;
141
142  const uint8_t kSyncPayload[] = { 's', 'y', 'n', 'c' };
143  int error = InsertPacketInternal(
144      rtp_header, kSyncPayload, sizeof(kSyncPayload), receive_timestamp, true);
145
146   if (error != 0) {
147     LOG_FERR1(LS_WARNING, InsertPacketInternal, error);
148     error_code_ = error;
149     return kFail;
150   }
151   return kOK;
152}
153
154int NetEqImpl::GetAudio(size_t max_length, int16_t* output_audio,
155                        int* samples_per_channel, int* num_channels,
156                        NetEqOutputType* type) {
157  CriticalSectionScoped lock(crit_sect_.get());
158  LOG(LS_VERBOSE) << "GetAudio";
159  int error = GetAudioInternal(max_length, output_audio, samples_per_channel,
160                               num_channels);
161  LOG(LS_VERBOSE) << "Produced " << *samples_per_channel <<
162      " samples/channel for " << *num_channels << " channel(s)";
163  if (error != 0) {
164    LOG_FERR1(LS_WARNING, GetAudioInternal, error);
165    error_code_ = error;
166    return kFail;
167  }
168  if (type) {
169    *type = LastOutputType();
170  }
171  return kOK;
172}
173
174int NetEqImpl::RegisterPayloadType(enum NetEqDecoder codec,
175                                   uint8_t rtp_payload_type) {
176  CriticalSectionScoped lock(crit_sect_.get());
177  LOG_API2(static_cast<int>(rtp_payload_type), codec);
178  int ret = decoder_database_->RegisterPayload(rtp_payload_type, codec);
179  if (ret != DecoderDatabase::kOK) {
180    LOG_FERR2(LS_WARNING, RegisterPayload, rtp_payload_type, codec);
181    switch (ret) {
182      case DecoderDatabase::kInvalidRtpPayloadType:
183        error_code_ = kInvalidRtpPayloadType;
184        break;
185      case DecoderDatabase::kCodecNotSupported:
186        error_code_ = kCodecNotSupported;
187        break;
188      case DecoderDatabase::kDecoderExists:
189        error_code_ = kDecoderExists;
190        break;
191      default:
192        error_code_ = kOtherError;
193    }
194    return kFail;
195  }
196  return kOK;
197}
198
199int NetEqImpl::RegisterExternalDecoder(AudioDecoder* decoder,
200                                       enum NetEqDecoder codec,
201                                       int sample_rate_hz,
202                                       uint8_t rtp_payload_type) {
203  CriticalSectionScoped lock(crit_sect_.get());
204  LOG_API2(static_cast<int>(rtp_payload_type), codec);
205  if (!decoder) {
206    LOG(LS_ERROR) << "Cannot register external decoder with NULL pointer";
207    assert(false);
208    return kFail;
209  }
210  int ret = decoder_database_->InsertExternal(rtp_payload_type, codec,
211                                              sample_rate_hz, decoder);
212  if (ret != DecoderDatabase::kOK) {
213    LOG_FERR2(LS_WARNING, InsertExternal, rtp_payload_type, codec);
214    switch (ret) {
215      case DecoderDatabase::kInvalidRtpPayloadType:
216        error_code_ = kInvalidRtpPayloadType;
217        break;
218      case DecoderDatabase::kCodecNotSupported:
219        error_code_ = kCodecNotSupported;
220        break;
221      case DecoderDatabase::kDecoderExists:
222        error_code_ = kDecoderExists;
223        break;
224      case DecoderDatabase::kInvalidSampleRate:
225        error_code_ = kInvalidSampleRate;
226        break;
227      case DecoderDatabase::kInvalidPointer:
228        error_code_ = kInvalidPointer;
229        break;
230      default:
231        error_code_ = kOtherError;
232    }
233    return kFail;
234  }
235  return kOK;
236}
237
238int NetEqImpl::RemovePayloadType(uint8_t rtp_payload_type) {
239  CriticalSectionScoped lock(crit_sect_.get());
240  LOG_API1(static_cast<int>(rtp_payload_type));
241  int ret = decoder_database_->Remove(rtp_payload_type);
242  if (ret == DecoderDatabase::kOK) {
243    return kOK;
244  } else if (ret == DecoderDatabase::kDecoderNotFound) {
245    error_code_ = kDecoderNotFound;
246  } else {
247    error_code_ = kOtherError;
248  }
249  LOG_FERR1(LS_WARNING, Remove, rtp_payload_type);
250  return kFail;
251}
252
253bool NetEqImpl::SetMinimumDelay(int delay_ms) {
254  CriticalSectionScoped lock(crit_sect_.get());
255  if (delay_ms >= 0 && delay_ms < 10000) {
256    assert(delay_manager_.get());
257    return delay_manager_->SetMinimumDelay(delay_ms);
258  }
259  return false;
260}
261
262bool NetEqImpl::SetMaximumDelay(int delay_ms) {
263  CriticalSectionScoped lock(crit_sect_.get());
264  if (delay_ms >= 0 && delay_ms < 10000) {
265    assert(delay_manager_.get());
266    return delay_manager_->SetMaximumDelay(delay_ms);
267  }
268  return false;
269}
270
271int NetEqImpl::LeastRequiredDelayMs() const {
272  CriticalSectionScoped lock(crit_sect_.get());
273  assert(delay_manager_.get());
274  return delay_manager_->least_required_delay_ms();
275}
276
277void NetEqImpl::SetPlayoutMode(NetEqPlayoutMode mode) {
278  CriticalSectionScoped lock(crit_sect_.get());
279  if (!decision_logic_.get() || mode != decision_logic_->playout_mode()) {
280    // The reset() method calls delete for the old object.
281    decision_logic_.reset(DecisionLogic::Create(fs_hz_, output_size_samples_,
282                                                mode,
283                                                decoder_database_.get(),
284                                                *packet_buffer_.get(),
285                                                delay_manager_.get(),
286                                                buffer_level_filter_.get()));
287  }
288}
289
290NetEqPlayoutMode NetEqImpl::PlayoutMode() const {
291  CriticalSectionScoped lock(crit_sect_.get());
292  assert(decision_logic_.get());
293  return decision_logic_->playout_mode();
294}
295
296int NetEqImpl::NetworkStatistics(NetEqNetworkStatistics* stats) {
297  CriticalSectionScoped lock(crit_sect_.get());
298  assert(decoder_database_.get());
299  const int total_samples_in_buffers = packet_buffer_->NumSamplesInBuffer(
300      decoder_database_.get(), decoder_frame_length_) +
301          static_cast<int>(sync_buffer_->FutureLength());
302  assert(delay_manager_.get());
303  assert(decision_logic_.get());
304  stats_.GetNetworkStatistics(fs_hz_, total_samples_in_buffers,
305                              decoder_frame_length_, *delay_manager_.get(),
306                              *decision_logic_.get(), stats);
307  return 0;
308}
309
310void NetEqImpl::WaitingTimes(std::vector<int>* waiting_times) {
311  CriticalSectionScoped lock(crit_sect_.get());
312  stats_.WaitingTimes(waiting_times);
313}
314
315void NetEqImpl::GetRtcpStatistics(RtcpStatistics* stats) {
316  CriticalSectionScoped lock(crit_sect_.get());
317  if (stats) {
318    rtcp_.GetStatistics(false, stats);
319  }
320}
321
322void NetEqImpl::GetRtcpStatisticsNoReset(RtcpStatistics* stats) {
323  CriticalSectionScoped lock(crit_sect_.get());
324  if (stats) {
325    rtcp_.GetStatistics(true, stats);
326  }
327}
328
329void NetEqImpl::EnableVad() {
330  CriticalSectionScoped lock(crit_sect_.get());
331  assert(vad_.get());
332  vad_->Enable();
333}
334
335void NetEqImpl::DisableVad() {
336  CriticalSectionScoped lock(crit_sect_.get());
337  assert(vad_.get());
338  vad_->Disable();
339}
340
341uint32_t NetEqImpl::PlayoutTimestamp() {
342  CriticalSectionScoped lock(crit_sect_.get());
343  return timestamp_scaler_->ToExternal(playout_timestamp_);
344}
345
346int NetEqImpl::LastError() {
347  CriticalSectionScoped lock(crit_sect_.get());
348  return error_code_;
349}
350
351int NetEqImpl::LastDecoderError() {
352  CriticalSectionScoped lock(crit_sect_.get());
353  return decoder_error_code_;
354}
355
356void NetEqImpl::FlushBuffers() {
357  CriticalSectionScoped lock(crit_sect_.get());
358  LOG_API0();
359  packet_buffer_->Flush();
360  assert(sync_buffer_.get());
361  assert(expand_.get());
362  sync_buffer_->Flush();
363  sync_buffer_->set_next_index(sync_buffer_->next_index() -
364                               expand_->overlap_length());
365  // Set to wait for new codec.
366  first_packet_ = true;
367}
368
369void NetEqImpl::PacketBufferStatistics(int* current_num_packets,
370                                       int* max_num_packets,
371                                       int* current_memory_size_bytes,
372                                       int* max_memory_size_bytes) const {
373  CriticalSectionScoped lock(crit_sect_.get());
374  packet_buffer_->BufferStat(current_num_packets, max_num_packets,
375                             current_memory_size_bytes, max_memory_size_bytes);
376}
377
378int NetEqImpl::DecodedRtpInfo(int* sequence_number, uint32_t* timestamp) const {
379  CriticalSectionScoped lock(crit_sect_.get());
380  if (decoded_packet_sequence_number_ < 0)
381    return -1;
382  *sequence_number = decoded_packet_sequence_number_;
383  *timestamp = decoded_packet_timestamp_;
384  return 0;
385}
386
387void NetEqImpl::SetBackgroundNoiseMode(NetEqBackgroundNoiseMode mode) {
388  CriticalSectionScoped lock(crit_sect_.get());
389  assert(background_noise_.get());
390  background_noise_->set_mode(mode);
391}
392
393NetEqBackgroundNoiseMode NetEqImpl::BackgroundNoiseMode() const {
394  CriticalSectionScoped lock(crit_sect_.get());
395  assert(background_noise_.get());
396  return background_noise_->mode();
397}
398
399// Methods below this line are private.
400
401int NetEqImpl::InsertPacketInternal(const WebRtcRTPHeader& rtp_header,
402                                    const uint8_t* payload,
403                                    int length_bytes,
404                                    uint32_t receive_timestamp,
405                                    bool is_sync_packet) {
406  if (!payload) {
407    LOG_F(LS_ERROR) << "payload == NULL";
408    return kInvalidPointer;
409  }
410  // Sanity checks for sync-packets.
411  if (is_sync_packet) {
412    if (decoder_database_->IsDtmf(rtp_header.header.payloadType) ||
413        decoder_database_->IsRed(rtp_header.header.payloadType) ||
414        decoder_database_->IsComfortNoise(rtp_header.header.payloadType)) {
415      LOG_F(LS_ERROR) << "Sync-packet with an unacceptable payload type "
416          << rtp_header.header.payloadType;
417      return kSyncPacketNotAccepted;
418    }
419    if (first_packet_ ||
420        rtp_header.header.payloadType != current_rtp_payload_type_ ||
421        rtp_header.header.ssrc != ssrc_) {
422      // Even if |current_rtp_payload_type_| is 0xFF, sync-packet isn't
423      // accepted.
424      LOG_F(LS_ERROR) << "Changing codec, SSRC or first packet "
425          "with sync-packet.";
426      return kSyncPacketNotAccepted;
427    }
428  }
429  PacketList packet_list;
430  RTPHeader main_header;
431  {
432    // Convert to Packet.
433    // Create |packet| within this separate scope, since it should not be used
434    // directly once it's been inserted in the packet list. This way, |packet|
435    // is not defined outside of this block.
436    Packet* packet = new Packet;
437    packet->header.markerBit = false;
438    packet->header.payloadType = rtp_header.header.payloadType;
439    packet->header.sequenceNumber = rtp_header.header.sequenceNumber;
440    packet->header.timestamp = rtp_header.header.timestamp;
441    packet->header.ssrc = rtp_header.header.ssrc;
442    packet->header.numCSRCs = 0;
443    packet->payload_length = length_bytes;
444    packet->primary = true;
445    packet->waiting_time = 0;
446    packet->payload = new uint8_t[packet->payload_length];
447    packet->sync_packet = is_sync_packet;
448    if (!packet->payload) {
449      LOG_F(LS_ERROR) << "Payload pointer is NULL.";
450    }
451    assert(payload);  // Already checked above.
452    memcpy(packet->payload, payload, packet->payload_length);
453    // Insert packet in a packet list.
454    packet_list.push_back(packet);
455    // Save main payloads header for later.
456    memcpy(&main_header, &packet->header, sizeof(main_header));
457  }
458
459  bool update_sample_rate_and_channels = false;
460  // Reinitialize NetEq if it's needed (changed SSRC or first call).
461  if ((main_header.ssrc != ssrc_) || first_packet_) {
462    rtcp_.Init(main_header.sequenceNumber);
463    first_packet_ = false;
464
465    // Flush the packet buffer and DTMF buffer.
466    packet_buffer_->Flush();
467    dtmf_buffer_->Flush();
468
469    // Store new SSRC.
470    ssrc_ = main_header.ssrc;
471
472    // Update audio buffer timestamp.
473    sync_buffer_->IncreaseEndTimestamp(main_header.timestamp - timestamp_);
474
475    // Update codecs.
476    timestamp_ = main_header.timestamp;
477    current_rtp_payload_type_ = main_header.payloadType;
478
479    // Set MCU to update codec on next SignalMCU call.
480    new_codec_ = true;
481
482    // Reset timestamp scaling.
483    timestamp_scaler_->Reset();
484
485    // Triger an update of sampling rate and the number of channels.
486    update_sample_rate_and_channels = true;
487  }
488
489  // Update RTCP statistics, only for regular packets.
490  if (!is_sync_packet)
491    rtcp_.Update(main_header, receive_timestamp);
492
493  // Check for RED payload type, and separate payloads into several packets.
494  if (decoder_database_->IsRed(main_header.payloadType)) {
495    assert(!is_sync_packet);  // We had a sanity check for this.
496    if (payload_splitter_->SplitRed(&packet_list) != PayloadSplitter::kOK) {
497      LOG_FERR1(LS_WARNING, SplitRed, packet_list.size());
498      PacketBuffer::DeleteAllPackets(&packet_list);
499      return kRedundancySplitError;
500    }
501    // Only accept a few RED payloads of the same type as the main data,
502    // DTMF events and CNG.
503    payload_splitter_->CheckRedPayloads(&packet_list, *decoder_database_);
504    // Update the stored main payload header since the main payload has now
505    // changed.
506    memcpy(&main_header, &packet_list.front()->header, sizeof(main_header));
507  }
508
509  // Check payload types.
510  if (decoder_database_->CheckPayloadTypes(packet_list) ==
511      DecoderDatabase::kDecoderNotFound) {
512    LOG_FERR1(LS_WARNING, CheckPayloadTypes, packet_list.size());
513    PacketBuffer::DeleteAllPackets(&packet_list);
514    return kUnknownRtpPayloadType;
515  }
516
517  // Scale timestamp to internal domain (only for some codecs).
518  timestamp_scaler_->ToInternal(&packet_list);
519
520  // Process DTMF payloads. Cycle through the list of packets, and pick out any
521  // DTMF payloads found.
522  PacketList::iterator it = packet_list.begin();
523  while (it != packet_list.end()) {
524    Packet* current_packet = (*it);
525    assert(current_packet);
526    assert(current_packet->payload);
527    if (decoder_database_->IsDtmf(current_packet->header.payloadType)) {
528      assert(!current_packet->sync_packet);  // We had a sanity check for this.
529      DtmfEvent event;
530      int ret = DtmfBuffer::ParseEvent(
531          current_packet->header.timestamp,
532          current_packet->payload,
533          current_packet->payload_length,
534          &event);
535      if (ret != DtmfBuffer::kOK) {
536        LOG_FERR2(LS_WARNING, ParseEvent, ret,
537                  current_packet->payload_length);
538        PacketBuffer::DeleteAllPackets(&packet_list);
539        return kDtmfParsingError;
540      }
541      if (dtmf_buffer_->InsertEvent(event) != DtmfBuffer::kOK) {
542        LOG_FERR0(LS_WARNING, InsertEvent);
543        PacketBuffer::DeleteAllPackets(&packet_list);
544        return kDtmfInsertError;
545      }
546      // TODO(hlundin): Let the destructor of Packet handle the payload.
547      delete [] current_packet->payload;
548      delete current_packet;
549      it = packet_list.erase(it);
550    } else {
551      ++it;
552    }
553  }
554
555  // Split payloads into smaller chunks. This also verifies that all payloads
556  // are of a known payload type. SplitAudio() method is protected against
557  // sync-packets.
558  int ret = payload_splitter_->SplitAudio(&packet_list, *decoder_database_);
559  if (ret != PayloadSplitter::kOK) {
560    LOG_FERR1(LS_WARNING, SplitAudio, packet_list.size());
561    PacketBuffer::DeleteAllPackets(&packet_list);
562    switch (ret) {
563      case PayloadSplitter::kUnknownPayloadType:
564        return kUnknownRtpPayloadType;
565      case PayloadSplitter::kFrameSplitError:
566        return kFrameSplitError;
567      default:
568        return kOtherError;
569    }
570  }
571
572  // Update bandwidth estimate, if the packet is not sync-packet.
573  if (!packet_list.empty() && !packet_list.front()->sync_packet) {
574    // The list can be empty here if we got nothing but DTMF payloads.
575    AudioDecoder* decoder =
576        decoder_database_->GetDecoder(main_header.payloadType);
577    assert(decoder);  // Should always get a valid object, since we have
578                      // already checked that the payload types are known.
579    decoder->IncomingPacket(packet_list.front()->payload,
580                            packet_list.front()->payload_length,
581                            packet_list.front()->header.sequenceNumber,
582                            packet_list.front()->header.timestamp,
583                            receive_timestamp);
584  }
585
586  // Insert packets in buffer.
587  int temp_bufsize = packet_buffer_->NumPacketsInBuffer();
588  ret = packet_buffer_->InsertPacketList(
589      &packet_list,
590      *decoder_database_,
591      &current_rtp_payload_type_,
592      &current_cng_rtp_payload_type_);
593  if (ret == PacketBuffer::kFlushed) {
594    // Reset DSP timestamp etc. if packet buffer flushed.
595    new_codec_ = true;
596    update_sample_rate_and_channels = true;
597    LOG_F(LS_WARNING) << "Packet buffer flushed";
598  } else if (ret == PacketBuffer::kOversizePacket) {
599    LOG_F(LS_WARNING) << "Packet larger than packet buffer";
600    return kOversizePacket;
601  } else if (ret != PacketBuffer::kOK) {
602    LOG_FERR1(LS_WARNING, InsertPacketList, packet_list.size());
603    PacketBuffer::DeleteAllPackets(&packet_list);
604    return kOtherError;
605  }
606  if (current_rtp_payload_type_ != 0xFF) {
607    const DecoderDatabase::DecoderInfo* dec_info =
608        decoder_database_->GetDecoderInfo(current_rtp_payload_type_);
609    if (!dec_info) {
610      assert(false);  // Already checked that the payload type is known.
611    }
612  }
613
614  if (update_sample_rate_and_channels && !packet_buffer_->Empty()) {
615    // We do not use |current_rtp_payload_type_| to |set payload_type|, but
616    // get the next RTP header from |packet_buffer_| to obtain the payload type.
617    // The reason for it is the following corner case. If NetEq receives a
618    // CNG packet with a sample rate different than the current CNG then it
619    // flushes its buffer, assuming send codec must have been changed. However,
620    // payload type of the hypothetically new send codec is not known.
621    const RTPHeader* rtp_header = packet_buffer_->NextRtpHeader();
622    assert(rtp_header);
623    int payload_type = rtp_header->payloadType;
624    AudioDecoder* decoder = decoder_database_->GetDecoder(payload_type);
625    assert(decoder);  // Payloads are already checked to be valid.
626    const DecoderDatabase::DecoderInfo* decoder_info =
627        decoder_database_->GetDecoderInfo(payload_type);
628    assert(decoder_info);
629    if (decoder_info->fs_hz != fs_hz_ ||
630        decoder->channels() != algorithm_buffer_->Channels())
631      SetSampleRateAndChannels(decoder_info->fs_hz, decoder->channels());
632  }
633
634  // TODO(hlundin): Move this code to DelayManager class.
635  const DecoderDatabase::DecoderInfo* dec_info =
636          decoder_database_->GetDecoderInfo(main_header.payloadType);
637  assert(dec_info);  // Already checked that the payload type is known.
638  delay_manager_->LastDecoderType(dec_info->codec_type);
639  if (delay_manager_->last_pack_cng_or_dtmf() == 0) {
640    // Calculate the total speech length carried in each packet.
641    temp_bufsize = packet_buffer_->NumPacketsInBuffer() - temp_bufsize;
642    temp_bufsize *= decoder_frame_length_;
643
644    if ((temp_bufsize > 0) &&
645        (temp_bufsize != decision_logic_->packet_length_samples())) {
646      decision_logic_->set_packet_length_samples(temp_bufsize);
647      delay_manager_->SetPacketAudioLength((1000 * temp_bufsize) / fs_hz_);
648    }
649
650    // Update statistics.
651    if ((int32_t) (main_header.timestamp - timestamp_) >= 0 &&
652        !new_codec_) {
653      // Only update statistics if incoming packet is not older than last played
654      // out packet, and if new codec flag is not set.
655      delay_manager_->Update(main_header.sequenceNumber, main_header.timestamp,
656                             fs_hz_);
657    }
658  } else if (delay_manager_->last_pack_cng_or_dtmf() == -1) {
659    // This is first "normal" packet after CNG or DTMF.
660    // Reset packet time counter and measure time until next packet,
661    // but don't update statistics.
662    delay_manager_->set_last_pack_cng_or_dtmf(0);
663    delay_manager_->ResetPacketIatCount();
664  }
665  return 0;
666}
667
668int NetEqImpl::GetAudioInternal(size_t max_length, int16_t* output,
669                                int* samples_per_channel, int* num_channels) {
670  PacketList packet_list;
671  DtmfEvent dtmf_event;
672  Operations operation;
673  bool play_dtmf;
674  int return_value = GetDecision(&operation, &packet_list, &dtmf_event,
675                                 &play_dtmf);
676  if (return_value != 0) {
677    LOG_FERR1(LS_WARNING, GetDecision, return_value);
678    assert(false);
679    last_mode_ = kModeError;
680    return return_value;
681  }
682  LOG(LS_VERBOSE) << "GetDecision returned operation=" << operation <<
683      " and " << packet_list.size() << " packet(s)";
684
685  AudioDecoder::SpeechType speech_type;
686  int length = 0;
687  int decode_return_value = Decode(&packet_list, &operation,
688                                   &length, &speech_type);
689
690  assert(vad_.get());
691  bool sid_frame_available =
692      (operation == kRfc3389Cng && !packet_list.empty());
693  vad_->Update(decoded_buffer_.get(), length, speech_type,
694               sid_frame_available, fs_hz_);
695
696  algorithm_buffer_->Clear();
697  switch (operation) {
698    case kNormal: {
699      DoNormal(decoded_buffer_.get(), length, speech_type, play_dtmf);
700      break;
701    }
702    case kMerge: {
703      DoMerge(decoded_buffer_.get(), length, speech_type, play_dtmf);
704      break;
705    }
706    case kExpand: {
707      return_value = DoExpand(play_dtmf);
708      break;
709    }
710    case kAccelerate: {
711      return_value = DoAccelerate(decoded_buffer_.get(), length, speech_type,
712                                  play_dtmf);
713      break;
714    }
715    case kPreemptiveExpand: {
716      return_value = DoPreemptiveExpand(decoded_buffer_.get(), length,
717                                        speech_type, play_dtmf);
718      break;
719    }
720    case kRfc3389Cng:
721    case kRfc3389CngNoPacket: {
722      return_value = DoRfc3389Cng(&packet_list, play_dtmf);
723      break;
724    }
725    case kCodecInternalCng: {
726      // This handles the case when there is no transmission and the decoder
727      // should produce internal comfort noise.
728      // TODO(hlundin): Write test for codec-internal CNG.
729      DoCodecInternalCng();
730      break;
731    }
732    case kDtmf: {
733      // TODO(hlundin): Write test for this.
734      return_value = DoDtmf(dtmf_event, &play_dtmf);
735      break;
736    }
737    case kAlternativePlc: {
738      // TODO(hlundin): Write test for this.
739      DoAlternativePlc(false);
740      break;
741    }
742    case kAlternativePlcIncreaseTimestamp: {
743      // TODO(hlundin): Write test for this.
744      DoAlternativePlc(true);
745      break;
746    }
747    case kAudioRepetitionIncreaseTimestamp: {
748      // TODO(hlundin): Write test for this.
749      sync_buffer_->IncreaseEndTimestamp(output_size_samples_);
750      // Skipping break on purpose. Execution should move on into the
751      // next case.
752    }
753    case kAudioRepetition: {
754      // TODO(hlundin): Write test for this.
755      // Copy last |output_size_samples_| from |sync_buffer_| to
756      // |algorithm_buffer|.
757      algorithm_buffer_->PushBackFromIndex(
758          *sync_buffer_, sync_buffer_->Size() - output_size_samples_);
759      expand_->Reset();
760      break;
761    }
762    case kUndefined: {
763      LOG_F(LS_ERROR) << "Invalid operation kUndefined.";
764      assert(false);  // This should not happen.
765      last_mode_ = kModeError;
766      return kInvalidOperation;
767    }
768  }  // End of switch.
769  if (return_value < 0) {
770    return return_value;
771  }
772
773  if (last_mode_ != kModeRfc3389Cng) {
774    comfort_noise_->Reset();
775  }
776
777  // Copy from |algorithm_buffer| to |sync_buffer_|.
778  sync_buffer_->PushBack(*algorithm_buffer_);
779
780  // Extract data from |sync_buffer_| to |output|.
781  size_t num_output_samples_per_channel = output_size_samples_;
782  size_t num_output_samples = output_size_samples_ * sync_buffer_->Channels();
783  if (num_output_samples > max_length) {
784    LOG(LS_WARNING) << "Output array is too short. " << max_length << " < " <<
785        output_size_samples_ << " * " << sync_buffer_->Channels();
786    num_output_samples = max_length;
787    num_output_samples_per_channel = static_cast<int>(
788        max_length / sync_buffer_->Channels());
789  }
790  int samples_from_sync = static_cast<int>(
791      sync_buffer_->GetNextAudioInterleaved(num_output_samples_per_channel,
792                                            output));
793  *num_channels = static_cast<int>(sync_buffer_->Channels());
794  LOG(LS_VERBOSE) << "Sync buffer (" << *num_channels << " channel(s)):" <<
795      " insert " << algorithm_buffer_->Size() << " samples, extract " <<
796      samples_from_sync << " samples";
797  if (samples_from_sync != output_size_samples_) {
798    LOG_F(LS_ERROR) << "samples_from_sync != output_size_samples_";
799    // TODO(minyue): treatment of under-run, filling zeros
800    memset(output, 0, num_output_samples * sizeof(int16_t));
801    *samples_per_channel = output_size_samples_;
802    return kSampleUnderrun;
803  }
804  *samples_per_channel = output_size_samples_;
805
806  // Should always have overlap samples left in the |sync_buffer_|.
807  assert(sync_buffer_->FutureLength() >= expand_->overlap_length());
808
809  if (play_dtmf) {
810    return_value = DtmfOverdub(dtmf_event, sync_buffer_->Channels(), output);
811  }
812
813  // Update the background noise parameters if last operation wrote data
814  // straight from the decoder to the |sync_buffer_|. That is, none of the
815  // operations that modify the signal can be followed by a parameter update.
816  if ((last_mode_ == kModeNormal) ||
817      (last_mode_ == kModeAccelerateFail) ||
818      (last_mode_ == kModePreemptiveExpandFail) ||
819      (last_mode_ == kModeRfc3389Cng) ||
820      (last_mode_ == kModeCodecInternalCng)) {
821    background_noise_->Update(*sync_buffer_, *vad_.get());
822  }
823
824  if (operation == kDtmf) {
825    // DTMF data was written the end of |sync_buffer_|.
826    // Update index to end of DTMF data in |sync_buffer_|.
827    sync_buffer_->set_dtmf_index(sync_buffer_->Size());
828  }
829
830  if ((last_mode_ != kModeExpand) && (last_mode_ != kModeRfc3389Cng)) {
831    // If last operation was neither expand, nor comfort noise, calculate the
832    // |playout_timestamp_| from the |sync_buffer_|. However, do not update the
833    // |playout_timestamp_| if it would be moved "backwards".
834    uint32_t temp_timestamp = sync_buffer_->end_timestamp() -
835        static_cast<uint32_t>(sync_buffer_->FutureLength());
836    if (static_cast<int32_t>(temp_timestamp - playout_timestamp_) > 0) {
837      playout_timestamp_ = temp_timestamp;
838    }
839  } else {
840    // Use dead reckoning to estimate the |playout_timestamp_|.
841    playout_timestamp_ += output_size_samples_;
842  }
843
844  if (decode_return_value) return decode_return_value;
845  return return_value;
846}
847
848int NetEqImpl::GetDecision(Operations* operation,
849                           PacketList* packet_list,
850                           DtmfEvent* dtmf_event,
851                           bool* play_dtmf) {
852  // Initialize output variables.
853  *play_dtmf = false;
854  *operation = kUndefined;
855
856  // Increment time counters.
857  packet_buffer_->IncrementWaitingTimes();
858  stats_.IncreaseCounter(output_size_samples_, fs_hz_);
859
860  assert(sync_buffer_.get());
861  uint32_t end_timestamp = sync_buffer_->end_timestamp();
862  if (!new_codec_) {
863    packet_buffer_->DiscardOldPackets(end_timestamp);
864  }
865  const RTPHeader* header = packet_buffer_->NextRtpHeader();
866
867  if (decision_logic_->CngRfc3389On()) {
868    // Because of timestamp peculiarities, we have to "manually" disallow using
869    // a CNG packet with the same timestamp as the one that was last played.
870    // This can happen when using redundancy and will cause the timing to shift.
871    while (header &&
872        decoder_database_->IsComfortNoise(header->payloadType) &&
873        end_timestamp >= header->timestamp) {
874      // Don't use this packet, discard it.
875      // TODO(hlundin): Write test for this case.
876      if (packet_buffer_->DiscardNextPacket() != PacketBuffer::kOK) {
877        assert(false);  // Must be ok by design.
878      }
879      // Check buffer again.
880      if (!new_codec_) {
881        packet_buffer_->DiscardOldPackets(end_timestamp);
882      }
883      header = packet_buffer_->NextRtpHeader();
884    }
885  }
886
887  assert(expand_.get());
888  const int samples_left = static_cast<int>(sync_buffer_->FutureLength() -
889      expand_->overlap_length());
890  if (last_mode_ == kModeAccelerateSuccess ||
891      last_mode_ == kModeAccelerateLowEnergy ||
892      last_mode_ == kModePreemptiveExpandSuccess ||
893      last_mode_ == kModePreemptiveExpandLowEnergy) {
894    // Subtract (samples_left + output_size_samples_) from sampleMemory.
895    decision_logic_->AddSampleMemory(-(samples_left + output_size_samples_));
896  }
897
898  // Check if it is time to play a DTMF event.
899  if (dtmf_buffer_->GetEvent(end_timestamp +
900                             decision_logic_->generated_noise_samples(),
901                             dtmf_event)) {
902    *play_dtmf = true;
903  }
904
905  // Get instruction.
906  assert(sync_buffer_.get());
907  assert(expand_.get());
908  *operation = decision_logic_->GetDecision(*sync_buffer_,
909                                            *expand_,
910                                            decoder_frame_length_,
911                                            header,
912                                            last_mode_,
913                                            *play_dtmf,
914                                            &reset_decoder_);
915
916  // Check if we already have enough samples in the |sync_buffer_|. If so,
917  // change decision to normal, unless the decision was merge, accelerate, or
918  // preemptive expand.
919  if (samples_left >= output_size_samples_ &&
920      *operation != kMerge &&
921      *operation != kAccelerate &&
922      *operation != kPreemptiveExpand) {
923    *operation = kNormal;
924    return 0;
925  }
926
927  decision_logic_->ExpandDecision(*operation == kExpand);
928
929  // Check conditions for reset.
930  if (new_codec_ || *operation == kUndefined) {
931    // The only valid reason to get kUndefined is that new_codec_ is set.
932    assert(new_codec_);
933    if (*play_dtmf && !header) {
934      timestamp_ = dtmf_event->timestamp;
935    } else {
936      assert(header);
937      if (!header) {
938        LOG_F(LS_ERROR) << "Packet missing where it shouldn't.";
939        return -1;
940      }
941      timestamp_ = header->timestamp;
942      if (*operation == kRfc3389CngNoPacket
943#ifndef LEGACY_BITEXACT
944          // Without this check, it can happen that a non-CNG packet is sent to
945          // the CNG decoder as if it was a SID frame. This is clearly a bug,
946          // but is kept for now to maintain bit-exactness with the test
947          // vectors.
948          && decoder_database_->IsComfortNoise(header->payloadType)
949#endif
950      ) {
951        // Change decision to CNG packet, since we do have a CNG packet, but it
952        // was considered too early to use. Now, use it anyway.
953        *operation = kRfc3389Cng;
954      } else if (*operation != kRfc3389Cng) {
955        *operation = kNormal;
956      }
957    }
958    // Adjust |sync_buffer_| timestamp before setting |end_timestamp| to the
959    // new value.
960    sync_buffer_->IncreaseEndTimestamp(timestamp_ - end_timestamp);
961    end_timestamp = timestamp_;
962    new_codec_ = false;
963    decision_logic_->SoftReset();
964    buffer_level_filter_->Reset();
965    delay_manager_->Reset();
966    stats_.ResetMcu();
967  }
968
969  int required_samples = output_size_samples_;
970  const int samples_10_ms = 80 * fs_mult_;
971  const int samples_20_ms = 2 * samples_10_ms;
972  const int samples_30_ms = 3 * samples_10_ms;
973
974  switch (*operation) {
975    case kExpand: {
976      timestamp_ = end_timestamp;
977      return 0;
978    }
979    case kRfc3389CngNoPacket:
980    case kCodecInternalCng: {
981      return 0;
982    }
983    case kDtmf: {
984      // TODO(hlundin): Write test for this.
985      // Update timestamp.
986      timestamp_ = end_timestamp;
987      if (decision_logic_->generated_noise_samples() > 0 &&
988          last_mode_ != kModeDtmf) {
989        // Make a jump in timestamp due to the recently played comfort noise.
990        uint32_t timestamp_jump = decision_logic_->generated_noise_samples();
991        sync_buffer_->IncreaseEndTimestamp(timestamp_jump);
992        timestamp_ += timestamp_jump;
993      }
994      decision_logic_->set_generated_noise_samples(0);
995      return 0;
996    }
997    case kAccelerate: {
998      // In order to do a accelerate we need at least 30 ms of audio data.
999      if (samples_left >= samples_30_ms) {
1000        // Already have enough data, so we do not need to extract any more.
1001        decision_logic_->set_sample_memory(samples_left);
1002        decision_logic_->set_prev_time_scale(true);
1003        return 0;
1004      } else if (samples_left >= samples_10_ms &&
1005          decoder_frame_length_ >= samples_30_ms) {
1006        // Avoid decoding more data as it might overflow the playout buffer.
1007        *operation = kNormal;
1008        return 0;
1009      } else if (samples_left < samples_20_ms &&
1010          decoder_frame_length_ < samples_30_ms) {
1011        // Build up decoded data by decoding at least 20 ms of audio data. Do
1012        // not perform accelerate yet, but wait until we only need to do one
1013        // decoding.
1014        required_samples = 2 * output_size_samples_;
1015        *operation = kNormal;
1016      }
1017      // If none of the above is true, we have one of two possible situations:
1018      // (1) 20 ms <= samples_left < 30 ms and decoder_frame_length_ < 30 ms; or
1019      // (2) samples_left < 10 ms and decoder_frame_length_ >= 30 ms.
1020      // In either case, we move on with the accelerate decision, and decode one
1021      // frame now.
1022      break;
1023    }
1024    case kPreemptiveExpand: {
1025      // In order to do a preemptive expand we need at least 30 ms of decoded
1026      // audio data.
1027      if ((samples_left >= samples_30_ms) ||
1028          (samples_left >= samples_10_ms &&
1029              decoder_frame_length_ >= samples_30_ms)) {
1030        // Already have enough data, so we do not need to extract any more.
1031        // Or, avoid decoding more data as it might overflow the playout buffer.
1032        // Still try preemptive expand, though.
1033        decision_logic_->set_sample_memory(samples_left);
1034        decision_logic_->set_prev_time_scale(true);
1035        return 0;
1036      }
1037      if (samples_left < samples_20_ms &&
1038          decoder_frame_length_ < samples_30_ms) {
1039        // Build up decoded data by decoding at least 20 ms of audio data.
1040        // Still try to perform preemptive expand.
1041        required_samples = 2 * output_size_samples_;
1042      }
1043      // Move on with the preemptive expand decision.
1044      break;
1045    }
1046    default: {
1047      // Do nothing.
1048    }
1049  }
1050
1051  // Get packets from buffer.
1052  int extracted_samples = 0;
1053  if (header &&
1054      *operation != kAlternativePlc &&
1055      *operation != kAlternativePlcIncreaseTimestamp &&
1056      *operation != kAudioRepetition &&
1057      *operation != kAudioRepetitionIncreaseTimestamp) {
1058    sync_buffer_->IncreaseEndTimestamp(header->timestamp - end_timestamp);
1059    if (decision_logic_->CngOff()) {
1060      // Adjustment of timestamp only corresponds to an actual packet loss
1061      // if comfort noise is not played. If comfort noise was just played,
1062      // this adjustment of timestamp is only done to get back in sync with the
1063      // stream timestamp; no loss to report.
1064      stats_.LostSamples(header->timestamp - end_timestamp);
1065    }
1066
1067    if (*operation != kRfc3389Cng) {
1068      // We are about to decode and use a non-CNG packet.
1069      decision_logic_->SetCngOff();
1070    }
1071    // Reset CNG timestamp as a new packet will be delivered.
1072    // (Also if this is a CNG packet, since playedOutTS is updated.)
1073    decision_logic_->set_generated_noise_samples(0);
1074
1075    extracted_samples = ExtractPackets(required_samples, packet_list);
1076    if (extracted_samples < 0) {
1077      LOG_F(LS_WARNING) << "Failed to extract packets from buffer.";
1078      return kPacketBufferCorruption;
1079    }
1080  }
1081
1082  if (*operation == kAccelerate ||
1083      *operation == kPreemptiveExpand) {
1084    decision_logic_->set_sample_memory(samples_left + extracted_samples);
1085    decision_logic_->set_prev_time_scale(true);
1086  }
1087
1088  if (*operation == kAccelerate) {
1089    // Check that we have enough data (30ms) to do accelerate.
1090    if (extracted_samples + samples_left < samples_30_ms) {
1091      // TODO(hlundin): Write test for this.
1092      // Not enough, do normal operation instead.
1093      *operation = kNormal;
1094    }
1095  }
1096
1097  timestamp_ = end_timestamp;
1098  return 0;
1099}
1100
1101int NetEqImpl::Decode(PacketList* packet_list, Operations* operation,
1102                      int* decoded_length,
1103                      AudioDecoder::SpeechType* speech_type) {
1104  *speech_type = AudioDecoder::kSpeech;
1105  AudioDecoder* decoder = NULL;
1106  if (!packet_list->empty()) {
1107    const Packet* packet = packet_list->front();
1108    int payload_type = packet->header.payloadType;
1109    if (!decoder_database_->IsComfortNoise(payload_type)) {
1110      decoder = decoder_database_->GetDecoder(payload_type);
1111      assert(decoder);
1112      if (!decoder) {
1113        LOG_FERR1(LS_WARNING, GetDecoder, payload_type);
1114        PacketBuffer::DeleteAllPackets(packet_list);
1115        return kDecoderNotFound;
1116      }
1117      bool decoder_changed;
1118      decoder_database_->SetActiveDecoder(payload_type, &decoder_changed);
1119      if (decoder_changed) {
1120        // We have a new decoder. Re-init some values.
1121        const DecoderDatabase::DecoderInfo* decoder_info = decoder_database_
1122            ->GetDecoderInfo(payload_type);
1123        assert(decoder_info);
1124        if (!decoder_info) {
1125          LOG_FERR1(LS_WARNING, GetDecoderInfo, payload_type);
1126          PacketBuffer::DeleteAllPackets(packet_list);
1127          return kDecoderNotFound;
1128        }
1129        // We should have correct sampling rate and number of channels. They
1130        // are set when packets are inserted.
1131        if (decoder_info->fs_hz != fs_hz_ ||
1132            decoder->channels() != algorithm_buffer_->Channels()) {
1133          LOG_F(LS_ERROR) << "Sampling rate or number of channels mismatch.";
1134          assert(false);
1135          SetSampleRateAndChannels(decoder_info->fs_hz, decoder->channels());
1136        }
1137        sync_buffer_->set_end_timestamp(timestamp_);
1138        playout_timestamp_ = timestamp_;
1139      }
1140    }
1141  }
1142
1143  if (reset_decoder_) {
1144    // TODO(hlundin): Write test for this.
1145    // Reset decoder.
1146    if (decoder) {
1147      decoder->Init();
1148    }
1149    // Reset comfort noise decoder.
1150    AudioDecoder* cng_decoder = decoder_database_->GetActiveCngDecoder();
1151    if (cng_decoder) {
1152      cng_decoder->Init();
1153    }
1154    reset_decoder_ = false;
1155  }
1156
1157#ifdef LEGACY_BITEXACT
1158  // Due to a bug in old SignalMCU, it could happen that CNG operation was
1159  // decided, but a speech packet was provided. The speech packet will be used
1160  // to update the comfort noise decoder, as if it was a SID frame, which is
1161  // clearly wrong.
1162  if (*operation == kRfc3389Cng) {
1163    return 0;
1164  }
1165#endif
1166
1167  *decoded_length = 0;
1168  // Update codec-internal PLC state.
1169  if ((*operation == kMerge) && decoder && decoder->HasDecodePlc()) {
1170    decoder->DecodePlc(1, &decoded_buffer_[*decoded_length]);
1171  }
1172
1173  int return_value = DecodeLoop(packet_list, operation, decoder,
1174                                decoded_length, speech_type);
1175
1176  if (*decoded_length < 0) {
1177    // Error returned from the decoder.
1178    *decoded_length = 0;
1179    sync_buffer_->IncreaseEndTimestamp(decoder_frame_length_);
1180    int error_code = 0;
1181    if (decoder)
1182      error_code = decoder->ErrorCode();
1183    if (error_code != 0) {
1184      // Got some error code from the decoder.
1185      decoder_error_code_ = error_code;
1186      return_value = kDecoderErrorCode;
1187    } else {
1188      // Decoder does not implement error codes. Return generic error.
1189      return_value = kOtherDecoderError;
1190    }
1191    LOG_FERR2(LS_WARNING, DecodeLoop, error_code, packet_list->size());
1192    *operation = kExpand;  // Do expansion to get data instead.
1193  }
1194  if (*speech_type != AudioDecoder::kComfortNoise) {
1195    // Don't increment timestamp if codec returned CNG speech type
1196    // since in this case, the we will increment the CNGplayedTS counter.
1197    // Increase with number of samples per channel.
1198    assert(*decoded_length == 0 ||
1199           (decoder && decoder->channels() == sync_buffer_->Channels()));
1200    sync_buffer_->IncreaseEndTimestamp(
1201        *decoded_length / static_cast<int>(sync_buffer_->Channels()));
1202  }
1203  return return_value;
1204}
1205
1206int NetEqImpl::DecodeLoop(PacketList* packet_list, Operations* operation,
1207                          AudioDecoder* decoder, int* decoded_length,
1208                          AudioDecoder::SpeechType* speech_type) {
1209  Packet* packet = NULL;
1210  if (!packet_list->empty()) {
1211    packet = packet_list->front();
1212  }
1213  // Do decoding.
1214  while (packet &&
1215      !decoder_database_->IsComfortNoise(packet->header.payloadType)) {
1216    assert(decoder);  // At this point, we must have a decoder object.
1217    // The number of channels in the |sync_buffer_| should be the same as the
1218    // number decoder channels.
1219    assert(sync_buffer_->Channels() == decoder->channels());
1220    assert(decoded_buffer_length_ >= kMaxFrameSize * decoder->channels());
1221    assert(*operation == kNormal || *operation == kAccelerate ||
1222           *operation == kMerge || *operation == kPreemptiveExpand);
1223    packet_list->pop_front();
1224    int payload_length = packet->payload_length;
1225    int16_t decode_length;
1226    if (packet->sync_packet) {
1227      // Decode to silence with the same frame size as the last decode.
1228      LOG(LS_VERBOSE) << "Decoding sync-packet: " <<
1229          " ts=" << packet->header.timestamp <<
1230          ", sn=" << packet->header.sequenceNumber <<
1231          ", pt=" << static_cast<int>(packet->header.payloadType) <<
1232          ", ssrc=" << packet->header.ssrc <<
1233          ", len=" << packet->payload_length;
1234      memset(&decoded_buffer_[*decoded_length], 0, decoder_frame_length_ *
1235             decoder->channels() * sizeof(decoded_buffer_[0]));
1236      decode_length = decoder_frame_length_;
1237    } else if (!packet->primary) {
1238      // This is a redundant payload; call the special decoder method.
1239      LOG(LS_VERBOSE) << "Decoding packet (redundant):" <<
1240          " ts=" << packet->header.timestamp <<
1241          ", sn=" << packet->header.sequenceNumber <<
1242          ", pt=" << static_cast<int>(packet->header.payloadType) <<
1243          ", ssrc=" << packet->header.ssrc <<
1244          ", len=" << packet->payload_length;
1245      decode_length = decoder->DecodeRedundant(
1246          packet->payload, packet->payload_length,
1247          &decoded_buffer_[*decoded_length], speech_type);
1248    } else {
1249      LOG(LS_VERBOSE) << "Decoding packet: ts=" << packet->header.timestamp <<
1250          ", sn=" << packet->header.sequenceNumber <<
1251          ", pt=" << static_cast<int>(packet->header.payloadType) <<
1252          ", ssrc=" << packet->header.ssrc <<
1253          ", len=" << packet->payload_length;
1254      decode_length = decoder->Decode(packet->payload,
1255                                      packet->payload_length,
1256                                      &decoded_buffer_[*decoded_length],
1257                                      speech_type);
1258    }
1259
1260    delete[] packet->payload;
1261    delete packet;
1262    packet = NULL;
1263    if (decode_length > 0) {
1264      *decoded_length += decode_length;
1265      // Update |decoder_frame_length_| with number of samples per channel.
1266      decoder_frame_length_ = decode_length /
1267          static_cast<int>(decoder->channels());
1268      LOG(LS_VERBOSE) << "Decoded " << decode_length << " samples (" <<
1269          decoder->channels() << " channel(s) -> " << decoder_frame_length_ <<
1270          " samples per channel)";
1271    } else if (decode_length < 0) {
1272      // Error.
1273      LOG_FERR2(LS_WARNING, Decode, decode_length, payload_length);
1274      *decoded_length = -1;
1275      PacketBuffer::DeleteAllPackets(packet_list);
1276      break;
1277    }
1278    if (*decoded_length > static_cast<int>(decoded_buffer_length_)) {
1279      // Guard against overflow.
1280      LOG_F(LS_WARNING) << "Decoded too much.";
1281      PacketBuffer::DeleteAllPackets(packet_list);
1282      return kDecodedTooMuch;
1283    }
1284    if (!packet_list->empty()) {
1285      packet = packet_list->front();
1286    } else {
1287      packet = NULL;
1288    }
1289  }  // End of decode loop.
1290
1291  // If the list is not empty at this point, either a decoding error terminated
1292  // the while-loop, or list must hold exactly one CNG packet.
1293  assert(packet_list->empty() || *decoded_length < 0 ||
1294         (packet_list->size() == 1 && packet &&
1295             decoder_database_->IsComfortNoise(packet->header.payloadType)));
1296  return 0;
1297}
1298
1299void NetEqImpl::DoNormal(const int16_t* decoded_buffer, size_t decoded_length,
1300                         AudioDecoder::SpeechType speech_type, bool play_dtmf) {
1301  assert(normal_.get());
1302  assert(mute_factor_array_.get());
1303  normal_->Process(decoded_buffer, decoded_length, last_mode_,
1304                   mute_factor_array_.get(), algorithm_buffer_.get());
1305  if (decoded_length != 0) {
1306    last_mode_ = kModeNormal;
1307  }
1308
1309  // If last packet was decoded as an inband CNG, set mode to CNG instead.
1310  if ((speech_type == AudioDecoder::kComfortNoise)
1311      || ((last_mode_ == kModeCodecInternalCng)
1312          && (decoded_length == 0))) {
1313    // TODO(hlundin): Remove second part of || statement above.
1314    last_mode_ = kModeCodecInternalCng;
1315  }
1316
1317  if (!play_dtmf) {
1318    dtmf_tone_generator_->Reset();
1319  }
1320}
1321
1322void NetEqImpl::DoMerge(int16_t* decoded_buffer, size_t decoded_length,
1323                        AudioDecoder::SpeechType speech_type, bool play_dtmf) {
1324  assert(mute_factor_array_.get());
1325  assert(merge_.get());
1326  int new_length = merge_->Process(decoded_buffer, decoded_length,
1327                                   mute_factor_array_.get(),
1328                                   algorithm_buffer_.get());
1329
1330  // Update in-call and post-call statistics.
1331  if (expand_->MuteFactor(0) == 0) {
1332    // Expand generates only noise.
1333    stats_.ExpandedNoiseSamples(new_length - static_cast<int>(decoded_length));
1334  } else {
1335    // Expansion generates more than only noise.
1336    stats_.ExpandedVoiceSamples(new_length - static_cast<int>(decoded_length));
1337  }
1338
1339  last_mode_ = kModeMerge;
1340  // If last packet was decoded as an inband CNG, set mode to CNG instead.
1341  if (speech_type == AudioDecoder::kComfortNoise) {
1342    last_mode_ = kModeCodecInternalCng;
1343  }
1344  expand_->Reset();
1345  if (!play_dtmf) {
1346    dtmf_tone_generator_->Reset();
1347  }
1348}
1349
1350int NetEqImpl::DoExpand(bool play_dtmf) {
1351  while ((sync_buffer_->FutureLength() - expand_->overlap_length()) <
1352      static_cast<size_t>(output_size_samples_)) {
1353    algorithm_buffer_->Clear();
1354    int return_value = expand_->Process(algorithm_buffer_.get());
1355    int length = static_cast<int>(algorithm_buffer_->Size());
1356
1357    // Update in-call and post-call statistics.
1358    if (expand_->MuteFactor(0) == 0) {
1359      // Expand operation generates only noise.
1360      stats_.ExpandedNoiseSamples(length);
1361    } else {
1362      // Expand operation generates more than only noise.
1363      stats_.ExpandedVoiceSamples(length);
1364    }
1365
1366    last_mode_ = kModeExpand;
1367
1368    if (return_value < 0) {
1369      return return_value;
1370    }
1371
1372    sync_buffer_->PushBack(*algorithm_buffer_);
1373    algorithm_buffer_->Clear();
1374  }
1375  if (!play_dtmf) {
1376    dtmf_tone_generator_->Reset();
1377  }
1378  return 0;
1379}
1380
1381int NetEqImpl::DoAccelerate(int16_t* decoded_buffer, size_t decoded_length,
1382                            AudioDecoder::SpeechType speech_type,
1383                            bool play_dtmf) {
1384  const size_t required_samples = 240 * fs_mult_;  // Must have 30 ms.
1385  size_t borrowed_samples_per_channel = 0;
1386  size_t num_channels = algorithm_buffer_->Channels();
1387  size_t decoded_length_per_channel = decoded_length / num_channels;
1388  if (decoded_length_per_channel < required_samples) {
1389    // Must move data from the |sync_buffer_| in order to get 30 ms.
1390    borrowed_samples_per_channel = static_cast<int>(required_samples -
1391        decoded_length_per_channel);
1392    memmove(&decoded_buffer[borrowed_samples_per_channel * num_channels],
1393            decoded_buffer,
1394            sizeof(int16_t) * decoded_length);
1395    sync_buffer_->ReadInterleavedFromEnd(borrowed_samples_per_channel,
1396                                         decoded_buffer);
1397    decoded_length = required_samples * num_channels;
1398  }
1399
1400  int16_t samples_removed;
1401  Accelerate::ReturnCodes return_code = accelerate_->Process(
1402      decoded_buffer, decoded_length, algorithm_buffer_.get(),
1403      &samples_removed);
1404  stats_.AcceleratedSamples(samples_removed);
1405  switch (return_code) {
1406    case Accelerate::kSuccess:
1407      last_mode_ = kModeAccelerateSuccess;
1408      break;
1409    case Accelerate::kSuccessLowEnergy:
1410      last_mode_ = kModeAccelerateLowEnergy;
1411      break;
1412    case Accelerate::kNoStretch:
1413      last_mode_ = kModeAccelerateFail;
1414      break;
1415    case Accelerate::kError:
1416      // TODO(hlundin): Map to kModeError instead?
1417      last_mode_ = kModeAccelerateFail;
1418      return kAccelerateError;
1419  }
1420
1421  if (borrowed_samples_per_channel > 0) {
1422    // Copy borrowed samples back to the |sync_buffer_|.
1423    size_t length = algorithm_buffer_->Size();
1424    if (length < borrowed_samples_per_channel) {
1425      // This destroys the beginning of the buffer, but will not cause any
1426      // problems.
1427      sync_buffer_->ReplaceAtIndex(*algorithm_buffer_,
1428                                   sync_buffer_->Size() -
1429                                   borrowed_samples_per_channel);
1430      sync_buffer_->PushFrontZeros(borrowed_samples_per_channel - length);
1431      algorithm_buffer_->PopFront(length);
1432      assert(algorithm_buffer_->Empty());
1433    } else {
1434      sync_buffer_->ReplaceAtIndex(*algorithm_buffer_,
1435                                   borrowed_samples_per_channel,
1436                                   sync_buffer_->Size() -
1437                                   borrowed_samples_per_channel);
1438      algorithm_buffer_->PopFront(borrowed_samples_per_channel);
1439    }
1440  }
1441
1442  // If last packet was decoded as an inband CNG, set mode to CNG instead.
1443  if (speech_type == AudioDecoder::kComfortNoise) {
1444    last_mode_ = kModeCodecInternalCng;
1445  }
1446  if (!play_dtmf) {
1447    dtmf_tone_generator_->Reset();
1448  }
1449  expand_->Reset();
1450  return 0;
1451}
1452
1453int NetEqImpl::DoPreemptiveExpand(int16_t* decoded_buffer,
1454                                  size_t decoded_length,
1455                                  AudioDecoder::SpeechType speech_type,
1456                                  bool play_dtmf) {
1457  const size_t required_samples = 240 * fs_mult_;  // Must have 30 ms.
1458  size_t num_channels = algorithm_buffer_->Channels();
1459  int borrowed_samples_per_channel = 0;
1460  int old_borrowed_samples_per_channel = 0;
1461  size_t decoded_length_per_channel = decoded_length / num_channels;
1462  if (decoded_length_per_channel < required_samples) {
1463    // Must move data from the |sync_buffer_| in order to get 30 ms.
1464    borrowed_samples_per_channel = static_cast<int>(required_samples -
1465        decoded_length_per_channel);
1466    // Calculate how many of these were already played out.
1467    old_borrowed_samples_per_channel = static_cast<int>(
1468        borrowed_samples_per_channel - sync_buffer_->FutureLength());
1469    old_borrowed_samples_per_channel = std::max(
1470        0, old_borrowed_samples_per_channel);
1471    memmove(&decoded_buffer[borrowed_samples_per_channel * num_channels],
1472            decoded_buffer,
1473            sizeof(int16_t) * decoded_length);
1474    sync_buffer_->ReadInterleavedFromEnd(borrowed_samples_per_channel,
1475                                         decoded_buffer);
1476    decoded_length = required_samples * num_channels;
1477  }
1478
1479  int16_t samples_added;
1480  PreemptiveExpand::ReturnCodes return_code = preemptive_expand_->Process(
1481      decoded_buffer, static_cast<int>(decoded_length),
1482      old_borrowed_samples_per_channel,
1483      algorithm_buffer_.get(), &samples_added);
1484  stats_.PreemptiveExpandedSamples(samples_added);
1485  switch (return_code) {
1486    case PreemptiveExpand::kSuccess:
1487      last_mode_ = kModePreemptiveExpandSuccess;
1488      break;
1489    case PreemptiveExpand::kSuccessLowEnergy:
1490      last_mode_ = kModePreemptiveExpandLowEnergy;
1491      break;
1492    case PreemptiveExpand::kNoStretch:
1493      last_mode_ = kModePreemptiveExpandFail;
1494      break;
1495    case PreemptiveExpand::kError:
1496      // TODO(hlundin): Map to kModeError instead?
1497      last_mode_ = kModePreemptiveExpandFail;
1498      return kPreemptiveExpandError;
1499  }
1500
1501  if (borrowed_samples_per_channel > 0) {
1502    // Copy borrowed samples back to the |sync_buffer_|.
1503    sync_buffer_->ReplaceAtIndex(
1504        *algorithm_buffer_, borrowed_samples_per_channel,
1505        sync_buffer_->Size() - borrowed_samples_per_channel);
1506    algorithm_buffer_->PopFront(borrowed_samples_per_channel);
1507  }
1508
1509  // If last packet was decoded as an inband CNG, set mode to CNG instead.
1510  if (speech_type == AudioDecoder::kComfortNoise) {
1511    last_mode_ = kModeCodecInternalCng;
1512  }
1513  if (!play_dtmf) {
1514    dtmf_tone_generator_->Reset();
1515  }
1516  expand_->Reset();
1517  return 0;
1518}
1519
1520int NetEqImpl::DoRfc3389Cng(PacketList* packet_list, bool play_dtmf) {
1521  if (!packet_list->empty()) {
1522    // Must have exactly one SID frame at this point.
1523    assert(packet_list->size() == 1);
1524    Packet* packet = packet_list->front();
1525    packet_list->pop_front();
1526    if (!decoder_database_->IsComfortNoise(packet->header.payloadType)) {
1527#ifdef LEGACY_BITEXACT
1528      // This can happen due to a bug in GetDecision. Change the payload type
1529      // to a CNG type, and move on. Note that this means that we are in fact
1530      // sending a non-CNG payload to the comfort noise decoder for decoding.
1531      // Clearly wrong, but will maintain bit-exactness with legacy.
1532      if (fs_hz_ == 8000) {
1533        packet->header.payloadType =
1534            decoder_database_->GetRtpPayloadType(kDecoderCNGnb);
1535      } else if (fs_hz_ == 16000) {
1536        packet->header.payloadType =
1537            decoder_database_->GetRtpPayloadType(kDecoderCNGwb);
1538      } else if (fs_hz_ == 32000) {
1539        packet->header.payloadType =
1540            decoder_database_->GetRtpPayloadType(kDecoderCNGswb32kHz);
1541      } else if (fs_hz_ == 48000) {
1542        packet->header.payloadType =
1543            decoder_database_->GetRtpPayloadType(kDecoderCNGswb48kHz);
1544      }
1545      assert(decoder_database_->IsComfortNoise(packet->header.payloadType));
1546#else
1547      LOG(LS_ERROR) << "Trying to decode non-CNG payload as CNG.";
1548      return kOtherError;
1549#endif
1550    }
1551    // UpdateParameters() deletes |packet|.
1552    if (comfort_noise_->UpdateParameters(packet) ==
1553        ComfortNoise::kInternalError) {
1554      LOG_FERR0(LS_WARNING, UpdateParameters);
1555      algorithm_buffer_->Zeros(output_size_samples_);
1556      return -comfort_noise_->internal_error_code();
1557    }
1558  }
1559  int cn_return = comfort_noise_->Generate(output_size_samples_,
1560                                           algorithm_buffer_.get());
1561  expand_->Reset();
1562  last_mode_ = kModeRfc3389Cng;
1563  if (!play_dtmf) {
1564    dtmf_tone_generator_->Reset();
1565  }
1566  if (cn_return == ComfortNoise::kInternalError) {
1567    LOG_FERR1(LS_WARNING, comfort_noise_->Generate, cn_return);
1568    decoder_error_code_ = comfort_noise_->internal_error_code();
1569    return kComfortNoiseErrorCode;
1570  } else if (cn_return == ComfortNoise::kUnknownPayloadType) {
1571    LOG_FERR1(LS_WARNING, comfort_noise_->Generate, cn_return);
1572    return kUnknownRtpPayloadType;
1573  }
1574  return 0;
1575}
1576
1577void NetEqImpl::DoCodecInternalCng() {
1578  int length = 0;
1579  // TODO(hlundin): Will probably need a longer buffer for multi-channel.
1580  int16_t decoded_buffer[kMaxFrameSize];
1581  AudioDecoder* decoder = decoder_database_->GetActiveDecoder();
1582  if (decoder) {
1583    const uint8_t* dummy_payload = NULL;
1584    AudioDecoder::SpeechType speech_type;
1585    length = decoder->Decode(dummy_payload, 0, decoded_buffer, &speech_type);
1586  }
1587  assert(mute_factor_array_.get());
1588  normal_->Process(decoded_buffer, length, last_mode_, mute_factor_array_.get(),
1589                   algorithm_buffer_.get());
1590  last_mode_ = kModeCodecInternalCng;
1591  expand_->Reset();
1592}
1593
1594int NetEqImpl::DoDtmf(const DtmfEvent& dtmf_event, bool* play_dtmf) {
1595  // This block of the code and the block further down, handling |dtmf_switch|
1596  // are commented out. Otherwise playing out-of-band DTMF would fail in VoE
1597  // test, DtmfTest.ManualSuccessfullySendsOutOfBandTelephoneEvents. This is
1598  // equivalent to |dtmf_switch| always be false.
1599  //
1600  // See http://webrtc-codereview.appspot.com/1195004/ for discussion
1601  // On this issue. This change might cause some glitches at the point of
1602  // switch from audio to DTMF. Issue 1545 is filed to track this.
1603  //
1604  //  bool dtmf_switch = false;
1605  //  if ((last_mode_ != kModeDtmf) && dtmf_tone_generator_->initialized()) {
1606  //    // Special case; see below.
1607  //    // We must catch this before calling Generate, since |initialized| is
1608  //    // modified in that call.
1609  //    dtmf_switch = true;
1610  //  }
1611
1612  int dtmf_return_value = 0;
1613  if (!dtmf_tone_generator_->initialized()) {
1614    // Initialize if not already done.
1615    dtmf_return_value = dtmf_tone_generator_->Init(fs_hz_, dtmf_event.event_no,
1616                                                   dtmf_event.volume);
1617  }
1618
1619  if (dtmf_return_value == 0) {
1620    // Generate DTMF signal.
1621    dtmf_return_value = dtmf_tone_generator_->Generate(output_size_samples_,
1622                                                       algorithm_buffer_.get());
1623  }
1624
1625  if (dtmf_return_value < 0) {
1626    algorithm_buffer_->Zeros(output_size_samples_);
1627    return dtmf_return_value;
1628  }
1629
1630  //  if (dtmf_switch) {
1631  //    // This is the special case where the previous operation was DTMF
1632  //    // overdub, but the current instruction is "regular" DTMF. We must make
1633  //    // sure that the DTMF does not have any discontinuities. The first DTMF
1634  //    // sample that we generate now must be played out immediately, therefore
1635  //    // it must be copied to the speech buffer.
1636  //    // TODO(hlundin): This code seems incorrect. (Legacy.) Write test and
1637  //    // verify correct operation.
1638  //    assert(false);
1639  //    // Must generate enough data to replace all of the |sync_buffer_|
1640  //    // "future".
1641  //    int required_length = sync_buffer_->FutureLength();
1642  //    assert(dtmf_tone_generator_->initialized());
1643  //    dtmf_return_value = dtmf_tone_generator_->Generate(required_length,
1644  //                                                       algorithm_buffer_);
1645  //    assert((size_t) required_length == algorithm_buffer_->Size());
1646  //    if (dtmf_return_value < 0) {
1647  //      algorithm_buffer_->Zeros(output_size_samples_);
1648  //      return dtmf_return_value;
1649  //    }
1650  //
1651  //    // Overwrite the "future" part of the speech buffer with the new DTMF
1652  //    // data.
1653  //    // TODO(hlundin): It seems that this overwriting has gone lost.
1654  //    // Not adapted for multi-channel yet.
1655  //    assert(algorithm_buffer_->Channels() == 1);
1656  //    if (algorithm_buffer_->Channels() != 1) {
1657  //      LOG(LS_WARNING) << "DTMF not supported for more than one channel";
1658  //      return kStereoNotSupported;
1659  //    }
1660  //    // Shuffle the remaining data to the beginning of algorithm buffer.
1661  //    algorithm_buffer_->PopFront(sync_buffer_->FutureLength());
1662  //  }
1663
1664  sync_buffer_->IncreaseEndTimestamp(output_size_samples_);
1665  expand_->Reset();
1666  last_mode_ = kModeDtmf;
1667
1668  // Set to false because the DTMF is already in the algorithm buffer.
1669  *play_dtmf = false;
1670  return 0;
1671}
1672
1673void NetEqImpl::DoAlternativePlc(bool increase_timestamp) {
1674  AudioDecoder* decoder = decoder_database_->GetActiveDecoder();
1675  int length;
1676  if (decoder && decoder->HasDecodePlc()) {
1677    // Use the decoder's packet-loss concealment.
1678    // TODO(hlundin): Will probably need a longer buffer for multi-channel.
1679    int16_t decoded_buffer[kMaxFrameSize];
1680    length = decoder->DecodePlc(1, decoded_buffer);
1681    if (length > 0) {
1682      algorithm_buffer_->PushBackInterleaved(decoded_buffer, length);
1683    } else {
1684      length = 0;
1685    }
1686  } else {
1687    // Do simple zero-stuffing.
1688    length = output_size_samples_;
1689    algorithm_buffer_->Zeros(length);
1690    // By not advancing the timestamp, NetEq inserts samples.
1691    stats_.AddZeros(length);
1692  }
1693  if (increase_timestamp) {
1694    sync_buffer_->IncreaseEndTimestamp(length);
1695  }
1696  expand_->Reset();
1697}
1698
1699int NetEqImpl::DtmfOverdub(const DtmfEvent& dtmf_event, size_t num_channels,
1700                           int16_t* output) const {
1701  size_t out_index = 0;
1702  int overdub_length = output_size_samples_;  // Default value.
1703
1704  if (sync_buffer_->dtmf_index() > sync_buffer_->next_index()) {
1705    // Special operation for transition from "DTMF only" to "DTMF overdub".
1706    out_index = std::min(
1707        sync_buffer_->dtmf_index() - sync_buffer_->next_index(),
1708        static_cast<size_t>(output_size_samples_));
1709    overdub_length = output_size_samples_ - static_cast<int>(out_index);
1710  }
1711
1712  AudioMultiVector dtmf_output(num_channels);
1713  int dtmf_return_value = 0;
1714  if (!dtmf_tone_generator_->initialized()) {
1715    dtmf_return_value = dtmf_tone_generator_->Init(fs_hz_, dtmf_event.event_no,
1716                                                   dtmf_event.volume);
1717  }
1718  if (dtmf_return_value == 0) {
1719    dtmf_return_value = dtmf_tone_generator_->Generate(overdub_length,
1720                                                       &dtmf_output);
1721    assert((size_t) overdub_length == dtmf_output.Size());
1722  }
1723  dtmf_output.ReadInterleaved(overdub_length, &output[out_index]);
1724  return dtmf_return_value < 0 ? dtmf_return_value : 0;
1725}
1726
1727int NetEqImpl::ExtractPackets(int required_samples, PacketList* packet_list) {
1728  bool first_packet = true;
1729  uint8_t prev_payload_type = 0;
1730  uint32_t prev_timestamp = 0;
1731  uint16_t prev_sequence_number = 0;
1732  bool next_packet_available = false;
1733
1734  const RTPHeader* header = packet_buffer_->NextRtpHeader();
1735  assert(header);
1736  if (!header) {
1737    return -1;
1738  }
1739  uint32_t first_timestamp = header->timestamp;
1740  int extracted_samples = 0;
1741
1742  // Packet extraction loop.
1743  do {
1744    timestamp_ = header->timestamp;
1745    int discard_count = 0;
1746    Packet* packet = packet_buffer_->GetNextPacket(&discard_count);
1747    // |header| may be invalid after the |packet_buffer_| operation.
1748    header = NULL;
1749    if (!packet) {
1750      LOG_FERR1(LS_ERROR, GetNextPacket, discard_count) <<
1751          "Should always be able to extract a packet here";
1752      assert(false);  // Should always be able to extract a packet here.
1753      return -1;
1754    }
1755    stats_.PacketsDiscarded(discard_count);
1756    // Store waiting time in ms; packets->waiting_time is in "output blocks".
1757    stats_.StoreWaitingTime(packet->waiting_time * kOutputSizeMs);
1758    assert(packet->payload_length > 0);
1759    packet_list->push_back(packet);  // Store packet in list.
1760
1761    if (first_packet) {
1762      first_packet = false;
1763      decoded_packet_sequence_number_ = prev_sequence_number =
1764          packet->header.sequenceNumber;
1765      decoded_packet_timestamp_ = prev_timestamp = packet->header.timestamp;
1766      prev_payload_type = packet->header.payloadType;
1767    }
1768
1769    // Store number of extracted samples.
1770    int packet_duration = 0;
1771    AudioDecoder* decoder = decoder_database_->GetDecoder(
1772        packet->header.payloadType);
1773    if (decoder) {
1774      packet_duration = packet->sync_packet ? decoder_frame_length_ :
1775          decoder->PacketDuration(packet->payload, packet->payload_length);
1776    } else {
1777      LOG_FERR1(LS_WARNING, GetDecoder, packet->header.payloadType) <<
1778          "Could not find a decoder for a packet about to be extracted.";
1779      assert(false);
1780    }
1781    if (packet_duration <= 0) {
1782      // Decoder did not return a packet duration. Assume that the packet
1783      // contains the same number of samples as the previous one.
1784      packet_duration = decoder_frame_length_;
1785    }
1786    extracted_samples = packet->header.timestamp - first_timestamp +
1787        packet_duration;
1788
1789    // Check what packet is available next.
1790    header = packet_buffer_->NextRtpHeader();
1791    next_packet_available = false;
1792    if (header && prev_payload_type == header->payloadType) {
1793      int16_t seq_no_diff = header->sequenceNumber - prev_sequence_number;
1794      int32_t ts_diff = header->timestamp - prev_timestamp;
1795      if (seq_no_diff == 1 ||
1796          (seq_no_diff == 0 && ts_diff == decoder_frame_length_)) {
1797        // The next sequence number is available, or the next part of a packet
1798        // that was split into pieces upon insertion.
1799        next_packet_available = true;
1800      }
1801      prev_sequence_number = header->sequenceNumber;
1802    }
1803  } while (extracted_samples < required_samples && next_packet_available);
1804
1805  return extracted_samples;
1806}
1807
1808void NetEqImpl::SetSampleRateAndChannels(int fs_hz, size_t channels) {
1809  LOG_API2(fs_hz, channels);
1810  // TODO(hlundin): Change to an enumerator and skip assert.
1811  assert(fs_hz == 8000 || fs_hz == 16000 || fs_hz ==  32000 || fs_hz == 48000);
1812  assert(channels > 0);
1813
1814  fs_hz_ = fs_hz;
1815  fs_mult_ = fs_hz / 8000;
1816  output_size_samples_ = kOutputSizeMs * 8 * fs_mult_;
1817  decoder_frame_length_ = 3 * output_size_samples_;  // Initialize to 30ms.
1818
1819  last_mode_ = kModeNormal;
1820
1821  // Create a new array of mute factors and set all to 1.
1822  mute_factor_array_.reset(new int16_t[channels]);
1823  for (size_t i = 0; i < channels; ++i) {
1824    mute_factor_array_[i] = 16384;  // 1.0 in Q14.
1825  }
1826
1827  // Reset comfort noise decoder, if there is one active.
1828  AudioDecoder* cng_decoder = decoder_database_->GetActiveCngDecoder();
1829  if (cng_decoder) {
1830    cng_decoder->Init();
1831  }
1832
1833  // Reinit post-decode VAD with new sample rate.
1834  assert(vad_.get());  // Cannot be NULL here.
1835  vad_->Init();
1836
1837  // Delete algorithm buffer and create a new one.
1838  algorithm_buffer_.reset(new AudioMultiVector(channels));
1839
1840  // Delete sync buffer and create a new one.
1841  sync_buffer_.reset(new SyncBuffer(channels, kSyncBufferSize * fs_mult_));
1842
1843
1844  // Delete BackgroundNoise object and create a new one, while preserving its
1845  // mode.
1846  NetEqBackgroundNoiseMode current_mode = kBgnOn;
1847  if (background_noise_.get())
1848    current_mode = background_noise_->mode();
1849  background_noise_.reset(new BackgroundNoise(channels));
1850  background_noise_->set_mode(current_mode);
1851
1852  // Reset random vector.
1853  random_vector_.Reset();
1854
1855  // Delete Expand object and create a new one.
1856  expand_.reset(new Expand(background_noise_.get(), sync_buffer_.get(),
1857                           &random_vector_, fs_hz, channels));
1858  // Move index so that we create a small set of future samples (all 0).
1859  sync_buffer_->set_next_index(sync_buffer_->next_index() -
1860                               expand_->overlap_length());
1861
1862  normal_.reset(new Normal(fs_hz, decoder_database_.get(), *background_noise_,
1863                           expand_.get()));
1864  merge_.reset(new Merge(fs_hz, channels, expand_.get(), sync_buffer_.get()));
1865  accelerate_.reset(new Accelerate(fs_hz, channels, *background_noise_));
1866  preemptive_expand_.reset(new PreemptiveExpand(fs_hz, channels,
1867                                                *background_noise_));
1868
1869  // Delete ComfortNoise object and create a new one.
1870  comfort_noise_.reset(new ComfortNoise(fs_hz, decoder_database_.get(),
1871                                        sync_buffer_.get()));
1872
1873  // Verify that |decoded_buffer_| is long enough.
1874  if (decoded_buffer_length_ < kMaxFrameSize * channels) {
1875    // Reallocate to larger size.
1876    decoded_buffer_length_ = kMaxFrameSize * channels;
1877    decoded_buffer_.reset(new int16_t[decoded_buffer_length_]);
1878  }
1879
1880  // Communicate new sample rate and output size to DecisionLogic object.
1881  assert(decision_logic_.get());
1882  decision_logic_->SetSampleRate(fs_hz_, output_size_samples_);
1883}
1884
1885NetEqOutputType NetEqImpl::LastOutputType() {
1886  assert(vad_.get());
1887  assert(expand_.get());
1888  if (last_mode_ == kModeCodecInternalCng || last_mode_ == kModeRfc3389Cng) {
1889    return kOutputCNG;
1890  } else if (vad_->running() && !vad_->active_speech()) {
1891    return kOutputVADPassive;
1892  } else if (last_mode_ == kModeExpand && expand_->MuteFactor(0) == 0) {
1893    // Expand mode has faded down to background noise only (very long expand).
1894    return kOutputPLCtoCNG;
1895  } else if (last_mode_ == kModeExpand) {
1896    return kOutputPLC;
1897  } else {
1898    return kOutputNormal;
1899  }
1900}
1901
1902}  // namespace webrtc
Note: See TracBrowser for help on using the repository browser.