Note: We no longer publish the latest version of our code here. We primarily use a kumc-bmi github organization. The heron ETL repository, in particular, is not public. Peers in the informatics community should see MultiSiteDev for details on requesting access.

source: webrtc/webrtc/modules/audio_coding/neteq4/neteq_impl.h @ 0:4bda6873e34c

pub_scrub_3792 tip
Last change on this file since 0:4bda6873e34c was 0:4bda6873e34c, checked in by Michael Prittie <mprittie@…>, 6 years ago

Scrubbed password for publication.

File size: 15.1 KB
Line 
1/*
2 *  Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
3 *
4 *  Use of this source code is governed by a BSD-style license
5 *  that can be found in the LICENSE file in the root of the source
6 *  tree. An additional intellectual property rights grant can be found
7 *  in the file PATENTS.  All contributing project authors may
8 *  be found in the AUTHORS file in the root of the source tree.
9 */
10
11#ifndef WEBRTC_MODULES_AUDIO_CODING_NETEQ4_NETEQ_IMPL_H_
12#define WEBRTC_MODULES_AUDIO_CODING_NETEQ4_NETEQ_IMPL_H_
13
14#include <vector>
15
16#include "webrtc/modules/audio_coding/neteq4/audio_multi_vector.h"
17#include "webrtc/modules/audio_coding/neteq4/defines.h"
18#include "webrtc/modules/audio_coding/neteq4/interface/neteq.h"
19#include "webrtc/modules/audio_coding/neteq4/packet.h"  // Declare PacketList.
20#include "webrtc/modules/audio_coding/neteq4/random_vector.h"
21#include "webrtc/modules/audio_coding/neteq4/rtcp.h"
22#include "webrtc/modules/audio_coding/neteq4/statistics_calculator.h"
23#include "webrtc/system_wrappers/interface/constructor_magic.h"
24#include "webrtc/system_wrappers/interface/scoped_ptr.h"
25#include "webrtc/typedefs.h"
26
27namespace webrtc {
28
29// Forward declarations.
30class Accelerate;
31class BackgroundNoise;
32class BufferLevelFilter;
33class ComfortNoise;
34class CriticalSectionWrapper;
35class DecisionLogic;
36class DecoderDatabase;
37class DelayManager;
38class DelayPeakDetector;
39class DtmfBuffer;
40class DtmfToneGenerator;
41class Expand;
42class Merge;
43class Normal;
44class PacketBuffer;
45class PayloadSplitter;
46class PostDecodeVad;
47class PreemptiveExpand;
48class RandomVector;
49class SyncBuffer;
50class TimestampScaler;
51struct DtmfEvent;
52
53class NetEqImpl : public webrtc::NetEq {
54 public:
55  // Creates a new NetEqImpl object. The object will assume ownership of all
56  // injected dependencies, and will delete them when done.
57  NetEqImpl(int fs,
58            BufferLevelFilter* buffer_level_filter,
59            DecoderDatabase* decoder_database,
60            DelayManager* delay_manager,
61            DelayPeakDetector* delay_peak_detector,
62            DtmfBuffer* dtmf_buffer,
63            DtmfToneGenerator* dtmf_tone_generator,
64            PacketBuffer* packet_buffer,
65            PayloadSplitter* payload_splitter,
66            TimestampScaler* timestamp_scaler);
67
68  virtual ~NetEqImpl();
69
70  // Inserts a new packet into NetEq. The |receive_timestamp| is an indication
71  // of the time when the packet was received, and should be measured with
72  // the same tick rate as the RTP timestamp of the current payload.
73  // Returns 0 on success, -1 on failure.
74  virtual int InsertPacket(const WebRtcRTPHeader& rtp_header,
75                           const uint8_t* payload,
76                           int length_bytes,
77                           uint32_t receive_timestamp);
78
79  // Inserts a sync-packet into packet queue. Sync-packets are decoded to
80  // silence and are intended to keep AV-sync intact in an event of long packet
81  // losses when Video NACK is enabled but Audio NACK is not. Clients of NetEq
82  // might insert sync-packet when they observe that buffer level of NetEq is
83  // decreasing below a certain threshold, defined by the application.
84  // Sync-packets should have the same payload type as the last audio payload
85  // type, i.e. they cannot have DTMF or CNG payload type, nor a codec change
86  // can be implied by inserting a sync-packet.
87  // Returns kOk on success, kFail on failure.
88  virtual int InsertSyncPacket(const WebRtcRTPHeader& rtp_header,
89                               uint32_t receive_timestamp);
90
91  // Instructs NetEq to deliver 10 ms of audio data. The data is written to
92  // |output_audio|, which can hold (at least) |max_length| elements.
93  // The number of channels that were written to the output is provided in
94  // the output variable |num_channels|, and each channel contains
95  // |samples_per_channel| elements. If more than one channel is written,
96  // the samples are interleaved.
97  // The speech type is written to |type|, if |type| is not NULL.
98  // Returns kOK on success, or kFail in case of an error.
99  virtual int GetAudio(size_t max_length, int16_t* output_audio,
100                       int* samples_per_channel, int* num_channels,
101                       NetEqOutputType* type);
102
103  // Associates |rtp_payload_type| with |codec| and stores the information in
104  // the codec database. Returns kOK on success, kFail on failure.
105  virtual int RegisterPayloadType(enum NetEqDecoder codec,
106                                  uint8_t rtp_payload_type);
107
108  // Provides an externally created decoder object |decoder| to insert in the
109  // decoder database. The decoder implements a decoder of type |codec| and
110  // associates it with |rtp_payload_type|. The decoder operates at the
111  // frequency |sample_rate_hz|. Returns kOK on success, kFail on failure.
112  virtual int RegisterExternalDecoder(AudioDecoder* decoder,
113                                      enum NetEqDecoder codec,
114                                      int sample_rate_hz,
115                                      uint8_t rtp_payload_type);
116
117  // Removes |rtp_payload_type| from the codec database. Returns 0 on success,
118  // -1 on failure.
119  virtual int RemovePayloadType(uint8_t rtp_payload_type);
120
121  virtual bool SetMinimumDelay(int delay_ms);
122
123  virtual bool SetMaximumDelay(int delay_ms);
124
125  virtual int LeastRequiredDelayMs() const;
126
127  virtual int SetTargetDelay() { return kNotImplemented; }
128
129  virtual int TargetDelay() { return kNotImplemented; }
130
131  virtual int CurrentDelay() { return kNotImplemented; }
132
133  // Sets the playout mode to |mode|.
134  virtual void SetPlayoutMode(NetEqPlayoutMode mode);
135
136  // Returns the current playout mode.
137  virtual NetEqPlayoutMode PlayoutMode() const;
138
139  // Writes the current network statistics to |stats|. The statistics are reset
140  // after the call.
141  virtual int NetworkStatistics(NetEqNetworkStatistics* stats);
142
143  // Writes the last packet waiting times (in ms) to |waiting_times|. The number
144  // of values written is no more than 100, but may be smaller if the interface
145  // is polled again before 100 packets has arrived.
146  virtual void WaitingTimes(std::vector<int>* waiting_times);
147
148  // Writes the current RTCP statistics to |stats|. The statistics are reset
149  // and a new report period is started with the call.
150  virtual void GetRtcpStatistics(RtcpStatistics* stats);
151
152  // Same as RtcpStatistics(), but does not reset anything.
153  virtual void GetRtcpStatisticsNoReset(RtcpStatistics* stats);
154
155  // Enables post-decode VAD. When enabled, GetAudio() will return
156  // kOutputVADPassive when the signal contains no speech.
157  virtual void EnableVad();
158
159  // Disables post-decode VAD.
160  virtual void DisableVad();
161
162  // Returns the RTP timestamp for the last sample delivered by GetAudio().
163  virtual uint32_t PlayoutTimestamp();
164
165  virtual int SetTargetNumberOfChannels() { return kNotImplemented; }
166
167  virtual int SetTargetSampleRate() { return kNotImplemented; }
168
169  // Returns the error code for the last occurred error. If no error has
170  // occurred, 0 is returned.
171  virtual int LastError();
172
173  // Returns the error code last returned by a decoder (audio or comfort noise).
174  // When LastError() returns kDecoderErrorCode or kComfortNoiseErrorCode, check
175  // this method to get the decoder's error code.
176  virtual int LastDecoderError();
177
178  // Flushes both the packet buffer and the sync buffer.
179  virtual void FlushBuffers();
180
181  virtual void PacketBufferStatistics(int* current_num_packets,
182                                      int* max_num_packets,
183                                      int* current_memory_size_bytes,
184                                      int* max_memory_size_bytes) const;
185
186  // Get sequence number and timestamp of the latest RTP.
187  // This method is to facilitate NACK.
188  virtual int DecodedRtpInfo(int* sequence_number, uint32_t* timestamp) const;
189
190  // Sets background noise mode.
191  virtual void SetBackgroundNoiseMode(NetEqBackgroundNoiseMode mode);
192
193  // Gets background noise mode.
194  virtual NetEqBackgroundNoiseMode BackgroundNoiseMode() const;
195
196 private:
197  static const int kOutputSizeMs = 10;
198  static const int kMaxFrameSize = 2880;  // 60 ms @ 48 kHz.
199  // TODO(hlundin): Provide a better value for kSyncBufferSize.
200  static const int kSyncBufferSize = 2 * kMaxFrameSize;
201
202  // Inserts a new packet into NetEq. This is used by the InsertPacket method
203  // above. Returns 0 on success, otherwise an error code.
204  // TODO(hlundin): Merge this with InsertPacket above?
205  int InsertPacketInternal(const WebRtcRTPHeader& rtp_header,
206                           const uint8_t* payload,
207                           int length_bytes,
208                           uint32_t receive_timestamp,
209                           bool is_sync_packet);
210
211
212  // Delivers 10 ms of audio data. The data is written to |output|, which can
213  // hold (at least) |max_length| elements. The number of channels that were
214  // written to the output is provided in the output variable |num_channels|,
215  // and each channel contains |samples_per_channel| elements. If more than one
216  // channel is written, the samples are interleaved.
217  // Returns 0 on success, otherwise an error code.
218  int GetAudioInternal(size_t max_length, int16_t* output,
219                       int* samples_per_channel, int* num_channels);
220
221
222  // Provides a decision to the GetAudioInternal method. The decision what to
223  // do is written to |operation|. Packets to decode are written to
224  // |packet_list|, and a DTMF event to play is written to |dtmf_event|. When
225  // DTMF should be played, |play_dtmf| is set to true by the method.
226  // Returns 0 on success, otherwise an error code.
227  int GetDecision(Operations* operation,
228                  PacketList* packet_list,
229                  DtmfEvent* dtmf_event,
230                  bool* play_dtmf);
231
232  // Decodes the speech packets in |packet_list|, and writes the results to
233  // |decoded_buffer|, which is allocated to hold |decoded_buffer_length|
234  // elements. The length of the decoded data is written to |decoded_length|.
235  // The speech type -- speech or (codec-internal) comfort noise -- is written
236  // to |speech_type|. If |packet_list| contains any SID frames for RFC 3389
237  // comfort noise, those are not decoded.
238  int Decode(PacketList* packet_list, Operations* operation,
239             int* decoded_length, AudioDecoder::SpeechType* speech_type);
240
241  // Sub-method to Decode(). Performs the actual decoding.
242  int DecodeLoop(PacketList* packet_list, Operations* operation,
243                 AudioDecoder* decoder, int* decoded_length,
244                 AudioDecoder::SpeechType* speech_type);
245
246  // Sub-method which calls the Normal class to perform the normal operation.
247  void DoNormal(const int16_t* decoded_buffer, size_t decoded_length,
248                AudioDecoder::SpeechType speech_type, bool play_dtmf);
249
250  // Sub-method which calls the Merge class to perform the merge operation.
251  void DoMerge(int16_t* decoded_buffer, size_t decoded_length,
252               AudioDecoder::SpeechType speech_type, bool play_dtmf);
253
254  // Sub-method which calls the Expand class to perform the expand operation.
255  int DoExpand(bool play_dtmf);
256
257  // Sub-method which calls the Accelerate class to perform the accelerate
258  // operation.
259  int DoAccelerate(int16_t* decoded_buffer, size_t decoded_length,
260                   AudioDecoder::SpeechType speech_type, bool play_dtmf);
261
262  // Sub-method which calls the PreemptiveExpand class to perform the
263  // preemtive expand operation.
264  int DoPreemptiveExpand(int16_t* decoded_buffer, size_t decoded_length,
265                         AudioDecoder::SpeechType speech_type, bool play_dtmf);
266
267  // Sub-method which calls the ComfortNoise class to generate RFC 3389 comfort
268  // noise. |packet_list| can either contain one SID frame to update the
269  // noise parameters, or no payload at all, in which case the previously
270  // received parameters are used.
271  int DoRfc3389Cng(PacketList* packet_list, bool play_dtmf);
272
273  // Calls the audio decoder to generate codec-internal comfort noise when
274  // no packet was received.
275  void DoCodecInternalCng();
276
277  // Calls the DtmfToneGenerator class to generate DTMF tones.
278  int DoDtmf(const DtmfEvent& dtmf_event, bool* play_dtmf);
279
280  // Produces packet-loss concealment using alternative methods. If the codec
281  // has an internal PLC, it is called to generate samples. Otherwise, the
282  // method performs zero-stuffing.
283  void DoAlternativePlc(bool increase_timestamp);
284
285  // Overdub DTMF on top of |output|.
286  int DtmfOverdub(const DtmfEvent& dtmf_event, size_t num_channels,
287                  int16_t* output) const;
288
289  // Extracts packets from |packet_buffer_| to produce at least
290  // |required_samples| samples. The packets are inserted into |packet_list|.
291  // Returns the number of samples that the packets in the list will produce, or
292  // -1 in case of an error.
293  int ExtractPackets(int required_samples, PacketList* packet_list);
294
295  // Resets various variables and objects to new values based on the sample rate
296  // |fs_hz| and |channels| number audio channels.
297  void SetSampleRateAndChannels(int fs_hz, size_t channels);
298
299  // Returns the output type for the audio produced by the latest call to
300  // GetAudio().
301  NetEqOutputType LastOutputType();
302
303  scoped_ptr<BackgroundNoise> background_noise_;
304  scoped_ptr<BufferLevelFilter> buffer_level_filter_;
305  scoped_ptr<DecoderDatabase> decoder_database_;
306  scoped_ptr<DelayManager> delay_manager_;
307  scoped_ptr<DelayPeakDetector> delay_peak_detector_;
308  scoped_ptr<DtmfBuffer> dtmf_buffer_;
309  scoped_ptr<DtmfToneGenerator> dtmf_tone_generator_;
310  scoped_ptr<PacketBuffer> packet_buffer_;
311  scoped_ptr<PayloadSplitter> payload_splitter_;
312  scoped_ptr<TimestampScaler> timestamp_scaler_;
313  scoped_ptr<DecisionLogic> decision_logic_;
314  scoped_ptr<PostDecodeVad> vad_;
315  scoped_ptr<AudioMultiVector> algorithm_buffer_;
316  scoped_ptr<SyncBuffer> sync_buffer_;
317  scoped_ptr<Expand> expand_;
318  scoped_ptr<Normal> normal_;
319  scoped_ptr<Merge> merge_;
320  scoped_ptr<Accelerate> accelerate_;
321  scoped_ptr<PreemptiveExpand> preemptive_expand_;
322  RandomVector random_vector_;
323  scoped_ptr<ComfortNoise> comfort_noise_;
324  Rtcp rtcp_;
325  StatisticsCalculator stats_;
326  int fs_hz_;
327  int fs_mult_;
328  int output_size_samples_;
329  int decoder_frame_length_;
330  Modes last_mode_;
331  scoped_array<int16_t> mute_factor_array_;
332  size_t decoded_buffer_length_;
333  scoped_array<int16_t> decoded_buffer_;
334  uint32_t playout_timestamp_;
335  bool new_codec_;
336  uint32_t timestamp_;
337  bool reset_decoder_;
338  uint8_t current_rtp_payload_type_;
339  uint8_t current_cng_rtp_payload_type_;
340  uint32_t ssrc_;
341  bool first_packet_;
342  int error_code_;  // Store last error code.
343  int decoder_error_code_;
344  scoped_ptr<CriticalSectionWrapper> crit_sect_;
345
346  // These values are used by NACK module to estimate time-to-play of
347  // a missing packet. Occasionally, NetEq might decide to decode more
348  // than one packet. Therefore, these values store sequence number and
349  // timestamp of the first packet pulled from the packet buffer. In
350  // such cases, these values do not exactly represent the sequence number
351  // or timestamp associated with a 10ms audio pulled from NetEq. NACK
352  // module is designed to compensate for this.
353  int decoded_packet_sequence_number_;
354  uint32_t decoded_packet_timestamp_;
355
356  DISALLOW_COPY_AND_ASSIGN(NetEqImpl);
357};
358
359}  // namespace webrtc
360#endif  // WEBRTC_MODULES_AUDIO_CODING_NETEQ4_NETEQ_IMPL_H_
Note: See TracBrowser for help on using the repository browser.