Note: We no longer publish the latest version of our code here. We primarily use a kumc-bmi github organization. The heron ETL repository, in particular, is not public. Peers in the informatics community should see MultiSiteDev for details on requesting access.

source: webrtc/talk/media/base/videoadapter.h @ 0:4bda6873e34c

pub_scrub_3792 tip
Last change on this file since 0:4bda6873e34c was 0:4bda6873e34c, checked in by Michael Prittie <mprittie@…>, 6 years ago

Scrubbed password for publication.

File size: 10.7 KB
Line 
1// libjingle
2// Copyright 2010 Google Inc.
3//
4// Redistribution and use in source and binary forms, with or without
5// modification, are permitted provided that the following conditions are met:
6//
7//  1. Redistributions of source code must retain the above copyright notice,
8//     this list of conditions and the following disclaimer.
9//  2. Redistributions in binary form must reproduce the above copyright notice,
10//     this list of conditions and the following disclaimer in the documentation
11//     and/or other materials provided with the distribution.
12//  3. The name of the author may not be used to endorse or promote products
13//     derived from this software without specific prior written permission.
14//
15// THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED
16// WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
17// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
18// EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
19// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
20// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
21// OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
22// WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
23// OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
24// ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
25
26#ifndef TALK_MEDIA_BASE_VIDEOADAPTER_H_  // NOLINT
27#define TALK_MEDIA_BASE_VIDEOADAPTER_H_
28
29#include "talk/base/common.h"  // For ASSERT
30#include "talk/base/criticalsection.h"
31#include "talk/base/logging.h"
32#include "talk/base/scoped_ptr.h"
33#include "talk/base/sigslot.h"
34#include "talk/media/base/videocommon.h"
35
36namespace cricket {
37
38class VideoFrame;
39
40// VideoAdapter adapts an input video frame to an output frame based on the
41// specified input and output formats. The adaptation includes dropping frames
42// to reduce frame rate and scaling frames. VideoAdapter is thread safe.
43class VideoAdapter {
44 public:
45  VideoAdapter();
46  virtual ~VideoAdapter();
47
48  virtual void SetInputFormat(const VideoFormat& format);
49  void SetOutputFormat(const VideoFormat& format);
50  // Constrain output resolution to this many pixels overall
51  void SetOutputNumPixels(int num_pixels);
52  int GetOutputNumPixels() const;
53
54  const VideoFormat& input_format();
55  const VideoFormat& output_format();
56  // If the parameter black is true, the adapted frames will be black.
57  void SetBlackOutput(bool black);
58
59  // Adapt the input frame from the input format to the output format. Return
60  // true and set the output frame to NULL if the input frame is dropped. Return
61  // true and set the out frame to output_frame_ if the input frame is adapted
62  // successfully. Return false otherwise.
63  // output_frame_ is owned by the VideoAdapter that has the best knowledge on
64  // the output frame.
65  bool AdaptFrame(const VideoFrame* in_frame, const VideoFrame** out_frame);
66
67  void set_scale_third(bool enable) {
68    LOG(LS_INFO) << "Video Adapter third scaling is now "
69                 << (enable ? "enabled" : "disabled");
70    scale_third_ = enable;
71  }
72  bool scale_third() const { return scale_third_; }
73
74 protected:
75  float FindClosestScale(int width, int height, int target_num_pixels);
76  float FindClosestViewScale(int width, int height, int target_num_pixels);
77  float FindLowerScale(int width, int height, int target_num_pixels);
78
79 private:
80  const float* GetViewScaleFactors() const;
81  float FindScale(const float* scale_factors,
82                  const float upbias, int width, int height,
83                  int target_num_pixels);
84  bool StretchToOutputFrame(const VideoFrame* in_frame);
85
86  VideoFormat input_format_;
87  VideoFormat output_format_;
88  int output_num_pixels_;
89  bool scale_third_;  // True if adapter allows scaling to 1/3 and 2/3.
90  int frames_;  // Number of input frames.
91  int adapted_frames_;  // Number of frames scaled.
92  int adaption_changes_;  // Number of changes in scale factor.
93  size_t previous_width;  // Previous adapter output width.
94  size_t previous_height;  // Previous adapter output height.
95  bool black_output_;  // Flag to tell if we need to black output_frame_.
96  bool is_black_;  // Flag to tell if output_frame_ is currently black.
97  int64 interval_next_frame_;
98  talk_base::scoped_ptr<VideoFrame> output_frame_;
99  // The critical section to protect the above variables.
100  talk_base::CriticalSection critical_section_;
101
102  DISALLOW_COPY_AND_ASSIGN(VideoAdapter);
103};
104
105// CoordinatedVideoAdapter adapts the video input to the encoder by coordinating
106// the format request from the server, the resolution request from the encoder,
107// and the CPU load.
108class CoordinatedVideoAdapter
109    : public VideoAdapter, public sigslot::has_slots<>  {
110 public:
111  enum AdaptRequest { UPGRADE, KEEP, DOWNGRADE };
112  enum AdaptReasonEnum {
113    ADAPTREASON_CPU = 1,
114    ADAPTREASON_BANDWIDTH = 2,
115    ADAPTREASON_VIEW = 4
116  };
117  typedef int AdaptReason;
118
119  CoordinatedVideoAdapter();
120  virtual ~CoordinatedVideoAdapter() {}
121
122  virtual void SetInputFormat(const VideoFormat& format);
123
124  // Enable or disable video adaptation due to the change of the CPU load.
125  void set_cpu_adaptation(bool enable) { cpu_adaptation_ = enable; }
126  bool cpu_adaptation() const { return cpu_adaptation_; }
127  // Enable or disable smoothing when doing CPU adaptation. When smoothing is
128  // enabled, system CPU load is tracked using an exponential weighted
129  // average.
130  void set_cpu_smoothing(bool enable) {
131    LOG(LS_INFO) << "CPU smoothing is now "
132                 << (enable ? "enabled" : "disabled");
133    cpu_smoothing_ = enable;
134  }
135  bool cpu_smoothing() const { return cpu_smoothing_; }
136  // Enable or disable video adaptation due to the change of the GD
137  void set_gd_adaptation(bool enable) { gd_adaptation_ = enable; }
138  bool gd_adaptation() const { return gd_adaptation_; }
139  // Enable or disable video adaptation due to the change of the View
140  void set_view_adaptation(bool enable) { view_adaptation_ = enable; }
141  bool view_adaptation() const { return view_adaptation_; }
142  // Enable or disable video adaptation to fast switch View
143  void set_view_switch(bool enable) { view_switch_ = enable; }
144  bool view_switch() const { return view_switch_; }
145
146  CoordinatedVideoAdapter::AdaptReason adapt_reason() const {
147    return adapt_reason_;
148  }
149
150  // When the video is decreased, set the waiting time for CPU adaptation to
151  // decrease video again.
152  void set_cpu_adapt_wait_time(uint32 cpu_adapt_wait_time) {
153    if (cpu_adapt_wait_time_ != static_cast<int>(cpu_adapt_wait_time)) {
154      LOG(LS_INFO) << "VAdapt Change Cpu Adapt Wait Time from: "
155                   << cpu_adapt_wait_time_ << " to "
156                   << cpu_adapt_wait_time;
157      cpu_adapt_wait_time_ = static_cast<int>(cpu_adapt_wait_time);
158    }
159  }
160  // CPU system load high threshold for reducing resolution.  e.g. 0.85f
161  void set_high_system_threshold(float high_system_threshold) {
162    ASSERT(high_system_threshold <= 1.0f);
163    ASSERT(high_system_threshold >= 0.0f);
164    if (high_system_threshold_ != high_system_threshold) {
165      LOG(LS_INFO) << "VAdapt Change High System Threshold from: "
166                   << high_system_threshold_ << " to " << high_system_threshold;
167      high_system_threshold_ = high_system_threshold;
168    }
169  }
170  float high_system_threshold() const { return high_system_threshold_; }
171  // CPU system load low threshold for increasing resolution.  e.g. 0.70f
172  void set_low_system_threshold(float low_system_threshold) {
173    ASSERT(low_system_threshold <= 1.0f);
174    ASSERT(low_system_threshold >= 0.0f);
175    if (low_system_threshold_ != low_system_threshold) {
176      LOG(LS_INFO) << "VAdapt Change Low System Threshold from: "
177                   << low_system_threshold_ << " to " << low_system_threshold;
178      low_system_threshold_ = low_system_threshold;
179    }
180  }
181  float low_system_threshold() const { return low_system_threshold_; }
182  // CPU process load threshold for reducing resolution.  e.g. 0.10f
183  void set_process_threshold(float process_threshold) {
184    ASSERT(process_threshold <= 1.0f);
185    ASSERT(process_threshold >= 0.0f);
186    if (process_threshold_ != process_threshold) {
187      LOG(LS_INFO) << "VAdapt Change High Process Threshold from: "
188                   << process_threshold_ << " to " << process_threshold;
189      process_threshold_ = process_threshold;
190    }
191  }
192  float process_threshold() const { return process_threshold_; }
193
194  // Handle the format request from the server via Jingle update message.
195  void OnOutputFormatRequest(const VideoFormat& format);
196  // Handle the resolution request from the encoder due to bandwidth changes.
197  void OnEncoderResolutionRequest(int width, int height, AdaptRequest request);
198  // Handle the resolution request for CPU overuse.
199  void OnCpuResolutionRequest(AdaptRequest request);
200  // Handle the CPU load provided by a CPU monitor.
201  void OnCpuLoadUpdated(int current_cpus, int max_cpus,
202                        float process_load, float system_load);
203
204  sigslot::signal0<> SignalCpuAdaptationUnable;
205
206 private:
207  // Adapt to the minimum of the formats the server requests, the CPU wants, and
208  // the encoder wants. Returns true if resolution changed.
209  bool AdaptToMinimumFormat(int* new_width, int* new_height);
210  bool IsMinimumFormat(int pixels);
211  void StepPixelCount(CoordinatedVideoAdapter::AdaptRequest request,
212                      int* num_pixels);
213  CoordinatedVideoAdapter::AdaptRequest FindCpuRequest(
214    int current_cpus, int max_cpus,
215    float process_load, float system_load);
216
217  bool cpu_adaptation_;  // True if cpu adaptation is enabled.
218  bool cpu_smoothing_;  // True if cpu smoothing is enabled (with adaptation).
219  bool gd_adaptation_;  // True if gd adaptation is enabled.
220  bool view_adaptation_;  // True if view adaptation is enabled.
221  bool view_switch_;  // True if view switch is enabled.
222  int cpu_downgrade_count_;
223  int cpu_adapt_wait_time_;
224  // cpu system load thresholds relative to max cpus.
225  float high_system_threshold_;
226  float low_system_threshold_;
227  // cpu process load thresholds relative to current cpus.
228  float process_threshold_;
229  // Video formats that the server view requests, the CPU wants, and the encoder
230  // wants respectively. The adapted output format is the minimum of these.
231  int view_desired_num_pixels_;
232  int64 view_desired_interval_;
233  int encoder_desired_num_pixels_;
234  int cpu_desired_num_pixels_;
235  CoordinatedVideoAdapter::AdaptReason adapt_reason_;
236  // The critical section to protect handling requests.
237  talk_base::CriticalSection request_critical_section_;
238
239  // The weighted average of cpu load over time. It's always updated (if cpu
240  // adaptation is on), but only used if cpu_smoothing_ is set.
241  float system_load_average_;
242
243  DISALLOW_COPY_AND_ASSIGN(CoordinatedVideoAdapter);
244};
245
246}  // namespace cricket
247
248#endif  // TALK_MEDIA_BASE_VIDEOADAPTER_H_  // NOLINT
Note: See TracBrowser for help on using the repository browser.