Note: We no longer publish the latest version of our code here. We primarily use a kumc-bmi github organization. The heron ETL repository, in particular, is not public. Peers in the informatics community should see MultiSiteDev for details on requesting access.

source: webrtc/webrtc/modules/video_coding/codecs/tools/video_quality_measurement.cc @ 0:4bda6873e34c

pub_scrub_3792 tip
Last change on this file since 0:4bda6873e34c was 0:4bda6873e34c, checked in by Michael Prittie <mprittie@…>, 6 years ago

Scrubbed password for publication.

File size: 22.0 KB
Line 
1/*
2 *  Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
3 *
4 *  Use of this source code is governed by a BSD-style license
5 *  that can be found in the LICENSE file in the root of the source
6 *  tree. An additional intellectual property rights grant can be found
7 *  in the file PATENTS.  All contributing project authors may
8 *  be found in the AUTHORS file in the root of the source tree.
9 */
10
11#include <assert.h>
12#include <stdio.h>
13#include <time.h>
14
15#include <stdarg.h>
16#include <sys/stat.h>  // To check for directory existence.
17
18#ifndef S_ISDIR  // Not defined in stat.h on Windows.
19#define S_ISDIR(mode) (((mode) & S_IFMT) == S_IFDIR)
20#endif
21
22#include "gflags/gflags.h"
23#include "webrtc/common_types.h"
24#include "webrtc/modules/video_coding/codecs/test/packet_manipulator.h"
25#include "webrtc/modules/video_coding/codecs/test/stats.h"
26#include "webrtc/modules/video_coding/codecs/test/videoprocessor.h"
27#include "webrtc/modules/video_coding/codecs/vp8/include/vp8.h"
28#include "webrtc/modules/video_coding/main/interface/video_coding.h"
29#include "webrtc/system_wrappers/interface/trace.h"
30#include "webrtc/test/testsupport/frame_reader.h"
31#include "webrtc/test/testsupport/frame_writer.h"
32#include "webrtc/test/testsupport/metrics/video_metrics.h"
33#include "webrtc/test/testsupport/packet_reader.h"
34
35DEFINE_string(test_name, "Quality test", "The name of the test to run. ");
36DEFINE_string(test_description, "", "A more detailed description about what "
37              "the current test is about.");
38DEFINE_string(input_filename, "", "Input file. "
39              "The source video file to be encoded and decoded. Must be in "
40              ".yuv format");
41DEFINE_int32(width, -1, "Width in pixels of the frames in the input file.");
42DEFINE_int32(height, -1, "Height in pixels of the frames in the input file.");
43DEFINE_int32(framerate, 30, "Frame rate of the input file, in FPS "
44             "(frames-per-second). ");
45DEFINE_string(output_dir, ".", "Output directory. "
46              "The directory where the output file will be put. Must already "
47              "exist.");
48DEFINE_bool(use_single_core, false, "Force using a single core. If set to "
49            "true, only one core will be used for processing. Using a single "
50            "core is necessary to get a deterministic behavior for the"
51            "encoded frames - using multiple cores will produce different "
52            "encoded frames since multiple cores are competing to consume the "
53            "byte budget for each frame in parallel. If set to false, "
54            "the maximum detected number of cores will be used. ");
55DEFINE_bool(disable_fixed_random_seed , false, "Set this flag to disable the"
56            "usage of a fixed random seed for the random generator used "
57            "for packet loss. Disabling this will cause consecutive runs "
58            "loose packets at different locations, which is bad for "
59            "reproducibility.");
60DEFINE_string(output_filename, "", "Output file. "
61              "The name of the output video file resulting of the processing "
62              "of the source file. By default this is the same name as the "
63              "input file with '_out' appended before the extension.");
64DEFINE_int32(bitrate, 500, "Bit rate in kilobits/second.");
65DEFINE_int32(keyframe_interval, 0, "Forces a keyframe every Nth frame. "
66             "0 means the encoder decides when to insert keyframes.  Note that "
67             "the encoder may create a keyframe in other locations in addition "
68             "to the interval that is set using this parameter.");
69DEFINE_int32(temporal_layers, 0, "The number of temporal layers to use "
70             "(VP8 specific codec setting). Must be 0-4.");
71DEFINE_int32(packet_size, 1500, "Simulated network packet size in bytes (MTU). "
72             "Used for packet loss simulation.");
73DEFINE_int32(max_payload_size, 1440, "Max payload size in bytes for the "
74             "encoder.");
75DEFINE_string(packet_loss_mode, "uniform", "Packet loss mode. Two different "
76              "packet loss models are supported: uniform or burst. This "
77              "setting has no effect unless packet_loss_rate is >0. ");
78DEFINE_double(packet_loss_probability, 0.0, "Packet loss probability. A value "
79              "between 0.0 and 1.0 that defines the probability of a packet "
80              "being lost. 0.1 means 10% and so on.");
81DEFINE_int32(packet_loss_burst_length, 1, "Packet loss burst length. Defines "
82             "how many packets will be lost in a burst when a packet has been "
83             "decided to be lost. Must be >=1.");
84DEFINE_bool(csv, false, "CSV output. Enabling this will output all frame "
85            "statistics at the end of execution. Recommended to run combined "
86            "with --noverbose to avoid mixing output.");
87DEFINE_bool(python, false, "Python output. Enabling this will output all frame "
88            "statistics as a Python script at the end of execution. "
89            "Recommended to run combine with --noverbose to avoid mixing "
90            "output.");
91DEFINE_bool(verbose, true, "Verbose mode. Prints a lot of debugging info. "
92            "Suitable for tracking progress but not for capturing output. "
93            "Disable with --noverbose flag.");
94
95// Custom log method that only prints if the verbose flag is given.
96// Supports all the standard printf parameters and formatting (just forwarded).
97int Log(const char *format, ...) {
98  int result = 0;
99  if (FLAGS_verbose) {
100    va_list args;
101    va_start(args, format);
102    result = vprintf(format, args);
103    va_end(args);
104  }
105  return result;
106}
107
108// Validates the arguments given as command line flags and fills in the
109// TestConfig struct with all configurations needed for video processing.
110// Returns 0 if everything is OK, otherwise an exit code.
111int HandleCommandLineFlags(webrtc::test::TestConfig* config) {
112  // Validate the mandatory flags:
113  if (FLAGS_input_filename == "" || FLAGS_width == -1 || FLAGS_height == -1) {
114    printf("%s\n", google::ProgramUsage());
115    return 1;
116  }
117  config->name = FLAGS_test_name;
118  config->description = FLAGS_test_description;
119
120  // Verify the input file exists and is readable.
121  FILE* test_file;
122  test_file = fopen(FLAGS_input_filename.c_str(), "rb");
123  if (test_file == NULL) {
124    fprintf(stderr, "Cannot read the specified input file: %s\n",
125            FLAGS_input_filename.c_str());
126    return 2;
127  }
128  fclose(test_file);
129  config->input_filename = FLAGS_input_filename;
130
131  // Verify the output dir exists.
132  struct stat dir_info;
133  if (!(stat(FLAGS_output_dir.c_str(), &dir_info) == 0 &&
134      S_ISDIR(dir_info.st_mode))) {
135    fprintf(stderr, "Cannot find output directory: %s\n",
136              FLAGS_output_dir.c_str());
137    return 3;
138  }
139  config->output_dir = FLAGS_output_dir;
140
141  // Manufacture an output filename if none was given.
142  if (FLAGS_output_filename == "") {
143    // Cut out the filename without extension from the given input file
144    // (which may include a path)
145    int startIndex = FLAGS_input_filename.find_last_of("/") + 1;
146    if (startIndex == 0) {
147      startIndex = 0;
148    }
149    FLAGS_output_filename =
150        FLAGS_input_filename.substr(startIndex,
151                                    FLAGS_input_filename.find_last_of(".")
152                                    - startIndex) + "_out.yuv";
153  }
154
155  // Verify output file can be written.
156  if (FLAGS_output_dir == ".") {
157    config->output_filename = FLAGS_output_filename;
158  } else {
159    config->output_filename = FLAGS_output_dir + "/"+ FLAGS_output_filename;
160  }
161  test_file = fopen(config->output_filename.c_str(), "wb");
162  if (test_file == NULL) {
163    fprintf(stderr, "Cannot write output file: %s\n",
164            config->output_filename.c_str());
165    return 4;
166  }
167  fclose(test_file);
168
169  // Check single core flag.
170  config->use_single_core = FLAGS_use_single_core;
171
172  // Get codec specific configuration.
173  webrtc::VideoCodingModule::Codec(webrtc::kVideoCodecVP8,
174                                   config->codec_settings);
175
176  // Check the temporal layers.
177  if (FLAGS_temporal_layers < 0 ||
178      FLAGS_temporal_layers > webrtc::kMaxTemporalStreams) {
179    fprintf(stderr, "Temporal layers number must be 0-4, was: %d\n",
180            FLAGS_temporal_layers);
181    return 13;
182  }
183  config->codec_settings->codecSpecific.VP8.numberOfTemporalLayers =
184      FLAGS_temporal_layers;
185
186  // Check the bit rate.
187  if (FLAGS_bitrate <= 0) {
188    fprintf(stderr, "Bit rate must be >0 kbps, was: %d\n", FLAGS_bitrate);
189    return 5;
190  }
191  config->codec_settings->startBitrate = FLAGS_bitrate;
192
193  // Check the keyframe interval.
194  if (FLAGS_keyframe_interval < 0) {
195    fprintf(stderr, "Keyframe interval must be >=0, was: %d\n",
196            FLAGS_keyframe_interval);
197    return 6;
198  }
199  config->keyframe_interval = FLAGS_keyframe_interval;
200
201  // Check packet size and max payload size.
202  if (FLAGS_packet_size <= 0) {
203    fprintf(stderr, "Packet size must be >0 bytes, was: %d\n",
204            FLAGS_packet_size);
205    return 7;
206  }
207  config->networking_config.packet_size_in_bytes = FLAGS_packet_size;
208
209  if (FLAGS_max_payload_size <= 0) {
210    fprintf(stderr, "Max payload size must be >0 bytes, was: %d\n",
211            FLAGS_max_payload_size);
212    return 8;
213  }
214  config->networking_config.max_payload_size_in_bytes =
215      FLAGS_max_payload_size;
216
217  // Check the width and height
218  if (FLAGS_width <= 0 || FLAGS_height <= 0) {
219    fprintf(stderr, "Width and height must be >0.");
220    return 9;
221  }
222  config->codec_settings->width = FLAGS_width;
223  config->codec_settings->height = FLAGS_height;
224  config->codec_settings->maxFramerate = FLAGS_framerate;
225
226  // Calculate the size of each frame to read (according to YUV spec).
227  config->frame_length_in_bytes =
228      3 * config->codec_settings->width * config->codec_settings->height / 2;
229
230  // Check packet loss settings
231  if (FLAGS_packet_loss_mode != "uniform" &&
232      FLAGS_packet_loss_mode != "burst") {
233    fprintf(stderr, "Unsupported packet loss mode, must be 'uniform' or "
234            "'burst'\n.");
235    return 10;
236  }
237  config->networking_config.packet_loss_mode = webrtc::test::kUniform;
238  if (FLAGS_packet_loss_mode == "burst") {
239    config->networking_config.packet_loss_mode =  webrtc::test::kBurst;
240  }
241
242  if (FLAGS_packet_loss_probability < 0.0 ||
243      FLAGS_packet_loss_probability > 1.0) {
244    fprintf(stderr, "Invalid packet loss probability. Must be 0.0 - 1.0, "
245            "was: %f\n", FLAGS_packet_loss_probability);
246    return 11;
247  }
248  config->networking_config.packet_loss_probability =
249      FLAGS_packet_loss_probability;
250
251  if (FLAGS_packet_loss_burst_length < 1) {
252    fprintf(stderr, "Invalid packet loss burst length, must be >=1, "
253            "was: %d\n", FLAGS_packet_loss_burst_length);
254    return 12;
255  }
256  config->networking_config.packet_loss_burst_length =
257      FLAGS_packet_loss_burst_length;
258  config->verbose = FLAGS_verbose;
259  return 0;
260}
261
262void CalculateSsimVideoMetrics(webrtc::test::TestConfig* config,
263                               webrtc::test::QualityMetricsResult* result) {
264  Log("Calculating SSIM...\n");
265  I420SSIMFromFiles(config->input_filename.c_str(),
266                    config->output_filename.c_str(),
267                    config->codec_settings->width,
268                    config->codec_settings->height, result);
269  Log("  Average: %3.2f\n", result->average);
270  Log("  Min    : %3.2f (frame %d)\n", result->min, result->min_frame_number);
271  Log("  Max    : %3.2f (frame %d)\n", result->max, result->max_frame_number);
272}
273
274void CalculatePsnrVideoMetrics(webrtc::test::TestConfig* config,
275                               webrtc::test::QualityMetricsResult* result) {
276  Log("Calculating PSNR...\n");
277  I420PSNRFromFiles(config->input_filename.c_str(),
278                    config->output_filename.c_str(),
279                    config->codec_settings->width,
280                    config->codec_settings->height, result);
281  Log("  Average: %3.2f\n", result->average);
282  Log("  Min    : %3.2f (frame %d)\n", result->min, result->min_frame_number);
283  Log("  Max    : %3.2f (frame %d)\n", result->max, result->max_frame_number);
284}
285
286void PrintConfigurationSummary(const webrtc::test::TestConfig& config) {
287  Log("Quality test with parameters:\n");
288  Log("  Test name        : %s\n", config.name.c_str());
289  Log("  Description      : %s\n", config.description.c_str());
290  Log("  Input filename   : %s\n", config.input_filename.c_str());
291  Log("  Output directory : %s\n", config.output_dir.c_str());
292  Log("  Output filename  : %s\n", config.output_filename.c_str());
293  Log("  Frame length       : %d bytes\n", config.frame_length_in_bytes);
294  Log("  Packet size      : %d bytes\n",
295      config.networking_config.packet_size_in_bytes);
296  Log("  Max payload size : %d bytes\n",
297      config.networking_config.max_payload_size_in_bytes);
298  Log("  Packet loss:\n");
299  Log("    Mode           : %s\n",
300      PacketLossModeToStr(config.networking_config.packet_loss_mode));
301  Log("    Probability    : %2.1f\n",
302      config.networking_config.packet_loss_probability);
303  Log("    Burst length   : %d packets\n",
304      config.networking_config.packet_loss_burst_length);
305}
306
307void PrintCsvOutput(const webrtc::test::Stats& stats,
308                    const webrtc::test::QualityMetricsResult& ssim_result,
309                    const webrtc::test::QualityMetricsResult& psnr_result) {
310  Log("\nCSV output (recommended to run with --noverbose to skip the "
311              "above output)\n");
312  printf("frame_number encoding_successful decoding_successful "
313      "encode_return_code decode_return_code "
314      "encode_time_in_us decode_time_in_us "
315      "bit_rate_in_kbps encoded_frame_length_in_bytes frame_type "
316      "packets_dropped total_packets "
317      "ssim psnr\n");
318
319  for (unsigned int i = 0; i < stats.stats_.size(); ++i) {
320    const webrtc::test::FrameStatistic& f = stats.stats_[i];
321    const webrtc::test::FrameResult& ssim = ssim_result.frames[i];
322    const webrtc::test::FrameResult& psnr = psnr_result.frames[i];
323    printf("%4d, %d, %d, %2d, %2d, %6d, %6d, %5d, %7d, %d, %2d, %2d, "
324           "%5.3f, %5.2f\n",
325           f.frame_number,
326           f.encoding_successful,
327           f.decoding_successful,
328           f.encode_return_code,
329           f.decode_return_code,
330           f.encode_time_in_us,
331           f.decode_time_in_us,
332           f.bit_rate_in_kbps,
333           f.encoded_frame_length_in_bytes,
334           f.frame_type,
335           f.packets_dropped,
336           f.total_packets,
337           ssim.value,
338           psnr.value);
339  }
340}
341
342void PrintPythonOutput(const webrtc::test::TestConfig& config,
343                       const webrtc::test::Stats& stats,
344                       const webrtc::test::QualityMetricsResult& ssim_result,
345                       const webrtc::test::QualityMetricsResult& psnr_result) {
346  Log("\nPython output (recommended to run with --noverbose to skip the "
347               "above output)\n");
348  printf("test_configuration = ["
349         "{'name': 'name',                      'value': '%s'},\n"
350         "{'name': 'description',               'value': '%s'},\n"
351         "{'name': 'test_number',               'value': '%d'},\n"
352         "{'name': 'input_filename',            'value': '%s'},\n"
353         "{'name': 'output_filename',           'value': '%s'},\n"
354         "{'name': 'output_dir',                'value': '%s'},\n"
355         "{'name': 'packet_size_in_bytes',      'value': '%d'},\n"
356         "{'name': 'max_payload_size_in_bytes', 'value': '%d'},\n"
357         "{'name': 'packet_loss_mode',          'value': '%s'},\n"
358         "{'name': 'packet_loss_probability',   'value': '%f'},\n"
359         "{'name': 'packet_loss_burst_length',  'value': '%d'},\n"
360         "{'name': 'exclude_frame_types',       'value': '%s'},\n"
361         "{'name': 'frame_length_in_bytes',     'value': '%d'},\n"
362         "{'name': 'use_single_core',           'value': '%s'},\n"
363         "{'name': 'keyframe_interval;',        'value': '%d'},\n"
364         "{'name': 'video_codec_type',          'value': '%s'},\n"
365         "{'name': 'width',                     'value': '%d'},\n"
366         "{'name': 'height',                    'value': '%d'},\n"
367         "{'name': 'bit_rate_in_kbps',          'value': '%d'},\n"
368         "]\n",
369         config.name.c_str(),
370         config.description.c_str(),
371         config.test_number,
372         config.input_filename.c_str(),
373         config.output_filename.c_str(),
374         config.output_dir.c_str(),
375         config.networking_config.packet_size_in_bytes,
376         config.networking_config.max_payload_size_in_bytes,
377         PacketLossModeToStr(config.networking_config.packet_loss_mode),
378         config.networking_config.packet_loss_probability,
379         config.networking_config.packet_loss_burst_length,
380         ExcludeFrameTypesToStr(config.exclude_frame_types),
381         config.frame_length_in_bytes,
382         config.use_single_core ? "True " : "False",
383         config.keyframe_interval,
384         webrtc::test::VideoCodecTypeToStr(config.codec_settings->codecType),
385         config.codec_settings->width,
386         config.codec_settings->height,
387         config.codec_settings->startBitrate);
388  printf("frame_data_types = {"
389         "'frame_number': ('number', 'Frame number'),\n"
390         "'encoding_successful': ('boolean', 'Encoding successful?'),\n"
391         "'decoding_successful': ('boolean', 'Decoding successful?'),\n"
392         "'encode_time': ('number', 'Encode time (us)'),\n"
393         "'decode_time': ('number', 'Decode time (us)'),\n"
394         "'encode_return_code': ('number', 'Encode return code'),\n"
395         "'decode_return_code': ('number', 'Decode return code'),\n"
396         "'bit_rate': ('number', 'Bit rate (kbps)'),\n"
397         "'encoded_frame_length': "
398         "('number', 'Encoded frame length (bytes)'),\n"
399         "'frame_type': ('string', 'Frame type'),\n"
400         "'packets_dropped': ('number', 'Packets dropped'),\n"
401         "'total_packets': ('number', 'Total packets'),\n"
402         "'ssim': ('number', 'SSIM'),\n"
403         "'psnr': ('number', 'PSNR (dB)'),\n"
404         "}\n");
405  printf("frame_data = [");
406  for (unsigned int i = 0; i < stats.stats_.size(); ++i) {
407    const webrtc::test::FrameStatistic& f = stats.stats_[i];
408    const webrtc::test::FrameResult& ssim = ssim_result.frames[i];
409    const webrtc::test::FrameResult& psnr = psnr_result.frames[i];
410    printf("{'frame_number': %d, "
411           "'encoding_successful': %s, 'decoding_successful': %s, "
412           "'encode_time': %d, 'decode_time': %d, "
413           "'encode_return_code': %d, 'decode_return_code': %d, "
414           "'bit_rate': %d, 'encoded_frame_length': %d, 'frame_type': %s, "
415           "'packets_dropped': %d, 'total_packets': %d, "
416           "'ssim': %f, 'psnr': %f},\n",
417           f.frame_number,
418           f.encoding_successful ? "True " : "False",
419           f.decoding_successful ? "True " : "False",
420           f.encode_time_in_us,
421           f.decode_time_in_us,
422           f.encode_return_code,
423           f.decode_return_code,
424           f.bit_rate_in_kbps,
425           f.encoded_frame_length_in_bytes,
426           f.frame_type == webrtc::kDeltaFrame ? "'Delta'" : "'Other'",
427           f.packets_dropped,
428           f.total_packets,
429           ssim.value,
430           psnr.value);
431  }
432  printf("]\n");
433}
434
435// Runs a quality measurement on the input file supplied to the program.
436// The input file must be in YUV format.
437int main(int argc, char* argv[]) {
438  std::string program_name = argv[0];
439  std::string usage = "Quality test application for video comparisons.\n"
440    "Run " + program_name + " --helpshort for usage.\n"
441    "Example usage:\n" + program_name +
442    " --input_filename=filename.yuv --width=352 --height=288\n";
443  google::SetUsageMessage(usage);
444
445  google::ParseCommandLineFlags(&argc, &argv, true);
446
447  // Create TestConfig and codec settings struct.
448  webrtc::test::TestConfig config;
449  webrtc::VideoCodec codec_settings;
450  config.codec_settings = &codec_settings;
451
452  int return_code = HandleCommandLineFlags(&config);
453  // Exit if an invalid argument is supplied.
454  if (return_code != 0) {
455    return return_code;
456  }
457
458  PrintConfigurationSummary(config);
459
460  webrtc::VP8Encoder* encoder = webrtc::VP8Encoder::Create();
461  webrtc::VP8Decoder* decoder = webrtc::VP8Decoder::Create();
462  webrtc::test::Stats stats;
463  webrtc::test::FrameReaderImpl frame_reader(config.input_filename,
464                                             config.frame_length_in_bytes);
465  webrtc::test::FrameWriterImpl frame_writer(config.output_filename,
466                                             config.frame_length_in_bytes);
467  frame_reader.Init();
468  frame_writer.Init();
469  webrtc::test::PacketReader packet_reader;
470
471  webrtc::test::PacketManipulatorImpl packet_manipulator(
472      &packet_reader, config.networking_config, config.verbose);
473  // By default the packet manipulator is seeded with a fixed random.
474  // If disabled we must generate a new seed.
475  if (FLAGS_disable_fixed_random_seed) {
476    packet_manipulator.InitializeRandomSeed(time(NULL));
477  }
478  webrtc::test::VideoProcessor* processor =
479      new webrtc::test::VideoProcessorImpl(encoder, decoder,
480                                           &frame_reader,
481                                           &frame_writer,
482                                           &packet_manipulator,
483                                           config, &stats);
484  processor->Init();
485
486  int frame_number = 0;
487  while (processor->ProcessFrame(frame_number)) {
488    if (frame_number % 80 == 0) {
489      Log("\n");  // make the output a bit nicer.
490    }
491    Log(".");
492    frame_number++;
493  }
494  Log("\n");
495  Log("Processed %d frames\n", frame_number);
496
497  // Release encoder and decoder to make sure they have finished processing.
498  encoder->Release();
499  decoder->Release();
500
501  // Verify statistics are correct:
502  assert(frame_number == static_cast<int>(stats.stats_.size()));
503
504  // Close the files before we start using them for SSIM/PSNR calculations.
505  frame_reader.Close();
506  frame_writer.Close();
507
508  stats.PrintSummary();
509
510  webrtc::test::QualityMetricsResult ssim_result;
511  CalculateSsimVideoMetrics(&config, &ssim_result);
512  webrtc::test::QualityMetricsResult psnr_result;
513  CalculatePsnrVideoMetrics(&config, &psnr_result);
514
515  if (FLAGS_csv) {
516    PrintCsvOutput(stats, ssim_result, psnr_result);
517  }
518  if (FLAGS_python) {
519    PrintPythonOutput(config, stats, ssim_result, psnr_result);
520  }
521  delete processor;
522  delete encoder;
523  delete decoder;
524  Log("Quality test finished!");
525  return 0;
526}
Note: See TracBrowser for help on using the repository browser.