signalapp/ringrtc
signalapp/ringrtc
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Call Sim: Add dred option to baseline tests

Browse Source
This commit is contained in:
Jim Gustafson 2026-05-04 10:21:19 -07:00
parent 1901a15a00
commit 082ecc87a8
2 changed files with 112 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
call_sim/.gitignore vendored
View File

@ -1,2 +1,3 @@
test_results
media
data

136
call_sim/src/main.rs
View File

@ -137,7 +137,7 @@ async fn run_minimal_example(test: &mut Test) -> Result<()> {
/// This is a test set to test the "normal phrasing" audio (and video) over various network
/// profiles. This sets packet time to 60ms to align with our production configuration.
async fn run_baseline(test: &mut Test, with_video: bool) -> Result<()> {
async fn run_baseline(test: &mut Test, with_video: bool, with_dred: bool) -> Result<()> {
let video = if with_video {
VideoConfig {
input_name: Some("ConferenceMotion_50fps@1280x720".to_string()),
@ -149,10 +149,11 @@ async fn run_baseline(test: &mut Test, with_video: bool) -> Result<()> {
test.run(
GroupConfig {
group_name: if !with_video {
"baseline".to_string()
} else {
"baseline_with_video".to_string()
group_name: match (with_video, with_dred) {
(false, false) => "baseline".to_string(),
(true, false) => "baseline_with_video".to_string(),
(false, true) => "baseline_with_dred".to_string(),
(true, true) => "baseline_with_video_and_dred".to_string(),
},
// Show all the different measurements in the summary columns. Hide video.
summary_report_columns: SummaryReportColumns {
@ -161,6 +162,8 @@ async fn run_baseline(test: &mut Test, with_video: bool) -> Result<()> {
show_pesq_mos: true,
show_plc_mos: true,
show_video: with_video,
show_send_stats: !with_dred,
show_dred_stats: with_dred,
..Default::default()
},
..Default::default()
@ -174,6 +177,13 @@ async fn run_baseline(test: &mut Test, with_video: bool) -> Result<()> {
// We don't look at analysis from client_a's point of view, so there
// is no need to generate anything for it.
generate_spectrogram: false,
enable_fec: !with_dred,
dred_duration: if with_dred { 100 } else { 0 },
dnn_weights_path: if with_dred {
"/data/deep_plc-dred-weights.bin".to_string()
} else {
"".to_string()
},
..Default::default()
},
video: video.clone(),
@ -188,6 +198,13 @@ async fn run_baseline(test: &mut Test, with_video: bool) -> Result<()> {
visqol_audio_analysis: true,
pesq_speech_analysis: true,
plc_speech_analysis: true,
enable_fec: !with_dred,
dred_duration: if with_dred { 100 } else { 0 },
dnn_weights_path: if with_dred {
"/data/deep_plc-dred-weights.bin".to_string()
} else {
"".to_string()
},
..Default::default()
},
video,
@ -220,7 +237,7 @@ async fn run_baseline(test: &mut Test, with_video: bool) -> Result<()> {
}
/// Test 60ms ptime over a range of loss using various bursty loss profiles.
async fn run_bursty_loss_test(test: &mut Test, with_video: bool) -> Result<()> {
async fn run_bursty_loss_test(test: &mut Test, with_video: bool, with_dred: bool) -> Result<()> {
let video = if with_video {
VideoConfig {
input_name: Some("ConferenceMotion_50fps@1280x720".to_string()),
@ -232,10 +249,11 @@ async fn run_bursty_loss_test(test: &mut Test, with_video: bool) -> Result<()> {
test.run(
GroupConfig {
group_name: if !with_video {
"bursty_loss_test".to_string()
} else {
"bursty_loss_test_with_video".to_string()
group_name: match (with_video, with_dred) {
(false, false) => "bursty_loss_test".to_string(),
(true, false) => "bursty_loss_test_with_video".to_string(),
(false, true) => "bursty_loss_test_with_dred".to_string(),
(true, true) => "bursty_loss_test_with_video_and_dred".to_string(),
},
// Show all the different measurements in the summary columns. Hide video.
summary_report_columns: SummaryReportColumns {
@ -244,6 +262,8 @@ async fn run_bursty_loss_test(test: &mut Test, with_video: bool) -> Result<()> {
show_pesq_mos: true,
show_plc_mos: true,
show_video: with_video,
show_send_stats: !with_dred,
show_dred_stats: with_dred,
..Default::default()
},
..Default::default()
@ -257,6 +277,13 @@ async fn run_bursty_loss_test(test: &mut Test, with_video: bool) -> Result<()> {
// We don't look at analysis from client_a's point of view, so there
// is no need to generate anything for it.
generate_spectrogram: false,
enable_fec: !with_dred,
dred_duration: if with_dred { 100 } else { 0 },
dnn_weights_path: if with_dred {
"/data/deep_plc-dred-weights.bin".to_string()
} else {
"".to_string()
},
..Default::default()
},
video: video.clone(),
@ -271,6 +298,13 @@ async fn run_bursty_loss_test(test: &mut Test, with_video: bool) -> Result<()> {
visqol_audio_analysis: true,
pesq_speech_analysis: true,
plc_speech_analysis: true,
enable_fec: !with_dred,
dred_duration: if with_dred { 100 } else { 0 },
dnn_weights_path: if with_dred {
"/data/deep_plc-dred-weights.bin".to_string()
} else {
"".to_string()
},
..Default::default()
},
video,
@ -310,7 +344,11 @@ async fn run_bursty_loss_test(test: &mut Test, with_video: bool) -> Result<()> {
/// Test 20ms and 60ms ptime over a range of deterministic loss, with and without dtx.
/// Note that deterministic loss is better than the SimpleLoss network profile, but
/// it is still not completely reliable.
async fn run_deterministic_loss_test(test: &mut Test, with_video: bool) -> Result<()> {
async fn run_deterministic_loss_test(
test: &mut Test,
with_video: bool,
with_dred: bool,
) -> Result<()> {
let video = if with_video {
VideoConfig {
input_name: Some("ConferenceMotion_50fps@1280x720".to_string()),
@ -334,6 +372,13 @@ async fn run_deterministic_loss_test(test: &mut Test, with_video: bool) -> Resul
initial_packet_size_ms,
enable_dtx,
generate_spectrogram: false,
enable_fec: !with_dred,
dred_duration: if with_dred { 100 } else { 0 },
dnn_weights_path: if with_dred {
"/data/deep_plc-dred-weights.bin".to_string()
} else {
"".to_string()
},
..Default::default()
},
video: video.clone(),
@ -349,6 +394,13 @@ async fn run_deterministic_loss_test(test: &mut Test, with_video: bool) -> Resul
visqol_audio_analysis: true,
pesq_speech_analysis: true,
plc_speech_analysis: true,
enable_fec: !with_dred,
dred_duration: if with_dred { 100 } else { 0 },
dnn_weights_path: if with_dred {
"/data/deep_plc-dred-weights.bin".to_string()
} else {
"".to_string()
},
..Default::default()
},
video: video.clone(),
@ -363,10 +415,11 @@ async fn run_deterministic_loss_test(test: &mut Test, with_video: bool) -> Resul
test.run(
GroupConfig {
group_name: if !with_video {
"deterministic_loss_test".to_string()
} else {
"deterministic_loss_test_with_video".to_string()
group_name: match (with_video, with_dred) {
(false, false) => "deterministic_loss_test".to_string(),
(true, false) => "deterministic_loss_test_with_video".to_string(),
(false, true) => "deterministic_loss_test_with_dred".to_string(),
(true, true) => "deterministic_loss_test_with_video_and_dred".to_string(),
},
summary_report_columns: SummaryReportColumns {
show_visqol_mos_speech: true,
@ -374,6 +427,8 @@ async fn run_deterministic_loss_test(test: &mut Test, with_video: bool) -> Resul
show_pesq_mos: true,
show_plc_mos: true,
show_video: with_video,
show_send_stats: !with_dred,
show_dred_stats: with_dred,
..Default::default()
},
..Default::default()
@ -746,7 +801,7 @@ async fn run_video_compare_vp8_vs_vp9(test: &mut Test, bitrate_values: &Vec<u16>
///
/// Uses a 12-second reference audio file so that the resulting 240-second session recording
/// can be chopped evenly and MOS calculated for each 12-second audio segment.
async fn run_changing_bandwidth_audio_test(test: &mut Test) -> Result<()> {
async fn run_changing_bandwidth_audio_test(test: &mut Test, with_dred: bool) -> Result<()> {
let test_cases = [20, 60, 120].map(|initial_packet_size_ms| TestCaseConfig {
test_case_name: format!("ptime_{initial_packet_size_ms}"),
length_seconds: 240,
@ -755,6 +810,13 @@ async fn run_changing_bandwidth_audio_test(test: &mut Test) -> Result<()> {
input_name: "normal_12s".to_string(),
initial_packet_size_ms,
generate_spectrogram: false,
enable_fec: !with_dred,
dred_duration: if with_dred { 100 } else { 0 },
dnn_weights_path: if with_dred {
"/data/deep_plc-dred-weights.bin".to_string()
} else {
"".to_string()
},
..Default::default()
},
..Default::default()
@ -769,6 +831,13 @@ async fn run_changing_bandwidth_audio_test(test: &mut Test) -> Result<()> {
visqol_audio_analysis: true,
pesq_speech_analysis: true,
plc_speech_analysis: true,
enable_fec: !with_dred,
dred_duration: if with_dred { 100 } else { 0 },
dnn_weights_path: if with_dred {
"/data/deep_plc-dred-weights.bin".to_string()
} else {
"".to_string()
},
..Default::default()
},
..Default::default()
@ -778,13 +847,18 @@ async fn run_changing_bandwidth_audio_test(test: &mut Test) -> Result<()> {
test.run(
GroupConfig {
group_name: "changing_bandwidth_audio_test".to_string(),
group_name: match with_dred {
false => "changing_bandwidth_audio_test".to_string(),
true => "changing_bandwidth_audio_test_with_dred".to_string(),
},
summary_report_columns: SummaryReportColumns {
show_visqol_mos_speech: true,
show_visqol_mos_audio: true,
show_pesq_mos: true,
show_plc_mos: true,
show_video: false,
show_send_stats: !with_dred,
show_dred_stats: with_dred,
..Default::default()
},
..Default::default()
@ -1124,7 +1198,7 @@ async fn main() -> Result<()> {
let mut test_sets = args.test_sets;
if test_sets.is_empty() {
// For quick testing, change this to the name of your test case.
test_sets.push("baseline".to_string());
test_sets.push("baseline_with_dred".to_string());
}
let direct_call_config = CallTypeConfig::Direct;
@ -1156,19 +1230,31 @@ async fn main() -> Result<()> {
)?;
match test_set_name.as_str() {
"minimal_example" => run_minimal_example(test).await?,
"baseline" => run_baseline(test, false).await?,
"baseline_with_video" => run_baseline(test, true).await?,
"bursty_loss_test" => run_bursty_loss_test(test, false).await?,
"bursty_loss_test_with_video" => run_bursty_loss_test(test, true).await?,
"deterministic_loss_test" => run_deterministic_loss_test(test, false).await?,
"deterministic_loss_test_with_video" => run_deterministic_loss_test(test, true).await?,
"baseline" => run_baseline(test, false, false).await?,
"baseline_with_dred" => run_baseline(test, false, true).await?,
"baseline_with_video" => run_baseline(test, true, false).await?,
"bursty_loss_test" => run_bursty_loss_test(test, false, false).await?,
"bursty_loss_test_with_dred" => run_bursty_loss_test(test, false, true).await?,
"bursty_loss_test_with_video" => run_bursty_loss_test(test, true, false).await?,
"deterministic_loss_test" => run_deterministic_loss_test(test, false, false).await?,
"deterministic_loss_test_with_dred" => {
run_deterministic_loss_test(test, false, true).await?
}
"deterministic_loss_test_with_video" => {
run_deterministic_loss_test(test, true, false).await?
}
"relay_tests" => run_relay_tests(test).await?,
"turn_long_tests" => run_turn_long_tests(test).await?,
"video_send_over_bandwidth" => run_video_send_over_bandwidth(test).await?,
"video_compare_vp8_vs_vp9" => {
run_video_compare_vp8_vs_vp9(test, &vec![1000, 2000]).await?
}
"changing_bandwidth_audio_test" => run_changing_bandwidth_audio_test(test).await?,
"changing_bandwidth_audio_test" => {
run_changing_bandwidth_audio_test(test, false).await?
}
"changing_bandwidth_audio_test_with_dred" => {
run_changing_bandwidth_audio_test(test, true).await?
}
"profiling_suite" => run_perf_test(test).await?,
"plc_tests" => run_plc_tests(test).await?,
"dred_tests" => run_dred_tests(test).await?,