margin test

This commit is contained in:
Brandon Thomas committed 2026-05-24 01:14:42 -04:00
1 parent 1f943bff01
commit 718d4ec4ee
1 file changed
+170
@@ -231,6 +231,176 @@ mod tests {
}
}
// ── Cross-provider margin: ArtCraft is consistently ~30% above GrokApi ──
mod small_margin {
use super::*;
use crate::api::common_aspect_ratio::CommonAspectRatio;
#[test]
fn small_margin() {
let resolutions = [
None,
Some(CommonResolution::FourEightyP),
Some(CommonResolution::SevenTwentyP),
Some(CommonResolution::TenEightyP), // clamps to 720p on both sides
];
let durations: &[u16] = &[1, 4, 5, 8, 10, 12, 15];
let aspect_ratios = [
None,
Some(CommonAspectRatio::Auto),
Some(CommonAspectRatio::Square),
Some(CommonAspectRatio::WideSixteenByNine),
Some(CommonAspectRatio::TallNineBySixteen),
Some(CommonAspectRatio::WideFourByThree),
Some(CommonAspectRatio::TallThreeByFour),
Some(CommonAspectRatio::WideThreeByTwo),
Some(CommonAspectRatio::TallTwoByThree),
];
// (has_start_frame, num_reference_images). At most one of the two is
// non-default — start_frame and reference_images are mutually exclusive
// on the GrokApi wire (image-to-video vs reference-to-video), so we
// don't mix them here to keep the comparison apples-to-apples.
let image_combos: &[(bool, usize)] = &[
(false, 0), // text-to-video
(true, 0), // image-to-video, 1 source image
(false, 1), // reference-to-video, 1 ref image
(false, 2),
(false, 3),
];
let mut total_cases = 0;
let mut failures = Vec::new();
for &res in &resolutions {
for &dur in durations {
for &ar in &aspect_ratios {
for &(has_start, num_refs) in image_combos {
total_cases += 1;
let artcraft = artcraft_cents(res, dur, ar, has_start, num_refs);
let grokapi = grokapi_cents(res, dur, ar, has_start, num_refs);
// Margin requirement: artcraft / max(grokapi - 1, 1) >= 1.29.
// The (-1, 1) floor accounts for one cent of GrokApi-side
// ceiling-rounding noise (see test doc comment above).
let grokapi_compare = grokapi.saturating_sub(1).max(1);
let lhs = artcraft.saturating_mul(100);
let rhs = grokapi_compare.saturating_mul(129);
if lhs < rhs {
let pct = if grokapi == 0 { 0.0 }
else { (artcraft as f64 / grokapi as f64 - 1.0) * 100.0 };
failures.push(format!(
"res={:?} dur={}s ar={:?} start={} refs={} → \
artcraft={}¢ vs grokapi={}¢ (margin {:.1}%, want ≥ 29% after 1¢ tolerance)",
res, dur, ar, has_start, num_refs, artcraft, grokapi, pct,
));
}
// Defense in depth: ArtCraft must always cost at least as
// much as GrokApi, regardless of rounding noise.
assert!(
artcraft >= grokapi,
"ArtCraft ({artcraft}¢) cheaper than GrokApi ({grokapi}¢) for \
res={res:?} dur={dur}s ar={ar:?} start={has_start} refs={num_refs}",
);
}
}
}
}
if !failures.is_empty() {
let summary = failures.iter().take(10).cloned().collect::<Vec<_>>().join("\n ");
panic!(
"{} / {} (res × duration × aspect × image_combo) tuples failed \
the ≥29% (with 1¢ tolerance) margin check.\nFirst 10:\n {}",
failures.len(), total_cases, summary,
);
}
}
// ── Cross-provider helpers ──
fn artcraft_cents(
resolution: Option<CommonResolution>,
duration_seconds: u16,
aspect_ratio: Option<CommonAspectRatio>,
has_start_frame: bool,
num_reference_images: usize,
) -> u64 {
let start_frame = if has_start_frame {
// ArtCraft's omni endpoint takes MediaFileTokens for image refs.
Some(ImageRef::MediaFileToken(MediaFileToken::new("mf_start".to_string())))
} else { None };
let reference_images = if num_reference_images > 0 {
Some(ImageListRef::MediaFileTokens(
(0..num_reference_images)
.map(|i| MediaFileToken::new(format!("mf_ref_{i}")))
.collect(),
))
} else { None };
let builder = GenerateVideoRequestBuilder {
model: CommonVideoModel::GrokImagineVideo,
provider: Provider::Artcraft,
resolution,
aspect_ratio,
duration_seconds: Some(duration_seconds),
// Batch is fixed at 1 because GrokApi's per-request cost calculator
// doesn't multiply by batch (Grok returns one video per call); the
// markup test would compare apples to oranges otherwise.
video_batch_count: Some(1),
start_frame,
reference_images,
..Default::default()
};
builder.build2()
.expect("build2 (artcraft) should succeed")
.estimate_cost()
.expect("estimate_cost (artcraft) should succeed")
.cost_in_usd_cents
.unwrap()
}
fn grokapi_cents(
resolution: Option<CommonResolution>,
duration_seconds: u16,
aspect_ratio: Option<CommonAspectRatio>,
has_start_frame: bool,
num_reference_images: usize,
) -> u64 {
let start_frame = if has_start_frame {
// GrokApi (direct xAI) takes URLs for image refs.
Some(ImageRef::Url("https://example.com/start.png".to_string()))
} else { None };
let reference_images = if num_reference_images > 0 {
Some(ImageListRef::Urls(
(0..num_reference_images)
.map(|i| format!("https://example.com/ref_{i}.png"))
.collect(),
))
} else { None };
let builder = GenerateVideoRequestBuilder {
model: CommonVideoModel::GrokImagineVideo,
provider: Provider::GrokApi,
resolution,
aspect_ratio,
duration_seconds: Some(duration_seconds),
video_batch_count: Some(1),
start_frame,
reference_images,
..Default::default()
};
builder.build2()
.expect("build2 (grokapi) should succeed")
.estimate_cost()
.expect("estimate_cost (grokapi) should succeed")
.cost_in_usd_cents
.unwrap()
}
}
// ── Helpers ──
fn build_cost(