mirror of
https://github.com/storytold/artcraft.git
synced 2026-11-01 16:08:30 +00:00
margin test
This commit is contained in:
1 parent
1f943bff01
commit
718d4ec4ee
1 file changed
+170
+170
@@ -231,6 +231,176 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
// ── Cross-provider margin: ArtCraft is consistently ~30% above GrokApi ──
|
||||
|
||||
mod small_margin {
|
||||
use super::*;
|
||||
use crate::api::common_aspect_ratio::CommonAspectRatio;
|
||||
|
||||
#[test]
|
||||
fn small_margin() {
|
||||
let resolutions = [
|
||||
None,
|
||||
Some(CommonResolution::FourEightyP),
|
||||
Some(CommonResolution::SevenTwentyP),
|
||||
Some(CommonResolution::TenEightyP), // clamps to 720p on both sides
|
||||
];
|
||||
let durations: &[u16] = &[1, 4, 5, 8, 10, 12, 15];
|
||||
let aspect_ratios = [
|
||||
None,
|
||||
Some(CommonAspectRatio::Auto),
|
||||
Some(CommonAspectRatio::Square),
|
||||
Some(CommonAspectRatio::WideSixteenByNine),
|
||||
Some(CommonAspectRatio::TallNineBySixteen),
|
||||
Some(CommonAspectRatio::WideFourByThree),
|
||||
Some(CommonAspectRatio::TallThreeByFour),
|
||||
Some(CommonAspectRatio::WideThreeByTwo),
|
||||
Some(CommonAspectRatio::TallTwoByThree),
|
||||
];
|
||||
// (has_start_frame, num_reference_images). At most one of the two is
|
||||
// non-default — start_frame and reference_images are mutually exclusive
|
||||
// on the GrokApi wire (image-to-video vs reference-to-video), so we
|
||||
// don't mix them here to keep the comparison apples-to-apples.
|
||||
let image_combos: &[(bool, usize)] = &[
|
||||
(false, 0), // text-to-video
|
||||
(true, 0), // image-to-video, 1 source image
|
||||
(false, 1), // reference-to-video, 1 ref image
|
||||
(false, 2),
|
||||
(false, 3),
|
||||
];
|
||||
|
||||
let mut total_cases = 0;
|
||||
let mut failures = Vec::new();
|
||||
|
||||
for &res in &resolutions {
|
||||
for &dur in durations {
|
||||
for &ar in &aspect_ratios {
|
||||
for &(has_start, num_refs) in image_combos {
|
||||
total_cases += 1;
|
||||
let artcraft = artcraft_cents(res, dur, ar, has_start, num_refs);
|
||||
let grokapi = grokapi_cents(res, dur, ar, has_start, num_refs);
|
||||
|
||||
// Margin requirement: artcraft / max(grokapi - 1, 1) >= 1.29.
|
||||
// The (-1, 1) floor accounts for one cent of GrokApi-side
|
||||
// ceiling-rounding noise (see test doc comment above).
|
||||
let grokapi_compare = grokapi.saturating_sub(1).max(1);
|
||||
let lhs = artcraft.saturating_mul(100);
|
||||
let rhs = grokapi_compare.saturating_mul(129);
|
||||
|
||||
if lhs < rhs {
|
||||
let pct = if grokapi == 0 { 0.0 }
|
||||
else { (artcraft as f64 / grokapi as f64 - 1.0) * 100.0 };
|
||||
failures.push(format!(
|
||||
"res={:?} dur={}s ar={:?} start={} refs={} → \
|
||||
artcraft={}¢ vs grokapi={}¢ (margin {:.1}%, want ≥ 29% after 1¢ tolerance)",
|
||||
res, dur, ar, has_start, num_refs, artcraft, grokapi, pct,
|
||||
));
|
||||
}
|
||||
|
||||
// Defense in depth: ArtCraft must always cost at least as
|
||||
// much as GrokApi, regardless of rounding noise.
|
||||
assert!(
|
||||
artcraft >= grokapi,
|
||||
"ArtCraft ({artcraft}¢) cheaper than GrokApi ({grokapi}¢) for \
|
||||
res={res:?} dur={dur}s ar={ar:?} start={has_start} refs={num_refs}",
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if !failures.is_empty() {
|
||||
let summary = failures.iter().take(10).cloned().collect::<Vec<_>>().join("\n ");
|
||||
panic!(
|
||||
"{} / {} (res × duration × aspect × image_combo) tuples failed \
|
||||
the ≥29% (with 1¢ tolerance) margin check.\nFirst 10:\n {}",
|
||||
failures.len(), total_cases, summary,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// ── Cross-provider helpers ──
|
||||
|
||||
fn artcraft_cents(
|
||||
resolution: Option<CommonResolution>,
|
||||
duration_seconds: u16,
|
||||
aspect_ratio: Option<CommonAspectRatio>,
|
||||
has_start_frame: bool,
|
||||
num_reference_images: usize,
|
||||
) -> u64 {
|
||||
let start_frame = if has_start_frame {
|
||||
// ArtCraft's omni endpoint takes MediaFileTokens for image refs.
|
||||
Some(ImageRef::MediaFileToken(MediaFileToken::new("mf_start".to_string())))
|
||||
} else { None };
|
||||
let reference_images = if num_reference_images > 0 {
|
||||
Some(ImageListRef::MediaFileTokens(
|
||||
(0..num_reference_images)
|
||||
.map(|i| MediaFileToken::new(format!("mf_ref_{i}")))
|
||||
.collect(),
|
||||
))
|
||||
} else { None };
|
||||
|
||||
let builder = GenerateVideoRequestBuilder {
|
||||
model: CommonVideoModel::GrokImagineVideo,
|
||||
provider: Provider::Artcraft,
|
||||
resolution,
|
||||
aspect_ratio,
|
||||
duration_seconds: Some(duration_seconds),
|
||||
// Batch is fixed at 1 because GrokApi's per-request cost calculator
|
||||
// doesn't multiply by batch (Grok returns one video per call); the
|
||||
// markup test would compare apples to oranges otherwise.
|
||||
video_batch_count: Some(1),
|
||||
start_frame,
|
||||
reference_images,
|
||||
..Default::default()
|
||||
};
|
||||
builder.build2()
|
||||
.expect("build2 (artcraft) should succeed")
|
||||
.estimate_cost()
|
||||
.expect("estimate_cost (artcraft) should succeed")
|
||||
.cost_in_usd_cents
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
fn grokapi_cents(
|
||||
resolution: Option<CommonResolution>,
|
||||
duration_seconds: u16,
|
||||
aspect_ratio: Option<CommonAspectRatio>,
|
||||
has_start_frame: bool,
|
||||
num_reference_images: usize,
|
||||
) -> u64 {
|
||||
let start_frame = if has_start_frame {
|
||||
// GrokApi (direct xAI) takes URLs for image refs.
|
||||
Some(ImageRef::Url("https://example.com/start.png".to_string()))
|
||||
} else { None };
|
||||
let reference_images = if num_reference_images > 0 {
|
||||
Some(ImageListRef::Urls(
|
||||
(0..num_reference_images)
|
||||
.map(|i| format!("https://example.com/ref_{i}.png"))
|
||||
.collect(),
|
||||
))
|
||||
} else { None };
|
||||
|
||||
let builder = GenerateVideoRequestBuilder {
|
||||
model: CommonVideoModel::GrokImagineVideo,
|
||||
provider: Provider::GrokApi,
|
||||
resolution,
|
||||
aspect_ratio,
|
||||
duration_seconds: Some(duration_seconds),
|
||||
video_batch_count: Some(1),
|
||||
start_frame,
|
||||
reference_images,
|
||||
..Default::default()
|
||||
};
|
||||
builder.build2()
|
||||
.expect("build2 (grokapi) should succeed")
|
||||
.estimate_cost()
|
||||
.expect("estimate_cost (grokapi) should succeed")
|
||||
.cost_in_usd_cents
|
||||
.unwrap()
|
||||
}
|
||||
}
|
||||
|
||||
// ── Helpers ──
|
||||
|
||||
fn build_cost(
|
||||
|
||||
Reference in new issue
Block a user