mirror of
https://github.com/storytold/craft-libs.git
synced 2026-11-16 23:42:27 +00:00
Merge remote-tracking branch 'origin/codex/fbx-ascii-budget' into fredapps/i778-fbx-batch
# Conflicts: # crates/mesh/src/tests/fbx.rs
This commit is contained in:
commit
029137bf87
2 files changed
+110
-83
No files matched your search
+99
-83
@@ -88,7 +88,8 @@ impl Budget {
|
||||
}
|
||||
|
||||
/// Bytes the values under `records` keep. Used for ASCII files, which hold array elements as one
|
||||
/// [`Prop`] each and are parsed before they can be counted (the text itself bounds them).
|
||||
/// [`Prop`] each.
|
||||
#[cfg(test)]
|
||||
pub(crate) fn value_bytes(records: &[Record]) -> Option<usize> {
|
||||
records.iter().try_fold(0usize, |acc, n| {
|
||||
let own = n.props.iter().try_fold(0usize, |a, p| {
|
||||
@@ -238,9 +239,16 @@ pub fn read_document(bytes: &[u8], limits: &Limits) -> Result<Document> {
|
||||
let (version, records) = if is_binary(bytes) {
|
||||
read_binary(bytes, &budget)?
|
||||
} else {
|
||||
let (version, records) = read_ascii(&String::from_utf8_lossy(bytes))?;
|
||||
budget.take(value_bytes(&records), "its values")?;
|
||||
(version, records)
|
||||
let text = match std::str::from_utf8(bytes) {
|
||||
Ok(text) => std::borrow::Cow::Borrowed(text),
|
||||
Err(_) => {
|
||||
if bytes.len() > limits.max_data_bytes {
|
||||
return Err(too_large(KIND, "invalid UTF-8 text exceeds the memory limit"));
|
||||
}
|
||||
String::from_utf8_lossy(bytes)
|
||||
}
|
||||
};
|
||||
read_ascii(&text, &budget)?
|
||||
};
|
||||
Ok(Document { version, records, left: budget.left.get() })
|
||||
}
|
||||
@@ -436,110 +444,118 @@ fn read_binary(bytes: &[u8], budget: &Budget) -> Result<(u32, Vec<Record>)> {
|
||||
// ---------------------------------------------------------------------------------------------
|
||||
// ASCII
|
||||
|
||||
#[derive(Clone, Debug, PartialEq)]
|
||||
enum Tok {
|
||||
Key(String),
|
||||
Val(Prop),
|
||||
/// `*N` array length (the values follow in an `a:` child).
|
||||
#[derive(Clone, Copy, Debug, PartialEq)]
|
||||
enum Tok<'a> {
|
||||
Key(&'a str),
|
||||
Value(&'a str, bool),
|
||||
Count,
|
||||
Comma,
|
||||
Open,
|
||||
Close,
|
||||
}
|
||||
|
||||
fn tokenize(text: &str) -> Vec<Tok> {
|
||||
let mut out = vec![];
|
||||
let mut it = text.char_indices().peekable();
|
||||
while let Some(&(i, c)) = it.peek() {
|
||||
match c {
|
||||
c if c.is_whitespace() => {
|
||||
it.next();
|
||||
/// Scan on demand so punctuation and ignored input cannot build an unbounded token vector.
|
||||
struct Ascii<'a> {
|
||||
text: &'a str,
|
||||
pos: usize,
|
||||
lookahead: Option<Tok<'a>>,
|
||||
}
|
||||
|
||||
impl<'a> Ascii<'a> {
|
||||
fn next_token(&mut self) -> Option<Tok<'a>> {
|
||||
loop {
|
||||
let rest = self.text.get(self.pos..)?;
|
||||
let c = rest.chars().next()?;
|
||||
if c.is_whitespace() {
|
||||
self.pos += c.len_utf8();
|
||||
continue;
|
||||
}
|
||||
';' => {
|
||||
for (_, c) in it.by_ref() {
|
||||
if c == '\n' {
|
||||
break;
|
||||
}
|
||||
if c == ';' {
|
||||
self.pos += rest.find('\n').map_or(rest.len(), |i| i + 1);
|
||||
continue;
|
||||
}
|
||||
match c {
|
||||
',' | '{' | '}' => {
|
||||
self.pos += 1;
|
||||
return Some(match c {
|
||||
',' => Tok::Comma,
|
||||
'{' => Tok::Open,
|
||||
_ => Tok::Close,
|
||||
});
|
||||
}
|
||||
}
|
||||
',' => {
|
||||
it.next();
|
||||
out.push(Tok::Comma);
|
||||
}
|
||||
'{' => {
|
||||
it.next();
|
||||
out.push(Tok::Open);
|
||||
}
|
||||
'}' => {
|
||||
it.next();
|
||||
out.push(Tok::Close);
|
||||
}
|
||||
'"' => {
|
||||
it.next();
|
||||
let start = i + 1;
|
||||
let mut end = text.len();
|
||||
for (j, c) in it.by_ref() {
|
||||
if c == '"' {
|
||||
end = j;
|
||||
break;
|
||||
}
|
||||
'"' => {
|
||||
let tail = rest.get(1..).unwrap_or("");
|
||||
let end = tail.find('"').unwrap_or(tail.len());
|
||||
self.pos += 1 + end + usize::from(end < tail.len());
|
||||
return Some(Tok::Value(tail.get(..end).unwrap_or(""), true));
|
||||
}
|
||||
out.push(Tok::Val(Prop::Str(text.get(start..end).unwrap_or("").replace(""", "\""))));
|
||||
}
|
||||
_ => {
|
||||
let start = i;
|
||||
let mut end = text.len();
|
||||
while let Some(&(j, c)) = it.peek() {
|
||||
if c.is_whitespace() || matches!(c, ',' | '{' | '}' | '"' | ';' | ':') {
|
||||
end = j;
|
||||
break;
|
||||
_ => {
|
||||
let end = rest.find(|c: char| c.is_whitespace() || matches!(c, ',' | '{' | '}' | '"' | ';' | ':')).unwrap_or(rest.len());
|
||||
let word = rest.get(..end).unwrap_or("");
|
||||
self.pos += end;
|
||||
if rest.as_bytes().get(end) == Some(&b':') {
|
||||
self.pos += 1;
|
||||
return Some(Tok::Key(word));
|
||||
}
|
||||
if word.starts_with('*') {
|
||||
return Some(Tok::Count);
|
||||
}
|
||||
if !word.is_empty() {
|
||||
return Some(Tok::Value(word, false));
|
||||
}
|
||||
it.next();
|
||||
}
|
||||
let word = text.get(start..end).unwrap_or("");
|
||||
if it.peek().is_some_and(|(_, c)| *c == ':') {
|
||||
it.next();
|
||||
out.push(Tok::Key(word.to_string()));
|
||||
} else if word.starts_with('*') {
|
||||
out.push(Tok::Count);
|
||||
} else if let Ok(i) = word.parse::<i64>() {
|
||||
out.push(Tok::Val(Prop::Int(i)));
|
||||
} else if let Ok(f) = word.parse::<f64>() {
|
||||
out.push(Tok::Val(Prop::Float(f)));
|
||||
} else if !word.is_empty() {
|
||||
out.push(Tok::Val(Prop::Str(word.to_string())));
|
||||
} else {
|
||||
// A stray `:` with no key.
|
||||
it.next();
|
||||
self.pos += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
out
|
||||
|
||||
fn peek(&mut self) -> Option<Tok<'a>> {
|
||||
if self.lookahead.is_none() {
|
||||
self.lookahead = self.next_token();
|
||||
}
|
||||
self.lookahead
|
||||
}
|
||||
|
||||
fn take(&mut self) -> Option<Tok<'a>> {
|
||||
self.lookahead.take().or_else(|| self.next_token())
|
||||
}
|
||||
}
|
||||
|
||||
fn ascii_list(toks: &[Tok], i: &mut usize, depth: usize) -> Result<Vec<Record>> {
|
||||
fn ascii_list(scan: &mut Ascii<'_>, budget: &Budget, records_budget: &Budget, depth: usize) -> Result<Vec<Record>> {
|
||||
if depth > MAX_DEPTH {
|
||||
return Err(too_large(KIND, "records nested too deeply"));
|
||||
}
|
||||
let mut out = vec![];
|
||||
while let Some(t) = toks.get(*i) {
|
||||
*i += 1;
|
||||
while let Some(t) = scan.take() {
|
||||
match t {
|
||||
Tok::Close if depth > 0 => return Ok(out),
|
||||
Tok::Key(name) => {
|
||||
let mut record = Record { name: name.clone(), ..Record::default() };
|
||||
records_budget.take(std::mem::size_of::<Record>().checked_add(name.len()), "its records")?;
|
||||
let mut record = Record { name: name.to_string(), ..Record::default() };
|
||||
loop {
|
||||
match toks.get(*i) {
|
||||
Some(Tok::Val(v)) => record.props.push(v.clone()),
|
||||
match scan.peek() {
|
||||
Some(Tok::Value(value, quoted)) => {
|
||||
let number = if quoted {
|
||||
None
|
||||
} else {
|
||||
value.parse::<i64>().ok().map(Prop::Int).or_else(|| value.parse::<f64>().ok().map(Prop::Float))
|
||||
};
|
||||
let size = std::mem::size_of::<Prop>().checked_add(if number.is_some() { 0 } else { value.len() });
|
||||
// Charge before copying a string or growing the property vector.
|
||||
budget.take(size, "its values")?;
|
||||
record
|
||||
.props
|
||||
.push(number.unwrap_or_else(|| Prop::Str(if quoted { value.replace(""", "\"") } else { value.to_string() })));
|
||||
}
|
||||
Some(Tok::Count | Tok::Comma) => {}
|
||||
_ => break,
|
||||
}
|
||||
*i += 1;
|
||||
scan.take();
|
||||
}
|
||||
if toks.get(*i) == Some(&Tok::Open) {
|
||||
*i += 1;
|
||||
record.children = ascii_list(toks, i, depth + 1)?;
|
||||
if scan.peek() == Some(Tok::Open) {
|
||||
scan.take();
|
||||
record.children = ascii_list(scan, budget, records_budget, depth + 1)?;
|
||||
}
|
||||
out.push(record);
|
||||
}
|
||||
@@ -549,10 +565,10 @@ fn ascii_list(toks: &[Tok], i: &mut usize, depth: usize) -> Result<Vec<Record>>
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
fn read_ascii(text: &str) -> Result<(u32, Vec<Record>)> {
|
||||
let toks = tokenize(text);
|
||||
let mut i = 0;
|
||||
let records = ascii_list(&toks, &mut i, 0)?;
|
||||
fn read_ascii(text: &str, budget: &Budget) -> Result<(u32, Vec<Record>)> {
|
||||
let mut scan = Ascii { text, pos: 0, lookahead: None };
|
||||
let records_budget = Budget::new(budget.limit);
|
||||
let records = ascii_list(&mut scan, budget, &records_budget, 0)?;
|
||||
let version = records
|
||||
.iter()
|
||||
.find(|n| n.name == "FBXHeaderExtension")
|
||||
|
||||
@@ -771,4 +771,15 @@ fn fbx_corner_colors_keep_seams_and_incomplete_layers_are_omitted() {
|
||||
assert!(colors.contains(&[1.0, 0.0, 1.0, 1.0]));
|
||||
let incomplete = text.replace(", 1,0,1,1\n}", "\n}");
|
||||
assert!(parse(incomplete.as_bytes()).unwrap().meshes[0].colors.is_empty());
|
||||
fn ascii_budget_applies_while_reading_values_and_records() {
|
||||
let limits = Limits { max_data_bytes: 512, ..Limits::default() };
|
||||
let many_values = format!("Objects: {{\n A: {}\n}}", "1,".repeat(1000));
|
||||
assert!(matches!(read_document(many_values.as_bytes(), &limits), Err(Error::TooLarge(_))));
|
||||
let many_records = format!("Objects: {{\n{}\n}}", "A: {}\n".repeat(1000));
|
||||
assert!(matches!(read_document(many_records.as_bytes(), &limits), Err(Error::TooLarge(_))));
|
||||
let giant_string = format!("Objects: {{ A: \"{}\" }}", "x".repeat(1024));
|
||||
assert!(matches!(read_document(giant_string.as_bytes(), &limits), Err(Error::TooLarge(_))));
|
||||
// Empty punctuation and comments do not consume memory in proportion to their length.
|
||||
let ignored = format!(";{}\nObjects: {{}}", ",".repeat(10_000));
|
||||
assert!(read_document(ignored.as_bytes(), &limits).is_ok());
|
||||
}
|
||||
Reference in new issue
Block a user