diff --git a/Cargo.lock b/Cargo.lock index c4cc46feaa..0e0f375a93 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6851,6 +6851,7 @@ dependencies = [ "symphonia", "tempdir", "tempfile", + "testing", "thumbnail_generator", "time", "tokens", diff --git a/crates/lib/files/filesys/src/lib.rs b/crates/lib/files/filesys/src/lib.rs index 05a67e9d43..2061b52018 100644 --- a/crates/lib/files/filesys/src/lib.rs +++ b/crates/lib/files/filesys/src/lib.rs @@ -30,4 +30,5 @@ pub mod file_size; pub mod filename_concat; pub mod is_filesystem_full_error; pub mod path_to_string; +pub mod read_lines_iterator; pub mod rename_across_devices; diff --git a/crates/lib/files/filesys/src/read_lines_iterator.rs b/crates/lib/files/filesys/src/read_lines_iterator.rs new file mode 100644 index 0000000000..3700115ec8 --- /dev/null +++ b/crates/lib/files/filesys/src/read_lines_iterator.rs @@ -0,0 +1,11 @@ +use std::fs::File; +use std::io::{self, BufRead}; +use std::path::Path; + +/// Read lines from a file in an iterator. +/// From Rust manual: https://doc.rust-lang.org/rust-by-example/std_misc/file/read_lines.html +pub fn read_lines_iterator

(filename: P) -> io::Result>> +where P: AsRef, { + let file = File::open(filename)?; + Ok(io::BufReader::new(file).lines()) +} diff --git a/crates/service/web/storyteller_web/Cargo.toml b/crates/service/web/storyteller_web/Cargo.toml index cb6c0bba78..be37e63770 100644 --- a/crates/service/web/storyteller_web/Cargo.toml +++ b/crates/service/web/storyteller_web/Cargo.toml @@ -183,7 +183,10 @@ actix = "0.13.5" diesel = { version = "2.2.1", features = ["mysql"] } [dev-dependencies] +# Testing/Dev Deps: Internal +testing = { path = "../../../lib/testing" } + +# Testing/Dev Deps: External speculoos = "0.11.0" # Fluent assertions strum = { version = "0.26", features = ["derive"] } tempfile = "3.10.1" # For testing - diff --git a/crates/service/web/storyteller_web/src/util/title_to_url_slug.rs b/crates/service/web/storyteller_web/src/util/title_to_url_slug.rs index 7461c9b86c..32a30f22b2 100644 --- a/crates/service/web/storyteller_web/src/util/title_to_url_slug.rs +++ b/crates/service/web/storyteller_web/src/util/title_to_url_slug.rs @@ -1,10 +1,18 @@ use once_cell::sync::Lazy; use regex::Regex; -const PARENS_REGEX : Lazy = Lazy::new(|| { - Regex::new(r"\([^\\()]*\)").unwrap() +const PARENS_AND_BRACKETS_REGEX: Lazy = Lazy::new(|| { + //Regex::new(r"\([^\\()]*\)").unwrap() + Regex::new(concat!( + r"[\(\[]", // Start delimiter: ( or [ + r"[^\\()\\[\\]]*", // In-between content + r"[\)\]]" // End delimiter: ) or ] + )).unwrap() }); +const INVALID_CHARACTER_REGEX: Lazy = Lazy::new(|| { + Regex::new(r"\s+").unwrap() +}); const SPACE_REGEX : Lazy = Lazy::new(|| { Regex::new(r"\s+").unwrap() }); @@ -25,9 +33,44 @@ const VERSION_REGEX: Lazy = Lazy::new(|| { /// Convert a model title to a URL slug. pub fn title_to_url_slug(title: &str) -> Option { let title = title.trim().to_lowercase(); - let title = PARENS_REGEX.replace_all(&title, "").to_string(); - + let title = PARENS_AND_BRACKETS_REGEX.replace_all(&title, "").to_string(); let title = VERSION_REGEX.replace_all(title.trim(), "").to_string(); + + // Dangerous and otherwise bad characters + // https://perishablepress.com/stop-using-unsafe-characters-in-urls/ + // { , } , | , \ , ^ , ~ , [ , ] , and ` + // Unsafe characters " < > % { } | \ ^ ` + // Reserved Characters : / ? # [ ] @ ! $ & ' ( ) * + , ; = + let title = title.replace(&[ + '!', + '#', + '%', + '&', + '(', + ')', + '+', + ',', + '.', + '/', + ':', + ';', + '<', + '=', + '>', + '?', + '@', + '[', + '\"', + '\'', + ']', + '^', + '`', + '{', + '|', + '}', + '~', + ][..], " "); + let title = SPACE_REGEX.replace_all(title.trim(), "-").to_string(); let title = ENDING_NOISE.replace_all(title.trim(), "").to_string(); @@ -41,43 +84,150 @@ pub fn title_to_url_slug(title: &str) -> Option { #[cfg(test)] mod tests { + use filesys::read_lines_iterator::read_lines_iterator; + use testing::test_file_path::test_file_path; + use crate::util::title_to_url_slug::title_to_url_slug; - fn assert_parsed(title: &str, expected: &str) { + fn assert_expected(expected: &str, title: &str) { assert_eq!(title_to_url_slug(title).as_deref(), Some(expected)); } + fn assert_none(title: &str) { + assert_eq!(title_to_url_slug(title).as_deref(), None); + } + + //#[test] + fn many_files() { + let path = test_file_path("model_weights_titles.txt").unwrap(); + let lines = read_lines_iterator(&path).unwrap(); + let mut i = 0; + for line in lines.flatten() { + if i > 1000 { + break; + } + let title = line.as_str(); + let slug = title_to_url_slug(title); + + match slug { + None => println!("Failed to parse: {}", title), + Some(slug) => println!("{:?} <- {:?}", slug, title), + } + + i += 1; + } + assert_eq!(1, 2); + } + + #[test] + fn empty_titles() { + assert_none(" "); + assert_none(""); + assert_none("..."); + } + #[test] fn simple_titles() { - assert_parsed("Bill Nye", "bill-nye"); + assert_expected("bill-nye", "Bill Nye"); + } + + #[test] + fn parens() { + assert_expected("foo", "Foo (Bar)"); + } + + #[test] + fn square_brackets() { + assert_expected("foo", "Foo [Bar]"); + } + + #[test] + pub fn dangerous_url_characters() { + assert_none(r" // { , } , | , \ , ^ , ~ , [ , ] , `"); + assert_none(r" \ < > % { } | \ ^ `"); + assert_none(r" / ? # [ ] @ ! $ & ' ( ) * + , ; = \ "); + assert_expected("at-t-natural-voices-rosa", "AT&T natural voices rosa"); + assert_expected("jaden-judai-yuki", "Jaden/Judai Yuki (JPN, V2)"); } #[test] fn removes_parens() { - assert_parsed("Donald Trump (Angry)", "donald-trump"); - assert_parsed("Lionel Messi. (Español 2020 - 2023.)", "lionel-messi"); - assert_parsed("Mariano Closs (Relator de fútbol Argentino)", "mariano-closs"); - assert_parsed("Mariano Closs (full version)", "mariano-closs"); - assert_parsed("Naruto Uzumaki (Part 1)", "naruto-uzumaki"); - assert_parsed("Waldemaro Martínez. (Locutor de DJ, Latin American Spanish.)", "waldemaro-martínez"); + assert_expected("donald-trump", "Donald Trump (Angry)" ); + assert_expected("lionel-messi", "Lionel Messi. (Español 2020 - 2023.)"); + assert_expected("mariano-closs", "Mariano Closs (Relator de fútbol Argentino)"); + assert_expected("mariano-closs","Mariano Closs (full version)"); + assert_expected("naruto-uzumaki", "Naruto Uzumaki (Part 1)"); + assert_expected("waldemaro-martínez", "Waldemaro Martínez. (Locutor de DJ, Latin American Spanish.)"); } #[test] - fn version_strings() { - assert_parsed("Cristiano Ronaldo. (Español) Versión 2.", "cristiano-ronaldo"); - assert_parsed("Dragonball Z Narrador (Latin, Version 1.0)", "dragonball-z-narrador"); - assert_parsed("Dragonball Z Narrador Version 1.0", "dragonball-z-narrador"); - assert_parsed("Peter Griffin (Classic, Version 2.0)", "peter-griffin"); - assert_parsed("Peter Griffin Version 2.0", "peter-griffin"); + fn removes_version_strings() { + assert_expected("cristiano-ronaldo", "Cristiano Ronaldo. (Español) Versión 2."); + assert_expected("dragonball-z-narrador", "Dragonball Z Narrador (Latin, Version 1.0)"); + assert_expected("dragonball-z-narrador", "Dragonball Z Narrador Version 1.0"); + assert_expected("peter-griffin", "Peter Griffin (Classic, Version 2.0)"); + assert_expected("peter-griffin", "Peter Griffin Version 2.0"); } #[test] fn removes_spaces() { - assert_parsed("Goku (Jose Antonio Gavira) (Castillian Spanish)", "goku"); + assert_expected("foo", " \t Foo \n "); + assert_expected("goku", "Goku (Jose Antonio Gavira) (Castillian Spanish)"); } #[test] fn removes_periods() { - assert_parsed("Vegeta. (IMITADORES.) (Dragon Ball, Latin American Spanish.)", "vegeta"); + assert_expected("vegeta", "Vegeta. (IMITADORES.) (Dragon Ball, Latin American Spanish.)"); } + + #[test] + fn complex_titles() { + assert_expected("anime-female-voice-jpn-01", "Anime Female Voice JPN 01"); + assert_expected("at-t-natural-voices-rosa", "AT&T natural voices rosa"); + assert_expected("batman", "Batman[Arkham Knight]"); + assert_expected("ed", "Ed (Plants Vs Zombies Plush) (Luigifan00001)"); + assert_expected("greg-page", "Greg Page (The Wiggles) [by: MasterRuby & wigglyfan2001]"); + assert_expected("jaden-judai-yuki", "Jaden/Judai Yuki (JPN, V2)"); + assert_expected("jeff-fatt", "Jeff Fatt (The Wiggles) [by: MasterRuby & wigglyfan2001]"); + assert_expected("king-charles-iii", "King Charles III"); + assert_expected("millie-vivian-nixon", "Millie [Helluva Boss] \"Vivian Nixon\""); + assert_expected("murray-cook", "Murray Cook (The Wiggles) [by: MasterRuby & wigglyfan2001]"); + assert_expected("pilou", "Pilou [D_1600] [Del]"); + assert_expected("shaquille-o-neal", "Shaquille O'Neal"); + assert_expected("tobey-maguire", "Tobey Maguire [Spider_Man]"); + assert_expected("vocal-planet-old-man-blues", "Vocal planet, old man blues"); + } + + //#[test] + //pub fn commas_long() { + // assert_expected("", "La voce del calcolatore 1978, delete the duplicate entry"); + // assert_expected("", "Don Evaristo corral, Version de Ernesto Alban"); + // assert_expected("", "Roland scat jaz vocals full version, use only for singing scat"); + //} + + #[test] + pub fn commas_short() { + assert_expected("dj-street-fighter-6-japan", "dj, street fighter 6 japan"); + } + + #[test] + pub fn slash() { + assert_expected("jaden-judai-yuki", "Jaden/Judai Yuki (JPN, V2)"); + assert_expected("jesse-johan-andersen", "Jesse/Johan Andersen (JPN, credit to RX-7Better)"); + //assert_expected("", "고세구 (노래) / gosegu (singing)"); + } + + #[test] + pub fn partial_parens() { + assert_expected("spider-man-ps4", "Spider man (PS4"); + assert_expected("black-cat-ps4", "Black Cat [PS4"); + } + + //#[test] + //pub fn international_titles() { + // assert_expected("", "ماريو-[فندق-ماريو"); // NB: Character printing direction + // assert_expected("", "İnek Şaban (Kemal Sunal)"); + // assert_expected("", "REİS SEDAT PEKER"); + // assert_expected("", "고세구 (노래) / gosegu (singing)"); + //} }