markdown with socials (needs performance optimizations)

Cargo.lock

@@ -3741,6 +3741,16 @@ dependencies = [
  "tiger",
 ]
 
+[[package]]
+name = "markdown"
+version = "0.0.1"
+dependencies = [
+ "errors",
+ "my-workspace-hack",
+ "pulldown-cmark",
+ "regex",
+]
+
 [[package]]
 name = "markup5ever"
 version = "0.11.0"
@@ -5331,9 +5341,9 @@ dependencies = [
 
 [[package]]
 name = "regex"
-version = "1.10.2"
+version = "1.10.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "380b951a9c5e80ddfd6136919eef32310721aa4aacd4889a8d39124b026ab343"
+checksum = "c117dbdfde9c8308975b6a18d71f3f385c89461f7b3fb054288ecf2a2058ba4c"
 dependencies = [
  "aho-corasick",
  "memchr",
@@ -5343,9 +5353,9 @@ dependencies = [
 
 [[package]]
 name = "regex-automata"
-version = "0.4.3"
+version = "0.4.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5f804c7828047e88b2d32e2d7fe5a105da8ee3264f01902f796c8e067dc2483f"
+checksum = "86b83b8b9847f9bf95ef68afb0b8e6cdb80f498442f5179a29fad448fcc1eaea"
 dependencies = [
  "aho-corasick",
  "memchr",
@@ -6597,6 +6607,7 @@ dependencies = [
  "log",
  "lru_time_cache",
  "magic-crypt",
+ "markdown",
  "md-5 0.9.1",
  "media",
  "memory_caching",

Cargo.toml

@@ -54,6 +54,7 @@ members = [
   "crates/lib/idempotency",
   "crates/lib/jobs_common",
   "crates/lib/logging",
+  "crates/lib/markdown",
   "crates/lib/migration",
   "crates/lib/mysql_queries",
   "crates/lib/password",

crates/lib/markdown/Cargo.toml

@@ -0,0 +1,31 @@
+[package]
+name = "markdown"
+edition = "2018"
+version = "0.0.1"
+authors = [
+    "Brandon Thomas <bt@brand.io>",
+    "Brandon Thomas <echelon@gmail.com>",
+]
+publish = false
+
+[lib]
+# NB: Public "markdown" crate is inferior to pulldown-cmark
+name = "markdown"
+path = "src/lib.rs"
+
+[package.metadata.cargo-udeps.ignore]
+normal = ["my-workspace-hack"]
+
+[dependencies]
+# Internal
+errors = { path = "../errors" }
+
+# Workspace hack (faster compile times)
+my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" }
+
+# External
+pulldown-cmark = "0.9.3" # markdown
+regex = "1.10.4"
+
+[dev-dependencies]
+# None yet

crates/lib/markdown/src/lib.rs

@@ -0,0 +1,22 @@
+//! markdown
+//!
+//! Our own internal markdown crate. Built on pulldown-cmark, but with added extras.
+//!
+
+// Never allow these
+#![forbid(private_bounds)]
+#![forbid(private_interfaces)]
+#![forbid(unused_must_use)] // NB: It's unsafe to not close/check some things
+
+// Okay to toggle
+#![forbid(unreachable_patterns)]
+#![forbid(unused_imports)]
+#![forbid(unused_mut)]
+#![forbid(unused_variables)]
+
+// Always allow
+#![allow(dead_code)]
+#![allow(non_snake_case)]
+
+pub mod simple_markdown_to_html;
+pub mod markdown_with_socials_to_html;

crates/lib/markdown/src/markdown_with_socials_to_html.rs

@@ -0,0 +1,240 @@
+use pulldown_cmark::{html, Options, Parser};
+use regex::{Captures, Regex};
+
+// TODO(bt,2024-05-25): In the future, we may want a builder class with features
+//  that can be turned on/off as well as comprehensive tests and sane defaults.
+
+pub fn markdown_with_socials_to_html(markdown_input: &str) -> String {
+  // NB: CommonMark allows for HTML, including <script>! WTF!
+  // Let's prevent that nonsense.
+  // https://cheatsheetseries.owasp.org/cheatsheets/Cross_Site_Scripting_Prevention_Cheat_Sheet.html
+  let markdown_input = markdown_input
+      .replace("&", "&amp;")
+      .replace("<", "&lt;")
+      .replace(">", "&gt;")
+      .replace("\"", "&quot;")
+      .replace("'", "&#x27;"); // NB: &apos; is not in the HTML spec
+
+  // TODO: Use copy on write
+  let markdown_input = replace_instagram_user(&markdown_input);
+  let markdown_input = replace_reddit_subreddit(&markdown_input);
+  let markdown_input = replace_reddit_user(&markdown_input);
+  let markdown_input = replace_tiktok_user(&markdown_input);
+  let markdown_input = replace_x_user(&markdown_input);
+  let markdown_input = replace_youtube_channel(&markdown_input);
+
+  let mut options = Options::empty();
+  options.insert(Options::ENABLE_STRIKETHROUGH);
+
+  let parser = Parser::new_ext(&markdown_input, options);
+
+  // NB: The buffer size can expand, but the math is a sizing heuristic I found in one of the
+  // examples that may prevent buffer resizing.
+  let mut html_output: String = String::with_capacity(markdown_input.len() * 3 / 2);
+  html::push_html(&mut html_output, parser);
+
+  html_output
+}
+
+// TODO(bt,2024-05-25): Not sure if the username parsing is right for each service.
+
+fn replace_instagram_user(input: &str) -> String {
+  let re = Regex::new(r"(insta(gram)?(.com)?/([\w\-_]+))").unwrap(); // TODO: once_cell
+  let result = re.replace(input, |caps: &Captures| {
+    format!("[{}](https://instagram.com/{})", &caps[1], &caps[4])
+  });
+  result.to_string()
+}
+
+fn replace_reddit_subreddit(input: &str) -> String {
+  let re = Regex::new(r"(r/([\w\-_]+))").unwrap(); // TODO: once_cell
+  let result = re.replace(input, |caps: &Captures| {
+    format!("[{}](https://www.reddit.com/r/{})", &caps[1], &caps[2])
+  });
+  result.to_string()
+}
+
+fn replace_reddit_user(input: &str) -> String {
+  let re = Regex::new(r"(u/([\w\-_]+))").unwrap(); // TODO: once_cell
+  let result = re.replace(input, |caps: &Captures| {
+    format!("[{}](https://www.reddit.com/user/{})", &caps[1], &caps[2])
+  });
+  result.to_string()
+}
+
+fn replace_tiktok_user(input: &str) -> String {
+  let re = Regex::new(r"(tiktok(.com)?/([@\w\-_]+))").unwrap(); // TODO: once_cell
+  let result = re.replace(input, |caps: &Captures| {
+    format!("[{}](https://www.tiktok.com/{})", &caps[1], &caps[3])
+  });
+  result.to_string()
+}
+
+fn replace_x_user(input: &str) -> String {
+  let re = Regex::new(r"(x(.com)?/([\w\-_]+))").unwrap(); // TODO: once_cell
+  let result = re.replace(input, |caps: &Captures| {
+    format!("[{}](https://x.com/{})", &caps[1], &caps[3])
+  });
+  result.to_string()
+}
+
+fn replace_youtube_channel(input: &str) -> String {
+  let re = Regex::new(r"((yt|youtube(.com)?)/([@\w\-_]+))").unwrap(); // TODO: once_cell
+  let result = re.replace(input, |caps: &Captures| {
+    format!("[{}](https://www.youtube.com/{})", &caps[1], &caps[4])
+  });
+  result.to_string()
+}
+
+#[cfg(test)]
+mod tests {
+  use crate::markdown_with_socials_to_html::markdown_with_socials_to_html;
+
+  mod basics {
+    use super::*;
+    
+    #[test]
+    fn handles_markdown() {
+      assert_eq!(&markdown_with_socials_to_html("*italics text*"),
+                 "<p><em>italics text</em></p>\n");
+
+      assert_eq!(&markdown_with_socials_to_html("**bold text**"),
+                 "<p><strong>bold text</strong></p>\n");
+    }
+
+    #[test]
+    fn do_not_allow_html_entities() {
+      assert_eq!(&markdown_with_socials_to_html("this > that"),
+                 "<p>this &gt; that</p>\n");
+
+      assert_eq!(&markdown_with_socials_to_html("&"),
+                 "<p>&amp;</p>\n");
+
+      // NB: We doubly entity decode because pulldown_cmark doesn't do enough!
+      assert_eq!(&markdown_with_socials_to_html("&gt;"),
+                 "<p>&amp;gt;</p>\n");
+    }
+
+    #[test]
+    fn do_not_allow_html_tags() {
+      assert_eq!(&markdown_with_socials_to_html("<script>alert();</script>"),
+                 "<p>&lt;script&gt;alert();&lt;/script&gt;</p>\n");
+    }
+  }
+
+  mod instagram {
+    use super::*;
+
+    #[test]
+    pub fn insta() {
+      assert_eq!(&markdown_with_socials_to_html("insta/storyteller_ai"),
+                 "<p><a href=\"https://instagram.com/storyteller_ai\">insta/storyteller_ai</a></p>\n");
+    }
+
+    #[test]
+    pub fn instagram() {
+      assert_eq!(&markdown_with_socials_to_html("instagram/storyteller_ai"),
+                 "<p><a href=\"https://instagram.com/storyteller_ai\">instagram/storyteller_ai</a></p>\n");
+    }
+
+    #[test]
+    pub fn instagram_dot_com() {
+      assert_eq!(&markdown_with_socials_to_html("instagram.com/storyteller_ai"),
+                 "<p><a href=\"https://instagram.com/storyteller_ai\">instagram.com/storyteller_ai</a></p>\n");
+    }
+
+    #[test]
+    pub fn insta_before_after() {
+      assert_eq!(&markdown_with_socials_to_html("creator insta/storyteller_ai posted it"),
+                 "<p>creator <a href=\"https://instagram.com/storyteller_ai\">insta/storyteller_ai</a> posted it</p>\n");
+    }
+  }
+
+  mod reddit {
+    use super::*;
+
+    #[test]
+    pub fn reddit_subreddit() {
+      assert_eq!(&markdown_with_socials_to_html("r/storyteller"),
+                 "<p><a href=\"https://www.reddit.com/r/storyteller\">r/storyteller</a></p>\n");
+    }
+
+    #[test]
+    pub fn reddit_user() {
+      assert_eq!(&markdown_with_socials_to_html("u/storyteller_ai"),
+                 "<p><a href=\"https://www.reddit.com/user/storyteller_ai\">u/storyteller_ai</a></p>\n");
+    }
+
+    #[test]
+    pub fn reddit_user_before_after() {
+      assert_eq!(&markdown_with_socials_to_html("creator u/storyteller_ai posted it"),
+                 "<p>creator <a href=\"https://www.reddit.com/user/storyteller_ai\">u/storyteller_ai</a> posted it</p>\n");
+    }
+  }
+
+  mod tiktok {
+    use super::*;
+
+    #[test]
+    pub fn tiktok() {
+      assert_eq!(&markdown_with_socials_to_html("tiktok/storyteller_ai"),
+                 "<p><a href=\"https://www.tiktok.com/storyteller_ai\">tiktok/storyteller_ai</a></p>\n");
+    }
+
+    #[test]
+    pub fn tiktok_dot_com() {
+      assert_eq!(&markdown_with_socials_to_html("tiktok.com/storyteller_ai"),
+                 "<p><a href=\"https://www.tiktok.com/storyteller_ai\">tiktok.com/storyteller_ai</a></p>\n");
+    }
+
+    #[test]
+    pub fn tiktok_before_after() {
+      assert_eq!(&markdown_with_socials_to_html("creator tiktok/storyteller_ai posted it"),
+                 "<p>creator <a href=\"https://www.tiktok.com/storyteller_ai\">tiktok/storyteller_ai</a> posted it</p>\n");
+    }
+  }
+
+  mod x {
+    use super::*;
+
+    #[test]
+    pub fn x() {
+      assert_eq!(&markdown_with_socials_to_html("x/storyteller_ai"),
+                 "<p><a href=\"https://x.com/storyteller_ai\">x/storyteller_ai</a></p>\n");
+    }
+
+    #[test]
+    pub fn x_dot_com() {
+      assert_eq!(&markdown_with_socials_to_html("x.com/storyteller_ai"),
+                 "<p><a href=\"https://x.com/storyteller_ai\">x.com/storyteller_ai</a></p>\n");
+    }
+
+    #[test]
+    pub fn x_before_after() {
+      assert_eq!(&markdown_with_socials_to_html("creator x/storyteller_ai posted it"),
+                 "<p>creator <a href=\"https://x.com/storyteller_ai\">x/storyteller_ai</a> posted it</p>\n");
+    }
+  }
+
+  mod youtube {
+    use super::*;
+
+    #[test]
+    pub fn youtube() {
+      assert_eq!(&markdown_with_socials_to_html("youtube/@storyteller_ai"),
+                 "<p><a href=\"https://www.youtube.com/@storyteller_ai\">youtube/@storyteller_ai</a></p>\n");
+    }
+
+    #[test]
+    pub fn youtube_dot_com() {
+      assert_eq!(&markdown_with_socials_to_html("youtube.com/@storyteller_ai"),
+                 "<p><a href=\"https://www.youtube.com/@storyteller_ai\">youtube.com/@storyteller_ai</a></p>\n");
+    }
+
+    #[test]
+    pub fn youtube_before_after() {
+      assert_eq!(&markdown_with_socials_to_html("creator youtube/@storyteller_ai posted it"),
+                 "<p>creator <a href=\"https://www.youtube.com/@storyteller_ai\">youtube/@storyteller_ai</a> posted it</p>\n");
+    }
+  }
+}

crates/lib/markdown/src/simple_markdown_to_html.rs

@@ -0,0 +1,58 @@
+use pulldown_cmark::{html, Options, Parser};
+
+pub fn simple_markdown_to_html(markdown_input: &str) -> String {
+  // NB: CommonMark allows for HTML, including <script>! WTF!
+  // Let's prevent that nonsense.
+  // https://cheatsheetseries.owasp.org/cheatsheets/Cross_Site_Scripting_Prevention_Cheat_Sheet.html
+  let markdown_input = markdown_input
+      .replace("&", "&amp;")
+      .replace("<", "&lt;")
+      .replace(">", "&gt;")
+      .replace("\"", "&quot;")
+      .replace("'", "&#x27;"); // NB: &apos; is not in the HTML spec
+
+  let mut options = Options::empty();
+  options.insert(Options::ENABLE_STRIKETHROUGH);
+
+  let parser = Parser::new_ext(&markdown_input, options);
+
+  // NB: The buffer size can expand, but the math is a sizing heuristic I found in one of the
+  // examples that may prevent buffer resizing.
+  let mut html_output: String = String::with_capacity(markdown_input.len() * 3 / 2);
+  html::push_html(&mut html_output, parser);
+
+  html_output
+}
+
+#[cfg(test)]
+mod tests {
+  use crate::simple_markdown_to_html::simple_markdown_to_html;
+
+  #[test]
+  fn handles_markdown() {
+    assert_eq!(&simple_markdown_to_html("*italics text*"),
+      "<p><em>italics text</em></p>\n");
+
+    assert_eq!(&simple_markdown_to_html("**bold text**"),
+      "<p><strong>bold text</strong></p>\n");
+  }
+
+  #[test]
+  fn do_not_allow_html_entities() {
+    assert_eq!(&simple_markdown_to_html("this > that"),
+      "<p>this &gt; that</p>\n");
+
+    assert_eq!(&simple_markdown_to_html("&"),
+      "<p>&amp;</p>\n");
+
+    // NB: We doubly entity decode because pulldown_cmark doesn't do enough!
+    assert_eq!(&simple_markdown_to_html("&gt;"),
+      "<p>&amp;gt;</p>\n");
+  }
+
+  #[test]
+  fn do_not_allow_html_tags() {
+    assert_eq!(&simple_markdown_to_html("<script>alert();</script>"),
+      "<p>&lt;script&gt;alert();&lt;/script&gt;</p>\n");
+  }
+}

crates/service/web/storyteller_web/Cargo.toml

@@ -47,6 +47,7 @@ errors = { path = "../../../lib/errors" }
 filesys = { path = "../../../lib/files/filesys" }
 hashing = { path = "../../../lib/files/hashing" }
 http_server_common = { path = "../../../lib/deprecated/http_server_common" }
+markdown = { path = "../../../lib/markdown" }
 media = { path = "../../../lib/files/media" }
 memory_caching = { path = "../../../lib/caching/memory_caching" }
 migration = { path = "../../../lib/migration" }

crates/service/web/storyteller_web/src/http_server/endpoints/beta_keys/list_beta_keys_handler.rs

@@ -7,6 +7,7 @@ use actix_web::web::Query;
 use chrono::{DateTime, Utc};
 use log::{debug, error, warn};
 use r2d2_redis::redis::Commands;
+use regex::Regex;
 use utoipa::{IntoParams, ToSchema};
 
 use buckets::public::media_files::bucket_file_path::MediaFileBucketPath;
@@ -71,6 +72,7 @@ pub struct BetaKeyItem {
   pub maybe_redeemer: Option<UserDetailsLight>,
   pub is_distributed: bool,
   pub maybe_note: Option<String>,
+  pub maybe_note_html: Option<String>,
   pub created_at: DateTime<Utc>,
   pub maybe_redeemed_at: Option<DateTime<Utc>>,
 }
@@ -195,6 +197,7 @@ pub async fn list_beta_keys_handler(
             beta_key.maybe_redeemer_gravatar_hash
           ),
           is_distributed: beta_key.is_distributed,
+          maybe_note_html: markdownify(beta_key.maybe_notes.as_deref()),
           maybe_note: beta_key.maybe_notes,
           created_at: beta_key.created_at,
           maybe_redeemed_at: beta_key.maybe_redeemed_at,
@@ -217,3 +220,22 @@ pub async fn list_beta_keys_handler(
       .content_type("application/json")
       .body(body))
 }
+
+/*
+TODO: Example comments:
+reddit.com/u/FallingKnifeFilms
+r/aivideo mod ZashManson
+
+tiktok.com/@lil.filth
+
+youtube.com/@papagamersby5865
+https://www.youtube.com/@WebsiteLearners (WHOA!)
+
+linktr.ee/virtualityxr
+
+https://x.com/redneopurp
+ */
+
+fn markdownify(maybe_note: Option<&str>) -> Option<String> {
+  None
+}