directory to persist html

crates/programs/aichatbot-sidecar/src/main.rs

@@ -70,7 +70,15 @@ pub async fn main() -> AnyhowResult<()> {
   //let _r = cnn_article_scraper("https://www.cnn.com/2023/02/02/tech/first-generation-iphone-auction/index.html").await?;
   let result = techcrunch_article_scraper("https://techcrunch.com/2023/02/04/elon-musk-says-twitter-will-provide-a-free-write-only-api-to-bots-providing-good-content/").await?;
 
-  println!("Result: {:?}", result);
+
+  let _ = dotenv::from_filename(".env-aichatbot-secrets").ok();
+  let startup_args = get_startup_args()?;
+  let save_directory = SaveDirectory::new(&startup_args.save_directory);
+
+  let directory = save_directory.for_webpage_url("https://techcrunch.com/2023/02/04/elon-musk-says-twitter-will-provide-a-free-write-only-api-to-bots-providing-good-content/")?;
+
+  println!("Directory: {}", directory);
+
 
 
   Ok(())

crates/programs/aichatbot-sidecar/src/persistence/hashed_directory_path.rs

@@ -0,0 +1,10 @@
+
+// NB: this is copied from crates/_types/buckets/src/util/hashed_directory_path_short_string.rs
+pub fn hashed_directory_path(file_hash: &str) -> String {
+  match file_hash.len() {
+    0 | 1 => "".to_string(),
+    2 => format!("{}/", &file_hash[0..1]),
+    3 => format!("{}/{}/", &file_hash[0..1], &file_hash[1..2]),
+    _ => format!("{}/{}/{}/", &file_hash[0..1], &file_hash[1..2], &file_hash[2..3]),
+  }
+}

crates/programs/aichatbot-sidecar/src/persistence/mod.rs

@@ -1 +1,2 @@
+pub mod hashed_directory_path;
 pub mod save_directory;

crates/programs/aichatbot-sidecar/src/persistence/save_directory.rs

@@ -1,7 +1,9 @@
 use std::path::{Path, PathBuf};
 use log::info;
 use path_absolutize::Absolutize;
-use errors::AnyhowResult;
+use errors::{anyhow, AnyhowResult};
+use hashing::sha256::sha256_hash_string::sha256_hash_string;
+use crate::persistence::hashed_directory_path::hashed_directory_path;
 
 #[derive(Clone)]
 pub struct SaveDirectory {
@@ -16,9 +18,19 @@ impl SaveDirectory {
     }
   }
 
-  pub fn for_url(url: &str) -> AnyhowResult<String> {
+  pub fn directory_for_webpage_url(&self, url: &str) -> AnyhowResult<String> {
+    let url_hash = sha256_hash_string(url)?;
+    let directory_path= hashed_directory_path(&url_hash);
 
-    Ok("todo".to_string())
+    let directory = self.directory.clone()
+        .join(directory_path)
+        .join(url_hash);
+
+    let full_path = directory.to_str()
+        .ok_or(anyhow!("could not construct path"))?
+        .to_string();
+
+    Ok(full_path)
   }
 
   /// This is just the first directory structure, which is sequential audio files.

crates/programs/aichatbot-sidecar/src/startup_args.rs

@@ -4,7 +4,7 @@ use log::info;
 use path_absolutize::Absolutize;
 use errors::{anyhow, AnyhowResult};
 
-const DEFAULT_SAVE_DIRECTORY : &'static str = "data";
+const DEFAULT_SAVE_DIRECTORY : &'static str = "runtime_data";
 
 #[derive(Clone, Default)]
 pub struct StartupArgs {