From adc52cd01ed6a2170a192daa328f2d9d1e3f299b Mon Sep 17 00:00:00 2001
From: Brandon Thomas <echelon@gmail.com>
Date: Mon, 6 Feb 2023 03:32:07 -0500
Subject: [PATCH] directory to persist html

---
 crates/programs/aichatbot-sidecar/src/main.rs  | 10 +++++++++-
 .../src/persistence/hashed_directory_path.rs   | 10 ++++++++++
 .../aichatbot-sidecar/src/persistence/mod.rs   |  1 +
 .../src/persistence/save_directory.rs          | 18 +++++++++++++++---
 .../aichatbot-sidecar/src/startup_args.rs      |  2 +-
 5 files changed, 36 insertions(+), 5 deletions(-)
 create mode 100644 crates/programs/aichatbot-sidecar/src/persistence/hashed_directory_path.rs

diff --git a/crates/programs/aichatbot-sidecar/src/main.rs b/crates/programs/aichatbot-sidecar/src/main.rs
index 116a3fe8b5..b6db5ecd99 100644
--- a/crates/programs/aichatbot-sidecar/src/main.rs
+++ b/crates/programs/aichatbot-sidecar/src/main.rs
@@ -70,7 +70,15 @@ pub async fn main() -> AnyhowResult<()> {
   //let _r = cnn_article_scraper("https://www.cnn.com/2023/02/02/tech/first-generation-iphone-auction/index.html").await?;
   let result = techcrunch_article_scraper("https://techcrunch.com/2023/02/04/elon-musk-says-twitter-will-provide-a-free-write-only-api-to-bots-providing-good-content/").await?;
 
-  println!("Result: {:?}", result);
+
+  let _ = dotenv::from_filename(".env-aichatbot-secrets").ok();
+  let startup_args = get_startup_args()?;
+  let save_directory = SaveDirectory::new(&startup_args.save_directory);
+
+  let directory = save_directory.for_webpage_url("https://techcrunch.com/2023/02/04/elon-musk-says-twitter-will-provide-a-free-write-only-api-to-bots-providing-good-content/")?;
+
+  println!("Directory: {}", directory);
+
 
 
   Ok(())
diff --git a/crates/programs/aichatbot-sidecar/src/persistence/hashed_directory_path.rs b/crates/programs/aichatbot-sidecar/src/persistence/hashed_directory_path.rs
new file mode 100644
index 0000000000..f6e7cadd68
--- /dev/null
+++ b/crates/programs/aichatbot-sidecar/src/persistence/hashed_directory_path.rs
@@ -0,0 +1,10 @@
+
+// NB: this is copied from crates/_types/buckets/src/util/hashed_directory_path_short_string.rs
+pub fn hashed_directory_path(file_hash: &str) -> String {
+  match file_hash.len() {
+    0 | 1 => "".to_string(),
+    2 => format!("{}/", &file_hash[0..1]),
+    3 => format!("{}/{}/", &file_hash[0..1], &file_hash[1..2]),
+    _ => format!("{}/{}/{}/", &file_hash[0..1], &file_hash[1..2], &file_hash[2..3]),
+  }
+}
diff --git a/crates/programs/aichatbot-sidecar/src/persistence/mod.rs b/crates/programs/aichatbot-sidecar/src/persistence/mod.rs
index 5808013875..187f9dc28d 100644
--- a/crates/programs/aichatbot-sidecar/src/persistence/mod.rs
+++ b/crates/programs/aichatbot-sidecar/src/persistence/mod.rs
@@ -1 +1,2 @@
+pub mod hashed_directory_path;
 pub mod save_directory;
\ No newline at end of file
diff --git a/crates/programs/aichatbot-sidecar/src/persistence/save_directory.rs b/crates/programs/aichatbot-sidecar/src/persistence/save_directory.rs
index ec018d5cab..16a7e16fa6 100644
--- a/crates/programs/aichatbot-sidecar/src/persistence/save_directory.rs
+++ b/crates/programs/aichatbot-sidecar/src/persistence/save_directory.rs
@@ -1,7 +1,9 @@
 use std::path::{Path, PathBuf};
 use log::info;
 use path_absolutize::Absolutize;
-use errors::AnyhowResult;
+use errors::{anyhow, AnyhowResult};
+use hashing::sha256::sha256_hash_string::sha256_hash_string;
+use crate::persistence::hashed_directory_path::hashed_directory_path;
 
 #[derive(Clone)]
 pub struct SaveDirectory {
@@ -16,9 +18,19 @@ impl SaveDirectory {
     }
   }
 
-  pub fn for_url(url: &str) -> AnyhowResult<String> {
+  pub fn directory_for_webpage_url(&self, url: &str) -> AnyhowResult<String> {
+    let url_hash = sha256_hash_string(url)?;
+    let directory_path= hashed_directory_path(&url_hash);
 
-    Ok("todo".to_string())
+    let directory = self.directory.clone()
+        .join(directory_path)
+        .join(url_hash);
+
+    let full_path = directory.to_str()
+        .ok_or(anyhow!("could not construct path"))?
+        .to_string();
+
+    Ok(full_path)
   }
 
   /// This is just the first directory structure, which is sequential audio files.
diff --git a/crates/programs/aichatbot-sidecar/src/startup_args.rs b/crates/programs/aichatbot-sidecar/src/startup_args.rs
index f86b539625..4044ef1ca4 100644
--- a/crates/programs/aichatbot-sidecar/src/startup_args.rs
+++ b/crates/programs/aichatbot-sidecar/src/startup_args.rs
@@ -4,7 +4,7 @@ use log::info;
 use path_absolutize::Absolutize;
 use errors::{anyhow, AnyhowResult};
 
-const DEFAULT_SAVE_DIRECTORY : &'static str = "data";
+const DEFAULT_SAVE_DIRECTORY : &'static str = "runtime_data";
 
 #[derive(Clone, Default)]
 pub struct StartupArgs {