subprocess stuff in its own lib

Cargo.lock

@@ -1637,6 +1637,7 @@ dependencies = [
  "sqlx",
  "storage_buckets_common",
  "subprocess",
+ "subprocess_common",
  "tempdir",
  "tokio 1.13.0",
  "tts_common",
@@ -1993,6 +1994,7 @@ dependencies = [
  "anyhow",
  "container_common",
  "log",
+ "subprocess_common",
  "tempdir",
 ]
 
@@ -4543,6 +4545,14 @@ dependencies = [
  "winapi 0.3.9",
 ]
 
+[[package]]
+name = "subprocess_common"
+version = "0.0.1"
+dependencies = [
+ "anyhow",
+ "container_common",
+]
+
 [[package]]
 name = "subtle"
 version = "2.4.0"

Cargo.toml

@@ -21,6 +21,7 @@ members = [
   "crates/lib/redis_common",
   "crates/lib/reusable_types",
   "crates/lib/storage_buckets_common",
+  "crates/lib/subprocess_common",
   "crates/lib/tts_common",
   "crates/lib/twitch_common",
   "crates/lib/url_config",

crates/lib/google_drive_common/Cargo.toml

@@ -16,6 +16,7 @@ path = "src/lib.rs"
 
 # Internal
 container_common = { path = "../container_common" }
+subprocess_common = { path = "../subprocess_common" }
 
 # External
 anyhow = "1.0.40"

crates/lib/google_drive_common/src/google_drive_download_command.rs

@@ -5,6 +5,7 @@ use log::{info, warn};
 use std::io::{BufReader, Read};
 use std::path::{PathBuf, Path};
 use std::process::Command;
+use subprocess_common::docker_options::DockerOptions;
 use tempdir::TempDir;
 
 /// This is a Python script that uses the `gdown` package to download from Google Drive.
@@ -24,18 +25,6 @@ pub struct GoogleDriveDownloadCommand {
   maybe_docker_options: Option<DockerOptions>,
 }
 
-#[derive(Clone)]
-pub struct DockerOptions {
-  pub image_name: String,
-  pub maybe_bind_mount: Option<DockerFilesystemMount>,
-}
-
-#[derive(Clone)]
-pub struct DockerFilesystemMount {
-  pub local_filesystem: String,
-  pub container_filesystem: String,
-}
-
 impl GoogleDriveDownloadCommand {
   pub fn new(download_script: &str) -> Self {
     Self {

crates/lib/subprocess_common/Cargo.toml

@@ -0,0 +1,27 @@
+[package]
+name = "subprocess_common"
+edition = "2018"
+version = "0.0.1"
+authors = [
+    "Brandon Thomas <bt@brand.io>",
+    "Brandon Thomas <echelon@gmail.com>",
+]
+publish = false
+
+[lib]
+name = "subprocess_common"
+path = "src/lib.rs"
+
+[dependencies]
+
+# Internal
+container_common = { path = "../container_common" }
+
+# External
+anyhow = "1.0.40"
+
+# Serialization
+#serde = "1.0.125"
+#serde_derive = "1.0.125"
+#serde_json = "1.0.64"
+#toml = "0.5.8"

crates/lib/subprocess_common/src/docker_options.rs

@@ -0,0 +1,60 @@
+/// Options for launching subprocesses under a docker container
+#[derive(Clone)]
+pub struct DockerOptions {
+  pub image_name: String,
+  pub maybe_bind_mount: Option<DockerFilesystemMount>,
+}
+
+#[derive(Clone)]
+pub struct DockerFilesystemMount {
+  pub local_filesystem: String,
+  pub container_filesystem: String,
+}
+
+impl DockerFilesystemMount {
+  pub fn tmp_to_tmp() -> Self {
+    Self {
+      local_filesystem: "/tmp".to_string(),
+      container_filesystem: "/tmp".to_string(),
+    }
+  }
+
+  pub fn to_fuse_option_string(&self) -> String {
+    // TODO: Handle spaces and special characters. Make sure this can't lead to injection.
+    format!(" --mount type=bind,source={},target={} ", &self.local_filesystem, &self.container_filesystem)
+  }
+}
+
+impl DockerOptions {
+  pub fn to_command_string(&self, container_command: &str) -> String {
+    let fuse_command = self.maybe_bind_mount
+        .as_ref()
+        .map(|mount| mount.to_fuse_option_string())
+        .unwrap_or("".to_string());
+
+    // TODO: Handle spaces and special characters. Make sure this can't lead to injection.
+    format!("docker run --rm {} {} /bin/bash -c \"{}\"",
+            &fuse_command,
+            &self.image_name,
+            container_command)
+  }
+}
+
+#[cfg(test)]
+mod tests {
+  use crate::docker_options::{DockerFilesystemMount, DockerOptions};
+
+  #[test]
+  fn test_command() {
+    let command = DockerOptions {
+      image_name: "MY_IMAAGE".to_string(),
+      maybe_bind_mount: Some(DockerFilesystemMount {
+        local_filesystem: "/local".to_string(),
+        container_filesystem: "/container".to_string(),
+      }),
+    };
+
+    assert_eq!("docker run --rm  --mount type=bind,source=/local,target=/container  MY_IMAAGE /bin/bash -c \"echo wat\"",
+               command.to_command_string("echo wat"));
+  }
+}

crates/lib/subprocess_common/src/lib.rs

@@ -0,0 +1,15 @@
+// Never allow these
+#![forbid(private_in_public)]
+#![forbid(unused_must_use)] // NB: It's unsafe to not close/check some things
+
+// Okay to toggle
+#![forbid(unreachable_patterns)]
+#![forbid(unused_imports)]
+#![forbid(unused_mut)]
+#![forbid(unused_variables)]
+
+// Always allow
+#![allow(dead_code)]
+#![allow(non_snake_case)]
+
+pub mod docker_options;

crates/service/download_job/Cargo.toml

@@ -22,6 +22,7 @@ google_drive_common = { path = "../../lib/google_drive_common" }
 jobs_common = { path = "../../lib/jobs_common" }
 reusable_types = { path = "../../lib/reusable_types" }
 storage_buckets_common = { path = "../../lib/storage_buckets_common" }
+subprocess_common = { path = "../../lib/subprocess_common" }
 tts_common = { path = "../../lib/tts_common" }
 
 # Vendored code

crates/service/download_job/src/main.rs

@@ -31,7 +31,7 @@ use crate::job_steps::main_loop::main_loop;
 use crate::job_types::hifigan::hifigan_model_check_command::HifiGanModelCheckCommand;
 use database_queries::mediators::badge_granter::BadgeGranter;
 use database_queries::mediators::firehose_publisher::FirehosePublisher;
-use google_drive_common::google_drive_download_command::{DockerFilesystemMount, DockerOptions, GoogleDriveDownloadCommand};
+use google_drive_common::google_drive_download_command::GoogleDriveDownloadCommand;
 use log::info;
 use r2d2_redis::RedisConnectionManager;
 use r2d2_redis::r2d2;
@@ -41,6 +41,7 @@ use std::path::PathBuf;
 use std::time::Duration;
 use storage_buckets_common::bucket_client::BucketClient;
 use storage_buckets_common::bucket_path_unifier::BucketPathUnifier;
+use subprocess_common::docker_options::{DockerFilesystemMount, DockerOptions};
 
 // Buckets
 const ENV_ACCESS_KEY : &'static str = "ACCESS_KEY";