diff --git a/Cargo.lock b/Cargo.lock index e83bb17b2e..9333793b96 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1637,6 +1637,7 @@ dependencies = [ "sqlx", "storage_buckets_common", "subprocess", + "subprocess_common", "tempdir", "tokio 1.13.0", "tts_common", @@ -1993,6 +1994,7 @@ dependencies = [ "anyhow", "container_common", "log", + "subprocess_common", "tempdir", ] @@ -4543,6 +4545,14 @@ dependencies = [ "winapi 0.3.9", ] +[[package]] +name = "subprocess_common" +version = "0.0.1" +dependencies = [ + "anyhow", + "container_common", +] + [[package]] name = "subtle" version = "2.4.0" diff --git a/Cargo.toml b/Cargo.toml index 87ad9fc148..cd7c1df962 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -21,6 +21,7 @@ members = [ "crates/lib/redis_common", "crates/lib/reusable_types", "crates/lib/storage_buckets_common", + "crates/lib/subprocess_common", "crates/lib/tts_common", "crates/lib/twitch_common", "crates/lib/url_config", diff --git a/crates/lib/google_drive_common/Cargo.toml b/crates/lib/google_drive_common/Cargo.toml index 734978a1e2..dc49d4582b 100644 --- a/crates/lib/google_drive_common/Cargo.toml +++ b/crates/lib/google_drive_common/Cargo.toml @@ -16,6 +16,7 @@ path = "src/lib.rs" # Internal container_common = { path = "../container_common" } +subprocess_common = { path = "../subprocess_common" } # External anyhow = "1.0.40" diff --git a/crates/lib/google_drive_common/src/google_drive_download_command.rs b/crates/lib/google_drive_common/src/google_drive_download_command.rs index b6509e2043..486d988a1c 100644 --- a/crates/lib/google_drive_common/src/google_drive_download_command.rs +++ b/crates/lib/google_drive_common/src/google_drive_download_command.rs @@ -5,6 +5,7 @@ use log::{info, warn}; use std::io::{BufReader, Read}; use std::path::{PathBuf, Path}; use std::process::Command; +use subprocess_common::docker_options::DockerOptions; use tempdir::TempDir; /// This is a Python script that uses the `gdown` package to download from Google Drive. @@ -24,18 +25,6 @@ pub struct GoogleDriveDownloadCommand { maybe_docker_options: Option, } -#[derive(Clone)] -pub struct DockerOptions { - pub image_name: String, - pub maybe_bind_mount: Option, -} - -#[derive(Clone)] -pub struct DockerFilesystemMount { - pub local_filesystem: String, - pub container_filesystem: String, -} - impl GoogleDriveDownloadCommand { pub fn new(download_script: &str) -> Self { Self { diff --git a/crates/lib/subprocess_common/Cargo.toml b/crates/lib/subprocess_common/Cargo.toml new file mode 100644 index 0000000000..d4e747c64f --- /dev/null +++ b/crates/lib/subprocess_common/Cargo.toml @@ -0,0 +1,27 @@ +[package] +name = "subprocess_common" +edition = "2018" +version = "0.0.1" +authors = [ + "Brandon Thomas ", + "Brandon Thomas ", +] +publish = false + +[lib] +name = "subprocess_common" +path = "src/lib.rs" + +[dependencies] + +# Internal +container_common = { path = "../container_common" } + +# External +anyhow = "1.0.40" + +# Serialization +#serde = "1.0.125" +#serde_derive = "1.0.125" +#serde_json = "1.0.64" +#toml = "0.5.8" diff --git a/crates/lib/subprocess_common/src/docker_options.rs b/crates/lib/subprocess_common/src/docker_options.rs new file mode 100644 index 0000000000..b28d5bcd5a --- /dev/null +++ b/crates/lib/subprocess_common/src/docker_options.rs @@ -0,0 +1,60 @@ +/// Options for launching subprocesses under a docker container +#[derive(Clone)] +pub struct DockerOptions { + pub image_name: String, + pub maybe_bind_mount: Option, +} + +#[derive(Clone)] +pub struct DockerFilesystemMount { + pub local_filesystem: String, + pub container_filesystem: String, +} + +impl DockerFilesystemMount { + pub fn tmp_to_tmp() -> Self { + Self { + local_filesystem: "/tmp".to_string(), + container_filesystem: "/tmp".to_string(), + } + } + + pub fn to_fuse_option_string(&self) -> String { + // TODO: Handle spaces and special characters. Make sure this can't lead to injection. + format!(" --mount type=bind,source={},target={} ", &self.local_filesystem, &self.container_filesystem) + } +} + +impl DockerOptions { + pub fn to_command_string(&self, container_command: &str) -> String { + let fuse_command = self.maybe_bind_mount + .as_ref() + .map(|mount| mount.to_fuse_option_string()) + .unwrap_or("".to_string()); + + // TODO: Handle spaces and special characters. Make sure this can't lead to injection. + format!("docker run --rm {} {} /bin/bash -c \"{}\"", + &fuse_command, + &self.image_name, + container_command) + } +} + +#[cfg(test)] +mod tests { + use crate::docker_options::{DockerFilesystemMount, DockerOptions}; + + #[test] + fn test_command() { + let command = DockerOptions { + image_name: "MY_IMAAGE".to_string(), + maybe_bind_mount: Some(DockerFilesystemMount { + local_filesystem: "/local".to_string(), + container_filesystem: "/container".to_string(), + }), + }; + + assert_eq!("docker run --rm --mount type=bind,source=/local,target=/container MY_IMAAGE /bin/bash -c \"echo wat\"", + command.to_command_string("echo wat")); + } +} diff --git a/crates/lib/subprocess_common/src/lib.rs b/crates/lib/subprocess_common/src/lib.rs new file mode 100644 index 0000000000..b24c11ddf1 --- /dev/null +++ b/crates/lib/subprocess_common/src/lib.rs @@ -0,0 +1,15 @@ +// Never allow these +#![forbid(private_in_public)] +#![forbid(unused_must_use)] // NB: It's unsafe to not close/check some things + +// Okay to toggle +#![forbid(unreachable_patterns)] +#![forbid(unused_imports)] +#![forbid(unused_mut)] +#![forbid(unused_variables)] + +// Always allow +#![allow(dead_code)] +#![allow(non_snake_case)] + +pub mod docker_options; \ No newline at end of file diff --git a/crates/service/download_job/Cargo.toml b/crates/service/download_job/Cargo.toml index 0ad5a378a2..7de6324f9c 100644 --- a/crates/service/download_job/Cargo.toml +++ b/crates/service/download_job/Cargo.toml @@ -22,6 +22,7 @@ google_drive_common = { path = "../../lib/google_drive_common" } jobs_common = { path = "../../lib/jobs_common" } reusable_types = { path = "../../lib/reusable_types" } storage_buckets_common = { path = "../../lib/storage_buckets_common" } +subprocess_common = { path = "../../lib/subprocess_common" } tts_common = { path = "../../lib/tts_common" } # Vendored code diff --git a/crates/service/download_job/src/main.rs b/crates/service/download_job/src/main.rs index 49b343706f..d179eb7743 100644 --- a/crates/service/download_job/src/main.rs +++ b/crates/service/download_job/src/main.rs @@ -31,7 +31,7 @@ use crate::job_steps::main_loop::main_loop; use crate::job_types::hifigan::hifigan_model_check_command::HifiGanModelCheckCommand; use database_queries::mediators::badge_granter::BadgeGranter; use database_queries::mediators::firehose_publisher::FirehosePublisher; -use google_drive_common::google_drive_download_command::{DockerFilesystemMount, DockerOptions, GoogleDriveDownloadCommand}; +use google_drive_common::google_drive_download_command::GoogleDriveDownloadCommand; use log::info; use r2d2_redis::RedisConnectionManager; use r2d2_redis::r2d2; @@ -41,6 +41,7 @@ use std::path::PathBuf; use std::time::Duration; use storage_buckets_common::bucket_client::BucketClient; use storage_buckets_common::bucket_path_unifier::BucketPathUnifier; +use subprocess_common::docker_options::{DockerFilesystemMount, DockerOptions}; // Buckets const ENV_ACCESS_KEY : &'static str = "ACCESS_KEY";