diff --git a/crates/bin/docs_rs_builder/src/docbuilder/rustwide_builder.rs b/crates/bin/docs_rs_builder/src/docbuilder/rustwide_builder.rs index ee3f052d5..aa24da5e3 100644 --- a/crates/bin/docs_rs_builder/src/docbuilder/rustwide_builder.rs +++ b/crates/bin/docs_rs_builder/src/docbuilder/rustwide_builder.rs @@ -19,8 +19,8 @@ use docs_rs_rustdoc_json::{ read_format_version_from_rustdoc_json, }; use docs_rs_storage::{ - AsyncStorage, Storage, add_path_into_database, add_path_into_remote_archive, compress, - file_list_to_json, get_file_list, rustdoc_archive_path, rustdoc_json_path, source_archive_path, + AsyncStorage, Storage, compress, file_list_to_json, get_file_list, rustdoc_archive_path, + rustdoc_json_path, source_archive_path, }; use docs_rs_types::{ BuildId, BuildStatus, CrateId, KrateName, ReleaseId, Version, @@ -454,17 +454,13 @@ impl RustwideBuilder { // available at --static-root-path, we add files from that subdirectory, if present. let static_files = dest.as_ref().join("static.files"); if static_files.try_exists()? { - self.runtime.block_on(add_path_into_database( - &self.storage, - RUSTDOC_STATIC_STORAGE_PREFIX, - &static_files, - ))?; + self.runtime.block_on( + self.storage + .store_all(RUSTDOC_STATIC_STORAGE_PREFIX, &static_files), + )?; } else { - self.runtime.block_on(add_path_into_database( - &self.storage, - RUSTDOC_STATIC_STORAGE_PREFIX, - &dest, - ))?; + self.runtime + .block_on(self.storage.store_all(RUSTDOC_STATIC_STORAGE_PREFIX, &dest))?; } self.runtime.block_on(async { @@ -629,8 +625,8 @@ impl RustwideBuilder { debug!("adding sources into database"); let files_list = { let (files_list, new_alg) = - self.runtime.block_on(add_path_into_remote_archive( - &self.storage, + self.runtime.block_on( + self.storage.store_all_in_archive( &source_archive_path(name, version), build.host_source_dir(), ))?; @@ -724,8 +720,8 @@ impl RustwideBuilder { target_build_logs.insert(target, target_res.build_log); } let (file_list, new_alg) = - self.runtime.block_on(add_path_into_remote_archive( - &self.storage, + self.runtime.block_on( + self.storage.store_all_in_archive( &rustdoc_archive_path(name, version), local_storage.path(), ))?; diff --git a/crates/lib/docs_rs_storage/src/file.rs b/crates/lib/docs_rs_storage/src/file.rs index 3c01b27a2..f744022af 100644 --- a/crates/lib/docs_rs_storage/src/file.rs +++ b/crates/lib/docs_rs_storage/src/file.rs @@ -1,20 +1,7 @@ -//! Simple module to store files in database. -//! -//! docs.rs supports two ways of storing files: in a postgres database and in an S3 bucket. -//! It does not support storing files on disk because of the sheer number of files: -//! doing so would quickly run into file descriptor limits when running the web server. -//! -//! It's recommended that you use the S3 bucket in production to avoid running out of disk space. -//! However, postgres is still available for testing and backwards compatibility. - -use crate::storage::non_blocking::AsyncStorage; -use anyhow::Result; use docs_rs_mimes::detect_mime; -use docs_rs_types::CompressionAlgorithm; use mime::Mime; use serde_json::Value; -use std::path::{Path, PathBuf}; -use tracing::instrument; +use std::path::PathBuf; /// represents a file path from our source or documentation builds. /// Used to return metadata about the file. @@ -30,35 +17,6 @@ impl FileEntry { } } -/// Store all files in a directory and return [[mimetype, filename]] as Json -/// -/// If there is an S3 Client configured, store files into an S3 bucket; -/// otherwise, stores files into the 'files' table of the local database. -/// -/// The mimetype is detected using `magic`. -/// -/// Note that this function is used for uploading both sources -/// and files generated by rustdoc. -pub async fn add_path_into_database>( - storage: &AsyncStorage, - prefix: impl AsRef, - path: P, -) -> Result<(Vec, CompressionAlgorithm)> { - storage.store_all(prefix.as_ref(), path.as_ref()).await -} - -#[instrument(skip(storage))] -pub async fn add_path_into_remote_archive + std::fmt::Debug>( - storage: &AsyncStorage, - archive_path: &str, - path: P, -) -> Result<(Vec, CompressionAlgorithm)> { - let (file_list, algorithm) = storage - .store_all_in_archive(archive_path, path.as_ref()) - .await?; - Ok((file_list, algorithm)) -} - pub fn file_list_to_json(files: impl IntoIterator) -> Value { Value::Array( files diff --git a/crates/lib/docs_rs_storage/src/lib.rs b/crates/lib/docs_rs_storage/src/lib.rs index d4f330b17..3c70b88b1 100644 --- a/crates/lib/docs_rs_storage/src/lib.rs +++ b/crates/lib/docs_rs_storage/src/lib.rs @@ -17,7 +17,7 @@ pub use compression::{compress, compress_async, decompress}; pub use config::Config; pub use errors::{PathNotFoundError, SizeLimitReached}; pub use file::FileEntry; -pub use file::{add_path_into_database, add_path_into_remote_archive, file_list_to_json}; +pub use file::file_list_to_json; pub use storage::blocking::Storage; pub use storage::non_blocking::AsyncStorage; pub use types::StorageKind; diff --git a/crates/lib/docs_rs_storage/src/storage/non_blocking.rs b/crates/lib/docs_rs_storage/src/storage/non_blocking.rs index 0785c4443..9731740d1 100644 --- a/crates/lib/docs_rs_storage/src/storage/non_blocking.rs +++ b/crates/lib/docs_rs_storage/src/storage/non_blocking.rs @@ -428,8 +428,9 @@ impl AsyncStorage { pub async fn store_all_in_archive( &self, archive_path: &str, - root_dir: &Path, + root_dir: impl AsRef + fmt::Debug, ) -> Result<(Vec, CompressionAlgorithm)> { + let root_dir = root_dir.as_ref(); let (mut zip_content, file_paths) = spawn_blocking({ let archive_path = archive_path.to_owned(); @@ -523,9 +524,11 @@ impl AsyncStorage { #[instrument(skip(self))] pub async fn store_all( &self, - prefix: &Path, - root_dir: &Path, + prefix: impl AsRef + fmt::Debug, + root_dir: impl AsRef + fmt::Debug, ) -> Result<(Vec, CompressionAlgorithm)> { + let prefix = prefix.as_ref(); + let root_dir = root_dir.as_ref(); let alg = CompressionAlgorithm::default(); let (blobs, file_paths_and_mimes) = spawn_blocking({ diff --git a/crates/lib/docs_rs_test_fakes/src/legacy.rs b/crates/lib/docs_rs_test_fakes/src/legacy.rs index 520bdec3d..cab64b3ff 100644 --- a/crates/lib/docs_rs_test_fakes/src/legacy.rs +++ b/crates/lib/docs_rs_test_fakes/src/legacy.rs @@ -9,8 +9,8 @@ use docs_rs_database::{ use docs_rs_registry_api::{CrateData, CrateOwner, ReleaseData}; use docs_rs_rustdoc_json::{RUSTDOC_JSON_COMPRESSION_ALGORITHMS, RustdocJsonFormatVersion}; use docs_rs_storage::{ - AsyncStorage, FileEntry, add_path_into_database, add_path_into_remote_archive, compress, - file_list_to_json, rustdoc_archive_path, rustdoc_json_path, source_archive_path, + AsyncStorage, FileEntry, compress, file_list_to_json, rustdoc_archive_path, rustdoc_json_path, + source_archive_path, }; use docs_rs_types::{ BuildId, BuildStatus, CompressionAlgorithm, DocCoverage, ReleaseId, Version, VersionReq, @@ -411,20 +411,20 @@ impl<'a> FakeRelease<'a> { FileKind::Sources => source_archive_path(&package.name, &package.version), }; debug!("store in archive: {:?}", archive); - let (files_list, new_alg) = - add_path_into_remote_archive(storage, &archive, source_directory).await?; - Ok((files_list, new_alg)) + Ok(storage + .store_all_in_archive(&archive, source_directory) + .await?) } else { let prefix = match kind { FileKind::Rustdoc => "rustdoc", FileKind::Sources => "sources", }; - add_path_into_database( - storage, - format!("{}/{}/{}/", prefix, package.name, package.version), - source_directory, - ) - .await + storage + .store_all( + format!("{}/{}/{}/", prefix, package.name, package.version), + source_directory, + ) + .await } }