remove unnecessary wrapper around storage method

crates/bin/docs_rs_builder/src/docbuilder/rustwide_builder.rs

@@ -19,8 +19,8 @@ use docs_rs_rustdoc_json::{
     read_format_version_from_rustdoc_json,
 };
 use docs_rs_storage::{
-    AsyncStorage, Storage, add_path_into_database, add_path_into_remote_archive, compress,
-    file_list_to_json, get_file_list, rustdoc_archive_path, rustdoc_json_path, source_archive_path,
+    AsyncStorage, Storage, compress, file_list_to_json, get_file_list, rustdoc_archive_path,
+    rustdoc_json_path, source_archive_path,
 };
 use docs_rs_types::{
     BuildId, BuildStatus, CrateId, KrateName, ReleaseId, Version,
@@ -454,17 +454,13 @@ impl RustwideBuilder {
                 // available at --static-root-path, we add files from that subdirectory, if present.
                 let static_files = dest.as_ref().join("static.files");
                 if static_files.try_exists()? {
-                    self.runtime.block_on(add_path_into_database(
-                        &self.storage,
-                        RUSTDOC_STATIC_STORAGE_PREFIX,
-                        &static_files,
-                    ))?;
+                    self.runtime.block_on(
+                        self.storage
+                            .store_all(RUSTDOC_STATIC_STORAGE_PREFIX, &static_files),
+                    )?;
                 } else {
-                    self.runtime.block_on(add_path_into_database(
-                        &self.storage,
-                        RUSTDOC_STATIC_STORAGE_PREFIX,
-                        &dest,
-                    ))?;
+                    self.runtime
+                        .block_on(self.storage.store_all(RUSTDOC_STATIC_STORAGE_PREFIX, &dest))?;
                 }
 
                 self.runtime.block_on(async {
@@ -629,8 +625,8 @@ impl RustwideBuilder {
                 debug!("adding sources into database");
                 let files_list = {
                     let (files_list, new_alg) =
-                        self.runtime.block_on(add_path_into_remote_archive(
-                            &self.storage,
+                        self.runtime.block_on(
+                        self.storage.store_all_in_archive(
                             &source_archive_path(name, version),
                             build.host_source_dir(),
                         ))?;
@@ -724,8 +720,8 @@ impl RustwideBuilder {
                         target_build_logs.insert(target, target_res.build_log);
                     }
                     let (file_list, new_alg) =
-                        self.runtime.block_on(add_path_into_remote_archive(
-                            &self.storage,
+                        self.runtime.block_on(
+                        self.storage.store_all_in_archive(
                             &rustdoc_archive_path(name, version),
                             local_storage.path(),
                         ))?;

crates/lib/docs_rs_storage/src/file.rs

@@ -1,20 +1,7 @@
-//! Simple module to store files in database.
-//!
-//! docs.rs supports two ways of storing files: in a postgres database and in an S3 bucket.
-//! It does not support storing files on disk because of the sheer number of files:
-//! doing so would quickly run into file descriptor limits when running the web server.
-//!
-//! It's recommended that you use the S3 bucket in production to avoid running out of disk space.
-//! However, postgres is still available for testing and backwards compatibility.
-
-use crate::storage::non_blocking::AsyncStorage;
-use anyhow::Result;
 use docs_rs_mimes::detect_mime;
-use docs_rs_types::CompressionAlgorithm;
 use mime::Mime;
 use serde_json::Value;
-use std::path::{Path, PathBuf};
-use tracing::instrument;
+use std::path::PathBuf;
 
 /// represents a file path from our source or documentation builds.
 /// Used to return metadata about the file.
@@ -30,35 +17,6 @@ impl FileEntry {
     }
 }
 
-/// Store all files in a directory and return [[mimetype, filename]] as Json
-///
-/// If there is an S3 Client configured, store files into an S3 bucket;
-/// otherwise, stores files into the 'files' table of the local database.
-///
-/// The mimetype is detected using `magic`.
-///
-/// Note that this function is used for uploading both sources
-/// and files generated by rustdoc.
-pub async fn add_path_into_database<P: AsRef<Path>>(
-    storage: &AsyncStorage,
-    prefix: impl AsRef<Path>,
-    path: P,
-) -> Result<(Vec<FileEntry>, CompressionAlgorithm)> {
-    storage.store_all(prefix.as_ref(), path.as_ref()).await
-}
-
-#[instrument(skip(storage))]
-pub async fn add_path_into_remote_archive<P: AsRef<Path> + std::fmt::Debug>(
-    storage: &AsyncStorage,
-    archive_path: &str,
-    path: P,
-) -> Result<(Vec<FileEntry>, CompressionAlgorithm)> {
-    let (file_list, algorithm) = storage
-        .store_all_in_archive(archive_path, path.as_ref())
-        .await?;
-    Ok((file_list, algorithm))
-}
-
 pub fn file_list_to_json(files: impl IntoIterator<Item = FileEntry>) -> Value {
     Value::Array(
         files

crates/lib/docs_rs_storage/src/lib.rs

@@ -17,7 +17,7 @@ pub use compression::{compress, compress_async, decompress};
 pub use config::Config;
 pub use errors::{PathNotFoundError, SizeLimitReached};
 pub use file::FileEntry;
-pub use file::{add_path_into_database, add_path_into_remote_archive, file_list_to_json};
+pub use file::file_list_to_json;
 pub use storage::blocking::Storage;
 pub use storage::non_blocking::AsyncStorage;
 pub use types::StorageKind;

crates/lib/docs_rs_storage/src/storage/non_blocking.rs

@@ -428,8 +428,9 @@ impl AsyncStorage {
     pub async fn store_all_in_archive(
         &self,
         archive_path: &str,
-        root_dir: &Path,
+        root_dir: impl AsRef<Path> + fmt::Debug,
     ) -> Result<(Vec<FileEntry>, CompressionAlgorithm)> {
+        let root_dir = root_dir.as_ref();
         let (mut zip_content,    file_paths) =
             spawn_blocking({
                 let archive_path = archive_path.to_owned();
@@ -523,9 +524,11 @@ impl AsyncStorage {
     #[instrument(skip(self))]
     pub async fn store_all(
         &self,
-        prefix: &Path,
-        root_dir: &Path,
+        prefix: impl AsRef<Path> + fmt::Debug,
+        root_dir: impl AsRef<Path> + fmt::Debug,
     ) -> Result<(Vec<FileEntry>, CompressionAlgorithm)> {
+        let prefix = prefix.as_ref();
+        let root_dir = root_dir.as_ref();
         let alg = CompressionAlgorithm::default();
 
         let (blobs, file_paths_and_mimes) = spawn_blocking({

crates/lib/docs_rs_test_fakes/src/legacy.rs

@@ -9,8 +9,8 @@ use docs_rs_database::{
 use docs_rs_registry_api::{CrateData, CrateOwner, ReleaseData};
 use docs_rs_rustdoc_json::{RUSTDOC_JSON_COMPRESSION_ALGORITHMS, RustdocJsonFormatVersion};
 use docs_rs_storage::{
-    AsyncStorage, FileEntry, add_path_into_database, add_path_into_remote_archive, compress,
-    file_list_to_json, rustdoc_archive_path, rustdoc_json_path, source_archive_path,
+    AsyncStorage, FileEntry, compress, file_list_to_json, rustdoc_archive_path, rustdoc_json_path,
+    source_archive_path,
 };
 use docs_rs_types::{
     BuildId, BuildStatus, CompressionAlgorithm, DocCoverage, ReleaseId, Version, VersionReq,
@@ -411,20 +411,20 @@ impl<'a> FakeRelease<'a> {
                     FileKind::Sources => source_archive_path(&package.name, &package.version),
                 };
                 debug!("store in archive: {:?}", archive);
-                let (files_list, new_alg) =
-                    add_path_into_remote_archive(storage, &archive, source_directory).await?;
-                Ok((files_list, new_alg))
+                Ok(storage
+                    .store_all_in_archive(&archive, source_directory)
+                    .await?)
             } else {
                 let prefix = match kind {
                     FileKind::Rustdoc => "rustdoc",
                     FileKind::Sources => "sources",
                 };
-                add_path_into_database(
-                    storage,
-                    format!("{}/{}/{}/", prefix, package.name, package.version),
-                    source_directory,
-                )
-                .await
+                storage
+                    .store_all(
+                        format!("{}/{}/{}/", prefix, package.name, package.version),
+                        source_directory,
+                    )
+                    .await
             }
         }