feat: update to latest changes from docbox

Author: jacobtread

Cargo.lock

@@ -7,24 +7,23 @@ members = [
 ]
 
 [workspace.dependencies]
-docbox-core = "0.7.0"
-docbox-database = "0.6.0"
-docbox-processing = "0.2.0"
-docbox-search = "0.6.0"
-docbox-secrets = "0.3.0"
-docbox-storage = "0.4.0"
-docbox-web-scraper = "0.4.0"
+docbox-core = "0.8.0"
+docbox-database = "0.7.0"
+docbox-processing = "0.3.0"
+docbox-search = "0.7.0"
+docbox-secrets = "0.4.0"
+docbox-storage = "0.5.0"
+docbox-web-scraper = "0.5.0"
 
 # Futures utilities
 futures = "=0.3.31"
 
-
 # Bytes type used for cheap shared bytes (and utilities)
-bytes = "=1.10.1"
+bytes = "=1.11.0"
 bytes-utils = "=0.1.4"
 
 # Logging
-tracing = "=0.1.41"
+tracing = "=0.1.44"
 
 # Error handling
 thiserror = "=2.0.17"
@@ -33,19 +32,18 @@ thiserror = "=2.0.17"
 chrono = { version = "=0.4.42", features = ["serde"] }
 
 # AWS configuration
-aws-config = { version = "=1.8.10", features = ["behavior-version-latest"] }
+aws-config = { version = "=1.8.12", features = ["behavior-version-latest"] }
 
 # URL parsing
 url = "=2.5.7"
 
 # UUID v4 support
-uuid = { version = "=1.18.1", features = ["v4", "serde"] }
-
+uuid = { version = "=1.19.0", features = ["v4", "serde"] }
 
 # Serialization and JSON
 serde = { version = "=1.0.228", features = ["derive"] }
-serde_json = "=1.0.145"
-serde_with = "=3.15.0"
+serde_json = "=1.0.148"
+serde_with = "=3.16.1"
 
 # Mime types, parsing, extension guessing, reverse mime lookup
 mime = "=0.3.17"
@@ -56,7 +54,7 @@ mime2ext = "=0.1.54"
 http = "=1.3.1"
 
 # Validation
-garde = { version = "=0.22.0", features = ["derive", "full"] }
+garde = { version = "=0.22.1", features = ["derive", "full"] }
 
 # OpenAPI spec generation
 utoipa = { version = "=5.4.0", features = [
@@ -66,4 +64,4 @@ utoipa = { version = "=5.4.0", features = [
     "axum_extras",
 ] }
 
-dotenvy = "0.15.7"
+dotenvy = "=0.15.7"

Cargo.toml

@@ -9,10 +9,10 @@ lambda_http = "1.0.0"
 tokio = { version = "1", features = ["macros"] }
 
 # HTTP server framework
-axum = { version = "=0.8.6" }
+axum = { version = "=0.8.8" }
 
 # HTTP layers for ratelimiting, CORS, and tracing
-tower-http = { version = "=0.6.6", features = ["limit", "cors", "trace"] }
+tower-http = { version = "=0.6.8", features = ["limit", "cors", "trace"] }
 tower = { version = "=0.5.2" }
 
 # Validation & Axum validation integration

lambdas/http/Cargo.toml

@@ -0,0 +1 @@
+pub mod purge_expired_presigned_tasks;

lambdas/http/src/background/mod.rs

@@ -0,0 +1,94 @@
+use chrono::Utc;
+use docbox_database::{
+    DatabasePoolCache, DbPool, DbResult,
+    models::{
+        presigned_upload_task::{PresignedTaskStatus, PresignedUploadTask},
+        tenant::Tenant,
+    },
+};
+use docbox_storage::{StorageLayerFactory, TenantStorageLayer};
+use std::sync::Arc;
+use thiserror::Error;
+
+#[derive(Debug, Error)]
+pub enum PurgeExpiredPresignedError {
+    #[error("failed to connect to database")]
+    ConnectDatabase,
+
+    #[error("failed to query available tenants")]
+    QueryTenants,
+}
+
+#[tracing::instrument(skip_all)]
+pub async fn purge_expired_presigned_tasks(
+    db_cache: Arc<DatabasePoolCache>,
+    storage: StorageLayerFactory,
+) -> Result<(), PurgeExpiredPresignedError> {
+    let db = db_cache.get_root_pool().await.map_err(|error| {
+        tracing::error!(?error, "failed to connect to root database");
+        PurgeExpiredPresignedError::ConnectDatabase
+    })?;
+
+    let tenants = Tenant::all(&db).await.map_err(|error| {
+        tracing::error!(?error, "failed to query available tenants");
+        PurgeExpiredPresignedError::QueryTenants
+    })?;
+
+    // Early drop the root database pool access
+    drop(db);
+
+    for tenant in tenants {
+        // Create the database connection pool
+        let db = db_cache.get_tenant_pool(&tenant).await.map_err(|error| {
+            tracing::error!(?error, "failed to connect to tenant database");
+            PurgeExpiredPresignedError::ConnectDatabase
+        })?;
+
+        let storage = storage.create_storage_layer(&tenant);
+
+        if let Err(error) = purge_expired_presigned_tasks_tenant(&db, &storage).await {
+            tracing::error!(
+                ?error,
+                ?tenant,
+                "failed to purge presigned tasks for tenant"
+            );
+        }
+    }
+
+    Ok(())
+}
+
+pub async fn purge_expired_presigned_tasks_tenant(
+    db: &DbPool,
+    storage: &TenantStorageLayer,
+) -> DbResult<()> {
+    let current_date = Utc::now();
+    let tasks = PresignedUploadTask::find_expired(db, current_date).await?;
+    if tasks.is_empty() {
+        return Ok(());
+    }
+
+    for task in tasks {
+        // Delete the task itself
+        if let Err(error) = PresignedUploadTask::delete(db, task.id).await {
+            tracing::error!(?error, "failed to delete presigned upload task");
+        }
+
+        // Delete incomplete file uploads
+        match task.status {
+            PresignedTaskStatus::Completed { .. } => {
+                // Upload completed, nothing to revert
+            }
+            PresignedTaskStatus::Failed { .. } | PresignedTaskStatus::Pending => {
+                if let Err(error) = storage.delete_file(&task.file_key).await {
+                    tracing::error!(
+                        ?error,
+                        "failed to delete expired presigned task file from tenant"
+                    );
+                }
+            }
+        }
+    }
+
+    Ok(())
+}

lambdas/http/src/background/purge_expired_presigned_tasks.rs

@@ -9,7 +9,6 @@ use std::{
     fmt::{Debug, Display},
 };
 use thiserror::Error;
-use tracing::error;
 use utoipa::ToSchema;
 
 /// Type alias for dynamic error handling and JSON responses

lambdas/http/src/error.rs

@@ -5,6 +5,7 @@ use axum::{Extension, Router};
 use docbox_core::{
     aws::{SqsClient, aws_config},
     events::{EventPublisherFactory, sqs::SqsEventPublisherFactory},
+    links::resolve_website::{ResolveWebsiteConfig, ResolveWebsiteService},
     tenant::tenant_cache::TenantCache,
 };
 use docbox_database::{DatabasePoolCache, DatabasePoolCacheConfig};
@@ -16,6 +17,7 @@ use lambda_http::{Error, run_with_streaming_response, tracing};
 use std::sync::Arc;
 use tower_http::trace::TraceLayer;
 
+mod background;
 pub mod docs;
 mod error;
 mod extensions;
@@ -48,13 +50,18 @@ async fn app() -> Result<Router, Box<dyn std::error::Error + Send + Sync>> {
         Err(_) => 100 * 1000 * 1024,
     };
 
+    // Load AWS configuration
+    let aws_config = aws_config().await;
+
     // Create website scraping service
     let website_meta_service_config = WebsiteMetaServiceConfig::from_env()?;
-    let website_meta_service = Arc::new(WebsiteMetaService::from_config(
-        website_meta_service_config,
-    )?);
+    let website_meta_service = WebsiteMetaService::from_config(website_meta_service_config)?;
+    let resolve_website_config = ResolveWebsiteConfig::from_env()?;
 
-    let aws_config = aws_config().await;
+    let caching_website_meta_service = Arc::new(ResolveWebsiteService::from_client_with_config(
+        website_meta_service,
+        resolve_website_config,
+    ));
 
     // Create secrets manager
     let secrets_config = SecretsManagerConfig::from_env()?;
@@ -96,8 +103,8 @@ async fn app() -> Result<Router, Box<dyn std::error::Error + Send + Sync>> {
     let mut app = router()
         .layer(Extension(search))
         .layer(Extension(storage))
-        .layer(Extension(db_cache.clone()))
-        .layer(Extension(website_meta_service))
+        .layer(Extension(db_cache))
+        .layer(Extension(caching_website_meta_service))
         .layer(Extension(events))
         .layer(Extension(tenant_cache))
         .layer(Extension(MaxFileSizeBytes(max_file_size_bytes)))

lambdas/http/src/main.rs

@@ -21,7 +21,9 @@ pub struct TenantDocumentBoxesRequest {
 
 #[derive(Debug, Serialize, ToSchema)]
 pub struct TenantDocumentBoxesResponse {
+    /// The document boxes
     pub results: Vec<DocumentBox>,
+    /// The total number of document boxes available to query
     pub total: i64,
 }
 

lambdas/http/src/models/admin.rs

@@ -14,7 +14,7 @@ use utoipa::ToSchema;
 /// Valid document box scope string, must be: A-Z, a-z, 0-9, ':', '-', '_', '.'
 #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, ToSchema, Serialize)]
 #[serde(transparent)]
-#[schema(example = "user:1:files", value_type = String)]
+#[schema(examples( "user:1:files"), value_type = String)]
 pub struct DocumentBoxScope(pub String);
 
 impl Display for DocumentBoxScope {

lambdas/http/src/models/document_box.rs

@@ -12,7 +12,7 @@ use garde::Validate;
 use mime::Mime;
 use serde::{Deserialize, Serialize};
 use serde_with::serde_as;
-use std::collections::HashMap;
+use std::{collections::HashMap, marker::PhantomData};
 use thiserror::Error;
 use utoipa::ToSchema;
 
@@ -25,23 +25,25 @@ pub struct CreatePresignedRequest {
     #[schema(min_length = 1, max_length = 255)]
     pub name: String,
 
-    /// Folder to store the file in
+    /// ID of the folder to store the file in
     #[garde(skip)]
     #[schema(value_type = Uuid)]
     pub folder_id: FolderId,
 
-    /// Size of the file being uploaded
+    /// Size of the file being uploaded in bytes. Must match the size of the
+    /// file being uploaded
     #[garde(range(min = 1))]
     #[schema(minimum = 1)]
     pub size: i32,
 
     /// Mime type of the file
     #[garde(skip)]
-    #[serde_as(as = "serde_with::DisplayFromStr")]
-    #[schema(value_type = String)]
-    pub mime: Mime,
+    #[serde_as(as = "Option<serde_with::DisplayFromStr>")]
+    #[schema(value_type = Option<String>)]
+    pub mime: Option<Mime>,
 
-    /// Optional parent file ID
+    /// Optional ID of the parent file if this file is associated as a child
+    /// of another file. Mainly used to associating attachments to email files
     #[garde(skip)]
     #[schema(value_type = Option<Uuid>)]
     pub parent_id: Option<FileId>,
@@ -57,25 +59,37 @@ pub struct CreatePresignedRequest {
     pub disable_mime_sniffing: Option<bool>,
 }
 
+/// Response describing how to upload the presigned file and the ID
+/// for polling the progress
 #[derive(Serialize, ToSchema)]
 pub struct PresignedUploadResponse {
+    /// ID of the file upload task to poll
     #[schema(value_type = Uuid)]
     pub task_id: PresignedUploadTaskId,
+    /// HTTP method to use when uploading the file
     pub method: String,
+    /// URL to upload the file to
     pub uri: String,
+    /// Headers to include on the file upload request
     pub headers: HashMap<String, String>,
 }
 
 #[derive(Serialize, ToSchema)]
 #[serde(tag = "status")]
 #[allow(clippy::large_enum_variant)]
 pub enum PresignedStatusResponse {
+    /// Presigned upload is currently pending
     Pending,
+    /// Presigned upload is completed
     Complete {
+        /// The uploaded file
         file: FileWithExtra,
+        /// The generated file
         generated: Vec<GeneratedFile>,
     },
+    /// Presigned upload failed
     Failed {
+        /// The error that occurred
         error: String,
     },
 }
@@ -131,6 +145,11 @@ pub struct PresignedDownloadResponse {
     pub expires_at: DateTime<Utc>,
 }
 
+/// Type hint type for Utoipa to indicate a binary response type
+#[derive(ToSchema)]
+#[schema(value_type = String, format = Binary)]
+pub struct BinaryResponse(PhantomData<Vec<u8>>);
+
 #[derive(Debug, Error)]
 pub enum HttpFileError {
     #[error("unknown file")]
@@ -141,6 +160,7 @@ pub enum HttpFileError {
 
     #[error("file size is larger than the maximum allowed size (requested: {0}, maximum: {1})")]
     FileTooLarge(i32, i32),
+
     #[error("no matching generated file")]
     NoMatchingGenerated,