From aa6fe2724b4907ad8eb24f1bfe50919dcd8aacec Mon Sep 17 00:00:00 2001 From: Manu Date: Wed, 10 Jun 2026 22:06:18 +0200 Subject: [PATCH 01/24] Add on-demand file integrity verification (#94) Implements on-demand and scheduled integrity verification for backups. - New verification service (src/services/storage/verification-service.ts) that reads .meta.json sidecars, attempts adapter-native verification (S3/Cloudflare R2/Hetzner via metadata, Google Drive MD5, OneDrive SHA-256, local filesystem via direct hash) and falls back to download+SHA-256 when needed. Writes verification results back to the .meta.json sidecar. - Exposes a POST API route /api/storage/[id]/verify to trigger manual verifications. - UI/UX: adds an Integrity column, a Verify action button, toasts and client handlers (dashboard columns, actions cell, storage client) to support manual verification and display results. - Adapters: added verifyChecksum implementations for S3 (uses HeadObject metadata), Google Drive (md5Checksum), OneDrive (sha256Hash) and local filesystem (direct hash). S3 upload now attaches dbackup-sha256 metadata when provided. - Core changes: added UploadOptions and optional verifyChecksum to StorageAdapter interface; upload flows now compute SHA-256+MD5 in a single pass and pass checksums to adapters; withStorageSession shim updated to forward options. - Runner: post-upload integrity verification integrated via verificationService; post-upload verification is opt-in via system setting (local filesystem remains verified). IntegrityService refactored to use verificationService for scheduled runs and now aggregates results. - StorageService and types updated to include verification info in returned file metadata. - Added calculateFileChecksums utility for efficient dual-hash computation. - Updated changelog to document vNEXT integrity features. --- docs/changelog.md | 19 ++ src/app/api/storage/[id]/verify/route.ts | 37 ++++ src/app/dashboard/storage/columns.tsx | 23 ++- src/app/dashboard/storage/storage-client.tsx | 34 +++- .../dashboard/storage/cells/actions-cell.tsx | 41 ++++- src/lib/adapters/storage/google-drive.ts | 20 +++ src/lib/adapters/storage/local.ts | 13 ++ src/lib/adapters/storage/onedrive.ts | 16 ++ src/lib/adapters/storage/s3.ts | 61 ++++++- src/lib/core/interfaces.ts | 37 +++- src/lib/crypto/checksum.ts | 22 +++ src/lib/runner/steps/03-upload.ts | 62 ++++--- src/lib/runner/steps/upload-helpers.ts | 4 +- src/services/backup/integrity-service.ts | 166 +++--------------- src/services/storage/storage-service.ts | 10 ++ src/services/storage/verification-service.ts | 153 ++++++++++++++++ 16 files changed, 529 insertions(+), 189 deletions(-) create mode 100644 src/app/api/storage/[id]/verify/route.ts create mode 100644 src/services/storage/verification-service.ts diff --git a/docs/changelog.md b/docs/changelog.md index 3cdf097d..7c837325 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -2,6 +2,25 @@ All notable changes to DBackup are documented here. +## vNEXT +*Release: In Progress* + +### ✨ Features + +- **storage**: Added on-demand integrity verification for backup files via a new Verify button in the Storage Explorer. Results are persisted in the `.meta.json` sidecar and shown as a green/red badge in the Integrity column. +- **storage**: Added adapter-native checksum verification for S3/R2/Hetzner (SHA-256 via object metadata), local filesystem (direct stream hash), Google Drive (MD5 via API), and OneDrive (SHA-256 via Graph API) - no download required for these adapters. +- **storage**: MD5 checksums are now computed alongside SHA-256 during backup upload and stored in `.meta.json`, enabling native verification on Google Drive. +- **integrity**: The scheduled integrity check now uses native verification where available and writes results back to `.meta.json` sidecars. +- **backup**: Post-upload integrity verification is now opt-in for all storage destinations via the `backup.postUploadVerify` system setting (local filesystem always verifies). + +### 🐳 Docker + +- **Image**: `skyfay/dbackup:vNEXT` +- **Also tagged as**: `latest`, `vNEXT` +- **CI Image**: `skyfay/dbackup:ci` +- **Platforms**: linux/amd64, linux/arm64 + + ## v2.6.0 - Security Update, Vault Credential Profiles, OAuth Improvements, and Multiple Bug Fixes *Released: June 6, 2026* diff --git a/src/app/api/storage/[id]/verify/route.ts b/src/app/api/storage/[id]/verify/route.ts new file mode 100644 index 00000000..6c4bb3a7 --- /dev/null +++ b/src/app/api/storage/[id]/verify/route.ts @@ -0,0 +1,37 @@ +import { NextRequest, NextResponse } from "next/server"; +import { verificationService } from "@/services/storage/verification-service"; +import { headers } from "next/headers"; +import { getAuthContext, checkPermissionWithContext } from "@/lib/auth/access-control"; +import { PERMISSIONS } from "@/lib/auth/permissions"; +import { logger } from "@/lib/logging/logger"; +import { wrapError, getErrorMessage } from "@/lib/logging/errors"; + +const log = logger.child({ route: "storage/verify" }); + +export async function POST(req: NextRequest, props: { params: Promise<{ id: string }> }) { + const ctx = await getAuthContext(await headers()); + + if (!ctx) { + return NextResponse.json({ error: "Unauthorized" }, { status: 401 }); + } + + const params = await props.params; + + try { + checkPermissionWithContext(ctx, PERMISSIONS.STORAGE.READ); + + const body = await req.json(); + const { file } = body; + + if (!file || typeof file !== 'string' || file.includes('..') || file.startsWith('/')) { + return NextResponse.json({ error: "Invalid file path" }, { status: 400 }); + } + + const result = await verificationService.verifyFile(params.id, file, 'manual'); + + return NextResponse.json({ success: true, data: result }); + } catch (error: unknown) { + log.error("Verify route error", { id: params.id }, wrapError(error)); + return NextResponse.json({ error: getErrorMessage(error) }, { status: 500 }); + } +} diff --git a/src/app/dashboard/storage/columns.tsx b/src/app/dashboard/storage/columns.tsx index ce1edb54..9c0a7acc 100644 --- a/src/app/dashboard/storage/columns.tsx +++ b/src/app/dashboard/storage/columns.tsx @@ -1,7 +1,7 @@ "use client"; import { ColumnDef } from "@tanstack/react-table"; -import { ArrowUpDown, Clock, HardDrive, KeyRound, MousePointerClick } from "lucide-react"; +import { ArrowUpDown, Clock, HardDrive, KeyRound, MousePointerClick, ShieldCheck, ShieldX } from "lucide-react"; import { AdapterIcon } from "@/components/adapter/adapter-icon"; import { Button } from "@/components/ui/button"; import { DateDisplay } from "@/components/utils/date-display"; @@ -28,6 +28,11 @@ export type FileInfo = { locked?: boolean; trigger?: { type: string; actor?: string }; storageClass?: string; + verification?: { + verifiedAt: string; + passed: boolean; + trigger: 'manual' | 'post-upload' | 'scheduled'; + }; }; interface ColumnsProps { @@ -36,12 +41,13 @@ interface ColumnsProps { onDelete: (file: FileInfo) => void; onToggleLock: (file: FileInfo) => void; onGenerateLink: (file: FileInfo) => void; + onVerify: (file: FileInfo) => void; canDownload: boolean; canRestore: boolean; canDelete: boolean; } -export const getColumns = ({ onRestore, onDownload, onDelete, onToggleLock, onGenerateLink, canDownload, canRestore, canDelete }: ColumnsProps): ColumnDef[] => [ +export const getColumns = ({ onRestore, onDownload, onDelete, onToggleLock, onGenerateLink, onVerify, canDownload, canRestore, canDelete }: ColumnsProps): ColumnDef[] => [ { accessorKey: "name", header: ({ column }) => { @@ -174,6 +180,18 @@ export const getColumns = ({ onRestore, onDownload, onDelete, onToggleLock, onGe ); } }, + { + id: "verification", + header: "Integrity", + cell: ({ row }) => { + const v = row.original.verification; + if (!v) return -; + if (v.passed) { + return Verified; + } + return Failed; + } + }, { accessorKey: "size", header: ({ column }) => { @@ -224,6 +242,7 @@ export const getColumns = ({ onRestore, onDownload, onDelete, onToggleLock, onGe onDelete={onDelete} onToggleLock={onToggleLock} onGenerateLink={onGenerateLink} + onVerify={onVerify} canDownload={canDownload} canRestore={canRestore} canDelete={canDelete} diff --git a/src/app/dashboard/storage/storage-client.tsx b/src/app/dashboard/storage/storage-client.tsx index 7f719e07..1d716eba 100644 --- a/src/app/dashboard/storage/storage-client.tsx +++ b/src/app/dashboard/storage/storage-client.tsx @@ -259,6 +259,37 @@ export function StorageClient({ canDownload, canRestore, canDelete }: StorageCli setDownloadLinkFile(file); }, [canDownload]); + const handleVerify = useCallback(async (file: FileInfo) => { + const toastId = toast.loading(`Verifying integrity of ${file.name}...`); + try { + const res = await fetch(`/api/storage/${selectedDestination}/verify`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ file: file.path }), + }); + const data = await res.json(); + if (!res.ok) { + toast.error(data.error || "Verification failed", { id: toastId }); + return; + } + const result = data.data; + if (result.status === "passed") { + toast.success("Integrity check passed", { id: toastId }); + } else if (result.status === "failed") { + toast.error("Integrity check FAILED - backup may be corrupted", { id: toastId }); + } else if (result.status === "no_checksum") { + toast.info("No checksum available (legacy backup)", { id: toastId }); + } else if (result.status === "no_metadata") { + toast.info("No metadata found for this file", { id: toastId }); + } else { + toast.error("Verification error: " + result.status, { id: toastId }); + } + fetchFiles(selectedDestination, showSystemConfigs); + } catch { + toast.error("Error verifying file", { id: toastId }); + } + }, [selectedDestination, showSystemConfigs]); // eslint-disable-line react-hooks/exhaustive-deps + const confirmDelete = async () => { if (!fileToDelete) return; setDeleting(true); @@ -290,10 +321,11 @@ export function StorageClient({ canDownload, canRestore, canDelete }: StorageCli onDelete: handleDeleteClick, onToggleLock: handleToggleLock, onGenerateLink: handleGenerateLink, + onVerify: handleVerify, canDownload, canRestore, canDelete - }), [handleRestoreClick, handleDownload, handleDeleteClick, handleToggleLock, handleGenerateLink, canDownload, canRestore, canDelete]); + }), [handleRestoreClick, handleDownload, handleDeleteClick, handleToggleLock, handleGenerateLink, handleVerify, canDownload, canRestore, canDelete]); const filterableColumns = useMemo(() => { const jobs = Array.from(new Set(files.map(f => f.jobName).filter(Boolean).filter(n => n !== "Unknown"))) as string[]; diff --git a/src/components/dashboard/storage/cells/actions-cell.tsx b/src/components/dashboard/storage/cells/actions-cell.tsx index 329990ed..3daebeab 100644 --- a/src/components/dashboard/storage/cells/actions-cell.tsx +++ b/src/components/dashboard/storage/cells/actions-cell.tsx @@ -1,7 +1,7 @@ import { Button } from "@/components/ui/button"; import { Tooltip, TooltipContent, TooltipProvider, TooltipTrigger } from "@/components/ui/tooltip"; import { DropdownMenu, DropdownMenuContent, DropdownMenuItem, DropdownMenuSeparator, DropdownMenuTrigger } from "@/components/ui/dropdown-menu"; -import { Download, RotateCcw, Trash2, Lock, FileLock2, FileCheck, Terminal } from "lucide-react"; +import { Download, RotateCcw, Trash2, Lock, FileLock2, FileCheck, Terminal, ShieldCheck, ShieldX, Shield } from "lucide-react"; import { FileInfo } from "@/app/dashboard/storage/columns"; const ARCHIVED_STORAGE_CLASSES = ["GLACIER", "DEEP_ARCHIVE"]; @@ -13,6 +13,7 @@ interface ActionsCellProps { onDelete: (file: FileInfo) => void; onToggleLock?: (file: FileInfo) => void; onGenerateLink?: (file: FileInfo) => void; + onVerify?: (file: FileInfo) => void; canDownload: boolean; canRestore: boolean; canDelete: boolean; @@ -25,6 +26,7 @@ export function ActionsCell({ onDelete, onToggleLock, onGenerateLink, + onVerify, canDownload, canRestore, canDelete @@ -32,8 +34,45 @@ export function ActionsCell({ const isArchived = ARCHIVED_STORAGE_CLASSES.includes(file.storageClass ?? ""); const archivedTooltip = "This backup is archived in S3 Glacier or Deep Archive. Restore it via the AWS Console first (S3 - select object - Actions - Initiate restore)."; + const verifyIcon = file.verification + ? file.verification.passed + ? + : + : ; + + const verifyClass = file.verification + ? file.verification.passed + ? "text-green-600 hover:text-green-700" + : "text-red-600 hover:text-red-700" + : "text-muted-foreground hover:text-foreground"; + + const verifyTooltip = file.verification + ? file.verification.passed + ? "Integrity verified - click to re-verify" + : "Integrity check failed - click to re-verify" + : "Verify integrity"; + return (
+ {onVerify && ( + + + + + + +

{verifyTooltip}

+ + + + )} {onToggleLock && canDelete && ( diff --git a/src/lib/adapters/storage/google-drive.ts b/src/lib/adapters/storage/google-drive.ts index 3513d4ae..cb111c45 100644 --- a/src/lib/adapters/storage/google-drive.ts +++ b/src/lib/adapters/storage/google-drive.ts @@ -344,6 +344,26 @@ export const GoogleDriveAdapter: StorageAdapter = { } }, + async verifyChecksum(config: GoogleDriveConfig, remotePath: string, checksums: { sha256?: string; md5?: string }): Promise<'passed' | 'failed' | 'unsupported'> { + if (!checksums.md5) return 'unsupported'; + try { + const drive = createDriveClient(config); + const fileName = path.basename(remotePath); + const dirPath = path.posix.dirname(remotePath); + const folderId = dirPath === '.' + ? (config.folderId || 'root') + : await resolveOrCreatePath(drive, config.folderId, dirPath + '/dummy'); + const file = await findFile(drive, folderId, fileName); + if (!file?.id) return 'unsupported'; + const details = await drive.files.get({ fileId: file.id, fields: 'md5Checksum' }); + const md5 = details.data.md5Checksum; + if (!md5) return 'unsupported'; + return md5.toLowerCase() === checksums.md5.toLowerCase() ? 'passed' : 'failed'; + } catch { + return 'unsupported'; + } + }, + async test(config: GoogleDriveConfig): Promise<{ success: boolean; message: string }> { try { const drive = createDriveClient(config); diff --git a/src/lib/adapters/storage/local.ts b/src/lib/adapters/storage/local.ts index 8f121287..59f4c59d 100644 --- a/src/lib/adapters/storage/local.ts +++ b/src/lib/adapters/storage/local.ts @@ -1,4 +1,5 @@ import { StorageAdapter, FileInfo } from "@/lib/core/interfaces"; +import { calculateFileChecksum } from "@/lib/crypto/checksum"; import { LogLevel, LogType } from "@/lib/core/logs"; import { LocalStorageSchema } from "@/lib/adapters/definitions"; import fs from "fs/promises"; @@ -188,6 +189,18 @@ export const LocalFileSystemAdapter: StorageAdapter = { } }, + async verifyChecksum(config: { basePath: string }, remotePath: string, checksums: { sha256?: string; md5?: string }): Promise<'passed' | 'failed' | 'unsupported'> { + if (!checksums.sha256) return 'unsupported'; + try { + const filePath = resolveSafePath(config.basePath, remotePath); + await fs.access(filePath); + const actual = await calculateFileChecksum(filePath); + return actual === checksums.sha256 ? 'passed' : 'failed'; + } catch { + return 'unsupported'; + } + }, + async test(config: { basePath: string }): Promise<{ success: boolean; message: string }> { const testFile = path.join(config.basePath, `.connection-test-${Date.now()}`); let written = false; diff --git a/src/lib/adapters/storage/onedrive.ts b/src/lib/adapters/storage/onedrive.ts index 1e681dc5..15348e03 100644 --- a/src/lib/adapters/storage/onedrive.ts +++ b/src/lib/adapters/storage/onedrive.ts @@ -423,6 +423,22 @@ export const OneDriveAdapter: StorageAdapter = { } }, + async verifyChecksum(config: OneDriveConfig, remotePath: string, checksums: { sha256?: string; md5?: string }): Promise<'passed' | 'failed' | 'unsupported'> { + if (!checksums.sha256) return 'unsupported'; + try { + const accessToken = await getAccessToken(config); + const client = createGraphClient(accessToken); + const drivePath = buildDrivePath(config.folderPath, remotePath); + const item = await client.api(driveItemPath(drivePath)).select('id,file').get(); + const sha256Base64: string | undefined = item?.file?.hashes?.sha256Hash; + if (!sha256Base64) return 'unsupported'; + const sha256Hex = Buffer.from(sha256Base64, 'base64').toString('hex').toLowerCase(); + return sha256Hex === checksums.sha256.toLowerCase() ? 'passed' : 'failed'; + } catch { + return 'unsupported'; + } + }, + async test(config: OneDriveConfig): Promise<{ success: boolean; message: string }> { try { const accessToken = await getAccessToken(config); diff --git a/src/lib/adapters/storage/s3.ts b/src/lib/adapters/storage/s3.ts index e4f0b88b..ffa7d2fe 100644 --- a/src/lib/adapters/storage/s3.ts +++ b/src/lib/adapters/storage/s3.ts @@ -1,6 +1,6 @@ -import { StorageAdapter, FileInfo } from "@/lib/core/interfaces"; +import { StorageAdapter, FileInfo, UploadOptions } from "@/lib/core/interfaces"; import { S3GenericSchema, S3AWSSchema, S3R2Schema, S3HetznerSchema } from "@/lib/adapters/definitions"; -import { S3Client, ListObjectsV2Command, GetObjectCommand, DeleteObjectCommand, PutObjectCommand, StorageClass } from "@aws-sdk/client-s3"; +import { S3Client, ListObjectsV2Command, GetObjectCommand, DeleteObjectCommand, PutObjectCommand, HeadObjectCommand, StorageClass } from "@aws-sdk/client-s3"; import { Upload } from "@aws-sdk/lib-storage"; import { createReadStream, createWriteStream } from "fs"; import { pipeline } from "stream/promises"; @@ -40,7 +40,7 @@ class S3ClientFactory { // --- Shared Implementation --- -async function s3Upload(internalConfig: S3InternalConfig, localPath: string, remotePath: string, onProgress?: (percent: number) => void, onLog?: (msg: string, level?: LogLevel, type?: LogType, details?: string) => void): Promise { +async function s3Upload(internalConfig: S3InternalConfig, localPath: string, remotePath: string, onProgress?: (percent: number) => void, onLog?: (msg: string, level?: LogLevel, type?: LogType, details?: string) => void, options?: UploadOptions): Promise { const client = S3ClientFactory.create(internalConfig); const targetKey = S3ClientFactory.getTargetKey(internalConfig, remotePath); @@ -55,6 +55,7 @@ async function s3Upload(internalConfig: S3InternalConfig, localPath: string, rem Key: targetKey, Body: fileStream, StorageClass: (internalConfig.storageClass as StorageClass) || undefined, + Metadata: options?.checksumSha256 ? { 'dbackup-sha256': options.checksumSha256 } : undefined, }, }); @@ -201,6 +202,24 @@ async function s3Delete( } } +async function s3VerifyChecksum( + internalConfig: S3InternalConfig, + remotePath: string, + checksums: { sha256?: string; md5?: string } +): Promise<'passed' | 'failed' | 'unsupported'> { + if (!checksums.sha256) return 'unsupported'; + const client = S3ClientFactory.create(internalConfig); + const targetKey = S3ClientFactory.getTargetKey(internalConfig, remotePath); + try { + const response = await client.send(new HeadObjectCommand({ Bucket: internalConfig.bucket, Key: targetKey })); + const stored = response.Metadata?.['dbackup-sha256']; + if (!stored) return 'unsupported'; + return stored === checksums.sha256 ? 'passed' : 'failed'; + } catch { + return 'unsupported'; + } +} + async function s3Test(internalConfig: S3InternalConfig): Promise<{ success: boolean; message: string }> { const client = S3ClientFactory.create(internalConfig); const testFile = `.backup-manager-test-${Date.now()}`; @@ -289,7 +308,15 @@ export const S3GenericAdapter: StorageAdapter = { credentials: { accessKeyId: config.accessKeyId, secretAccessKey: config.secretAccessKey }, forcePathStyle: config.forcePathStyle, pathPrefix: config.pathPrefix - }, ...args) + }, ...args), + verifyChecksum: (config, remotePath, checksums) => s3VerifyChecksum({ + endpoint: config.endpoint, + region: config.region, + bucket: config.bucket, + credentials: { accessKeyId: config.accessKeyId, secretAccessKey: config.secretAccessKey }, + forcePathStyle: config.forcePathStyle, + pathPrefix: config.pathPrefix + }, remotePath, checksums), }; // 2. AWS S3 @@ -332,7 +359,13 @@ export const S3AWSAdapter: StorageAdapter = { bucket: config.bucket, credentials: { accessKeyId: config.accessKeyId, secretAccessKey: config.secretAccessKey }, pathPrefix: config.pathPrefix - }, ...args) + }, ...args), + verifyChecksum: (config, remotePath, checksums) => s3VerifyChecksum({ + region: config.region, + bucket: config.bucket, + credentials: { accessKeyId: config.accessKeyId, secretAccessKey: config.secretAccessKey }, + pathPrefix: config.pathPrefix + }, remotePath, checksums), }; function r2Endpoint(accountId: string, jurisdiction?: string): string { @@ -386,7 +419,14 @@ export const S3R2Adapter: StorageAdapter = { bucket: config.bucket, credentials: { accessKeyId: config.accessKeyId, secretAccessKey: config.secretAccessKey }, pathPrefix: config.pathPrefix - }, ...args) + }, ...args), + verifyChecksum: (config, remotePath, checksums) => s3VerifyChecksum({ + endpoint: r2Endpoint(config.accountId, config.jurisdiction), + region: "auto", + bucket: config.bucket, + credentials: { accessKeyId: config.accessKeyId, secretAccessKey: config.secretAccessKey }, + pathPrefix: config.pathPrefix + }, remotePath, checksums), }; // 4. Hetzner Object Storage @@ -434,5 +474,12 @@ export const S3HetznerAdapter: StorageAdapter = { bucket: config.bucket, credentials: { accessKeyId: config.accessKeyId, secretAccessKey: config.secretAccessKey }, pathPrefix: config.pathPrefix - }, ...args) + }, ...args), + verifyChecksum: (config, remotePath, checksums) => s3VerifyChecksum({ + endpoint: `https://${config.region}.your-objectstorage.com`, + region: config.region, + bucket: config.bucket, + credentials: { accessKeyId: config.accessKeyId, secretAccessKey: config.secretAccessKey }, + pathPrefix: config.pathPrefix + }, remotePath, checksums), }; diff --git a/src/lib/core/interfaces.ts b/src/lib/core/interfaces.ts index 49fc9508..eb2f84ec 100644 --- a/src/lib/core/interfaces.ts +++ b/src/lib/core/interfaces.ts @@ -46,6 +46,15 @@ export interface BackupMetadata { }; /** SHA-256 checksum of the final backup file (after compression/encryption) */ checksum?: string; + /** MD5 checksum of the final backup file - enables native verification for Google Drive and OneDrive */ + checksumMd5?: string; + /** Result of the most recent integrity verification */ + verification?: { + verifiedAt: string; + passed: boolean; + trigger: 'manual' | 'post-upload' | 'scheduled'; + actualChecksum?: string; + }; /** Trigger information - what initiated the backup */ trigger?: { type: "Manual" | "Scheduler" | "Api"; @@ -220,6 +229,12 @@ export type FileInfo = { storageClass?: string; }; +/** Optional options passed to upload() for adapters that support native checksum storage. */ +export interface UploadOptions { + checksumSha256?: string; + checksumMd5?: string; +} + /** * A persistent upload session that reuses a single connection for multiple uploads. * Returned by `StorageAdapter.openSession()` when an adapter supports connection reuse. @@ -233,7 +248,8 @@ export interface StorageSession { localPath: string, remotePath: string, onProgress?: (percent: number) => void, - onLog?: (msg: string, level?: LogLevel, type?: LogType, details?: string) => void + onLog?: (msg: string, level?: LogLevel, type?: LogType, details?: string) => void, + options?: UploadOptions ): Promise; close(): Promise; } @@ -241,9 +257,24 @@ export interface StorageSession { export interface StorageAdapter extends BaseAdapter { type: 'storage'; /** - * Uploads a local file to the storage destination + * Uploads a local file to the storage destination. + * Pass `options.checksumSha256` / `options.checksumMd5` so adapters that support + * native checksum storage (S3, etc.) can attach the hash during the upload request. */ - upload(config: AdapterConfig, localPath: string, remotePath: string, onProgress?: (percent: number) => void, onLog?: (msg: string, level?: LogLevel, type?: LogType, details?: string) => void): Promise; + upload(config: AdapterConfig, localPath: string, remotePath: string, onProgress?: (percent: number) => void, onLog?: (msg: string, level?: LogLevel, type?: LogType, details?: string) => void, options?: UploadOptions): Promise; + + /** + * Optional: Verifies the integrity of a remote file without downloading it. + * Adapters implement this when the storage API provides native checksum access + * (S3 custom metadata, Google Drive md5Checksum, OneDrive file.hashes.sha256Hash). + * Returns 'unsupported' when the adapter cannot verify natively (e.g. SFTP, FTP). + * The VerificationService calls this first and falls back to download+hash if unsupported. + */ + verifyChecksum?( + config: AdapterConfig, + remotePath: string, + checksums: { sha256?: string; md5?: string } + ): Promise<'passed' | 'failed' | 'unsupported'>; /** * Optional: Opens a persistent session for multiple uploads on a single connection. diff --git a/src/lib/crypto/checksum.ts b/src/lib/crypto/checksum.ts index 9dd4ad3c..3b70e0eb 100644 --- a/src/lib/crypto/checksum.ts +++ b/src/lib/crypto/checksum.ts @@ -32,6 +32,28 @@ export function calculateChecksum(data: Buffer | string): string { return crypto.createHash(ALGORITHM).update(data).digest("hex"); } +/** + * Calculates both SHA-256 and MD5 checksums for a file in a single streaming pass. + * More efficient than calling calculateFileChecksum() twice for large backup files. + * + * @param filePath - Absolute path to the file + * @returns Object with hex-encoded sha256 and md5 hash strings + */ +export function calculateFileChecksums(filePath: string): Promise<{ sha256: string; md5: string }> { + return new Promise((resolve, reject) => { + const sha256 = crypto.createHash("sha256"); + const md5 = crypto.createHash("md5"); + const stream = fs.createReadStream(filePath, { highWaterMark: 1024 * 1024 }); + + stream.on("data", (chunk) => { + sha256.update(chunk); + md5.update(chunk); + }); + stream.on("end", () => resolve({ sha256: sha256.digest("hex"), md5: md5.digest("hex") })); + stream.on("error", (err) => reject(err)); + }); +} + /** * Verifies a file against an expected checksum. * diff --git a/src/lib/runner/steps/03-upload.ts b/src/lib/runner/steps/03-upload.ts index 661345f4..d5551b09 100644 --- a/src/lib/runner/steps/03-upload.ts +++ b/src/lib/runner/steps/03-upload.ts @@ -10,10 +10,10 @@ import { createEncryptionStream } from "@/lib/crypto/stream"; import { getCompressionStream, getCompressionExtension, CompressionType } from "@/lib/crypto/compression"; import { ProgressMonitorStream } from "@/lib/streams/progress-monitor"; import { formatBytes } from "@/lib/utils"; -import { calculateFileChecksum, verifyFileChecksum } from "@/lib/crypto/checksum"; -import { getTempDir } from "@/lib/temp-dir"; +import { calculateFileChecksums } from "@/lib/crypto/checksum"; import { PIPELINE_STAGES } from "@/lib/core/logs"; import { withStorageSession } from "./upload-helpers"; +import { verificationService } from "@/services/storage/verification-service"; export async function stepUpload(ctx: RunnerContext) { if (!ctx.job || ctx.destinations.length === 0 || !ctx.tempFile) throw new Error("Context not ready for upload"); @@ -117,10 +117,10 @@ export async function stepUpload(ctx: RunnerContext) { } } - // --- CHECKSUM CALCULATION --- - ctx.log("Calculating SHA-256 checksum..."); - const checksum = await calculateFileChecksum(ctx.tempFile); - ctx.log(`Checksum: ${checksum}`); + // --- CHECKSUM CALCULATION (SHA-256 + MD5 in single stream pass) --- + ctx.log("Calculating checksums..."); + const { sha256: checksum, md5: checksumMd5 } = await calculateFileChecksums(ctx.tempFile); + ctx.log(`SHA-256: ${checksum}`); // --- PRIVACY SETTING: include actor in metadata? --- const privacySetting = await prisma.systemSetting.findUnique({ where: { key: "privacy.includeActorInMetadata" } }); @@ -153,6 +153,7 @@ export async function stepUpload(ctx: RunnerContext) { compression: compressionMeta, encryption: encryptionMeta, checksum, + checksumMd5, multiDb: ctx.metadata?.multiDb, trigger, locked: ctx.lock === true, @@ -167,9 +168,6 @@ export async function stepUpload(ctx: RunnerContext) { ctx.setStage(PIPELINE_STAGES.UPLOADING); - // Collect destinations that need post-upload integrity verification - const verifyQueue: { dest: typeof ctx.destinations[number]; destLabel: string }[] = []; - for (let i = 0; i < totalDests; i++) { const dest = ctx.destinations[i]; const destLabel = `[${dest.configName}]`; @@ -206,12 +204,13 @@ export async function stepUpload(ctx: RunnerContext) { sessionLog ); - // Upload main backup file + // Upload main backup file (pass checksums so adapters can store them natively) const uploadSuccess = await session.upload( ctx.tempFile!, remotePath, destProgress, - sessionLog + sessionLog, + { checksumSha256: checksum, checksumMd5 } ); if (!uploadSuccess) { @@ -222,11 +221,6 @@ export async function stepUpload(ctx: RunnerContext) { dest.uploadResult = { success: true, path: remotePath }; ctx.log(`${destLabel} Upload complete: ${remotePath}`); - // Queue integrity verification for local storage - if (dest.adapterId === "local-filesystem") { - verifyQueue.push({ dest, destLabel }); - } - } catch (e: unknown) { const message = e instanceof Error ? e.message : String(e); dest.uploadResult = { success: false, error: message }; @@ -238,30 +232,34 @@ export async function stepUpload(ctx: RunnerContext) { await fs.unlink(metaPath).catch(() => {}); // --- POST-UPLOAD VERIFICATION --- - ctx.setStage(PIPELINE_STAGES.VERIFYING); + const postVerifySetting = await prisma.systemSetting.findUnique({ where: { key: "backup.postUploadVerify" } }); + const postVerifyEnabled = postVerifySetting?.value === 'true'; - if (verifyQueue.length > 0) { - for (const { dest, destLabel } of verifyQueue) { + // Always verify local destinations (native, no bandwidth cost); verify remote only if setting enabled + const destinationsToVerify = ctx.destinations.filter(d => + d.uploadResult?.success && (d.adapterId === "local-filesystem" || postVerifyEnabled) + ); + + if (destinationsToVerify.length > 0) { + ctx.setStage(PIPELINE_STAGES.VERIFYING); + + for (const dest of destinationsToVerify) { + const destLabel = `[${dest.configName}]`; try { ctx.log(`${destLabel} Verifying upload integrity...`); - const verifyPath = path.join(getTempDir(), `verify_${Date.now()}_${path.basename(ctx.tempFile)}`); - const downloadOk = await dest.adapter.download(dest.config, remotePath, verifyPath); - if (downloadOk) { - const result = await verifyFileChecksum(verifyPath, checksum); - if (result.valid) { - ctx.log(`${destLabel} Integrity check passed ✓`, 'success'); - } else { - ctx.log(`${destLabel} WARNING: Integrity check FAILED! Expected: ${result.expected}, Got: ${result.actual}`, 'warning'); - } - await fs.unlink(verifyPath).catch(() => {}); + const verifyResult = await verificationService.verifyFile(dest.configId, remotePath, 'post-upload'); + if (verifyResult.status === 'passed') { + ctx.log(`${destLabel} Integrity check passed`, 'success'); + } else if (verifyResult.status === 'failed') { + ctx.log(`${destLabel} WARNING: Integrity check FAILED. Expected: ${verifyResult.expectedChecksum}, Got: ${verifyResult.actualChecksum}`, 'warning'); + } else { + ctx.log(`${destLabel} Integrity verification skipped: ${verifyResult.status}`, 'info'); } } catch (e: unknown) { const message = e instanceof Error ? e.message : String(e); - ctx.log(`${destLabel} Integrity verification skipped: ${message}`, 'warning'); + ctx.log(`${destLabel} Integrity verification error: ${message}`, 'warning'); } } - } else { - ctx.log("No local destinations - skipping integrity verification"); } // --- EVALUATE RESULTS --- diff --git a/src/lib/runner/steps/upload-helpers.ts b/src/lib/runner/steps/upload-helpers.ts index 30eaadc4..9618d493 100644 --- a/src/lib/runner/steps/upload-helpers.ts +++ b/src/lib/runner/steps/upload-helpers.ts @@ -32,8 +32,8 @@ export async function withStorageSession( function createStatelessSessionShim(adapter: StorageAdapter, config: AdapterConfig): StorageSession { return { - upload: (localPath, remotePath, onProgress, onLog) => - adapter.upload(config, localPath, remotePath, onProgress, onLog), + upload: (localPath, remotePath, onProgress, onLog, options) => + adapter.upload(config, localPath, remotePath, onProgress, onLog, options), close: async () => { }, }; } diff --git a/src/services/backup/integrity-service.ts b/src/services/backup/integrity-service.ts index a1f252e7..c4068a0a 100644 --- a/src/services/backup/integrity-service.ts +++ b/src/services/backup/integrity-service.ts @@ -1,19 +1,14 @@ import prisma from "@/lib/prisma"; import { registry } from "@/lib/core/registry"; import { registerAdapters } from "@/lib/adapters"; -import { StorageAdapter, BackupMetadata } from "@/lib/core/interfaces"; +import { StorageAdapter } from "@/lib/core/interfaces"; import { resolveAdapterConfig } from "@/lib/adapters/config-resolver"; -import { getTempDir } from "@/lib/temp-dir"; -import { verifyFileChecksum } from "@/lib/crypto/checksum"; +import { verificationService } from "@/services/storage/verification-service"; import { logger } from "@/lib/logging/logger"; import { wrapError } from "@/lib/logging/errors"; -import path from "path"; -import fs from "fs"; -import crypto from "crypto"; const log = logger.child({ service: "IntegrityService" }); -// Ensure adapters are loaded registerAdapters(); export interface IntegrityCheckResult { @@ -30,16 +25,7 @@ export interface IntegrityCheckResult { }>; } -/** - * Service for verifying the integrity of backup files on storage. - * Iterates over all storage destinations, downloads each backup file, - * computes its SHA-256 checksum, and compares it against the stored metadata. - */ export class IntegrityService { - /** - * Runs an integrity check on all backup files across all storage destinations. - * Only checks files that have a checksum in their metadata sidecar. - */ async runFullIntegrityCheck(): Promise { log.info("Starting full backup integrity check"); @@ -52,7 +38,6 @@ export class IntegrityService { errors: [], }; - // Get all storage adapters const storageConfigs = await prisma.adapterConfig.findMany({ where: { type: "storage" }, }); @@ -86,16 +71,12 @@ export class IntegrityService { ) { const adapter = registry.get(storageConfig.adapterId) as StorageAdapter; if (!adapter) { - log.warn("Storage adapter not found", { - adapterId: storageConfig.adapterId, - }); + log.warn("Storage adapter not found", { adapterId: storageConfig.adapterId }); return; } const config = await resolveAdapterConfig(storageConfig); - // List all top-level folders in storage (not just active jobs). - // This ensures backups from deleted jobs are also verified. let folders: string[] = []; try { const topLevel = await adapter.list(config, ""); @@ -108,7 +89,6 @@ export class IntegrityService { { destination: storageConfig.name }, wrapError(e) ); - // Fallback: use known job names const jobs = await prisma.job.findMany({ where: { destinations: { some: { configId: storageConfig.id } } }, select: { name: true }, @@ -118,25 +98,35 @@ export class IntegrityService { for (const folder of folders) { try { - // List files in folder const files = await adapter.list(config, folder); - - // Filter to only backup files (exclude .meta.json) - const backupFiles = files.filter( - (f) => !f.name.endsWith(".meta.json") - ); + const backupFiles = files.filter((f) => !f.name.endsWith(".meta.json")); for (const file of backupFiles) { result.totalFiles++; + const remotePath = `${folder}/${file.name}`; try { - await this.verifyFile( - adapter, - config, - `${folder}/${file.name}`, - storageConfig.name, - result + const verifyResult = await verificationService.verifyFile( + storageConfig.id, + remotePath, + "scheduled" ); + + if (verifyResult.status === "passed") { + result.verified++; + result.passed++; + } else if (verifyResult.status === "failed") { + result.verified++; + result.failed++; + result.errors.push({ + file: file.name, + destination: storageConfig.name, + expected: verifyResult.expectedChecksum ?? "", + actual: verifyResult.actualChecksum ?? "", + }); + } else { + result.skipped++; + } } catch (e: unknown) { log.error( "Failed to verify file", @@ -155,112 +145,6 @@ export class IntegrityService { } } } - - private async verifyFile( - adapter: StorageAdapter, - config: any, - remotePath: string, - destinationName: string, - result: IntegrityCheckResult - ) { - const fileName = path.basename(remotePath); - - // 1. Try to read metadata sidecar - let metadata: BackupMetadata | null = null; - - if (adapter.read) { - try { - const metaContent = await adapter.read( - config, - remotePath + ".meta.json" - ); - if (metaContent) { - metadata = JSON.parse(metaContent); - } - } catch { - // No metadata file, skip - } - } - - if (!metadata) { - // Try download-based approach - const tempMetaPath = path.join( - getTempDir(), - `integrity_meta_${crypto.randomUUID()}.json` - ); - try { - const ok = await adapter.download( - config, - remotePath + ".meta.json", - tempMetaPath - ); - if (ok) { - const content = await fs.promises.readFile(tempMetaPath, "utf-8"); - metadata = JSON.parse(content); - } - } catch { - // No metadata - } finally { - await fs.promises.unlink(tempMetaPath).catch(() => {}); - } - } - - if (!metadata?.checksum) { - log.debug("No checksum in metadata, skipping", { file: fileName }); - result.skipped++; - return; - } - - // 2. Download the backup file to temp - const tempFilePath = path.join( - getTempDir(), - `integrity_${crypto.randomUUID()}_${fileName}` - ); - - try { - const downloadOk = await adapter.download( - config, - remotePath, - tempFilePath - ); - if (!downloadOk) { - log.warn("Could not download file for verification", { - file: fileName, - }); - result.skipped++; - return; - } - - // 3. Verify checksum - const verification = await verifyFileChecksum( - tempFilePath, - metadata.checksum - ); - result.verified++; - - if (verification.valid) { - log.debug("File integrity OK", { file: fileName }); - result.passed++; - } else { - log.error("INTEGRITY FAILURE", { - file: fileName, - destination: destinationName, - expected: verification.expected, - actual: verification.actual, - }); - result.failed++; - result.errors.push({ - file: fileName, - destination: destinationName, - expected: verification.expected, - actual: verification.actual, - }); - } - } finally { - // Always cleanup temp file - await fs.promises.unlink(tempFilePath).catch(() => {}); - } - } } export const integrityService = new IntegrityService(); diff --git a/src/services/storage/storage-service.ts b/src/services/storage/storage-service.ts index 904a9495..70113aac 100644 --- a/src/services/storage/storage-service.ts +++ b/src/services/storage/storage-service.ts @@ -31,6 +31,11 @@ export type RichFileInfo = FileInfo & { compression?: string; locked?: boolean; trigger?: { type: string; actor?: string }; + verification?: { + verifiedAt: string; + passed: boolean; + trigger: 'manual' | 'post-upload' | 'scheduled'; + }; }; export class StorageService { @@ -249,6 +254,11 @@ export class StorageService { compression, locked: sidecar.locked, trigger: sidecar.trigger as { type: string; actor?: string } | undefined, + verification: sidecar.verification ? { + verifiedAt: sidecar.verification.verifiedAt, + passed: sidecar.verification.passed, + trigger: sidecar.verification.trigger, + } : undefined, }; } diff --git a/src/services/storage/verification-service.ts b/src/services/storage/verification-service.ts new file mode 100644 index 00000000..ee87248b --- /dev/null +++ b/src/services/storage/verification-service.ts @@ -0,0 +1,153 @@ +import prisma from "@/lib/prisma"; +import { registry } from "@/lib/core/registry"; +import { StorageAdapter, BackupMetadata } from "@/lib/core/interfaces"; +import { resolveAdapterConfig } from "@/lib/adapters/config-resolver"; +import { calculateFileChecksum } from "@/lib/crypto/checksum"; +import { getTempDir } from "@/lib/temp-dir"; +import { registerAdapters } from "@/lib/adapters"; +import { logger } from "@/lib/logging/logger"; +import { wrapError } from "@/lib/logging/errors"; +import path from "path"; +import fs from "fs/promises"; +import crypto from "crypto"; + +const log = logger.child({ service: "VerificationService" }); + +registerAdapters(); + +export type VerifyStatus = 'passed' | 'failed' | 'no_checksum' | 'no_metadata' | 'download_error'; + +export interface FileVerificationResult { + status: VerifyStatus; + expectedChecksum?: string; + actualChecksum?: string; + verifiedAt: string; +} + +export class VerificationService { + async verifyFile( + adapterConfigId: string, + remotePath: string, + trigger: 'manual' | 'post-upload' | 'scheduled' + ): Promise { + const verifiedAt = new Date().toISOString(); + + const adapterConfig = await prisma.adapterConfig.findUnique({ where: { id: adapterConfigId } }); + if (!adapterConfig) throw new Error("Storage configuration not found"); + + const adapter = registry.get(adapterConfig.adapterId) as StorageAdapter; + if (!adapter) throw new Error(`Adapter '${adapterConfig.adapterId}' not found`); + + const config = await resolveAdapterConfig(adapterConfig); + + // 1. Read .meta.json sidecar + const metaPath = remotePath + ".meta.json"; + let metadata: BackupMetadata | null = null; + + try { + if (adapter.read) { + const content = await adapter.read(config, metaPath); + if (content) metadata = JSON.parse(content) as BackupMetadata; + } + + if (!metadata) { + const tempMeta = path.join(getTempDir(), `verify_meta_${crypto.randomUUID()}.json`); + try { + const ok = await adapter.download(config, metaPath, tempMeta); + if (ok) { + const content = await fs.readFile(tempMeta, 'utf-8'); + metadata = JSON.parse(content) as BackupMetadata; + } + } finally { + await fs.unlink(tempMeta).catch(() => {}); + } + } + } catch (e: unknown) { + log.warn("Could not read metadata for verification", { remotePath }, wrapError(e)); + } + + if (!metadata) { + return { status: 'no_metadata', verifiedAt }; + } + + if (!metadata.checksum && !metadata.checksumMd5) { + return { status: 'no_checksum', verifiedAt }; + } + + // 2. Try native adapter verification (no download needed for S3, local, GDrive, OneDrive) + if (adapter.verifyChecksum) { + try { + const nativeResult = await adapter.verifyChecksum(config, remotePath, { + sha256: metadata.checksum, + md5: metadata.checksumMd5, + }); + + if (nativeResult !== 'unsupported') { + const passed = nativeResult === 'passed'; + await this.writeVerificationResult(adapter, config, metaPath, metadata, { + verifiedAt, + passed, + trigger, + }); + return { status: nativeResult, expectedChecksum: metadata.checksum, verifiedAt }; + } + } catch (e: unknown) { + log.warn("Native checksum verification error, falling back to download", { remotePath }, wrapError(e)); + } + } + + // 3. Fallback: download + compute SHA-256 + if (!metadata.checksum) { + return { status: 'no_checksum', verifiedAt }; + } + + const tempFile = path.join(getTempDir(), `verify_${crypto.randomUUID()}_${path.basename(remotePath)}`); + try { + const downloadOk = await adapter.download(config, remotePath, tempFile); + if (!downloadOk) { + return { status: 'download_error', verifiedAt }; + } + + const actual = await calculateFileChecksum(tempFile); + const passed = actual === metadata.checksum; + + await this.writeVerificationResult(adapter, config, metaPath, metadata, { + verifiedAt, + passed, + trigger, + actualChecksum: passed ? undefined : actual, + }); + + return { + status: passed ? 'passed' : 'failed', + expectedChecksum: metadata.checksum, + actualChecksum: passed ? undefined : actual, + verifiedAt, + }; + } catch (e: unknown) { + log.error("Download-based verification failed", { remotePath }, wrapError(e)); + return { status: 'download_error', verifiedAt }; + } finally { + await fs.unlink(tempFile).catch(() => {}); + } + } + + private async writeVerificationResult( + adapter: StorageAdapter, + config: any, + metaPath: string, + metadata: BackupMetadata, + verification: NonNullable + ) { + metadata.verification = verification; + const tempPath = path.join(getTempDir(), `meta-verify-${Date.now()}.json`); + await fs.writeFile(tempPath, JSON.stringify(metadata, null, 2)); + try { + await adapter.upload(config, tempPath, metaPath); + } finally { + await fs.unlink(tempPath).catch(() => {}); + } + } +} + +export const verificationService = new VerificationService(); From b4627be5e8e69522385005944bafb6e92030b1f8 Mon Sep 17 00:00:00 2001 From: Manu Date: Wed, 10 Jun 2026 22:11:58 +0200 Subject: [PATCH 02/24] Allow horizontal scrolling on data tables Add horizontal scrolling to the DataTable component by applying overflow-x-auto and responsive max-width constraints to the table container so columns can be scrolled when they overflow. Also update the changelog to document the UI improvement: all data tables (Storage, Jobs, Sources, Destinations, Notifications) now support horizontal scrolling to match the Database Explorer behavior. --- docs/changelog.md | 4 ++++ src/components/ui/data-table.tsx | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/changelog.md b/docs/changelog.md index 7c837325..710d41aa 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -5,6 +5,10 @@ All notable changes to DBackup are documented here. ## vNEXT *Release: In Progress* +### 🎨 Improvements + +- **ui**: All data tables (Storage, Jobs, Sources, Destinations, Notifications) now support horizontal scrolling when columns overflow, matching the Database Explorer behavior. + ### ✨ Features - **storage**: Added on-demand integrity verification for backup files via a new Verify button in the Storage Explorer. Results are persisted in the `.meta.json` sidecar and shown as a green/red badge in the Integrity column. diff --git a/src/components/ui/data-table.tsx b/src/components/ui/data-table.tsx index eed5e3a9..24ab8090 100644 --- a/src/components/ui/data-table.tsx +++ b/src/components/ui/data-table.tsx @@ -245,7 +245,7 @@ export function DataTable({ )}
-
+
{table.getHeaderGroups().map((headerGroup) => ( From c0c000d1c100ef3f75cd24918e61951dca172a55 Mon Sep 17 00:00:00 2001 From: Manu Date: Wed, 10 Jun 2026 22:19:55 +0200 Subject: [PATCH 03/24] Add Backup Verification docs and nav link Add a new docs page describing the Backup Verification feature (docs/user-guide/features/backup-verification.md). The page explains checksum generation, adapter-specific verification methods, triggers (manual, post-upload, scheduled), scheduler behavior, interpretation of results, and limitations. Also update the VitePress sidebar (docs/.vitepress/config.mts) to include a link to the new Backup Verification page under Storage Explorer so it appears in the documentation navigation. --- docs/.vitepress/config.mts | 1 + .../features/backup-verification.md | 120 ++++++++++++++++++ 2 files changed, 121 insertions(+) create mode 100644 docs/user-guide/features/backup-verification.md diff --git a/docs/.vitepress/config.mts b/docs/.vitepress/config.mts index 0e444571..64e50c86 100644 --- a/docs/.vitepress/config.mts +++ b/docs/.vitepress/config.mts @@ -171,6 +171,7 @@ export default defineConfig({ collapsed: false, items: [ { text: 'Storage Explorer', link: '/user-guide/features/storage-explorer' }, + { text: 'Backup Verification', link: '/user-guide/features/backup-verification' }, { text: 'Restore', link: '/user-guide/features/restore' }, { text: 'Notifications', link: '/user-guide/features/notifications' }, { text: 'System Backup', link: '/user-guide/features/system-backup' }, diff --git a/docs/user-guide/features/backup-verification.md b/docs/user-guide/features/backup-verification.md new file mode 100644 index 00000000..ea033c9b --- /dev/null +++ b/docs/user-guide/features/backup-verification.md @@ -0,0 +1,120 @@ +# Backup Verification + +Detect corrupted or tampered backup files before you need to restore them. + +## Why Verification Matters + +A backup that silently corrupts mid-upload is worse than no backup at all - you only discover the problem at the worst possible moment, during a restore. Verification catches this early by comparing a cryptographic checksum of the backup file against the original value recorded immediately after the dump. + +## How It Works + +### Checksum Generation (at upload time) + +Every backup file gets a SHA-256 checksum computed from the final processed file - after compression and encryption if enabled. For adapters that natively support MD5 (Google Drive), an MD5 checksum is computed in the same pass at no extra cost. + +Both values are stored in the `.meta.json` sidecar file alongside the backup: + +```json +{ + "checksum": "a3f1b2c4...", + "checksumMd5": "d8e9f0a1...", + "verification": { + "verifiedAt": "2026-06-10T14:32:00Z", + "passed": true, + "trigger": "post-upload" + } +} +``` + +### Verification Logic + +When a verification is triggered, DBackup uses the best available method for each storage adapter: + +| Adapter | Method | No Download Needed | +| :--- | :--- | :--- | +| **Local Filesystem** | Direct file hash via stream | Yes | +| **S3 / R2 / Hetzner** | `HeadObject` - reads SHA-256 from object metadata | Yes | +| **Google Drive** | `files.get` API - reads native MD5 field | Yes | +| **OneDrive** | Graph API - reads native SHA-256 hash | Yes | +| SFTP / FTP | Download + compute SHA-256 | No | +| SMB | Download + compute SHA-256 | No | +| WebDAV | Download + compute SHA-256 | No | +| Dropbox | Download + compute SHA-256 | No | +| Rsync | Download + compute SHA-256 | No | + +For adapters without native checksum APIs, DBackup downloads the full file, recomputes the hash, and compares it against the stored value. + +The result is written back into the `.meta.json` sidecar so it persists across sessions and appears immediately in the Storage Explorer without re-verifying. + +## Triggering a Verification + +### Manual Verification (on-demand) + +In the **Storage Explorer**, each backup row has a shield icon in the Actions column: + +- **Gray shield**: Never verified +- **Green shield**: Last check passed +- **Red shield**: Last check failed + +Click the icon to trigger a verification. A loading toast appears while the check runs, then the result is shown and the badge in the table updates automatically. + +::: tip Re-verify anytime +You can re-verify a backup at any time - clicking the green shield on an already-verified backup runs a fresh check. +::: + +The **Integrity** column shows the same status at a glance without opening the actions menu. + +### Post-Upload Verification (automatic) + +DBackup can verify each backup immediately after it finishes uploading. This is controlled by the `backup.postUploadVerify` system setting. + +**Local filesystem destinations always verify**, regardless of this setting - the check is a direct file read with near-zero overhead. + +For remote destinations (S3, Google Drive, SFTP, etc.), post-upload verification is **opt-in** and off by default. Enable it in **Settings - System** if you want automatic verification for all destinations. + +::: warning Bandwidth for download-based adapters +For SFTP, FTP, SMB, WebDAV, Dropbox, and Rsync, automatic post-upload verification downloads the full backup file a second time to recompute the hash. For large backups on slow or metered connections this adds significant time and transfer costs. For S3, Google Drive, OneDrive, and local storage, verification has near-zero overhead and is safe to enable freely. +::: + +### Scheduled Integrity Check + +DBackup includes a **Scheduled Integrity Check** job that periodically verifies all backups across all storage destinations. It runs through each destination, reads the `.meta.json` sidecar for each backup file, and runs the same native-first verification logic as the manual check. + +Results are written back to the sidecars as they complete, so the Storage Explorer badges stay up to date without any manual action. + +The scheduler can be configured under **Settings - Scheduler** (cron expression). A weekly or monthly check on your full archive is a reasonable default for most setups. + +::: info What counts as "scheduled" +The `trigger` field in `.meta.json` records how each check was triggered: `manual`, `post-upload`, or `scheduled`. This lets you distinguish between a fresh post-upload check and an older scheduled check in the metadata. +::: + +## Interpreting Results + +| Status | Meaning | +| :--- | :--- | +| **Verified (green)** | File matches its checksum - no corruption or tampering detected | +| **Failed (red)** | Hash mismatch - file may be corrupted or was modified after upload | +| **No checksum** | Backup predates checksum support - no baseline to compare against | +| **No metadata** | File has no `.meta.json` sidecar (manually placed file) | +| **-** (dash) | Never been verified | + +A **Failed** result does not necessarily mean the backup is unrestorable - it means something changed since the original upload. Possible causes: + +- Storage provider-level corruption (rare but documented on HDD-backed storage) +- Partial re-upload or file replacement +- Ransomware or accidental modification +- Network error during original upload that was not caught + +If you see a failed check on a backup you intend to use for a restore, treat it as suspect and try an older backup if available. + +## Limitations + +- **Encrypted backups**: The checksum covers the encrypted file, not the plaintext. A passing integrity check confirms the encrypted bytes are intact, but does not guarantee the decryption key is still available. +- **Legacy backups**: Files uploaded before checksum support was added have no baseline. Verification will report "No checksum" and no comparison is possible. +- **Native SHA-256 on S3**: The SHA-256 is stored as S3 custom metadata (`dbackup-sha256`) during upload. If the object was copied or re-uploaded via external tools without preserving this metadata, native verification will fall back gracefully and report "unsupported" - which triggers a download-based check instead. + +## Next Steps + +- [Storage Explorer](/user-guide/features/storage-explorer) - Browse and manage backup files +- [Restore](/user-guide/features/restore) - Restore a backup to a database +- [Encryption](/user-guide/security/encryption) - Encrypt backup files at rest From dc9552ac8c11ac6636388d34387417b40406e9ab Mon Sep 17 00:00:00 2001 From: Manu Date: Wed, 10 Jun 2026 22:37:57 +0200 Subject: [PATCH 04/24] Add integrity modal and checksum support Introduce a dedicated Integrity modal for storage items and wire up checksum support across the stack. UI: new IntegrityModal component (copyable SHA-256/MD5 display, last verification status, Verify Now flow) and the StorageClient now opens the modal instead of performing immediate fetches. Data model: FileInfo/RichFileInfo extended with checksum and checksumMd5, and StorageService surfaces these values from sidecar files. Runner: upload step now logs MD5 alongside SHA-256. Also removed the previous inline Integrity column from the table and updated the changelog entry to reflect the modal and MD5 history logging. The Verify action refreshes the file list after verification completes. --- docs/changelog.md | 3 +- src/app/dashboard/storage/columns.tsx | 16 +- src/app/dashboard/storage/storage-client.tsx | 48 ++-- .../dashboard/storage/integrity-modal.tsx | 210 ++++++++++++++++++ src/lib/runner/steps/03-upload.ts | 1 + src/services/storage/storage-service.ts | 4 + 6 files changed, 238 insertions(+), 44 deletions(-) create mode 100644 src/components/dashboard/storage/integrity-modal.tsx diff --git a/docs/changelog.md b/docs/changelog.md index 710d41aa..ad569344 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -11,11 +11,12 @@ All notable changes to DBackup are documented here. ### ✨ Features -- **storage**: Added on-demand integrity verification for backup files via a new Verify button in the Storage Explorer. Results are persisted in the `.meta.json` sidecar and shown as a green/red badge in the Integrity column. +- **storage**: Added on-demand integrity verification for backup files. The shield button in the Storage Explorer opens an Integrity modal showing stored SHA-256 and MD5 checksums, last verification result, and a Verify Now button. Results are persisted in the `.meta.json` sidecar and reflected in the button color (gray/green/red). - **storage**: Added adapter-native checksum verification for S3/R2/Hetzner (SHA-256 via object metadata), local filesystem (direct stream hash), Google Drive (MD5 via API), and OneDrive (SHA-256 via Graph API) - no download required for these adapters. - **storage**: MD5 checksums are now computed alongside SHA-256 during backup upload and stored in `.meta.json`, enabling native verification on Google Drive. - **integrity**: The scheduled integrity check now uses native verification where available and writes results back to `.meta.json` sidecars. - **backup**: Post-upload integrity verification is now opt-in for all storage destinations via the `backup.postUploadVerify` system setting (local filesystem always verifies). +- **history**: MD5 checksum is now logged alongside SHA-256 in execution history during the upload stage. ### 🐳 Docker diff --git a/src/app/dashboard/storage/columns.tsx b/src/app/dashboard/storage/columns.tsx index 9c0a7acc..5a6ad1fa 100644 --- a/src/app/dashboard/storage/columns.tsx +++ b/src/app/dashboard/storage/columns.tsx @@ -1,7 +1,7 @@ "use client"; import { ColumnDef } from "@tanstack/react-table"; -import { ArrowUpDown, Clock, HardDrive, KeyRound, MousePointerClick, ShieldCheck, ShieldX } from "lucide-react"; +import { ArrowUpDown, Clock, HardDrive, KeyRound, MousePointerClick } from "lucide-react"; import { AdapterIcon } from "@/components/adapter/adapter-icon"; import { Button } from "@/components/ui/button"; import { DateDisplay } from "@/components/utils/date-display"; @@ -28,6 +28,8 @@ export type FileInfo = { locked?: boolean; trigger?: { type: string; actor?: string }; storageClass?: string; + checksum?: string; + checksumMd5?: string; verification?: { verifiedAt: string; passed: boolean; @@ -180,18 +182,6 @@ export const getColumns = ({ onRestore, onDownload, onDelete, onToggleLock, onGe ); } }, - { - id: "verification", - header: "Integrity", - cell: ({ row }) => { - const v = row.original.verification; - if (!v) return -; - if (v.passed) { - return Verified; - } - return Failed; - } - }, { accessorKey: "size", header: ({ column }) => { diff --git a/src/app/dashboard/storage/storage-client.tsx b/src/app/dashboard/storage/storage-client.tsx index 1d716eba..bdc3068d 100644 --- a/src/app/dashboard/storage/storage-client.tsx +++ b/src/app/dashboard/storage/storage-client.tsx @@ -37,6 +37,7 @@ import { Label } from "@/components/ui/label"; import { getColumns, FileInfo } from "./columns"; import { lockBackup } from "@/app/actions/storage/lock"; import { DownloadLinkModal } from "@/components/dashboard/storage/download-link-modal"; +import { IntegrityModal } from "@/components/dashboard/storage/integrity-modal"; import { StorageHistoryTab } from "@/components/dashboard/storage/storage-history-tab"; import { StorageSettingsTab } from "@/components/dashboard/storage/storage-settings-tab"; import { Skeleton } from "@/components/ui/skeleton"; @@ -82,6 +83,9 @@ export function StorageClient({ canDownload, canRestore, canDelete }: StorageCli // Download Link Modal State const [downloadLinkFile, setDownloadLinkFile] = useState(null); + // Integrity Modal State + const [verifyModalFile, setVerifyModalFile] = useState(null); + // Encryption Key Resolution Dialog State (decrypted download fallback) const [decryptKeyDialogOpen, setDecryptKeyDialogOpen] = useState(false); const [pendingDecryptFile, setPendingDecryptFile] = useState(null); @@ -259,36 +263,9 @@ export function StorageClient({ canDownload, canRestore, canDelete }: StorageCli setDownloadLinkFile(file); }, [canDownload]); - const handleVerify = useCallback(async (file: FileInfo) => { - const toastId = toast.loading(`Verifying integrity of ${file.name}...`); - try { - const res = await fetch(`/api/storage/${selectedDestination}/verify`, { - method: "POST", - headers: { "Content-Type": "application/json" }, - body: JSON.stringify({ file: file.path }), - }); - const data = await res.json(); - if (!res.ok) { - toast.error(data.error || "Verification failed", { id: toastId }); - return; - } - const result = data.data; - if (result.status === "passed") { - toast.success("Integrity check passed", { id: toastId }); - } else if (result.status === "failed") { - toast.error("Integrity check FAILED - backup may be corrupted", { id: toastId }); - } else if (result.status === "no_checksum") { - toast.info("No checksum available (legacy backup)", { id: toastId }); - } else if (result.status === "no_metadata") { - toast.info("No metadata found for this file", { id: toastId }); - } else { - toast.error("Verification error: " + result.status, { id: toastId }); - } - fetchFiles(selectedDestination, showSystemConfigs); - } catch { - toast.error("Error verifying file", { id: toastId }); - } - }, [selectedDestination, showSystemConfigs]); // eslint-disable-line react-hooks/exhaustive-deps + const handleVerify = useCallback((file: FileInfo) => { + setVerifyModalFile(file); + }, []); const confirmDelete = async () => { if (!fileToDelete) return; @@ -573,6 +550,17 @@ export function StorageClient({ canDownload, canRestore, canDelete }: StorageCli /> )} + {/* Integrity Modal */} + {verifyModalFile && ( + { if (!o) setVerifyModalFile(null); }} + file={verifyModalFile} + storageConfigId={selectedDestination} + onVerifyComplete={() => fetchFiles(selectedDestination, showSystemConfigs)} + /> + )} + {/* Encryption Key Resolution Dialog (decrypted download fallback) */} void; + file: FileInfo; + storageConfigId: string; + onVerifyComplete: () => void; +} + +type LiveResult = { + status: 'passed' | 'failed' | 'no_checksum' | 'no_metadata' | 'download_error'; + verifiedAt: string; + actualChecksum?: string; +}; + +function CopyButton({ value }: { value: string }) { + const [copied, setCopied] = useState(false); + + const handleCopy = async () => { + try { + await navigator.clipboard.writeText(value); + setCopied(true); + setTimeout(() => setCopied(false), 1500); + } catch {} + }; + + return ( + + ); +} + +function ChecksumRow({ label, value }: { label: string; value?: string }) { + if (!value) return null; + return ( +
+ {label} + {value} + +
+ ); +} + +function VerificationStatus({ verification, liveResult }: { + verification: FileInfo['verification']; + liveResult: LiveResult | null; +}) { + const status = liveResult ?? (verification ? { status: verification.passed ? 'passed' : 'failed', verifiedAt: verification.verifiedAt } : null); + + if (!status) { + return ( +
+ + Not yet verified +
+ ); + } + + if (status.status === 'passed') { + return ( +
+
+ + Passed +
+

+ Verified + {liveResult ? null : verification?.trigger ? ` via ${verification.trigger}` : null} +

+
+ ); + } + + if (status.status === 'failed') { + return ( +
+
+ + Failed - backup may be corrupted +
+

+ Checked +

+ {(status as any).actualChecksum && ( +
+

Actual hash on storage:

+

{(status as any).actualChecksum}

+
+ )} +
+ ); + } + + if (status.status === 'no_checksum') { + return ( +
+ + No checksum stored - legacy backup cannot be verified +
+ ); + } + + if (status.status === 'no_metadata') { + return ( +
+ + No metadata file found for this backup +
+ ); + } + + return ( +
+ + Verification error: {status.status} +
+ ); +} + +export function IntegrityModal({ open, onOpenChange, file, storageConfigId, onVerifyComplete }: IntegrityModalProps) { + const [verifying, setVerifying] = useState(false); + const [liveResult, setLiveResult] = useState(null); + + const hasChecksums = !!(file.checksum || file.checksumMd5); + + const handleVerify = async () => { + setVerifying(true); + try { + const res = await fetch(`/api/storage/${storageConfigId}/verify`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ file: file.path }), + }); + const data = await res.json(); + if (!res.ok) { + setLiveResult({ status: 'download_error', verifiedAt: new Date().toISOString() }); + return; + } + const result = data.data; + setLiveResult({ + status: result.status, + verifiedAt: result.verifiedAt, + actualChecksum: result.actualChecksum, + }); + if (result.status === 'passed' || result.status === 'failed') { + onVerifyComplete(); + } + } catch { + setLiveResult({ status: 'download_error', verifiedAt: new Date().toISOString() }); + } finally { + setVerifying(false); + } + }; + + return ( + { if (!o && !verifying) { setLiveResult(null); onOpenChange(false); } }}> + + + Integrity Check + {file.name} + + +
+ {hasChecksums ? ( +
+

Stored Checksums

+
+ + +
+
+ ) : ( +
+

No checksums available. This is a legacy backup that was uploaded before checksum support was added.

+
+ )} + +
+

Last Verification

+ +
+
+ +
+ +
+ + + ); +} diff --git a/src/lib/runner/steps/03-upload.ts b/src/lib/runner/steps/03-upload.ts index d5551b09..2baabe65 100644 --- a/src/lib/runner/steps/03-upload.ts +++ b/src/lib/runner/steps/03-upload.ts @@ -121,6 +121,7 @@ export async function stepUpload(ctx: RunnerContext) { ctx.log("Calculating checksums..."); const { sha256: checksum, md5: checksumMd5 } = await calculateFileChecksums(ctx.tempFile); ctx.log(`SHA-256: ${checksum}`); + ctx.log(`MD5: ${checksumMd5}`); // --- PRIVACY SETTING: include actor in metadata? --- const privacySetting = await prisma.systemSetting.findUnique({ where: { key: "privacy.includeActorInMetadata" } }); diff --git a/src/services/storage/storage-service.ts b/src/services/storage/storage-service.ts index 70113aac..45e54d0f 100644 --- a/src/services/storage/storage-service.ts +++ b/src/services/storage/storage-service.ts @@ -31,6 +31,8 @@ export type RichFileInfo = FileInfo & { compression?: string; locked?: boolean; trigger?: { type: string; actor?: string }; + checksum?: string; + checksumMd5?: string; verification?: { verifiedAt: string; passed: boolean; @@ -254,6 +256,8 @@ export class StorageService { compression, locked: sidecar.locked, trigger: sidecar.trigger as { type: string; actor?: string } | undefined, + checksum: sidecar.checksum, + checksumMd5: sidecar.checksumMd5, verification: sidecar.verification ? { verifiedAt: sidecar.verification.verifiedAt, passed: sidecar.verification.passed, From ef31be2e716de65c50aa31d66195e25c09cf9ca2 Mon Sep 17 00:00:00 2001 From: Manu Date: Wed, 10 Jun 2026 22:41:08 +0200 Subject: [PATCH 05/24] Truncate long storage names with tooltip Truncate long backup names in the Storage Explorer and show the full name on hover. Update NameCell to add min-w-0 and shrink-0 layout tweaks, wrap the name in a Tooltip with a truncated max width (max-w-64), and keep the path display unchanged. Also update the changelog to document the improvement. --- docs/changelog.md | 1 + .../dashboard/storage/cells/name-cell.tsx | 13 ++++++++++--- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/docs/changelog.md b/docs/changelog.md index ad569344..fd387f6b 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -7,6 +7,7 @@ All notable changes to DBackup are documented here. ### 🎨 Improvements +- **storage**: Long backup names in the Storage Explorer are now truncated with a tooltip showing the full name on hover. - **ui**: All data tables (Storage, Jobs, Sources, Destinations, Notifications) now support horizontal scrolling when columns overflow, matching the Database Explorer behavior. ### ✨ Features diff --git a/src/components/dashboard/storage/cells/name-cell.tsx b/src/components/dashboard/storage/cells/name-cell.tsx index 3552981c..491deec0 100644 --- a/src/components/dashboard/storage/cells/name-cell.tsx +++ b/src/components/dashboard/storage/cells/name-cell.tsx @@ -10,18 +10,25 @@ interface NameCellProps { export function NameCell({ name, path, isEncrypted }: NameCellProps) { return (
-
+
{isEncrypted && ( - + Encrypted Backup )} - {name} + + + + {name} + + {name} + +
{path} From 389f2a00c1d73f8909b56b548629251b11044d70 Mon Sep 17 00:00:00 2001 From: Manu Date: Wed, 10 Jun 2026 23:11:01 +0200 Subject: [PATCH 06/24] Add configurable integrity check filters Enable filtering for scheduled integrity checks (skip already-passed, max age, max file size). Adds a server action to persist settings (src/app/actions/settings/integrity-settings.ts), a client modal to edit them (IntegrityCheckSettingsModal), and a gear button on the Integrity Check task row in SystemTasksSettings to open the modal. The IntegrityService now reads these settings and applies age/size/skip filters when enumerating files, and VerificationService gains an option to skip verification for already-passed files and a new 'skipped' status. Also load initial integrity settings in the Settings page and update the changelog ordering. --- docs/changelog.md | 11 +- .../actions/settings/integrity-settings.ts | 54 +++++++ src/app/dashboard/settings/page.tsx | 11 +- .../integrity-check-settings-modal.tsx | 135 ++++++++++++++++++ .../settings/system-tasks-settings.tsx | 40 +++++- src/services/backup/integrity-service.ts | 47 +++++- src/services/storage/verification-service.ts | 9 +- 7 files changed, 293 insertions(+), 14 deletions(-) create mode 100644 src/app/actions/settings/integrity-settings.ts create mode 100644 src/components/settings/integrity-check-settings-modal.tsx diff --git a/docs/changelog.md b/docs/changelog.md index fd387f6b..62795bbd 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -5,20 +5,21 @@ All notable changes to DBackup are documented here. ## vNEXT *Release: In Progress* -### 🎨 Improvements - -- **storage**: Long backup names in the Storage Explorer are now truncated with a tooltip showing the full name on hover. -- **ui**: All data tables (Storage, Jobs, Sources, Destinations, Notifications) now support horizontal scrolling when columns overflow, matching the Database Explorer behavior. - ### ✨ Features - **storage**: Added on-demand integrity verification for backup files. The shield button in the Storage Explorer opens an Integrity modal showing stored SHA-256 and MD5 checksums, last verification result, and a Verify Now button. Results are persisted in the `.meta.json` sidecar and reflected in the button color (gray/green/red). - **storage**: Added adapter-native checksum verification for S3/R2/Hetzner (SHA-256 via object metadata), local filesystem (direct stream hash), Google Drive (MD5 via API), and OneDrive (SHA-256 via Graph API) - no download required for these adapters. - **storage**: MD5 checksums are now computed alongside SHA-256 during backup upload and stored in `.meta.json`, enabling native verification on Google Drive. - **integrity**: The scheduled integrity check now uses native verification where available and writes results back to `.meta.json` sidecars. +- **integrity**: Scheduled integrity checks can now be filtered by configurable rules (skip already-passed backups, max backup age, max file size) accessible via the gear icon on the Integrity Check task row in Settings. - **backup**: Post-upload integrity verification is now opt-in for all storage destinations via the `backup.postUploadVerify` system setting (local filesystem always verifies). - **history**: MD5 checksum is now logged alongside SHA-256 in execution history during the upload stage. +### 🎨 Improvements + +- **storage**: Long backup names in the Storage Explorer are now truncated with a tooltip showing the full name on hover. +- **ui**: All data tables (Storage, Jobs, Sources, Destinations, Notifications) now support horizontal scrolling when columns overflow, matching the Database Explorer behavior. + ### 🐳 Docker - **Image**: `skyfay/dbackup:vNEXT` diff --git a/src/app/actions/settings/integrity-settings.ts b/src/app/actions/settings/integrity-settings.ts new file mode 100644 index 00000000..2fd384bb --- /dev/null +++ b/src/app/actions/settings/integrity-settings.ts @@ -0,0 +1,54 @@ +"use server" + +import prisma from "@/lib/prisma"; +import { revalidatePath } from "next/cache"; +import { z } from "zod"; +import { checkPermission } from "@/lib/auth/access-control"; +import { PERMISSIONS } from "@/lib/auth/permissions"; +import { logger } from "@/lib/logging/logger"; +import { wrapError } from "@/lib/logging/errors"; + +const log = logger.child({ action: "integrity-settings" }); + +const schema = z.object({ + skipPassed: z.boolean(), + maxAgeDays: z.coerce.number().min(0).max(3650), + maxFileSizeMb: z.coerce.number().min(0).max(1_000_000), +}); + +export async function saveIntegritySettings(data: z.infer) { + await checkPermission(PERMISSIONS.SETTINGS.WRITE); + + const result = schema.safeParse(data); + if (!result.success) { + return { success: false, error: result.error.issues[0].message }; + } + + try { + const { skipPassed, maxAgeDays, maxFileSizeMb } = result.data; + + await prisma.systemSetting.upsert({ + where: { key: "integrity.skipPassed" }, + update: { value: String(skipPassed) }, + create: { key: "integrity.skipPassed", value: String(skipPassed) }, + }); + + await prisma.systemSetting.upsert({ + where: { key: "integrity.maxAgeDays" }, + update: { value: String(maxAgeDays) }, + create: { key: "integrity.maxAgeDays", value: String(maxAgeDays) }, + }); + + await prisma.systemSetting.upsert({ + where: { key: "integrity.maxFileSizeMb" }, + update: { value: String(maxFileSizeMb) }, + create: { key: "integrity.maxFileSizeMb", value: String(maxFileSizeMb) }, + }); + + revalidatePath("/dashboard/settings"); + return { success: true }; + } catch (e: unknown) { + log.error("Failed to save integrity settings", {}, wrapError(e)); + return { success: false, error: "Failed to save settings" }; + } +} diff --git a/src/app/dashboard/settings/page.tsx b/src/app/dashboard/settings/page.tsx index ada25e36..840b5b5d 100644 --- a/src/app/dashboard/settings/page.tsx +++ b/src/app/dashboard/settings/page.tsx @@ -92,6 +92,15 @@ export default async function SettingsPage() { retention: configRetention ? parseInt(configRetention.value) : 10, }; + const integritySkipPassed = await prisma.systemSetting.findUnique({ where: { key: 'integrity.skipPassed' } }); + const integrityMaxAgeDays = await prisma.systemSetting.findUnique({ where: { key: 'integrity.maxAgeDays' } }); + const integrityMaxFileSizeMb = await prisma.systemSetting.findUnique({ where: { key: 'integrity.maxFileSizeMb' } }); + const integritySettings = { + skipPassed: integritySkipPassed?.value === 'true', + maxAgeDays: integrityMaxAgeDays ? parseInt(integrityMaxAgeDays.value) || 0 : 0, + maxFileSizeMb: integrityMaxFileSizeMb ? parseInt(integrityMaxFileSizeMb.value) || 0 : 0, + }; + // Load Rate Limit Settings const rateLimitConfig = await getRateLimitConfig(); @@ -138,7 +147,7 @@ export default async function SettingsPage() { - + void; + initialSettings: IntegritySettings; + onSaved: (settings: IntegritySettings) => void; +} + +export function IntegrityCheckSettingsModal({ open, onOpenChange, initialSettings, onSaved }: IntegrityCheckSettingsModalProps) { + const [skipPassed, setSkipPassed] = useState(initialSettings.skipPassed); + const [maxAgeDays, setMaxAgeDays] = useState(String(initialSettings.maxAgeDays)); + const [maxFileSizeMb, setMaxFileSizeMb] = useState(String(initialSettings.maxFileSizeMb)); + const [saving, setSaving] = useState(false); + + const handleSave = async () => { + setSaving(true); + try { + const result = await saveIntegritySettings({ + skipPassed, + maxAgeDays: parseInt(maxAgeDays) || 0, + maxFileSizeMb: parseInt(maxFileSizeMb) || 0, + }); + if (result.success) { + toast.success("Integrity check settings saved"); + onSaved({ + skipPassed, + maxAgeDays: parseInt(maxAgeDays) || 0, + maxFileSizeMb: parseInt(maxFileSizeMb) || 0, + }); + onOpenChange(false); + } else { + toast.error(result.error || "Failed to save settings"); + } + } catch { + toast.error("Failed to save settings"); + } finally { + setSaving(false); + } + }; + + return ( + { if (!saving) onOpenChange(o); }}> + + + Integrity Check Settings + + Configure which backups are included in scheduled integrity checks. + + + +
+
+
+ +

+ Only check backups that have never been verified or previously failed. + Backups with a passing result are skipped. +

+
+ +
+ +
+ +
+ setMaxAgeDays(e.target.value)} + /> + 0 = no limit +
+

+ Only verify backups newer than this many days. Older backups are counted as skipped. +

+
+ +
+ +
+ setMaxFileSizeMb(e.target.value)} + /> + 0 = no limit +
+

+ Skip files larger than this threshold. Mainly relevant for remote adapters + (SFTP, SMB, FTP, WebDAV, Dropbox) that must download the full file to verify. +

+
+
+ + + + + + + + ); +} diff --git a/src/components/settings/system-tasks-settings.tsx b/src/components/settings/system-tasks-settings.tsx index 067facab..5a2b2e1b 100644 --- a/src/components/settings/system-tasks-settings.tsx +++ b/src/components/settings/system-tasks-settings.tsx @@ -5,10 +5,12 @@ import { Button } from "@/components/ui/button"; import { Card, CardContent, CardDescription, CardHeader, CardTitle } from "@/components/ui/card"; import { Input } from "@/components/ui/input"; import { toast } from "sonner"; -import { Play, Loader2, Clock, CalendarClock } from "lucide-react"; +import { Play, Loader2, Clock, CalendarClock, Settings2 } from "lucide-react"; import { Switch } from "@/components/ui/switch"; import { Label } from "@/components/ui/label"; import { DateDisplay } from "@/components/utils/date-display"; +import { IntegrityCheckSettingsModal, type IntegritySettings } from "@/components/settings/integrity-check-settings-modal"; +import { Tooltip, TooltipContent, TooltipProvider, TooltipTrigger } from "@/components/ui/tooltip"; interface SystemTask { id: string; @@ -22,12 +24,20 @@ interface SystemTask { timezone: string; } -export function SystemTasksSettings() { +interface SystemTasksSettingsProps { + initialIntegritySettings?: IntegritySettings; +} + +export function SystemTasksSettings({ initialIntegritySettings }: SystemTasksSettingsProps) { const [tasks, setTasks] = useState([]); const [loading, setLoading] = useState(false); const [editing, setEditing] = useState>({}); const [saving, setSaving] = useState>({}); const [running, setRunning] = useState>({}); + const [integritySettingsOpen, setIntegritySettingsOpen] = useState(false); + const [integritySettings, setIntegritySettings] = useState( + initialIntegritySettings ?? { skipPassed: false, maxAgeDays: 0, maxFileSizeMb: 0 } + ); useEffect(() => { fetchTasks(); @@ -145,6 +155,7 @@ export function SystemTasksSettings() { }; return ( + <> System Tasks @@ -205,7 +216,22 @@ export function SystemTasksSettings() { )} -
+ {task.id === 'system.integrity_check' && ( +
+ + + + + + Configure check filters + + +
+ )} + +
+ + + ); } diff --git a/src/services/backup/integrity-service.ts b/src/services/backup/integrity-service.ts index c4068a0a..3cf8ac56 100644 --- a/src/services/backup/integrity-service.ts +++ b/src/services/backup/integrity-service.ts @@ -25,6 +25,12 @@ export interface IntegrityCheckResult { }>; } +interface IntegrityFilters { + skipPassed: boolean; + maxAgeDays: number; + maxFileSizeBytes: number; +} + export class IntegrityService { async runFullIntegrityCheck(): Promise { log.info("Starting full backup integrity check"); @@ -38,13 +44,31 @@ export class IntegrityService { errors: [], }; + const [skipPassedSetting, maxAgeSetting, maxSizeSetting] = await Promise.all([ + prisma.systemSetting.findUnique({ where: { key: 'integrity.skipPassed' } }), + prisma.systemSetting.findUnique({ where: { key: 'integrity.maxAgeDays' } }), + prisma.systemSetting.findUnique({ where: { key: 'integrity.maxFileSizeMb' } }), + ]); + + const filters: IntegrityFilters = { + skipPassed: skipPassedSetting?.value === 'true', + maxAgeDays: parseInt(maxAgeSetting?.value ?? '0') || 0, + maxFileSizeBytes: (parseInt(maxSizeSetting?.value ?? '0') || 0) * 1024 * 1024, + }; + + log.info("Integrity check filters", { + skipPassed: filters.skipPassed, + maxAgeDays: filters.maxAgeDays, + maxFileSizeMb: filters.maxFileSizeBytes / 1024 / 1024, + }); + const storageConfigs = await prisma.adapterConfig.findMany({ where: { type: "storage" }, }); for (const storageConfig of storageConfigs) { try { - await this.checkDestination(storageConfig, result); + await this.checkDestination(storageConfig, result, filters); } catch (e: unknown) { log.error( "Failed to check storage destination", @@ -67,7 +91,8 @@ export class IntegrityService { private async checkDestination( storageConfig: any, - result: IntegrityCheckResult + result: IntegrityCheckResult, + filters: IntegrityFilters ) { const adapter = registry.get(storageConfig.adapterId) as StorageAdapter; if (!adapter) { @@ -105,11 +130,27 @@ export class IntegrityService { result.totalFiles++; const remotePath = `${folder}/${file.name}`; + // Age filter: skip files older than maxAgeDays + if (filters.maxAgeDays > 0 && file.lastModified) { + const ageDays = (Date.now() - new Date(file.lastModified).getTime()) / 86_400_000; + if (ageDays > filters.maxAgeDays) { + result.skipped++; + continue; + } + } + + // Size filter: skip files larger than maxFileSizeBytes + if (filters.maxFileSizeBytes > 0 && file.size > filters.maxFileSizeBytes) { + result.skipped++; + continue; + } + try { const verifyResult = await verificationService.verifyFile( storageConfig.id, remotePath, - "scheduled" + "scheduled", + { skipIfPassed: filters.skipPassed } ); if (verifyResult.status === "passed") { diff --git a/src/services/storage/verification-service.ts b/src/services/storage/verification-service.ts index ee87248b..8c018db2 100644 --- a/src/services/storage/verification-service.ts +++ b/src/services/storage/verification-service.ts @@ -15,7 +15,7 @@ const log = logger.child({ service: "VerificationService" }); registerAdapters(); -export type VerifyStatus = 'passed' | 'failed' | 'no_checksum' | 'no_metadata' | 'download_error'; +export type VerifyStatus = 'passed' | 'failed' | 'no_checksum' | 'no_metadata' | 'download_error' | 'skipped'; export interface FileVerificationResult { status: VerifyStatus; @@ -28,7 +28,8 @@ export class VerificationService { async verifyFile( adapterConfigId: string, remotePath: string, - trigger: 'manual' | 'post-upload' | 'scheduled' + trigger: 'manual' | 'post-upload' | 'scheduled', + options?: { skipIfPassed?: boolean } ): Promise { const verifiedAt = new Date().toISOString(); @@ -74,6 +75,10 @@ export class VerificationService { return { status: 'no_checksum', verifiedAt }; } + if (options?.skipIfPassed && metadata.verification?.passed === true) { + return { status: 'skipped', verifiedAt }; + } + // 2. Try native adapter verification (no download needed for S3, local, GDrive, OneDrive) if (adapter.verifyChecksum) { try { From 9c2fb2da582a0523346247319f0b45c26efc2c23 Mon Sep 17 00:00:00 2001 From: Manu Date: Thu, 11 Jun 2026 15:16:23 +0200 Subject: [PATCH 07/24] Add system task runner and integrity check support Introduce a SystemTaskRunner and integrate integrity check execution and UI. Added src/lib/runner/system-task-runner.ts to manage system-task executions, streaming logs, stages, and progress. Extended core logs with INTEGRITY_CHECK stages, order and progress map. Updated IntegrityService to perform a two-pass scan/verify flow, expose progress callbacks, and return work items for verification. Wire runner into SystemTaskService to start, stream, and finalize IntegrityCheck executions (including handling trigger type/label). Frontend: add system task columns and a System Tasks tab in history, handle IntegrityCheck labeling and proper live-updating of logs. Also pass the invoking user name from the settings API when manually triggering a task. Minor adjustments to log viewer to support integrity check stage ordering. --- src/app/api/settings/system-tasks/route.ts | 4 +- src/app/dashboard/history/columns.tsx | 90 ++++++++++ src/app/dashboard/history/page.tsx | 105 ++++++++---- src/components/execution/log-viewer.tsx | 7 +- src/lib/core/logs.ts | 26 +++ src/lib/runner/system-task-runner.ts | 181 +++++++++++++++++++++ src/services/backup/integrity-service.ts | 166 ++++++++++++------- src/services/system/system-task-service.ts | 46 +++++- 8 files changed, 526 insertions(+), 99 deletions(-) create mode 100644 src/lib/runner/system-task-runner.ts diff --git a/src/app/api/settings/system-tasks/route.ts b/src/app/api/settings/system-tasks/route.ts index a683d592..b87727dc 100644 --- a/src/app/api/settings/system-tasks/route.ts +++ b/src/app/api/settings/system-tasks/route.ts @@ -107,8 +107,10 @@ export async function PUT(req: NextRequest) { if (!taskId) return NextResponse.json({ error: "Missing taskId" }, { status: 400 }); + const user = await prisma.user.findUnique({ where: { id: ctx.userId }, select: { name: true } }); + // Run async - systemTaskService.runTask(taskId); + systemTaskService.runTask(taskId, "Manual", user?.name ?? "Manual"); await auditService.log( ctx.userId, diff --git a/src/app/dashboard/history/columns.tsx b/src/app/dashboard/history/columns.tsx index 86f94718..318e25a0 100644 --- a/src/app/dashboard/history/columns.tsx +++ b/src/app/dashboard/history/columns.tsx @@ -24,6 +24,10 @@ export interface Execution { triggerLabel?: string | null; } +const SYSTEM_TASK_TYPE_LABELS: Record = { + IntegrityCheck: "Backup Integrity Check", +}; + export const createColumns = (onViewLogs: (execution: Execution) => void): ColumnDef[] => [ { id: "jobName", @@ -163,3 +167,89 @@ export const createColumns = (onViewLogs: (execution: Execution) => void): Colum } } ]; + +export const createSystemTaskColumns = (onViewLogs: (execution: Execution) => void): ColumnDef[] => [ + { + id: "taskName", + accessorFn: (row) => SYSTEM_TASK_TYPE_LABELS[row.type ?? ""] ?? row.type ?? "System Task", + header: "Task", + cell: ({ row }) => ( + {row.getValue("taskName")} + ), + }, + { + id: "trigger", + accessorFn: (row) => row.triggerType ?? "", + header: "Trigger", + filterFn: (row, _id, value) => { + return value.includes(row.original.triggerType ?? ""); + }, + cell: ({ row }) => { + const triggerType = row.original.triggerType; + const triggerLabel = row.original.triggerLabel; + + if (!triggerType) { + return -; + } + + const iconClass = "h-3.5 w-3.5 shrink-0"; + let icon: React.ReactNode; + let badgeClass: string; + + if (triggerType === "Scheduler") { + icon = ; + badgeClass = "bg-violet-100 text-violet-700 border-violet-200 dark:bg-violet-950 dark:text-violet-300 dark:border-violet-800"; + } else { + icon = ; + badgeClass = "bg-sky-100 text-sky-700 border-sky-200 dark:bg-sky-950 dark:text-sky-300 dark:border-sky-800"; + } + + return ( + + {icon} + {triggerLabel || triggerType} + + ); + }, + }, + { + accessorKey: "status", + header: "Status", + cell: ({ row }) => { + const status = row.getValue("status") as string; + if (status === "Success") { + return Success; + } else if (status === "Failed") { + return Failed; + } else if (status === "Running") { + return Running; + } + return {status}; + }, + filterFn: (row, id, value) => value.includes(row.getValue(id)), + }, + { + accessorKey: "startedAt", + header: "Started At", + cell: ({ row }) => , + }, + { + accessorKey: "endedAt", + header: "Duration", + cell: ({ row }) => { + const start = new Date(row.original.startedAt); + const end = row.original.endedAt ? new Date(row.original.endedAt) : null; + if (!end) return Running...; + return {formatDuration(end.getTime() - start.getTime())}; + }, + }, + { + id: "actions", + cell: ({ row }) => ( + + ), + }, +]; diff --git a/src/app/dashboard/history/page.tsx b/src/app/dashboard/history/page.tsx index 1cc1d575..d0dafb06 100644 --- a/src/app/dashboard/history/page.tsx +++ b/src/app/dashboard/history/page.tsx @@ -9,7 +9,7 @@ import { DialogTitle, } from "@/components/ui/dialog" import { DataTable } from "@/components/ui/data-table"; -import { createColumns, Execution } from "./columns"; +import { createColumns, createSystemTaskColumns, Execution } from "./columns"; import { createNotificationLogColumns, NotificationLogRow } from "./notification-log-columns"; import { NotificationPreview } from "./notification-preview"; import { useSearchParams, useRouter } from "next/navigation"; @@ -32,6 +32,7 @@ export default function HistoryPage() { function HistoryContent() { const [executions, setExecutions] = useState([]); + const [systemTasks, setSystemTasks] = useState([]); const [systemTimezone, setSystemTimezone] = useState("UTC"); const [selectedLog, setSelectedLog] = useState(null); const [activeTab, setActiveTab] = useState("activity"); @@ -50,26 +51,23 @@ function HistoryContent() { // Sync selectedLog with latest executions data to enable live updates in modal useEffect(() => { if (selectedLog) { - const updatedLog = executions.find(e => e.id === selectedLog.id); - // Only update if the content has actually changed to prevent loops + const allExecs = [...executions, ...systemTasks]; + const updatedLog = allExecs.find(e => e.id === selectedLog.id); if (updatedLog && JSON.stringify(updatedLog) !== JSON.stringify(selectedLog)) { setSelectedLog(updatedLog); } } - }, [executions, selectedLog]); + }, [executions, systemTasks, selectedLog]); useEffect(() => { - if (executionId && executions.length > 0) { - // Check if we are already viewing it or explicitly closed it (not easily tracked here without ref, but let's assume if query param exists we want to open) - // To prevent re-opening, we remove the query param immediately after finding the log - const found = executions.find(e => e.id === executionId); + if (executionId && (executions.length > 0 || systemTasks.length > 0)) { + const found = [...executions, ...systemTasks].find(e => e.id === executionId); if (found && !selectedLog) { setSelectedLog(found); - // Clear the query param so it doesn't re-trigger on close router.replace("/dashboard/history", { scroll: false }); } } - }, [executions, executionId, selectedLog, router]); + }, [executions, systemTasks, executionId, selectedLog, router]); const fetchInFlight = useRef(false); @@ -80,7 +78,9 @@ function HistoryContent() { const res = await fetch("/api/history"); if (res.ok) { const data = await res.json(); - setExecutions(data.executions); + const systemTypes = ["IntegrityCheck"]; + setSystemTasks(data.executions.filter((e: Execution) => systemTypes.includes(e.type ?? ""))); + setExecutions(data.executions.filter((e: Execution) => !systemTypes.includes(e.type ?? ""))); setSystemTimezone(data.systemTimezone); } } catch (_e) { @@ -102,10 +102,10 @@ function HistoryContent() { } }, []); - // Poll history: 5s default, 2s when a job is running for live feel + // Poll history: 5s default, 2s when any job or system task is running for live feel const hasRunningJob = useMemo( - () => executions.some(e => e.status === "Running" || e.status === "Pending"), - [executions] + () => [...executions, ...systemTasks].some(e => e.status === "Running" || e.status === "Pending"), + [executions, systemTasks] ); useEffect(() => { @@ -152,6 +152,7 @@ function HistoryContent() { }, [fetchHistory]); const columns = useMemo(() => createColumns(setSelectedLog), []); + const systemTaskColumns = useMemo(() => createSystemTaskColumns(setSelectedLog), []); const notificationColumns = useMemo( () => createNotificationLogColumns(setSelectedNotification), [] @@ -187,6 +188,26 @@ function HistoryContent() { }, ], []); + const systemTaskFilterableColumns = useMemo(() => [ + { + id: "status", + title: "Status", + options: [ + { label: "Success", value: "Success" }, + { label: "Failed", value: "Failed" }, + { label: "Running", value: "Running" }, + ], + }, + { + id: "trigger", + title: "Trigger", + options: [ + { label: "Manual", value: "Manual" }, + { label: "Scheduler", value: "Scheduler" }, + ], + }, + ], []); + const notificationFilterableColumns = useMemo(() => [ { id: "adapterId", @@ -239,9 +260,8 @@ function HistoryContent() { Activity Logs - - Notification Logs - + System Tasks + Notification Logs @@ -263,6 +283,25 @@ function HistoryContent() { + + + + System Tasks + History of automated system operations such as integrity checks. + + + + + + + @@ -291,7 +330,7 @@ function HistoryContent() { {selectedLog?.status === "Running" && } - {selectedLog?.job?.name || selectedLog?.type || "Manual Job"} + {selectedLog?.job?.name || (selectedLog?.type === "IntegrityCheck" ? "Backup Integrity Check" : selectedLog?.type) || "Manual Job"} {selectedLog?.status && ( {selectedLog.status} @@ -311,20 +350,22 @@ function HistoryContent() { {detail && - {detail}} {selectedLog?.status === "Running" && progress > 0 && !detail && {progress}%}
- + {["Backup", "Restore"].includes(selectedLog?.type ?? "Backup") && ( + + )}
{selectedLog?.status === "Running" && ( progress > 0 ? ( diff --git a/src/components/execution/log-viewer.tsx b/src/components/execution/log-viewer.tsx index d491c13b..99f6f2ec 100644 --- a/src/components/execution/log-viewer.tsx +++ b/src/components/execution/log-viewer.tsx @@ -13,7 +13,7 @@ import { Clock } from "lucide-react"; import { cn, formatDuration } from "@/lib/utils"; -import { LogEntry, BACKUP_STAGE_ORDER, RESTORE_STAGE_ORDER } from "@/lib/core/logs"; +import { LogEntry, BACKUP_STAGE_ORDER, RESTORE_STAGE_ORDER, INTEGRITY_CHECK_STAGE_ORDER } from "@/lib/core/logs"; import { Accordion, AccordionContent, @@ -68,10 +68,11 @@ export function LogViewer({ logs, className, autoScroll = true, status, executio // Grouping Logic - group by stage, sort by stage order, fill pending stages const groupedLogs = useMemo(() => { - // Detect execution type: explicit prop, or infer from stage names const stageOrder = executionType === "Restore" ? RESTORE_STAGE_ORDER - : BACKUP_STAGE_ORDER; + : executionType === "IntegrityCheck" + ? INTEGRITY_CHECK_STAGE_ORDER + : BACKUP_STAGE_ORDER; // Build a map of stage → logs const stageMap = new Map(); diff --git a/src/lib/core/logs.ts b/src/lib/core/logs.ts index 2c8bdff6..e73a3c41 100644 --- a/src/lib/core/logs.ts +++ b/src/lib/core/logs.ts @@ -66,6 +66,32 @@ export const RESTORE_STAGE_ORDER: string[] = [ RESTORE_STAGES.COMPLETED, ]; +/** Integrity Check stages */ +export const INTEGRITY_CHECK_STAGES = { + INITIALIZING: "Initializing", + SCANNING: "Scanning Storage", + VERIFYING_CHECKSUMS: "Verifying Checksums", + COMPLETED: "Completed", + FAILED: "Failed", +} as const; + +/** Ordered list of integrity check stages for frontend rendering */ +export const INTEGRITY_CHECK_STAGE_ORDER: string[] = [ + "Initializing", + "Scanning Storage", + "Verifying Checksums", + "Completed", +]; + +/** Progress ranges [min, max] for integrity check stages */ +export const INTEGRITY_CHECK_STAGE_PROGRESS_MAP: Record = { + "Initializing": [0, 5], + "Scanning Storage": [5, 20], + "Verifying Checksums": [20, 95], + "Completed": [100, 100], + "Failed": [100, 100], +}; + /** @deprecated Use BACKUP_STAGE_ORDER instead */ export const STAGE_ORDER: PipelineStage[] = BACKUP_STAGE_ORDER as PipelineStage[]; diff --git a/src/lib/runner/system-task-runner.ts b/src/lib/runner/system-task-runner.ts new file mode 100644 index 00000000..dcad940b --- /dev/null +++ b/src/lib/runner/system-task-runner.ts @@ -0,0 +1,181 @@ +import prisma from "@/lib/prisma"; +import { LogEntry, LogLevel, LogType, INTEGRITY_CHECK_STAGE_PROGRESS_MAP } from "@/lib/core/logs"; +import { logger } from "@/lib/logging/logger"; +import { wrapError } from "@/lib/logging/errors"; +import { formatDuration } from "@/lib/utils"; + +const log = logger.child({ module: "SystemTaskRunner" }); + +export class SystemTaskRunner { + private executionId: string; + private logs: LogEntry[] = []; + private currentStage = "Initializing"; + private currentProgress = 0; + private currentDetail = ""; + private stageStartTimes = new Map(); + private lastLogFlush = 0; + private isFlushing = false; + private hasPendingFlush = false; + private stageProgressMap: Record; + + private constructor( + executionId: string, + stageProgressMap: Record + ) { + this.executionId = executionId; + this.stageProgressMap = stageProgressMap; + } + + static async create( + taskType: string, + triggerType?: "Manual" | "Scheduler", + triggerLabel?: string, + stageProgressMap: Record = INTEGRITY_CHECK_STAGE_PROGRESS_MAP + ): Promise { + const initialLog: LogEntry = { + timestamp: new Date().toISOString(), + level: "info", + type: "general", + message: "Task queued", + stage: "Initializing", + }; + + const execution = await prisma.execution.create({ + data: { + jobId: null, + type: taskType, + status: "Pending", + logs: JSON.stringify([initialLog]), + metadata: JSON.stringify({ progress: 0, stage: "Initializing" }), + triggerType: triggerType ?? null, + triggerLabel: triggerLabel ?? null, + }, + }); + + return new SystemTaskRunner(execution.id, stageProgressMap); + } + + get id(): string { + return this.executionId; + } + + async start(): Promise { + const claimed = await prisma.execution.updateMany({ + where: { id: this.executionId, status: "Pending" }, + data: { status: "Running", startedAt: new Date() }, + }); + + if (claimed.count === 0) { + throw new Error("Execution already claimed by a concurrent call"); + } + } + + async finish(status: "Success" | "Failed"): Promise { + await this.flushLogs(true); + await prisma.execution.update({ + where: { id: this.executionId }, + data: { + status, + endedAt: new Date(), + logs: JSON.stringify(this.logs), + metadata: JSON.stringify({ + progress: this.currentProgress, + stage: this.currentStage, + }), + }, + }); + } + + logEntry(message: string, level: LogLevel = "info", type: LogType = "general", details?: string): void { + const entry: LogEntry = { + timestamp: new Date().toISOString(), + level, + type, + message, + stage: this.currentStage, + details, + }; + this.logs.push(entry); + void this.flushLogs(); + } + + setStage(stage: string): void { + const prevStart = this.stageStartTimes.get(this.currentStage); + if (prevStart && this.currentStage !== stage) { + const durationMs = Date.now() - prevStart; + this.logs.push({ + timestamp: new Date().toISOString(), + level: "success", + type: "general", + message: `${this.currentStage} completed (${formatDuration(durationMs)})`, + stage: this.currentStage, + durationMs, + }); + } + + this.currentStage = stage; + this.currentDetail = ""; + this.stageStartTimes.set(stage, Date.now()); + + const range = this.stageProgressMap[stage]; + this.currentProgress = range ? range[0] : this.currentProgress; + + void this.flushLogs(true); + } + + updateStageProgress(internalPercent: number): void { + const range = this.stageProgressMap[this.currentStage]; + if (range) { + const [min, max] = range; + const clamped = Math.max(0, Math.min(100, internalPercent)); + this.currentProgress = Math.round(min + (max - min) * (clamped / 100)); + } + void this.flushLogs(); + } + + updateDetail(detail: string): void { + this.currentDetail = detail; + void this.flushLogs(); + } + + async flushLogs(force = false): Promise { + const now = Date.now(); + const shouldRun = force || now - this.lastLogFlush > 1000; + if (!shouldRun) return; + + if (this.isFlushing) { + this.hasPendingFlush = true; + return; + } + + this.isFlushing = true; + const performUpdate = async () => { + try { + this.lastLogFlush = Date.now(); + await prisma.execution.update({ + where: { id: this.executionId }, + data: { + logs: JSON.stringify(this.logs), + metadata: JSON.stringify({ + progress: this.currentProgress, + stage: this.currentStage, + detail: this.currentDetail, + }), + }, + }); + } catch (error) { + log.error("Failed to flush logs", { executionId: this.executionId }, wrapError(error)); + } + }; + + try { + await performUpdate(); + if (this.hasPendingFlush) { + this.hasPendingFlush = false; + await performUpdate(); + } + } finally { + this.isFlushing = false; + } + } +} diff --git a/src/services/backup/integrity-service.ts b/src/services/backup/integrity-service.ts index 3cf8ac56..1f439294 100644 --- a/src/services/backup/integrity-service.ts +++ b/src/services/backup/integrity-service.ts @@ -6,6 +6,7 @@ import { resolveAdapterConfig } from "@/lib/adapters/config-resolver"; import { verificationService } from "@/services/storage/verification-service"; import { logger } from "@/lib/logging/logger"; import { wrapError } from "@/lib/logging/errors"; +import { INTEGRITY_CHECK_STAGES } from "@/lib/core/logs"; const log = logger.child({ service: "IntegrityService" }); @@ -25,14 +26,27 @@ export interface IntegrityCheckResult { }>; } +export interface IntegrityProgressCallbacks { + onLog: (message: string, level?: "info" | "success" | "warning" | "error", details?: string) => void; + onStage: (stage: string) => void; + onFileProgress: (done: number, total: number, currentFile?: string) => void; +} + interface IntegrityFilters { skipPassed: boolean; maxAgeDays: number; maxFileSizeBytes: number; } +interface WorkItem { + storageConfigId: string; + destinationName: string; + remotePath: string; + fileName: string; +} + export class IntegrityService { - async runFullIntegrityCheck(): Promise { + async runFullIntegrityCheck(callbacks?: IntegrityProgressCallbacks): Promise { log.info("Starting full backup integrity check"); const result: IntegrityCheckResult = { @@ -62,20 +76,91 @@ export class IntegrityService { maxFileSizeMb: filters.maxFileSizeBytes / 1024 / 1024, }); + callbacks?.onStage(INTEGRITY_CHECK_STAGES.INITIALIZING); + + const filterParts: string[] = []; + if (filters.maxAgeDays > 0) filterParts.push(`max age ${filters.maxAgeDays} days`); + if (filters.maxFileSizeBytes > 0) filterParts.push(`max size ${filters.maxFileSizeBytes / 1024 / 1024} MB`); + if (filters.skipPassed) filterParts.push("skip already-passed files"); + callbacks?.onLog( + filterParts.length > 0 ? `Filters: ${filterParts.join(", ")}` : "No filters configured - checking all files" + ); + const storageConfigs = await prisma.adapterConfig.findMany({ where: { type: "storage" }, }); + callbacks?.onLog(`Found ${storageConfigs.length} storage destination${storageConfigs.length !== 1 ? "s" : ""}`); + + // Pass 1: Scan all destinations and collect work items + callbacks?.onStage(INTEGRITY_CHECK_STAGES.SCANNING); + + const allWorkItems: WorkItem[] = []; for (const storageConfig of storageConfigs) { try { - await this.checkDestination(storageConfig, result, filters); + const items = await this.gatherFilesFromDestination(storageConfig, filters, callbacks); + allWorkItems.push(...items); } catch (e: unknown) { - log.error( - "Failed to check storage destination", - { destination: storageConfig.name }, - wrapError(e) + log.error("Failed to scan storage destination", { destination: storageConfig.name }, wrapError(e)); + callbacks?.onLog(`Failed to scan ${storageConfig.name}`, "error"); + } + } + + result.totalFiles = allWorkItems.length; + callbacks?.onLog( + `Total: ${allWorkItems.length} file${allWorkItems.length !== 1 ? "s" : ""} to verify`, + "info" + ); + + if (allWorkItems.length === 0) { + callbacks?.onLog("No files to verify - check filters or storage contents", "info"); + return result; + } + + // Pass 2: Verify all collected files + callbacks?.onStage(INTEGRITY_CHECK_STAGES.VERIFYING_CHECKSUMS); + + for (let i = 0; i < allWorkItems.length; i++) { + const item = allWorkItems[i]; + callbacks?.onFileProgress(i, allWorkItems.length, item.fileName); + + try { + const verifyResult = await verificationService.verifyFile( + item.storageConfigId, + item.remotePath, + "scheduled", + { skipIfPassed: filters.skipPassed } ); + + if (verifyResult.status === "passed") { + result.verified++; + result.passed++; + callbacks?.onLog(`${item.fileName}`, "success"); + } else if (verifyResult.status === "failed") { + result.verified++; + result.failed++; + result.errors.push({ + file: item.fileName, + destination: item.destinationName, + expected: verifyResult.expectedChecksum ?? "", + actual: verifyResult.actualChecksum ?? "", + }); + callbacks?.onLog( + `${item.fileName} - checksum mismatch`, + "error", + `Expected: ${verifyResult.expectedChecksum ?? "unknown"}\nActual: ${verifyResult.actualChecksum ?? "unknown"}` + ); + } else { + result.skipped++; + callbacks?.onLog(`${item.fileName} skipped (${verifyResult.status})`, "info"); + } + } catch (e: unknown) { + log.error("Failed to verify file", { file: item.fileName, destination: item.destinationName }, wrapError(e)); + callbacks?.onLog(`Failed to verify ${item.fileName}`, "error"); + result.skipped++; } + + callbacks?.onFileProgress(i + 1, allWorkItems.length, item.fileName); } log.info("Integrity check completed", { @@ -89,15 +174,16 @@ export class IntegrityService { return result; } - private async checkDestination( + private async gatherFilesFromDestination( storageConfig: any, - result: IntegrityCheckResult, - filters: IntegrityFilters - ) { + filters: IntegrityFilters, + callbacks?: IntegrityProgressCallbacks + ): Promise { + const workItems: WorkItem[] = []; const adapter = registry.get(storageConfig.adapterId) as StorageAdapter; if (!adapter) { log.warn("Storage adapter not found", { adapterId: storageConfig.adapterId }); - return; + return []; } const config = await resolveAdapterConfig(storageConfig); @@ -127,55 +213,19 @@ export class IntegrityService { const backupFiles = files.filter((f) => !f.name.endsWith(".meta.json")); for (const file of backupFiles) { - result.totalFiles++; - const remotePath = `${folder}/${file.name}`; - - // Age filter: skip files older than maxAgeDays if (filters.maxAgeDays > 0 && file.lastModified) { const ageDays = (Date.now() - new Date(file.lastModified).getTime()) / 86_400_000; - if (ageDays > filters.maxAgeDays) { - result.skipped++; - continue; - } + if (ageDays > filters.maxAgeDays) continue; } - // Size filter: skip files larger than maxFileSizeBytes - if (filters.maxFileSizeBytes > 0 && file.size > filters.maxFileSizeBytes) { - result.skipped++; - continue; - } + if (filters.maxFileSizeBytes > 0 && file.size > filters.maxFileSizeBytes) continue; - try { - const verifyResult = await verificationService.verifyFile( - storageConfig.id, - remotePath, - "scheduled", - { skipIfPassed: filters.skipPassed } - ); - - if (verifyResult.status === "passed") { - result.verified++; - result.passed++; - } else if (verifyResult.status === "failed") { - result.verified++; - result.failed++; - result.errors.push({ - file: file.name, - destination: storageConfig.name, - expected: verifyResult.expectedChecksum ?? "", - actual: verifyResult.actualChecksum ?? "", - }); - } else { - result.skipped++; - } - } catch (e: unknown) { - log.error( - "Failed to verify file", - { file: file.name, destination: storageConfig.name }, - wrapError(e) - ); - result.skipped++; - } + workItems.push({ + storageConfigId: storageConfig.id, + destinationName: storageConfig.name, + remotePath: `${folder}/${file.name}`, + fileName: file.name, + }); } } catch (e: unknown) { log.warn( @@ -185,6 +235,12 @@ export class IntegrityService { ); } } + + callbacks?.onLog( + `${storageConfig.name}: found ${workItems.length} file${workItems.length !== 1 ? "s" : ""} to verify` + ); + + return workItems; } } diff --git a/src/services/system/system-task-service.ts b/src/services/system/system-task-service.ts index fa8afa5b..606ae18a 100644 --- a/src/services/system/system-task-service.ts +++ b/src/services/system/system-task-service.ts @@ -214,7 +214,7 @@ export class SystemTaskService { }); } - async runTask(taskId: string) { + async runTask(taskId: string, triggerType?: "Manual" | "Scheduler", triggerLabel?: string) { log.info("Running system task", { taskId }); await this.setTaskLastRunAt(taskId); @@ -242,13 +242,43 @@ export class SystemTaskService { } case SYSTEM_TASKS.INTEGRITY_CHECK: { const { integrityService } = await import("@/services/backup/integrity-service"); - const result = await integrityService.runFullIntegrityCheck(); - log.info("Integrity check results", { - total: result.totalFiles, - passed: result.passed, - failed: result.failed, - skipped: result.skipped - }); + const { SystemTaskRunner } = await import("@/lib/runner/system-task-runner"); + const { INTEGRITY_CHECK_STAGES } = await import("@/lib/core/logs"); + const { getErrorMessage } = await import("@/lib/logging/errors"); + + const runner = await SystemTaskRunner.create( + "IntegrityCheck", + triggerType ?? "Scheduler", + triggerLabel ?? "Scheduler" + ); + + try { + await runner.start(); + const result = await integrityService.runFullIntegrityCheck({ + onLog: (msg, level, details) => runner.logEntry(msg, level ?? "info", "general", details), + onStage: (stage) => runner.setStage(stage), + onFileProgress: (done, total) => { + if (total > 0) runner.updateStageProgress((done / total) * 100); + }, + }); + runner.setStage(INTEGRITY_CHECK_STAGES.COMPLETED); + runner.logEntry( + `${result.passed} passed, ${result.failed} failed, ${result.skipped} skipped of ${result.totalFiles} total`, + result.failed > 0 ? "warning" : "success" + ); + await runner.finish("Success"); + log.info("Integrity check completed", { + total: result.totalFiles, + passed: result.passed, + failed: result.failed, + skipped: result.skipped, + }); + } catch (e: unknown) { + runner.logEntry(getErrorMessage(e), "error"); + runner.setStage(INTEGRITY_CHECK_STAGES.FAILED); + await runner.finish("Failed"); + log.error("Integrity check failed", {}, wrapError(e)); + } break; } case SYSTEM_TASKS.REFRESH_STORAGE_STATS: { From 72d8628a172e493d26ffd13b2bdaa2fcba498417 Mon Sep 17 00:00:00 2001 From: Manu Date: Thu, 11 Jun 2026 20:54:16 +0200 Subject: [PATCH 08/24] Use file.path for listings; clarify skip log Use adapter-provided file.path (full relative path) as remotePath so downloads/read operations resolve correctly for adapters that return flat recursive listings. Replace folder-based listing logic with a single aggregated file list (falling back to per-job listings when root listing fails) and filter out .meta.json files. Add human-friendly skip reason mapping for verifyResult.status to improve log messages and keep existing age/size filtering and error handling. --- src/services/backup/integrity-service.ts | 69 ++++++++++++------------ 1 file changed, 36 insertions(+), 33 deletions(-) diff --git a/src/services/backup/integrity-service.ts b/src/services/backup/integrity-service.ts index 1f439294..63a80a98 100644 --- a/src/services/backup/integrity-service.ts +++ b/src/services/backup/integrity-service.ts @@ -152,7 +152,14 @@ export class IntegrityService { ); } else { result.skipped++; - callbacks?.onLog(`${item.fileName} skipped (${verifyResult.status})`, "info"); + const skipReasons: Record = { + skipped: "already verified", + no_metadata: "no metadata file", + no_checksum: "no checksum stored", + download_error: "download failed", + }; + const reason = skipReasons[verifyResult.status] ?? verifyResult.status; + callbacks?.onLog(`${item.fileName} skipped (${reason})`, "info"); } } catch (e: unknown) { log.error("Failed to verify file", { file: item.fileName, destination: item.destinationName }, wrapError(e)); @@ -188,12 +195,12 @@ export class IntegrityService { const config = await resolveAdapterConfig(storageConfig); - let folders: string[] = []; + // Collect all files from storage. Adapters return flat recursive listings where + // file.name is the basename and file.path is the full relative path — use file.path + // as remotePath so downstream download/read calls resolve the correct location. + let allFiles: Awaited> = []; try { - const topLevel = await adapter.list(config, ""); - folders = topLevel - .filter((f) => f.name && !f.name.endsWith(".meta.json")) - .map((f) => f.name); + allFiles = await adapter.list(config, ""); } catch (e: unknown) { log.warn( "Could not list storage root, falling back to active jobs", @@ -204,36 +211,32 @@ export class IntegrityService { where: { destinations: { some: { configId: storageConfig.id } } }, select: { name: true }, }); - folders = jobs.map((j) => j.name); + for (const job of jobs) { + try { + const files = await adapter.list(config, job.name); + allFiles.push(...files); + } catch { + // skip unreachable job folders + } + } } - for (const folder of folders) { - try { - const files = await adapter.list(config, folder); - const backupFiles = files.filter((f) => !f.name.endsWith(".meta.json")); - - for (const file of backupFiles) { - if (filters.maxAgeDays > 0 && file.lastModified) { - const ageDays = (Date.now() - new Date(file.lastModified).getTime()) / 86_400_000; - if (ageDays > filters.maxAgeDays) continue; - } - - if (filters.maxFileSizeBytes > 0 && file.size > filters.maxFileSizeBytes) continue; - - workItems.push({ - storageConfigId: storageConfig.id, - destinationName: storageConfig.name, - remotePath: `${folder}/${file.name}`, - fileName: file.name, - }); - } - } catch (e: unknown) { - log.warn( - "Failed to list files for folder", - { folder, destination: storageConfig.name }, - wrapError(e) - ); + const backupFiles = allFiles.filter((f) => !f.name.endsWith(".meta.json")); + + for (const file of backupFiles) { + if (filters.maxAgeDays > 0 && file.lastModified) { + const ageDays = (Date.now() - new Date(file.lastModified).getTime()) / 86_400_000; + if (ageDays > filters.maxAgeDays) continue; } + + if (filters.maxFileSizeBytes > 0 && file.size > filters.maxFileSizeBytes) continue; + + workItems.push({ + storageConfigId: storageConfig.id, + destinationName: storageConfig.name, + remotePath: file.path, + fileName: file.name, + }); } callbacks?.onLog( From dd542300c66df1d38979d85d893cbf93f088bec4 Mon Sep 17 00:00:00 2001 From: Manu Date: Thu, 11 Jun 2026 21:29:26 +0200 Subject: [PATCH 09/24] Run verification and integrity checks asynchronously Introduce async execution for single-file verification and scheduled integrity checks so callers receive an executionId immediately and the work runs in the background with live progress and history entries. Added a new POST /api/storage/[id]/verify-async route and VERIFICATION stage order/progress mapping. systemTaskService.runTask now returns the execution id and runs IntegrityCheck tasks asynchronously (returns runner.id). UI changes: Integrity Modal now uses verify-async and respects the "Auto-redirect on job start" preference (redirects to live History view), System Tasks settings will redirect if executionId is returned, History view treats "Verification" as a system task type, and LogViewer supports VERIFICATION stages and auto-expands the active running stage. Updated docs/changelog and small preference copy tweak to include system tasks. --- docs/changelog.md | 3 + src/app/api/settings/system-tasks/route.ts | 5 +- .../api/storage/[id]/verify-async/route.ts | 92 +++++++++++++++++++ src/app/dashboard/history/page.tsx | 4 +- .../dashboard/storage/integrity-modal.tsx | 20 ++-- src/components/execution/log-viewer.tsx | 36 ++++---- src/components/settings/preferences-form.tsx | 2 +- .../settings/system-tasks-settings.tsx | 11 ++- src/lib/core/logs.ts | 6 ++ src/services/system/system-task-service.ts | 62 +++++++------ 10 files changed, 179 insertions(+), 62 deletions(-) create mode 100644 src/app/api/storage/[id]/verify-async/route.ts diff --git a/docs/changelog.md b/docs/changelog.md index 62795bbd..7872d0ba 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -12,6 +12,8 @@ All notable changes to DBackup are documented here. - **storage**: MD5 checksums are now computed alongside SHA-256 during backup upload and stored in `.meta.json`, enabling native verification on Google Drive. - **integrity**: The scheduled integrity check now uses native verification where available and writes results back to `.meta.json` sidecars. - **integrity**: Scheduled integrity checks can now be filtered by configurable rules (skip already-passed backups, max backup age, max file size) accessible via the gear icon on the Integrity Check task row in Settings. +- **integrity**: Manual system task runs now auto-redirect to the live history view when "Auto-redirect on job start" is enabled. +- **integrity**: Single-file "Verify Now" in the Storage Explorer now runs as a tracked async execution with live progress, history entry, and cancel support. - **backup**: Post-upload integrity verification is now opt-in for all storage destinations via the `backup.postUploadVerify` system setting (local filesystem always verifies). - **history**: MD5 checksum is now logged alongside SHA-256 in execution history during the upload stage. @@ -19,6 +21,7 @@ All notable changes to DBackup are documented here. - **storage**: Long backup names in the Storage Explorer are now truncated with a tooltip showing the full name on hover. - **ui**: All data tables (Storage, Jobs, Sources, Destinations, Notifications) now support horizontal scrolling when columns overflow, matching the Database Explorer behavior. +- **preferences**: "Auto-redirect on job start" description updated to include system tasks. ### 🐳 Docker diff --git a/src/app/api/settings/system-tasks/route.ts b/src/app/api/settings/system-tasks/route.ts index b87727dc..19c356fc 100644 --- a/src/app/api/settings/system-tasks/route.ts +++ b/src/app/api/settings/system-tasks/route.ts @@ -109,8 +109,7 @@ export async function PUT(req: NextRequest) { const user = await prisma.user.findUnique({ where: { id: ctx.userId }, select: { name: true } }); - // Run async - systemTaskService.runTask(taskId, "Manual", user?.name ?? "Manual"); + const executionId = await systemTaskService.runTask(taskId, "Manual", user?.name ?? "Manual"); await auditService.log( ctx.userId, @@ -120,5 +119,5 @@ export async function PUT(req: NextRequest) { taskId ); - return NextResponse.json({ success: true, message: "Task started" }); + return NextResponse.json({ success: true, ...(executionId ? { executionId } : {}) }); } diff --git a/src/app/api/storage/[id]/verify-async/route.ts b/src/app/api/storage/[id]/verify-async/route.ts new file mode 100644 index 00000000..1f50eef2 --- /dev/null +++ b/src/app/api/storage/[id]/verify-async/route.ts @@ -0,0 +1,92 @@ +import { NextRequest, NextResponse } from "next/server"; +import { verificationService } from "@/services/storage/verification-service"; +import { SystemTaskRunner } from "@/lib/runner/system-task-runner"; +import prisma from "@/lib/prisma"; +import { headers } from "next/headers"; +import { getAuthContext, checkPermissionWithContext } from "@/lib/auth/access-control"; +import { PERMISSIONS } from "@/lib/auth/permissions"; +import { logger } from "@/lib/logging/logger"; +import { wrapError, getErrorMessage } from "@/lib/logging/errors"; + +const log = logger.child({ route: "storage/verify-async" }); + +const VERIFICATION_STAGE_PROGRESS_MAP: Record = { + "Initializing": [0, 5], + "Verifying": [5, 95], + "Completed": [100, 100], + "Failed": [100, 100], +}; + +export async function POST(req: NextRequest, props: { params: Promise<{ id: string }> }) { + const ctx = await getAuthContext(await headers()); + if (!ctx) return NextResponse.json({ error: "Unauthorized" }, { status: 401 }); + + const params = await props.params; + + try { + checkPermissionWithContext(ctx, PERMISSIONS.STORAGE.READ); + + const body = await req.json(); + const { file } = body; + + if (!file || typeof file !== "string" || file.includes("..") || file.startsWith("/")) { + return NextResponse.json({ error: "Invalid file path" }, { status: 400 }); + } + + const user = await prisma.user.findUnique({ where: { id: ctx.userId }, select: { name: true } }); + + const runner = await SystemTaskRunner.create( + "Verification", + "Manual", + user?.name ?? "Manual", + VERIFICATION_STAGE_PROGRESS_MAP + ); + + // Run verification in the background — return executionId immediately. + (async () => { + try { + await runner.start(); + runner.setStage("Initializing"); + runner.logEntry(`Verifying ${file}`, "info"); + + runner.setStage("Verifying"); + const result = await verificationService.verifyFile(params.id, file, "manual"); + + if (result.status === "passed") { + runner.setStage("Completed"); + runner.logEntry("Checksum verified successfully", "success"); + await runner.finish("Success"); + } else if (result.status === "failed") { + runner.setStage("Completed"); + runner.logEntry( + "Checksum mismatch", + "error", + `Expected: ${result.expectedChecksum ?? "unknown"}\nActual: ${result.actualChecksum ?? "unknown"}` + ); + await runner.finish("Failed"); + } else { + const skipReasons: Record = { + no_metadata: "No metadata file found", + no_checksum: "No checksum stored in metadata", + download_error: "Download failed", + skipped: "Already verified", + }; + const reason = skipReasons[result.status] ?? result.status; + runner.setStage("Completed"); + runner.logEntry(`Skipped: ${reason}`, "info"); + await runner.finish("Success"); + } + } catch (e: unknown) { + log.error("Async verification failed", { file, storageConfigId: params.id }, wrapError(e)); + runner.logEntry(getErrorMessage(e), "error"); + runner.setStage("Failed"); + await runner.finish("Failed"); + } + })(); + + return NextResponse.json({ success: true, executionId: runner.id }); + } catch (error: unknown) { + log.error("Verify-async route error", { id: params.id }, wrapError(error)); + return NextResponse.json({ error: getErrorMessage(error) }, { status: 500 }); + } +} diff --git a/src/app/dashboard/history/page.tsx b/src/app/dashboard/history/page.tsx index d0dafb06..7ebdc769 100644 --- a/src/app/dashboard/history/page.tsx +++ b/src/app/dashboard/history/page.tsx @@ -78,7 +78,7 @@ function HistoryContent() { const res = await fetch("/api/history"); if (res.ok) { const data = await res.json(); - const systemTypes = ["IntegrityCheck"]; + const systemTypes = ["IntegrityCheck", "Verification"]; setSystemTasks(data.executions.filter((e: Execution) => systemTypes.includes(e.type ?? ""))); setExecutions(data.executions.filter((e: Execution) => !systemTypes.includes(e.type ?? ""))); setSystemTimezone(data.systemTimezone); @@ -350,7 +350,7 @@ function HistoryContent() { {detail && - {detail}} {selectedLog?.status === "Running" && progress > 0 && !detail && {progress}%}
- {["Backup", "Restore"].includes(selectedLog?.type ?? "Backup") && ( + {( + + + +
diff --git a/src/components/settings/system-tasks-settings.tsx b/src/components/settings/system-tasks-settings.tsx index bf290379..407d1300 100644 --- a/src/components/settings/system-tasks-settings.tsx +++ b/src/components/settings/system-tasks-settings.tsx @@ -38,7 +38,7 @@ export function SystemTasksSettings({ initialIntegritySettings }: SystemTasksSet const [running, setRunning] = useState>({}); const [integritySettingsOpen, setIntegritySettingsOpen] = useState(false); const [integritySettings, setIntegritySettings] = useState( - initialIntegritySettings ?? { skipPassed: false, maxAgeDays: 0, maxFileSizeMb: 0 } + initialIntegritySettings ?? { skipPassed: false, maxAgeDays: 0, maxFileSizeMb: 0, scanMode: "jobs" as const } ); const router = useRouter(); const { autoRedirectOnJobStart } = useUserPreferences(); diff --git a/src/services/backup/integrity-service.ts b/src/services/backup/integrity-service.ts index 63a80a98..9e8d46bb 100644 --- a/src/services/backup/integrity-service.ts +++ b/src/services/backup/integrity-service.ts @@ -1,4 +1,5 @@ import prisma from "@/lib/prisma"; +import path from "path"; import { registry } from "@/lib/core/registry"; import { registerAdapters } from "@/lib/adapters"; import { StorageAdapter } from "@/lib/core/interfaces"; @@ -58,10 +59,11 @@ export class IntegrityService { errors: [], }; - const [skipPassedSetting, maxAgeSetting, maxSizeSetting] = await Promise.all([ + const [skipPassedSetting, maxAgeSetting, maxSizeSetting, scanModeSetting] = await Promise.all([ prisma.systemSetting.findUnique({ where: { key: 'integrity.skipPassed' } }), prisma.systemSetting.findUnique({ where: { key: 'integrity.maxAgeDays' } }), prisma.systemSetting.findUnique({ where: { key: 'integrity.maxFileSizeMb' } }), + prisma.systemSetting.findUnique({ where: { key: 'integrity.scanMode' } }), ]); const filters: IntegrityFilters = { @@ -70,7 +72,10 @@ export class IntegrityService { maxFileSizeBytes: (parseInt(maxSizeSetting?.value ?? '0') || 0) * 1024 * 1024, }; - log.info("Integrity check filters", { + const isJobsMode = !scanModeSetting || scanModeSetting.value !== 'destinations'; + + log.info("Integrity check config", { + mode: isJobsMode ? 'jobs' : 'destinations', skipPassed: filters.skipPassed, maxAgeDays: filters.maxAgeDays, maxFileSizeMb: filters.maxFileSizeBytes / 1024 / 1024, @@ -86,23 +91,36 @@ export class IntegrityService { filterParts.length > 0 ? `Filters: ${filterParts.join(", ")}` : "No filters configured - checking all files" ); - const storageConfigs = await prisma.adapterConfig.findMany({ - where: { type: "storage" }, - }); - - callbacks?.onLog(`Found ${storageConfigs.length} storage destination${storageConfigs.length !== 1 ? "s" : ""}`); + callbacks?.onLog(`Scan mode: ${isJobsMode ? 'Jobs (job-linked files only)' : 'All destinations (full storage scan)'}`); - // Pass 1: Scan all destinations and collect work items + // Pass 1: Collect work items according to scan mode callbacks?.onStage(INTEGRITY_CHECK_STAGES.SCANNING); const allWorkItems: WorkItem[] = []; - for (const storageConfig of storageConfigs) { + + if (isJobsMode) { try { - const items = await this.gatherFilesFromDestination(storageConfig, filters, callbacks); + const items = await this.gatherFilesFromJobs(filters, callbacks); allWorkItems.push(...items); } catch (e: unknown) { - log.error("Failed to scan storage destination", { destination: storageConfig.name }, wrapError(e)); - callbacks?.onLog(`Failed to scan ${storageConfig.name}`, "error"); + log.error("Failed to gather files from jobs", {}, wrapError(e)); + callbacks?.onLog("Failed to scan job-linked files", "error"); + } + } else { + const storageConfigs = await prisma.adapterConfig.findMany({ + where: { type: "storage" }, + }); + + callbacks?.onLog(`Found ${storageConfigs.length} storage destination${storageConfigs.length !== 1 ? "s" : ""}`); + + for (const storageConfig of storageConfigs) { + try { + const items = await this.gatherFilesFromDestination(storageConfig, filters, callbacks); + allWorkItems.push(...items); + } catch (e: unknown) { + log.error("Failed to scan storage destination", { destination: storageConfig.name }, wrapError(e)); + callbacks?.onLog(`Failed to scan ${storageConfig.name}`, "error"); + } } } @@ -181,11 +199,82 @@ export class IntegrityService { return result; } + private async gatherFilesFromJobs( + filters: IntegrityFilters, + callbacks?: IntegrityProgressCallbacks + ): Promise { + const jobs = await prisma.job.findMany({ + where: { enabled: true, skipVerification: false }, + include: { + destinations: { + include: { config: true }, + }, + }, + }); + + callbacks?.onLog(`Found ${jobs.length} job${jobs.length !== 1 ? "s" : ""} to scan`); + + const cutoffDate = filters.maxAgeDays > 0 + ? new Date(Date.now() - filters.maxAgeDays * 86_400_000) + : null; + + const seen = new Set(); + const workItems: WorkItem[] = []; + + for (const job of jobs) { + const executions = await prisma.execution.findMany({ + where: { + jobId: job.id, + type: "Backup", + status: "Completed", + path: { not: null }, + ...(cutoffDate ? { endedAt: { gte: cutoffDate } } : {}), + }, + select: { path: true, size: true }, + }); + + for (const dest of job.destinations) { + const meta = dest.config.metadata ? JSON.parse(dest.config.metadata) : {}; + if (meta.skipVerification === true) continue; + + for (const exec of executions) { + if (!exec.path) continue; + + if (filters.maxFileSizeBytes > 0 && exec.size !== null && exec.size !== undefined) { + if (exec.size > filters.maxFileSizeBytes) continue; + } + + const key = `${dest.configId}:${exec.path}`; + if (seen.has(key)) continue; + seen.add(key); + + workItems.push({ + storageConfigId: dest.configId, + destinationName: dest.config.name, + remotePath: exec.path, + fileName: path.basename(exec.path), + }); + } + } + } + + callbacks?.onLog(`Jobs scan: found ${workItems.length} file${workItems.length !== 1 ? "s" : ""} to verify`); + + return workItems; + } + private async gatherFilesFromDestination( storageConfig: any, filters: IntegrityFilters, callbacks?: IntegrityProgressCallbacks ): Promise { + // Check if this destination has verification disabled + const meta = storageConfig.metadata ? JSON.parse(storageConfig.metadata) : {}; + if (meta.skipVerification === true) { + callbacks?.onLog(`${storageConfig.name}: verification disabled - skipping`, "info"); + return []; + } + const workItems: WorkItem[] = []; const adapter = registry.get(storageConfig.adapterId) as StorageAdapter; if (!adapter) { diff --git a/src/services/jobs/job-service.ts b/src/services/jobs/job-service.ts index 21f4d1af..8badd1eb 100644 --- a/src/services/jobs/job-service.ts +++ b/src/services/jobs/job-service.ts @@ -26,6 +26,7 @@ export interface CreateJobInput { notificationEvents?: string; namingTemplateId?: string | null; schedulePresetId?: string | null; + skipVerification?: boolean; } export interface UpdateJobInput { @@ -42,6 +43,7 @@ export interface UpdateJobInput { notificationEvents?: string; namingTemplateId?: string | null; schedulePresetId?: string | null; + skipVerification?: boolean; } const jobInclude = { @@ -84,7 +86,7 @@ export class JobService { } async createJob(input: CreateJobInput) { - const { name, schedule, sourceId, databases, destinations, notificationIds, enabled, encryptionProfileId, compression, pgCompression, notificationEvents } = input; + const { name, schedule, sourceId, databases, destinations, notificationIds, enabled, encryptionProfileId, compression, pgCompression, notificationEvents, skipVerification } = input; // Check name uniqueness const existingByName = await prisma.job.findFirst({ where: { name } }); @@ -105,6 +107,7 @@ export class JobService { compression: compression || "NONE", pgCompression: pgCompression ?? "", notificationEvents: notificationEvents || "ALWAYS", + skipVerification: skipVerification ?? false, notifications: { connect: notificationIds?.map((id) => ({ id })) || [] }, @@ -126,7 +129,7 @@ export class JobService { } async updateJob(id: string, input: UpdateJobInput) { - const { name, schedule, sourceId, databases, destinations, notificationIds, enabled, encryptionProfileId, compression, pgCompression, notificationEvents, namingTemplateId } = input; + const { name, schedule, sourceId, databases, destinations, notificationIds, enabled, encryptionProfileId, compression, pgCompression, notificationEvents, namingTemplateId, skipVerification } = input; // Check name uniqueness (excluding current job) if (name) { @@ -167,6 +170,7 @@ export class JobService { namingTemplateId: namingTemplateId !== undefined ? (namingTemplateId ?? null) : undefined, schedulePresetId: input.schedulePresetId !== undefined ? (input.schedulePresetId ?? null) : undefined, encryptionProfileId: encryptionProfileId === "" ? null : encryptionProfileId, + skipVerification: skipVerification !== undefined ? skipVerification : undefined, notifications: { set: [], connect: notificationIds?.map((id) => ({ id })) || [] From ba9f8afc99de47feda1615c26d5545511284ddfc Mon Sep 17 00:00:00 2001 From: Manu Date: Fri, 12 Jun 2026 20:49:03 +0200 Subject: [PATCH 14/24] Run prisma generate in dev script Add prisma generate to the dev script so pnpm dev runs migrations, regenerates the Prisma client, then starts Next. Also update the changelog entry to note that pnpm dev now regenerates the Prisma client on startup. --- docs/changelog.md | 2 +- package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/changelog.md b/docs/changelog.md index 7d5dde0b..05f7aa4a 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -15,7 +15,7 @@ All notable changes to DBackup are documented here. ### 🎨 Improvements -- **dev**: `pnpm dev` now automatically applies pending Prisma migrations on startup via `prisma migrate deploy`. +- **dev**: `pnpm dev` now automatically applies pending Prisma migrations and regenerates the Prisma client on startup. - **storage**: Storage Explorer cache uses surgical updates - create, delete, lock, and verify each patch only the affected cache entry instead of invalidating the full cache. - **storage**: A new "Pre-warm Storage Cache" system task (enabled by default, hourly) reconciles caches against remote storage and pre-populates the cache for adapters not yet visited. - **storage**: Stale cache entries trigger a background reconciliation via `adapter.list()` to detect files deleted outside DBackup without re-reading sidecars. diff --git a/package.json b/package.json index 08149ef2..7c55c9a6 100644 --- a/package.json +++ b/package.json @@ -3,7 +3,7 @@ "version": "2.6.0", "private": true, "scripts": { - "dev": "prisma migrate deploy && next dev", + "dev": "prisma migrate deploy && prisma generate && next dev", "build": "next build", "start": "next start", "lint": "eslint", From 9eca4be64ac82de401252219f8477650613a8d59 Mon Sep 17 00:00:00 2001 From: Manu Date: Sun, 14 Jun 2026 11:24:33 +0200 Subject: [PATCH 15/24] Validate OAuth tokens & scan improvements Add live token validation and re-authorize UI for OAuth adapters, fix a Google Drive path bug, and improve integrity scanning. - Add POST validation endpoints: /api/adapters/{dropbox,google-drive,onedrive}/validate-token to verify stored refresh tokens and return a friendly expiry message when needed. - Update Dropbox, Google Drive and OneDrive OAuth buttons to perform live validation on Connection tab open, show a loading state while checking, display an expiry alert with a Re-authorize button when expired, and otherwise keep the green authorized state. Import AlertTriangle and useEffect for these flows; network failures fail-open to avoid blocking UX. - Fix Google Drive adapter bug where resolveOrCreatePath used the wrong rootFolderId argument when navigating into a subdir. - Revamp integrity-service: deduplicate destinations across jobs, list each destination once, filter listed files by job prefix, apply max-age and max-file-size filters per-file, and build work items without duplicate entries. Improved logging and error handling for adapter/config resolution and listing failures. - Update changelog to document the bug fix: OAuth token expiry now shows a re-authorize option instead of a misleading authorized state. These changes make OAuth status more accurate for users and make integrity scans more efficient and correct. --- docs/changelog.md | 4 + .../adapters/dropbox/validate-token/route.ts | 58 ++++++++++++ .../google-drive/validate-token/route.ts | 57 ++++++++++++ .../adapters/onedrive/validate-token/route.ts | 67 ++++++++++++++ .../adapter/dropbox-oauth-button.tsx | 55 ++++++++++-- .../adapter/google-drive-oauth-button.tsx | 53 ++++++++++- .../adapter/onedrive-oauth-button.tsx | 55 ++++++++++-- src/lib/adapters/storage/google-drive.ts | 2 +- src/services/backup/integrity-service.ts | 89 ++++++++++++------- 9 files changed, 395 insertions(+), 45 deletions(-) create mode 100644 src/app/api/adapters/dropbox/validate-token/route.ts create mode 100644 src/app/api/adapters/google-drive/validate-token/route.ts create mode 100644 src/app/api/adapters/onedrive/validate-token/route.ts diff --git a/docs/changelog.md b/docs/changelog.md index 05f7aa4a..0ef49a8a 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -13,6 +13,10 @@ All notable changes to DBackup are documented here. - **integrity**: Scheduled integrity checks support two scan modes - **Jobs** (default, only verifies files linked to backup jobs) and **All Files** (full storage scan) - plus configurable filters (skip already-passed, max age, max file size). Jobs and storage destinations can individually opt out via a "Skip Verification" toggle. - **storage**: Storage Explorer file listings are now cached in SQLite for instant repeat visits. Cache is invalidated on backup create, delete, verify, and lock changes. +### 🐛 Bug Fixes + +- **storage**: Google Drive, OneDrive, and Dropbox now live-validate their stored OAuth token when the Connection tab is opened, showing an expiry warning with a Re-authorize button instead of a misleading green "authorized" status. + ### 🎨 Improvements - **dev**: `pnpm dev` now automatically applies pending Prisma migrations and regenerates the Prisma client on startup. diff --git a/src/app/api/adapters/dropbox/validate-token/route.ts b/src/app/api/adapters/dropbox/validate-token/route.ts new file mode 100644 index 00000000..d1553060 --- /dev/null +++ b/src/app/api/adapters/dropbox/validate-token/route.ts @@ -0,0 +1,58 @@ +import { NextRequest, NextResponse } from "next/server"; +import { headers } from "next/headers"; +import { getAuthContext } from "@/lib/auth/access-control"; +import { getDecryptedCredentialData } from "@/services/auth/credential-service"; +import type { OAuthData } from "@/lib/core/credentials"; +import { Dropbox } from "dropbox"; +import { logger } from "@/lib/logging/logger"; + +const log = logger.child({ route: "adapters/dropbox/validate-token" }); + +/** + * POST /api/adapters/dropbox/validate-token + * Validates whether the stored refresh token for an OAuth credential profile is still accepted by Dropbox. + * Body: { credentialId: string } + * Response: { valid: boolean, message?: string } + */ +export async function POST(req: NextRequest) { + const ctx = await getAuthContext(await headers()); + if (!ctx) { + return NextResponse.json({ error: "Unauthorized" }, { status: 401 }); + } + + try { + const { credentialId } = await req.json(); + if (!credentialId) { + return NextResponse.json({ valid: false, message: "Missing credentialId" }, { status: 400 }); + } + + const profile = (await getDecryptedCredentialData(credentialId, "OAUTH")) as OAuthData; + + if (!profile.clientId || !profile.clientSecret || !profile.refreshToken) { + return NextResponse.json({ valid: false, message: "Incomplete credentials" }); + } + + const dbx = new Dropbox({ + clientId: profile.clientId, + clientSecret: profile.clientSecret, + refreshToken: profile.refreshToken, + }); + + await dbx.usersGetCurrentAccount(); + + return NextResponse.json({ valid: true }); + } catch (error: unknown) { + const message = error instanceof Error ? error.message : String(error); + const isExpired = + message.includes("invalid_access_token") || + message.includes("expired_access_token") || + message.includes("invalid_grant"); + log.warn("Dropbox token validation failed", { message }); + return NextResponse.json({ + valid: false, + message: isExpired + ? "Authorization expired. Please re-authorize with Dropbox." + : message, + }); + } +} diff --git a/src/app/api/adapters/google-drive/validate-token/route.ts b/src/app/api/adapters/google-drive/validate-token/route.ts new file mode 100644 index 00000000..33b4767d --- /dev/null +++ b/src/app/api/adapters/google-drive/validate-token/route.ts @@ -0,0 +1,57 @@ +import { NextRequest, NextResponse } from "next/server"; +import { google } from "googleapis"; +import { headers } from "next/headers"; +import { getAuthContext } from "@/lib/auth/access-control"; +import { getDecryptedCredentialData } from "@/services/auth/credential-service"; +import type { OAuthData } from "@/lib/core/credentials"; +import { logger } from "@/lib/logging/logger"; + +const log = logger.child({ route: "adapters/google-drive/validate-token" }); + +/** + * POST /api/adapters/google-drive/validate-token + * Validates whether the stored refresh token for an OAuth credential profile is still accepted by Google. + * Body: { credentialId: string } + * Response: { valid: boolean, message?: string } + */ +export async function POST(req: NextRequest) { + const ctx = await getAuthContext(await headers()); + if (!ctx) { + return NextResponse.json({ error: "Unauthorized" }, { status: 401 }); + } + + try { + const { credentialId } = await req.json(); + if (!credentialId) { + return NextResponse.json({ valid: false, message: "Missing credentialId" }, { status: 400 }); + } + + const profile = (await getDecryptedCredentialData(credentialId, "OAUTH")) as OAuthData; + + if (!profile.clientId || !profile.clientSecret || !profile.refreshToken) { + return NextResponse.json({ valid: false, message: "Incomplete credentials" }); + } + + const origin = process.env.BETTER_AUTH_URL || req.nextUrl.origin; + const redirectUri = `${origin}/api/adapters/google-drive/callback`; + + const oauth2Client = new google.auth.OAuth2( + profile.clientId, + profile.clientSecret, + redirectUri + ); + oauth2Client.setCredentials({ refresh_token: profile.refreshToken }); + + await oauth2Client.getAccessToken(); + + return NextResponse.json({ valid: true }); + } catch (error: unknown) { + const message = error instanceof Error ? error.message : String(error); + const isExpired = message.includes("invalid_grant") || message.includes("Token has been expired"); + log.warn("Google Drive token validation failed", { message }); + return NextResponse.json({ + valid: false, + message: isExpired ? "Authorization expired. Please re-authorize with Google." : message, + }); + } +} diff --git a/src/app/api/adapters/onedrive/validate-token/route.ts b/src/app/api/adapters/onedrive/validate-token/route.ts new file mode 100644 index 00000000..843eaaf3 --- /dev/null +++ b/src/app/api/adapters/onedrive/validate-token/route.ts @@ -0,0 +1,67 @@ +import { NextRequest, NextResponse } from "next/server"; +import { headers } from "next/headers"; +import { getAuthContext } from "@/lib/auth/access-control"; +import { getDecryptedCredentialData } from "@/services/auth/credential-service"; +import type { OAuthData } from "@/lib/core/credentials"; +import { logger } from "@/lib/logging/logger"; + +const log = logger.child({ route: "adapters/onedrive/validate-token" }); +const TOKEN_URL = "https://login.microsoftonline.com/common/oauth2/v2.0/token"; + +/** + * POST /api/adapters/onedrive/validate-token + * Validates whether the stored refresh token for an OAuth credential profile is still accepted by Microsoft. + * Body: { credentialId: string } + * Response: { valid: boolean, message?: string } + */ +export async function POST(req: NextRequest) { + const ctx = await getAuthContext(await headers()); + if (!ctx) { + return NextResponse.json({ error: "Unauthorized" }, { status: 401 }); + } + + try { + const { credentialId } = await req.json(); + if (!credentialId) { + return NextResponse.json({ valid: false, message: "Missing credentialId" }, { status: 400 }); + } + + const profile = (await getDecryptedCredentialData(credentialId, "OAUTH")) as OAuthData; + + if (!profile.clientId || !profile.clientSecret || !profile.refreshToken) { + return NextResponse.json({ valid: false, message: "Incomplete credentials" }); + } + + const body = new URLSearchParams({ + client_id: profile.clientId, + client_secret: profile.clientSecret, + refresh_token: profile.refreshToken, + grant_type: "refresh_token", + scope: "Files.ReadWrite.All offline_access", + }); + + const res = await fetch(TOKEN_URL, { + method: "POST", + headers: { "Content-Type": "application/x-www-form-urlencoded" }, + body, + }); + + if (!res.ok) { + const data = await res.json().catch(() => ({})); + const isExpired = data.error === "invalid_grant" || data.error === "interaction_required"; + log.warn("OneDrive token validation failed", { error: data.error }); + return NextResponse.json({ + valid: false, + message: isExpired + ? "Authorization expired. Please re-authorize with Microsoft." + : `Token refresh failed: ${data.error_description ?? res.statusText}`, + }); + } + + return NextResponse.json({ valid: true }); + } catch (error: unknown) { + const message = error instanceof Error ? error.message : String(error); + log.warn("OneDrive token validation error", { message }); + return NextResponse.json({ valid: false, message }); + } +} diff --git a/src/components/adapter/dropbox-oauth-button.tsx b/src/components/adapter/dropbox-oauth-button.tsx index 811aafd7..d37fd9fb 100644 --- a/src/components/adapter/dropbox-oauth-button.tsx +++ b/src/components/adapter/dropbox-oauth-button.tsx @@ -1,8 +1,8 @@ "use client"; -import { useState } from "react"; +import { useState, useEffect } from "react"; import { Button } from "@/components/ui/button"; -import { ExternalLink, CheckCircle2, AlertCircle, Loader2 } from "lucide-react"; +import { ExternalLink, CheckCircle2, AlertCircle, Loader2, AlertTriangle } from "lucide-react"; import { toast } from "sonner"; import { Alert, AlertDescription } from "@/components/ui/alert"; @@ -21,6 +21,23 @@ interface DropboxOAuthButtonProps { */ export function DropboxOAuthButton({ credentialId, authorized, onAuthorized }: DropboxOAuthButtonProps) { const [isLoading, setIsLoading] = useState(false); + const [tokenState, setTokenState] = useState<"checking" | "valid" | "expired" | null>(null); + + useEffect(() => { + if (!authorized || !credentialId) { + setTokenState(null); + return; + } + setTokenState("checking"); + fetch("/api/adapters/dropbox/validate-token", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ credentialId }), + }) + .then((r) => r.json()) + .then((data) => setTokenState(data.valid ? "valid" : "expired")) + .catch(() => setTokenState("valid")); // Fail open - don't block UX on network error + }, [authorized, credentialId]); if (!credentialId) { return ( @@ -33,7 +50,37 @@ export function DropboxOAuthButton({ credentialId, authorized, onAuthorized }: D ); } - if (authorized) { + if (authorized && tokenState === "checking") { + return ( + + + Verifying Dropbox authorization... + + ); + } + + if (authorized && tokenState === "expired") { + return ( + + + + Authorization expired. Please re-authorize with Dropbox. + + + + ); + } + + if (authorized && (tokenState === "valid" || tokenState === null)) { return ( @@ -73,7 +120,6 @@ export function DropboxOAuthButton({ credentialId, authorized, onAuthorized }: D ); if (!popup) { - // Popup blocked - fall back to full navigation. window.location.href = data.data.authUrl; return; } @@ -94,7 +140,6 @@ export function DropboxOAuthButton({ credentialId, authorized, onAuthorized }: D window.addEventListener("message", handleMessage); - // Fallback: detect when the popup is closed without a message. const pollClosed = setInterval(() => { if (popup.closed) { clearInterval(pollClosed); diff --git a/src/components/adapter/google-drive-oauth-button.tsx b/src/components/adapter/google-drive-oauth-button.tsx index 1419d828..3843927f 100644 --- a/src/components/adapter/google-drive-oauth-button.tsx +++ b/src/components/adapter/google-drive-oauth-button.tsx @@ -1,8 +1,8 @@ "use client"; -import { useState } from "react"; +import { useState, useEffect } from "react"; import { Button } from "@/components/ui/button"; -import { ExternalLink, CheckCircle2, AlertCircle, Loader2 } from "lucide-react"; +import { ExternalLink, CheckCircle2, AlertCircle, Loader2, AlertTriangle } from "lucide-react"; import { toast } from "sonner"; import { Alert, AlertDescription } from "@/components/ui/alert"; @@ -21,6 +21,23 @@ interface GoogleDriveOAuthButtonProps { */ export function GoogleDriveOAuthButton({ credentialId, authorized, onAuthorized }: GoogleDriveOAuthButtonProps) { const [isLoading, setIsLoading] = useState(false); + const [tokenState, setTokenState] = useState<"checking" | "valid" | "expired" | null>(null); + + useEffect(() => { + if (!authorized || !credentialId) { + setTokenState(null); + return; + } + setTokenState("checking"); + fetch("/api/adapters/google-drive/validate-token", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ credentialId }), + }) + .then((r) => r.json()) + .then((data) => setTokenState(data.valid ? "valid" : "expired")) + .catch(() => setTokenState("valid")); // Fail open - don't block UX on network error + }, [authorized, credentialId]); if (!credentialId) { return ( @@ -33,7 +50,37 @@ export function GoogleDriveOAuthButton({ credentialId, authorized, onAuthorized ); } - if (authorized) { + if (authorized && tokenState === "checking") { + return ( + + + Verifying Google Drive authorization... + + ); + } + + if (authorized && tokenState === "expired") { + return ( + + + + Authorization expired. Please re-authorize with Google. + + + + ); + } + + if (authorized && (tokenState === "valid" || tokenState === null)) { return ( diff --git a/src/components/adapter/onedrive-oauth-button.tsx b/src/components/adapter/onedrive-oauth-button.tsx index da6b39ca..5e2aee43 100644 --- a/src/components/adapter/onedrive-oauth-button.tsx +++ b/src/components/adapter/onedrive-oauth-button.tsx @@ -1,8 +1,8 @@ "use client"; -import { useState } from "react"; +import { useState, useEffect } from "react"; import { Button } from "@/components/ui/button"; -import { ExternalLink, CheckCircle2, AlertCircle, Loader2 } from "lucide-react"; +import { ExternalLink, CheckCircle2, AlertCircle, Loader2, AlertTriangle } from "lucide-react"; import { toast } from "sonner"; import { Alert, AlertDescription } from "@/components/ui/alert"; @@ -21,6 +21,23 @@ interface OneDriveOAuthButtonProps { */ export function OneDriveOAuthButton({ credentialId, authorized, onAuthorized }: OneDriveOAuthButtonProps) { const [isLoading, setIsLoading] = useState(false); + const [tokenState, setTokenState] = useState<"checking" | "valid" | "expired" | null>(null); + + useEffect(() => { + if (!authorized || !credentialId) { + setTokenState(null); + return; + } + setTokenState("checking"); + fetch("/api/adapters/onedrive/validate-token", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ credentialId }), + }) + .then((r) => r.json()) + .then((data) => setTokenState(data.valid ? "valid" : "expired")) + .catch(() => setTokenState("valid")); // Fail open - don't block UX on network error + }, [authorized, credentialId]); if (!credentialId) { return ( @@ -33,7 +50,37 @@ export function OneDriveOAuthButton({ credentialId, authorized, onAuthorized }: ); } - if (authorized) { + if (authorized && tokenState === "checking") { + return ( + + + Verifying OneDrive authorization... + + ); + } + + if (authorized && tokenState === "expired") { + return ( + + + + Authorization expired. Please re-authorize with Microsoft. + + + + ); + } + + if (authorized && (tokenState === "valid" || tokenState === null)) { return ( @@ -73,7 +120,6 @@ export function OneDriveOAuthButton({ credentialId, authorized, onAuthorized }: ); if (!popup) { - // Popup blocked - fall back to full navigation. window.location.href = data.data.authUrl; return; } @@ -94,7 +140,6 @@ export function OneDriveOAuthButton({ credentialId, authorized, onAuthorized }: window.addEventListener("message", handleMessage); - // Fallback: detect when the popup is closed without a message. const pollClosed = setInterval(() => { if (popup.closed) { clearInterval(pollClosed); diff --git a/src/lib/adapters/storage/google-drive.ts b/src/lib/adapters/storage/google-drive.ts index cb111c45..8150dbae 100644 --- a/src/lib/adapters/storage/google-drive.ts +++ b/src/lib/adapters/storage/google-drive.ts @@ -313,7 +313,7 @@ export const GoogleDriveAdapter: StorageAdapter = { // Navigate to subdir if specified if (dir && dir !== ".") { - rootFolderId = await resolveOrCreatePath(drive, config.folderId, dir + "/dummy"); + rootFolderId = await resolveOrCreatePath(drive, rootFolderId, dir + "/dummy"); } return await listFilesRecursive(drive, rootFolderId, dir || ""); diff --git a/src/services/backup/integrity-service.ts b/src/services/backup/integrity-service.ts index 9e8d46bb..cf8bd4bf 100644 --- a/src/services/backup/integrity-service.ts +++ b/src/services/backup/integrity-service.ts @@ -214,47 +214,74 @@ export class IntegrityService { callbacks?.onLog(`Found ${jobs.length} job${jobs.length !== 1 ? "s" : ""} to scan`); - const cutoffDate = filters.maxAgeDays > 0 - ? new Date(Date.now() - filters.maxAgeDays * 86_400_000) - : null; - const seen = new Set(); const workItems: WorkItem[] = []; + // Collect all unique destinations across eligible jobs, then list each once + const destJobMap = new Map(); for (const job of jobs) { - const executions = await prisma.execution.findMany({ - where: { - jobId: job.id, - type: "Backup", - status: "Completed", - path: { not: null }, - ...(cutoffDate ? { endedAt: { gte: cutoffDate } } : {}), - }, - select: { path: true, size: true }, - }); - for (const dest of job.destinations) { - const meta = dest.config.metadata ? JSON.parse(dest.config.metadata) : {}; - if (meta.skipVerification === true) continue; + const destMeta = dest.config.metadata ? JSON.parse(dest.config.metadata) : {}; + if (destMeta.skipVerification === true) continue; + const existing = destJobMap.get(dest.configId); + if (existing) { + existing.jobNames.push(job.name); + } else { + destJobMap.set(dest.configId, { dest, jobNames: [job.name] }); + } + } + } + + for (const { dest, jobNames } of destJobMap.values()) { + const adapter = registry.get(dest.config.adapterId) as StorageAdapter; + if (!adapter) { + callbacks?.onLog(`${dest.config.name}: adapter '${dest.config.adapterId}' not found`, "error"); + continue; + } + + let config: Awaited>; + try { + config = await resolveAdapterConfig(dest.config); + } catch (e: unknown) { + const msg = e instanceof Error ? e.message : String(e); + callbacks?.onLog(`${dest.config.name}: config resolution failed - ${msg}`, "error"); + continue; + } - for (const exec of executions) { - if (!exec.path) continue; + let allFiles: Awaited> = []; + try { + allFiles = await adapter.list(config, ""); + } catch (e: unknown) { + const msg = e instanceof Error ? e.message : String(e); + log.warn("Could not list destination", { destination: dest.config.name }, wrapError(e)); + callbacks?.onLog(`${dest.config.name}: listing failed - ${msg}`, "error"); + continue; + } - if (filters.maxFileSizeBytes > 0 && exec.size !== null && exec.size !== undefined) { - if (exec.size > filters.maxFileSizeBytes) continue; - } + // Keep only files that belong to one of the eligible jobs (path starts with jobName/) + const backupFiles = allFiles.filter( + (f) => !f.name.endsWith(".meta.json") && jobNames.some((n) => f.path.startsWith(n + "/")) + ); - const key = `${dest.configId}:${exec.path}`; - if (seen.has(key)) continue; - seen.add(key); - workItems.push({ - storageConfigId: dest.configId, - destinationName: dest.config.name, - remotePath: exec.path, - fileName: path.basename(exec.path), - }); + for (const file of backupFiles) { + if (filters.maxAgeDays > 0 && file.lastModified) { + const ageDays = (Date.now() - new Date(file.lastModified).getTime()) / 86_400_000; + if (ageDays > filters.maxAgeDays) continue; } + + if (filters.maxFileSizeBytes > 0 && file.size > filters.maxFileSizeBytes) continue; + + const key = `${dest.configId}:${file.path}`; + if (seen.has(key)) continue; + seen.add(key); + + workItems.push({ + storageConfigId: dest.configId, + destinationName: dest.config.name, + remotePath: file.path, + fileName: file.name, + }); } } From 36b480126b1944d361317155280fb65fa5b52585 Mon Sep 17 00:00:00 2001 From: Manu Date: Sun, 14 Jun 2026 11:35:50 +0200 Subject: [PATCH 16/24] Log skipped jobs and per-job file counts Stop filtering skipVerification at the DB query and instead fetch all enabled jobs so we can report which jobs have verification disabled and skip them explicitly. Introduces eligibleJobs/skippedJobs separation and logs skipped jobs. Tracks per-job file counts (jobCounts) while building work items and logs counts per job. Also restricts destination/job collection to eligible jobs and performs minor cleanup. --- src/services/backup/integrity-service.ts | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/src/services/backup/integrity-service.ts b/src/services/backup/integrity-service.ts index cf8bd4bf..69fa2a43 100644 --- a/src/services/backup/integrity-service.ts +++ b/src/services/backup/integrity-service.ts @@ -204,7 +204,7 @@ export class IntegrityService { callbacks?: IntegrityProgressCallbacks ): Promise { const jobs = await prisma.job.findMany({ - where: { enabled: true, skipVerification: false }, + where: { enabled: true }, include: { destinations: { include: { config: true }, @@ -214,12 +214,23 @@ export class IntegrityService { callbacks?.onLog(`Found ${jobs.length} job${jobs.length !== 1 ? "s" : ""} to scan`); + const eligibleJobs = jobs.filter((j) => !j.skipVerification); + const skippedJobs = jobs.filter((j) => j.skipVerification); + + for (const job of skippedJobs) { + callbacks?.onLog(`${job.name}: verification disabled - skipping`, "info"); + } + const seen = new Set(); const workItems: WorkItem[] = []; + // Track per-job file counts for logging + const jobCounts = new Map(); + for (const job of eligibleJobs) jobCounts.set(job.name, 0); + // Collect all unique destinations across eligible jobs, then list each once const destJobMap = new Map(); - for (const job of jobs) { + for (const job of eligibleJobs) { for (const dest of job.destinations) { const destMeta = dest.config.metadata ? JSON.parse(dest.config.metadata) : {}; if (destMeta.skipVerification === true) continue; @@ -263,7 +274,6 @@ export class IntegrityService { (f) => !f.name.endsWith(".meta.json") && jobNames.some((n) => f.path.startsWith(n + "/")) ); - for (const file of backupFiles) { if (filters.maxAgeDays > 0 && file.lastModified) { const ageDays = (Date.now() - new Date(file.lastModified).getTime()) / 86_400_000; @@ -282,10 +292,15 @@ export class IntegrityService { remotePath: file.path, fileName: file.name, }); + + const matchingJob = jobNames.find((n) => file.path.startsWith(n + "/")); + if (matchingJob) jobCounts.set(matchingJob, (jobCounts.get(matchingJob) ?? 0) + 1); } } - callbacks?.onLog(`Jobs scan: found ${workItems.length} file${workItems.length !== 1 ? "s" : ""} to verify`); + for (const [jobName, count] of jobCounts.entries()) { + callbacks?.onLog(`${jobName}: found ${count} file${count !== 1 ? "s" : ""} to verify`); + } return workItems; } From 3815e0465d286a066e5b474bc2918788ccfc6155 Mon Sep 17 00:00:00 2001 From: Manu Date: Sun, 14 Jun 2026 11:55:16 +0200 Subject: [PATCH 17/24] Add log export (copy/download) to History modal Enable copying and downloading execution logs from the History dialog. Updated docs/changelog.md and modified the history page to add Copy and Download buttons (hidden for Running/Pending logs) and handlers that sanitize and format logs before export. Added src/lib/logs/sanitize.ts to redact IPs, connection strings and keys from SENSITIVE_KEYS, and src/lib/logs/format.ts to produce a human-readable .log text (metadata header, UTC timestamps, stage grouping, durations, details) and to generate a filename from job name and start time. Export uses navigator.clipboard for copy and a Blob/URL download for .log files; handlers rely on existing parseLogs. --- docs/changelog.md | 1 + src/app/dashboard/history/page.tsx | 86 +++++++++++++++++++---- src/lib/logs/format.ts | 108 +++++++++++++++++++++++++++++ src/lib/logs/sanitize.ts | 38 ++++++++++ 4 files changed, 220 insertions(+), 13 deletions(-) create mode 100644 src/lib/logs/format.ts create mode 100644 src/lib/logs/sanitize.ts diff --git a/docs/changelog.md b/docs/changelog.md index 0ef49a8a..b81b7617 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -12,6 +12,7 @@ All notable changes to DBackup are documented here. - **integrity**: Post-upload verification is now opt-in for all storage destinations via system settings (local filesystem always verifies). - **integrity**: Scheduled integrity checks support two scan modes - **Jobs** (default, only verifies files linked to backup jobs) and **All Files** (full storage scan) - plus configurable filters (skip already-passed, max age, max file size). Jobs and storage destinations can individually opt out via a "Skip Verification" toggle. - **storage**: Storage Explorer file listings are now cached in SQLite for instant repeat visits. Cache is invalidated on backup create, delete, verify, and lock changes. +- **history**: Execution logs can now be copied to the clipboard or downloaded as a `.log` file directly from the log dialog. Sensitive data (IPs, credentials, connection strings) is automatically redacted before export. ### 🐛 Bug Fixes diff --git a/src/app/dashboard/history/page.tsx b/src/app/dashboard/history/page.tsx index 7ebdc769..d66af9f8 100644 --- a/src/app/dashboard/history/page.tsx +++ b/src/app/dashboard/history/page.tsx @@ -13,12 +13,14 @@ import { createColumns, createSystemTaskColumns, Execution } from "./columns"; import { createNotificationLogColumns, NotificationLogRow } from "./notification-log-columns"; import { NotificationPreview } from "./notification-preview"; import { useSearchParams, useRouter } from "next/navigation"; -import { Loader2, Square } from "lucide-react"; +import { Loader2, Square, Copy, Download } from "lucide-react"; import { Progress } from "@/components/ui/progress"; import { DateDisplay } from "@/components/utils/date-display"; import { Card, CardContent, CardHeader, CardTitle, CardDescription } from "@/components/ui/card"; import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs"; import { LogViewer } from "@/components/execution/log-viewer"; +import { sanitizeLogs } from "@/lib/logs/sanitize"; +import { formatLogsAsText, generateLogFilename } from "@/lib/logs/format"; import { ScrollArea } from "@/components/ui/scroll-area"; import { Badge } from "@/components/ui/badge"; import { Button } from "@/components/ui/button"; @@ -151,6 +153,48 @@ function HistoryContent() { } }, [fetchHistory]); + const handleCopyLogs = useCallback(() => { + if (!selectedLog) return; + const logs = sanitizeLogs(parseLogs(selectedLog.logs)); + const text = formatLogsAsText(logs, { + jobName: selectedLog.job?.name ?? selectedLog.type ?? "Unknown", + type: selectedLog.type ?? "Backup", + status: selectedLog.status, + startedAt: selectedLog.startedAt, + endedAt: selectedLog.endedAt, + triggerType: selectedLog.triggerType, + triggerLabel: selectedLog.triggerLabel, + }); + navigator.clipboard.writeText(text) + .then(() => toast.success("Logs copied to clipboard")) + .catch(() => toast.error("Failed to copy logs")); + }, [selectedLog]); + + const handleDownloadLog = useCallback(() => { + if (!selectedLog) return; + const logs = sanitizeLogs(parseLogs(selectedLog.logs)); + const text = formatLogsAsText(logs, { + jobName: selectedLog.job?.name ?? selectedLog.type ?? "Unknown", + type: selectedLog.type ?? "Backup", + status: selectedLog.status, + startedAt: selectedLog.startedAt, + endedAt: selectedLog.endedAt, + triggerType: selectedLog.triggerType, + triggerLabel: selectedLog.triggerLabel, + }); + const filename = generateLogFilename( + selectedLog.job?.name ?? selectedLog.type ?? "log", + selectedLog.startedAt, + ); + const blob = new Blob([text], { type: "text/plain" }); + const url = URL.createObjectURL(blob); + const a = document.createElement("a"); + a.href = url; + a.download = filename; + a.click(); + URL.revokeObjectURL(url); + }, [selectedLog]); + const columns = useMemo(() => createColumns(setSelectedLog), []); const systemTaskColumns = useMemo(() => createSystemTaskColumns(setSelectedLog), []); const notificationColumns = useMemo( @@ -328,18 +372,34 @@ function HistoryContent() { { if(!open) setSelectedLog(null); }}> - - {selectedLog?.status === "Running" && } - {selectedLog?.job?.name || (selectedLog?.type === "IntegrityCheck" ? "Backup Integrity Check" : selectedLog?.type) || "Manual Job"} - {selectedLog?.status && ( - - {selectedLog.status} - - )} - - - {selectedLog?.startedAt && } - +
+
+ + {selectedLog?.status === "Running" && } + {selectedLog?.job?.name || (selectedLog?.type === "IntegrityCheck" ? "Backup Integrity Check" : selectedLog?.type) || "Manual Job"} + {selectedLog?.status && ( + + {selectedLog.status} + + )} + + + {selectedLog?.startedAt && } + +
+ {selectedLog?.status !== "Running" && selectedLog?.status !== "Pending" && ( +
+ + +
+ )} +
{(selectedLog?.status === "Running" || selectedLog?.status === "Pending") && ( diff --git a/src/lib/logs/format.ts b/src/lib/logs/format.ts new file mode 100644 index 00000000..7dfd8bd3 --- /dev/null +++ b/src/lib/logs/format.ts @@ -0,0 +1,108 @@ +import { LogEntry } from "@/lib/core/logs"; + +export interface ExportMeta { + jobName: string; + type: string; + status: string; + startedAt: string; + endedAt?: string | null; + triggerType?: string | null; + triggerLabel?: string | null; +} + +function formatUtcTimestamp(iso: string): string { + const d = new Date(iso); + if (isNaN(d.getTime())) return iso; + const hh = String(d.getUTCHours()).padStart(2, "0"); + const mm = String(d.getUTCMinutes()).padStart(2, "0"); + const ss = String(d.getUTCSeconds()).padStart(2, "0"); + return `${hh}:${mm}:${ss}`; +} + +function formatUtcDatetime(iso: string): string { + const d = new Date(iso); + if (isNaN(d.getTime())) return iso; + return d.toISOString().replace("T", " ").replace(/\.\d+Z$/, " UTC"); +} + +function formatDurationMs(ms: number): string { + if (ms < 1000) return `${ms}ms`; + const s = (ms / 1000).toFixed(1); + return `${s}s`; +} + +function levelPad(level: string): string { + return level.toUpperCase().padEnd(7); +} + +export function formatLogsAsText(logs: LogEntry[], meta: ExportMeta): string { + const lines: string[] = []; + + lines.push("DBackup Execution Log"); + lines.push("=".repeat(48)); + lines.push(`Job: ${meta.jobName}`); + lines.push(`Type: ${meta.type}`); + lines.push(`Status: ${meta.status}`); + lines.push(`Started: ${formatUtcDatetime(meta.startedAt)}`); + if (meta.endedAt) { + lines.push(`Ended: ${formatUtcDatetime(meta.endedAt)}`); + } + if (meta.triggerType) { + lines.push(`Trigger: ${meta.triggerType}`); + } + lines.push("=".repeat(48)); + + // Group by stage + const stageMap = new Map(); + for (const entry of logs) { + const stage = entry.stage ?? "General"; + if (!stageMap.has(stage)) stageMap.set(stage, []); + stageMap.get(stage)!.push(entry); + } + + for (const [stage, entries] of stageMap) { + lines.push(""); + + let stageHeader = `[STAGE: ${stage}]`; + if (entries.length >= 2) { + const first = new Date(entries[0].timestamp).getTime(); + const last = new Date(entries[entries.length - 1].timestamp).getTime(); + const durationMs = last - first; + if (!isNaN(durationMs) && durationMs >= 0) { + stageHeader += ` (${formatDurationMs(durationMs)})`; + } + } + lines.push(stageHeader); + + for (const entry of entries) { + const ts = formatUtcTimestamp(entry.timestamp); + const lvl = levelPad(entry.level); + const durationSuffix = entry.durationMs != null ? ` [${formatDurationMs(entry.durationMs)}]` : ""; + lines.push(`${ts} ${lvl} ${entry.message}${durationSuffix}`); + if (entry.details) { + lines.push(" [DETAILS]"); + for (const detailLine of entry.details.split("\n")) { + lines.push(` ${detailLine}`); + } + } + } + } + + lines.push(""); + return lines.join("\n"); +} + +export function generateLogFilename(jobName: string, startedAt: string): string { + const slug = jobName + .toLowerCase() + .replace(/[^a-z0-9]+/g, "-") + .replace(/^-+|-+$/g, "") + .slice(0, 40); + + const d = new Date(startedAt); + if (isNaN(d.getTime())) return `dbackup-${slug}.log`; + + const date = d.toISOString().slice(0, 10); + const hhmm = `${String(d.getUTCHours()).padStart(2, "0")}-${String(d.getUTCMinutes()).padStart(2, "0")}`; + return `dbackup-${slug}-${date}-${hhmm}.log`; +} diff --git a/src/lib/logs/sanitize.ts b/src/lib/logs/sanitize.ts new file mode 100644 index 00000000..a72f947c --- /dev/null +++ b/src/lib/logs/sanitize.ts @@ -0,0 +1,38 @@ +import { LogEntry } from "@/lib/core/logs"; +import { SENSITIVE_KEYS } from "@/lib/crypto"; + +const IPV4_PATTERN = /\b(?:\d{1,3}\.){3}\d{1,3}\b/g; +const IPV6_PATTERN = /\b(?:[0-9a-fA-F]{0,4}:){2,7}[0-9a-fA-F]{0,4}\b/g; +const CONNECTION_STRING_PATTERN = /((?:mongodb(?:\+srv)?|mysql|postgres(?:ql)?|redis|ftp|sftp):\/\/)[^@\s]+(@)/gi; + +function redactString(value: string): string { + return value + .replace(CONNECTION_STRING_PATTERN, "$1[CREDENTIALS REDACTED]$2") + .replace(IPV4_PATTERN, "[IP REDACTED]") + .replace(IPV6_PATTERN, "[IP REDACTED]"); +} + +function redactContext(ctx: Record): Record { + const result: Record = {}; + for (const [key, value] of Object.entries(ctx)) { + if (SENSITIVE_KEYS.includes(key)) { + result[key] = "[REDACTED]"; + } else if (value && typeof value === "object" && !Array.isArray(value)) { + result[key] = redactContext(value as Record); + } else if (typeof value === "string") { + result[key] = redactString(value); + } else { + result[key] = value; + } + } + return result; +} + +export function sanitizeLogs(logs: LogEntry[]): LogEntry[] { + return logs.map((entry) => ({ + ...entry, + message: redactString(entry.message), + details: entry.details ? redactString(entry.details) : entry.details, + context: entry.context ? redactContext(entry.context) : entry.context, + })); +} From a47e3dfbb5f8f53568213763bebdf7893641a8bb Mon Sep 17 00:00:00 2001 From: Manu Date: Sun, 14 Jun 2026 12:30:11 +0200 Subject: [PATCH 18/24] Detect and highlight DB version downgrades Add downgrade detection and presentation for database version history. Introduces an isDowngrade flag in notification types and recordVersionIfChanged; uses compareVersions to mark and persist downgrades. UI: explorer shows up/down arrows and distinct warning styling for downgrades. Notifications: templates now emit separate Upgrade/Downgrade titles, badges, and colors. Tests and changelog updated accordingly. --- docs/changelog.md | 1 + .../explorer/source-version-history.tsx | 35 +++++++++++-------- .../email/system-notification-template.tsx | 8 ++--- src/lib/notifications/templates.ts | 25 +++++++++++-- src/lib/notifications/types.ts | 1 + src/services/system/db-version-service.ts | 14 +++++--- src/services/system/system-task-service.ts | 1 + .../unit/lib/notifications/templates.test.ts | 6 ++-- .../unit/services/db-version-service.test.ts | 6 ++-- .../unit/services/system-task-service.test.ts | 3 ++ 10 files changed, 70 insertions(+), 30 deletions(-) diff --git a/docs/changelog.md b/docs/changelog.md index b81b7617..c8f5cd79 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -27,6 +27,7 @@ All notable changes to DBackup are documented here. - **storage**: Long backup names in the Storage Explorer are truncated with a tooltip showing the full name on hover. - **integrity**: Manual system task runs auto-redirect to the live execution history when "Auto-redirect on job start" is enabled. - **ui**: All data tables (Storage, Jobs, Sources, Destinations, Notifications) now support horizontal scrolling when columns overflow. +- **explorer**: Database version history now detects and visually distinguishes downgrades - the change log shows an orange downward arrow for downgrades vs. green upward arrow for upgrades, and downgrade notifications are sent with a distinct "Downgrade" label and warning color. ### 🧪 Tests diff --git a/src/components/dashboard/explorer/source-version-history.tsx b/src/components/dashboard/explorer/source-version-history.tsx index 82693783..c383ebf6 100644 --- a/src/components/dashboard/explorer/source-version-history.tsx +++ b/src/components/dashboard/explorer/source-version-history.tsx @@ -8,8 +8,9 @@ import { Badge } from "@/components/ui/badge"; import { ChartContainer, ChartTooltip, ChartTooltipContent } from "@/components/ui/chart"; import type { ChartConfig } from "@/components/ui/chart"; import { CartesianGrid, Line, LineChart, XAxis, YAxis } from "recharts"; -import { ArrowRight, History, AlertTriangle, Server } from "lucide-react"; +import { ArrowUp, ArrowDown, History, AlertTriangle, Server } from "lucide-react"; import { useDateFormatter } from "@/hooks/use-date-formatter"; +import { compareVersions } from "@/lib/utils"; interface VersionHistoryEntry { id: string; @@ -72,12 +73,9 @@ export function SourceVersionHistory({ sourceId, currentVersion }: SourceVersion // Each unique `newVersion` is mapped to a numeric Y position (0..n-1) // because recharts cannot plot string values on a numeric axis. const ordered = [...history].reverse(); - const versionOrder: string[] = []; - for (const entry of ordered) { - if (!versionOrder.includes(entry.newVersion)) { - versionOrder.push(entry.newVersion); - } - } + const versionSet = new Set(); + for (const entry of ordered) versionSet.add(entry.newVersion); + const versionOrder = [...versionSet].sort((a, b) => compareVersions(a, b)); const chartData = ordered.map((entry) => ({ detectedAt: entry.detectedAt, versionIndex: versionOrder.indexOf(entry.newVersion), @@ -212,13 +210,22 @@ export function SourceVersionHistory({ sourceId, currentVersion }: SourceVersion
- {entry.previousVersion ? ( - <> - {entry.previousVersion} - - {entry.newVersion} - - ) : ( + {entry.previousVersion ? (() => { + const isDowngrade = compareVersions(entry.previousVersion, entry.newVersion) > 0; + return isDowngrade ? ( + <> + {entry.previousVersion} + + {entry.newVersion} + + ) : ( + <> + {entry.previousVersion} + + {entry.newVersion} + + ); + })() : ( <> Initial: {entry.newVersion} diff --git a/src/components/email/system-notification-template.tsx b/src/components/email/system-notification-template.tsx index b3f7f5e2..693f69fe 100644 --- a/src/components/email/system-notification-template.tsx +++ b/src/components/email/system-notification-template.tsx @@ -74,14 +74,14 @@ function getStatusStyle(color: string | undefined, success: boolean, badge?: str // Color-based variations for success-type events switch (color) { case "#3b82f6": // blue – informational (login, etc.) - return { label: "Info", icon: "ℹ", accent: tokens.blue, bg: "#eff6ff", border: "#bfdbfe" }; + return { label: badge ?? "Info", icon: "ℹ", accent: tokens.blue, bg: "#eff6ff", border: "#bfdbfe" }; case "#8b5cf6": // purple – config backup - return { label: "Completed", icon: "✓", accent: tokens.purple, bg: "#f5f3ff", border: "#ddd6fe" }; + return { label: badge ?? "Completed", icon: "✓", accent: tokens.purple, bg: "#f5f3ff", border: "#ddd6fe" }; case "#f59e0b": // amber – warning - return { label: "Warning", icon: "⚠", accent: tokens.amber, bg: "#fffbeb", border: "#fde68a" }; + return { label: badge ?? "Warning", icon: "⚠", accent: tokens.amber, bg: "#fffbeb", border: "#fde68a" }; default: // green – success return { - label: "Success", + label: badge ?? "Success", icon: "✓", accent: tokens.success, bg: tokens.successBg, diff --git a/src/lib/notifications/templates.ts b/src/lib/notifications/templates.ts index 3bd3ab13..047bc426 100644 --- a/src/lib/notifications/templates.ts +++ b/src/lib/notifications/templates.ts @@ -384,9 +384,28 @@ function dbVersionChangedTemplate( data: DbVersionChangedData ): NotificationPayload { const from = data.previousVersion ?? "unknown"; + if (data.isDowngrade) { + return { + title: `Database Version Downgraded: ${data.sourceName}`, + message: `Source '${data.sourceName}' was downgraded (${from} → ${data.newVersion}).`, + fields: [ + { name: "Source", value: data.sourceName, inline: true }, + { name: "Adapter", value: data.adapterId, inline: true }, + { name: "Previous Version", value: from, inline: true }, + { name: "New Version", value: data.newVersion, inline: true }, + ...(data.edition + ? [{ name: "Edition", value: data.edition, inline: true }] + : []), + { name: "Time", value: data.timestamp, inline: true }, + ], + color: "#f59e0b", // amber - warning + success: false, + badge: "Downgrade", + }; + } return { - title: `Database Version Changed: ${data.sourceName}`, - message: `Source '${data.sourceName}' reported a new engine version (${from} → ${data.newVersion}).`, + title: `Database Version Upgraded: ${data.sourceName}`, + message: `Source '${data.sourceName}' was upgraded (${from} → ${data.newVersion}).`, fields: [ { name: "Source", value: data.sourceName, inline: true }, { name: "Adapter", value: data.adapterId, inline: true }, @@ -399,7 +418,7 @@ function dbVersionChangedTemplate( ], color: "#3b82f6", // blue success: true, - badge: "Version", + badge: "Upgrade", }; } diff --git a/src/lib/notifications/types.ts b/src/lib/notifications/types.ts index f3699665..c56af1f9 100644 --- a/src/lib/notifications/types.ts +++ b/src/lib/notifications/types.ts @@ -181,6 +181,7 @@ export interface DbVersionChangedData { newVersion: string; edition?: string; timestamp: string; + isDowngrade: boolean; } /** Union of all event data types for type-safe template dispatch */ diff --git a/src/services/system/db-version-service.ts b/src/services/system/db-version-service.ts index 7085790b..0389ebb0 100644 --- a/src/services/system/db-version-service.ts +++ b/src/services/system/db-version-service.ts @@ -1,6 +1,7 @@ import prisma from "@/lib/prisma"; import { logger } from "@/lib/logging/logger"; import { wrapError } from "@/lib/logging/errors"; +import { compareVersions } from "@/lib/utils"; const log = logger.child({ service: "DbVersionService" }); @@ -11,6 +12,8 @@ export interface RecordVersionResult { previousVersion: string | null; /** The version stored as `newVersion` on the latest entry. */ newVersion: string; + /** True when the new version is numerically lower than the previous version. */ + isDowngrade: boolean; } /** @@ -38,7 +41,7 @@ export async function recordVersionIfChanged( ): Promise { const normalized = normalize(version); if (!normalized) { - return { changed: false, previousVersion: null, newVersion: version }; + return { changed: false, previousVersion: null, newVersion: version, isDowngrade: false }; } const latest = await prisma.dbVersionHistory.findFirst({ @@ -52,9 +55,11 @@ export async function recordVersionIfChanged( // No change when version (and edition, if present) match the last entry. if (latest && previousVersion === normalized && previousEdition === normalizedEdition) { - return { changed: false, previousVersion, newVersion: normalized }; + return { changed: false, previousVersion, newVersion: normalized, isDowngrade: false }; } + const isDowngrade = previousVersion !== null && compareVersions(previousVersion, normalized) > 0; + try { await prisma.dbVersionHistory.create({ data: { @@ -68,13 +73,14 @@ export async function recordVersionIfChanged( adapterConfigId, previousVersion, newVersion: normalized, + isDowngrade, }); } catch (e: unknown) { log.error("Failed to record version history", { adapterConfigId }, wrapError(e)); - return { changed: false, previousVersion, newVersion: normalized }; + return { changed: false, previousVersion, newVersion: normalized, isDowngrade: false }; } - return { changed: true, previousVersion, newVersion: normalized }; + return { changed: true, previousVersion, newVersion: normalized, isDowngrade }; } /** diff --git a/src/services/system/system-task-service.ts b/src/services/system/system-task-service.ts index b743e1b7..f5c5936b 100644 --- a/src/services/system/system-task-service.ts +++ b/src/services/system/system-task-service.ts @@ -535,6 +535,7 @@ export class SystemTaskService { newVersion: change.newVersion, edition, timestamp: new Date().toISOString(), + isDowngrade: change.isDowngrade, }, }); } diff --git a/tests/unit/lib/notifications/templates.test.ts b/tests/unit/lib/notifications/templates.test.ts index 1817856f..38d172f2 100644 --- a/tests/unit/lib/notifications/templates.test.ts +++ b/tests/unit/lib/notifications/templates.test.ts @@ -655,16 +655,17 @@ describe("Notification Templates", () => { newVersion: "15.0.4360.2", edition: "Enterprise Edition", timestamp: "2026-05-31T10:00:00Z", + isDowngrade: false, }, }); - expect(payload.title).toBe("Database Version Changed: prod-mssql"); + expect(payload.title).toBe("Database Version Upgraded: prod-mssql"); expect(payload.message).toContain("prod-mssql"); expect(payload.message).toContain("15.0.4280.7"); expect(payload.message).toContain("15.0.4360.2"); expect(payload.success).toBe(true); expect(payload.color).toBe("#3b82f6"); - expect(payload.badge).toBe("Version"); + expect(payload.badge).toBe("Upgrade"); const fieldNames = payload.fields?.map((f) => f.name) ?? []; expect(fieldNames).toEqual( @@ -689,6 +690,7 @@ describe("Notification Templates", () => { previousVersion: null, newVersion: "8.0.31", timestamp: "2026-05-31T10:00:00Z", + isDowngrade: false, }, }); diff --git a/tests/unit/services/db-version-service.test.ts b/tests/unit/services/db-version-service.test.ts index f4ba291f..e8914628 100644 --- a/tests/unit/services/db-version-service.test.ts +++ b/tests/unit/services/db-version-service.test.ts @@ -43,7 +43,7 @@ describe("db-version-service", () => { const result = await recordVersionIfChanged(SOURCE_ID, "8.0.36"); - expect(result).toEqual({ changed: true, previousVersion: null, newVersion: "8.0.36" }); + expect(result).toEqual({ changed: true, previousVersion: null, newVersion: "8.0.36", isDowngrade: false }); expect(prismaMock.dbVersionHistory.create).toHaveBeenCalledWith({ data: { adapterConfigId: SOURCE_ID, @@ -59,7 +59,7 @@ describe("db-version-service", () => { const result = await recordVersionIfChanged(SOURCE_ID, "8.0.36"); - expect(result).toEqual({ changed: false, previousVersion: "8.0.36", newVersion: "8.0.36" }); + expect(result).toEqual({ changed: false, previousVersion: "8.0.36", newVersion: "8.0.36", isDowngrade: false }); expect(prismaMock.dbVersionHistory.create).not.toHaveBeenCalled(); }); @@ -69,7 +69,7 @@ describe("db-version-service", () => { const result = await recordVersionIfChanged(SOURCE_ID, "8.0.37"); - expect(result).toEqual({ changed: true, previousVersion: "8.0.36", newVersion: "8.0.37" }); + expect(result).toEqual({ changed: true, previousVersion: "8.0.36", newVersion: "8.0.37", isDowngrade: false }); expect(prismaMock.dbVersionHistory.create).toHaveBeenCalledWith({ data: { adapterConfigId: SOURCE_ID, diff --git a/tests/unit/services/system-task-service.test.ts b/tests/unit/services/system-task-service.test.ts index a7ccd3b4..67b5eae0 100644 --- a/tests/unit/services/system-task-service.test.ts +++ b/tests/unit/services/system-task-service.test.ts @@ -651,6 +651,7 @@ describe('SystemTaskService', () => { changed: true, previousVersion: '15.0.4280.7', newVersion: '15.0.4360.2', + isDowngrade: false, }); await service.runTask(SYSTEM_TASKS.UPDATE_DB_VERSIONS); @@ -688,6 +689,7 @@ describe('SystemTaskService', () => { changed: true, previousVersion: null, newVersion: '8.0.31', + isDowngrade: false, }); await service.runTask(SYSTEM_TASKS.UPDATE_DB_VERSIONS); @@ -715,6 +717,7 @@ describe('SystemTaskService', () => { changed: false, previousVersion: '16.2', newVersion: '16.2', + isDowngrade: false, }); await service.runTask(SYSTEM_TASKS.UPDATE_DB_VERSIONS); From 163c59de885636bc871b8a8caac0a2302fe11fb4 Mon Sep 17 00:00:00 2001 From: Manu Date: Sun, 14 Jun 2026 13:46:39 +0200 Subject: [PATCH 19/24] Add unit tests for logs, verification, and runner Add comprehensive unit tests covering log formatting and sanitization, the SystemTaskRunner lifecycle, and the VerificationService (native and download-based checksum paths). Extend IntegrityService tests to cover jobs-mode scanning and maxAge filtering, and fix JobService test payload to include skipVerification. Update docs/changelog.md to mention the new/updated tests. --- docs/changelog.md | 2 + tests/unit/lib/logs/format.test.ts | 161 ++++++++ tests/unit/lib/logs/sanitize.test.ts | 115 ++++++ .../lib/runner/system-task-runner.test.ts | 278 +++++++++++++ tests/unit/services/integrity-service.test.ts | 204 +++++++++- tests/unit/services/job-service.test.ts | 1 + .../services/verification-service.test.ts | 378 ++++++++++++++++++ 7 files changed, 1138 insertions(+), 1 deletion(-) create mode 100644 tests/unit/lib/logs/format.test.ts create mode 100644 tests/unit/lib/logs/sanitize.test.ts create mode 100644 tests/unit/lib/runner/system-task-runner.test.ts create mode 100644 tests/unit/services/verification-service.test.ts diff --git a/docs/changelog.md b/docs/changelog.md index c8f5cd79..801cbcae 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -32,6 +32,8 @@ All notable changes to DBackup are documented here. ### 🧪 Tests - **tests**: Updated unit tests for integrity service, upload step, storage service, and system task service to match the refactored verification interfaces. +- **tests**: Fixed integrity service tests to use destinations scan mode explicitly and added `skipVerification` field to job service create test. +- **tests**: Added unit tests for `VerificationService` (all verification code paths), `SystemTaskRunner` (execution lifecycle), `logs/sanitize` (credential and IP redaction), and `logs/format` (log text export). Extended `IntegrityService` tests to cover the jobs-mode scan path. ### 🐳 Docker diff --git a/tests/unit/lib/logs/format.test.ts b/tests/unit/lib/logs/format.test.ts new file mode 100644 index 00000000..423b7af9 --- /dev/null +++ b/tests/unit/lib/logs/format.test.ts @@ -0,0 +1,161 @@ +import { describe, it, expect } from 'vitest'; +import { formatLogsAsText, generateLogFilename, type ExportMeta } from '@/lib/logs/format'; +import type { LogEntry } from '@/lib/core/logs'; + +function makeEntry(msg: string, overrides: Partial = {}): LogEntry { + return { + timestamp: '2024-01-01T10:00:00.000Z', + level: 'info', + type: 'general', + message: msg, + ...overrides, + }; +} + +const baseMeta: ExportMeta = { + jobName: 'Test Job', + type: 'Backup', + status: 'Success', + startedAt: '2024-01-01T10:00:00.000Z', +}; + +describe('generateLogFilename', () => { + it('generates filename with slugified job name and UTC timestamp', () => { + const name = generateLogFilename('My Backup Job', '2024-03-15T14:30:00.000Z'); + expect(name).toBe('dbackup-my-backup-job-2024-03-15-14-30.log'); + }); + + it('replaces special characters with hyphens in slug', () => { + const name = generateLogFilename('Job/With:Special*Chars', '2024-01-01T00:00:00.000Z'); + expect(name).toMatch(/^dbackup-job-with-special-chars-/); + }); + + it('lowercases the job name', () => { + const name = generateLogFilename('UPPERCASE JOB', '2024-01-01T00:00:00.000Z'); + expect(name).toMatch(/^dbackup-uppercase-job-/); + }); + + it('truncates slug at 40 characters', () => { + const longName = 'This is a very long job name that exceeds the forty character limit by far'; + const name = generateLogFilename(longName, '2024-01-01T00:00:00.000Z'); + const slug = name + .replace(/^dbackup-/, '') + .replace(/-\d{4}-\d{2}-\d{2}-\d{2}-\d{2}\.log$/, ''); + expect(slug.length).toBeLessThanOrEqual(40); + }); + + it('returns fallback filename for invalid date', () => { + const name = generateLogFilename('My Job', 'not-a-date'); + expect(name).toBe('dbackup-my-job.log'); + }); + + it('uses UTC hours and minutes', () => { + const name = generateLogFilename('Job', '2024-06-01T23:45:00.000Z'); + expect(name).toContain('23-45'); + }); + + it('pads single-digit hours and minutes with leading zero', () => { + const name = generateLogFilename('Job', '2024-01-01T08:05:00.000Z'); + expect(name).toContain('08-05'); + }); +}); + +describe('formatLogsAsText', () => { + it('includes job name, type, and status in header', () => { + const output = formatLogsAsText([], baseMeta); + expect(output).toContain('Job: Test Job'); + expect(output).toContain('Type: Backup'); + expect(output).toContain('Status: Success'); + }); + + it('includes started timestamp in header', () => { + const output = formatLogsAsText([], baseMeta); + expect(output).toContain('Started:'); + expect(output).toContain('2024-01-01'); + }); + + it('includes ended timestamp when provided', () => { + const meta = { ...baseMeta, endedAt: '2024-01-01T10:05:00.000Z' }; + const output = formatLogsAsText([], meta); + expect(output).toContain('Ended:'); + }); + + it('omits ended line when endedAt is not provided', () => { + const output = formatLogsAsText([], baseMeta); + expect(output).not.toContain('Ended:'); + }); + + it('includes trigger type when provided', () => { + const meta = { ...baseMeta, triggerType: 'Scheduler' }; + const output = formatLogsAsText([], meta); + expect(output).toContain('Trigger: Scheduler'); + }); + + it('omits trigger line when triggerType is absent', () => { + const output = formatLogsAsText([], baseMeta); + expect(output).not.toContain('Trigger:'); + }); + + it('groups log entries by stage', () => { + const logs = [ + makeEntry('Starting up', { stage: 'Initializing' }), + makeEntry('Pushing file', { stage: 'Uploading' }), + ]; + const output = formatLogsAsText(logs, baseMeta); + expect(output).toContain('[STAGE: Initializing]'); + expect(output).toContain('[STAGE: Uploading]'); + }); + + it('includes log messages within their stage section', () => { + const logs = [makeEntry('Backup completed', { stage: 'Done' })]; + const output = formatLogsAsText(logs, baseMeta); + expect(output).toContain('Backup completed'); + }); + + it('appends duration suffix for entries with durationMs >= 1000', () => { + const logs = [makeEntry('Stage done', { stage: 'Done', durationMs: 2500 })]; + const output = formatLogsAsText(logs, baseMeta); + expect(output).toContain('[2.5s]'); + }); + + it('appends ms for durations under 1 second', () => { + const logs = [makeEntry('Fast op', { stage: 'Done', durationMs: 150 })]; + const output = formatLogsAsText(logs, baseMeta); + expect(output).toContain('[150ms]'); + }); + + it('renders details block when present', () => { + const logs = [makeEntry('Output', { stage: 'Done', details: 'line1\nline2' })]; + const output = formatLogsAsText(logs, baseMeta); + expect(output).toContain('[DETAILS]'); + expect(output).toContain('line1'); + expect(output).toContain('line2'); + }); + + it('renders UTC timestamps in HH:MM:SS format', () => { + const logs = [makeEntry('msg', { timestamp: '2024-01-15T14:30:45.000Z', stage: 'S' })]; + const output = formatLogsAsText(logs, baseMeta); + expect(output).toContain('14:30:45'); + }); + + it('assigns entries without stage to "General" group', () => { + const logs = [makeEntry('no stage', { stage: undefined })]; + const output = formatLogsAsText(logs, baseMeta); + expect(output).toContain('[STAGE: General]'); + }); + + it('includes stage duration in header when multiple entries share the same stage', () => { + const logs = [ + makeEntry('start', { timestamp: '2024-01-01T10:00:00.000Z', stage: 'Upload' }), + makeEntry('end', { timestamp: '2024-01-01T10:00:05.000Z', stage: 'Upload' }), + ]; + const output = formatLogsAsText(logs, baseMeta); + expect(output).toContain('[STAGE: Upload]'); + expect(output).toMatch(/\(5\.0s\)/); + }); + + it('includes the DBackup header line', () => { + const output = formatLogsAsText([], baseMeta); + expect(output).toContain('DBackup Execution Log'); + }); +}); diff --git a/tests/unit/lib/logs/sanitize.test.ts b/tests/unit/lib/logs/sanitize.test.ts new file mode 100644 index 00000000..7058b6ba --- /dev/null +++ b/tests/unit/lib/logs/sanitize.test.ts @@ -0,0 +1,115 @@ +import { describe, it, expect } from 'vitest'; +import { sanitizeLogs } from '@/lib/logs/sanitize'; +import type { LogEntry } from '@/lib/core/logs'; + +function makeEntry(overrides: Partial = {}): LogEntry { + return { + timestamp: '2024-01-01T00:00:00.000Z', + level: 'info', + type: 'general', + message: 'test message', + ...overrides, + }; +} + +describe('sanitizeLogs', () => { + it('redacts IPv4 addresses in message', () => { + const result = sanitizeLogs([makeEntry({ message: 'Connected to 192.168.1.100 successfully' })]); + expect(result[0].message).not.toContain('192.168.1.100'); + expect(result[0].message).toContain('[IP REDACTED]'); + }); + + it('redacts IPv6 addresses in message', () => { + const result = sanitizeLogs([makeEntry({ message: 'Server 2001:db8::1 timed out' })]); + expect(result[0].message).toContain('[IP REDACTED]'); + expect(result[0].message).not.toContain('2001:db8::1'); + }); + + it('redacts mongodb connection string credentials', () => { + const result = sanitizeLogs([makeEntry({ message: 'mongodb://admin:s3cr3t@host:27017/db' })]); + expect(result[0].message).not.toContain('s3cr3t'); + expect(result[0].message).toContain('[CREDENTIALS REDACTED]'); + }); + + it('redacts postgresql connection string credentials', () => { + const result = sanitizeLogs([makeEntry({ message: 'Connecting via postgresql://user:pass@host/mydb' })]); + expect(result[0].message).not.toContain('pass'); + expect(result[0].message).toContain('[CREDENTIALS REDACTED]'); + }); + + it('redacts mysql connection string credentials', () => { + const result = sanitizeLogs([makeEntry({ message: 'mysql://root:topsecret@db/schema' })]); + expect(result[0].message).not.toContain('topsecret'); + expect(result[0].message).toContain('[CREDENTIALS REDACTED]'); + }); + + it('redacts redis connection string credentials', () => { + const result = sanitizeLogs([makeEntry({ message: 'redis://user:pass@localhost:6379' })]); + expect(result[0].message).not.toContain('pass'); + expect(result[0].message).toContain('[CREDENTIALS REDACTED]'); + }); + + it('redacts credentials in details field', () => { + const result = sanitizeLogs([makeEntry({ details: 'sftp://user:secret@host/path' })]); + expect(result[0].details).not.toContain('secret'); + expect(result[0].details).toContain('[CREDENTIALS REDACTED]'); + }); + + it('redacts IPv4 addresses in details field', () => { + const result = sanitizeLogs([makeEntry({ details: 'Timeout on server 172.16.0.5' })]); + expect(result[0].details).not.toContain('172.16.0.5'); + expect(result[0].details).toContain('[IP REDACTED]'); + }); + + it('redacts SENSITIVE_KEYS in context', () => { + const result = sanitizeLogs([makeEntry({ context: { password: 'my-secret', jobId: 'job-1' } })]); + expect(result[0].context!.password).toBe('[REDACTED]'); + expect(result[0].context!.jobId).toBe('job-1'); + }); + + it('redacts apiKey in context', () => { + const result = sanitizeLogs([makeEntry({ context: { apiKey: 'sk-secret-key' } })]); + expect(result[0].context!.apiKey).toBe('[REDACTED]'); + }); + + it('redacts IP addresses inside non-sensitive context string values', () => { + const result = sanitizeLogs([makeEntry({ context: { info: 'server at 10.10.10.10 responded' } })]); + expect(result[0].context!.info).toContain('[IP REDACTED]'); + expect(result[0].context!.info).not.toContain('10.10.10.10'); + }); + + it('redacts nested sensitive keys in context objects', () => { + const result = sanitizeLogs([makeEntry({ context: { db: { password: 'nested-pass', port: 5432 } } })]); + expect(result[0].context!.db.password).toBe('[REDACTED]'); + expect(result[0].context!.db.port).toBe(5432); + }); + + it('preserves non-string non-object context values unchanged', () => { + const result = sanitizeLogs([makeEntry({ context: { count: 42, active: true } })]); + expect(result[0].context!.count).toBe(42); + expect(result[0].context!.active).toBe(true); + }); + + it('leaves entries without details or context unchanged', () => { + const result = sanitizeLogs([makeEntry({ details: undefined, context: undefined })]); + expect(result[0].details).toBeUndefined(); + expect(result[0].context).toBeUndefined(); + }); + + it('leaves clean messages without sensitive content unchanged', () => { + const result = sanitizeLogs([makeEntry({ message: 'Backup completed successfully' })]); + expect(result[0].message).toBe('Backup completed successfully'); + }); + + it('processes multiple entries independently', () => { + const logs = [ + makeEntry({ message: 'Server: 10.0.0.1 responded' }), + makeEntry({ message: 'No sensitive data here' }), + makeEntry({ message: 'mongodb://user:pass@host/db' }), + ]; + const result = sanitizeLogs(logs); + expect(result[0].message).toContain('[IP REDACTED]'); + expect(result[1].message).toBe('No sensitive data here'); + expect(result[2].message).toContain('[CREDENTIALS REDACTED]'); + }); +}); diff --git a/tests/unit/lib/runner/system-task-runner.test.ts b/tests/unit/lib/runner/system-task-runner.test.ts new file mode 100644 index 00000000..46bad46d --- /dev/null +++ b/tests/unit/lib/runner/system-task-runner.test.ts @@ -0,0 +1,278 @@ +import { describe, it, expect, vi, beforeEach } from 'vitest'; +import { prismaMock } from '@/lib/testing/prisma-mock'; +import { SystemTaskRunner } from '@/lib/runner/system-task-runner'; + +vi.mock('@/lib/logging/logger', () => ({ + logger: { + child: vi.fn().mockReturnValue({ + info: vi.fn(), + warn: vi.fn(), + error: vi.fn(), + debug: vi.fn(), + }), + }, +})); + +vi.mock('@/lib/logging/errors', () => ({ + wrapError: vi.fn((e) => e), +})); + +const STAGE_MAP: Record = { + Initializing: [0, 10], + Running: [10, 90], + Completed: [90, 100], +}; + +async function makeRunner(): Promise { + prismaMock.execution.create.mockResolvedValue({ id: 'exec-1' } as any); + return SystemTaskRunner.create('IntegrityCheck', 'Manual', 'Test trigger', STAGE_MAP); +} + +describe('SystemTaskRunner', () => { + beforeEach(() => { + vi.clearAllMocks(); + prismaMock.execution.update.mockResolvedValue({} as any); + prismaMock.execution.updateMany.mockResolvedValue({ count: 1 }); + }); + + describe('create()', () => { + it('creates an execution record with status Pending', async () => { + prismaMock.execution.create.mockResolvedValue({ id: 'exec-1' } as any); + + await SystemTaskRunner.create('IntegrityCheck'); + + expect(prismaMock.execution.create).toHaveBeenCalledWith( + expect.objectContaining({ + data: expect.objectContaining({ + status: 'Pending', + type: 'IntegrityCheck', + }), + }) + ); + }); + + it('stores the initial log entry in the execution record', async () => { + prismaMock.execution.create.mockResolvedValue({ id: 'exec-1' } as any); + + await SystemTaskRunner.create('IntegrityCheck'); + + const call = prismaMock.execution.create.mock.calls[0][0]; + const logs = JSON.parse(call.data.logs); + expect(logs).toHaveLength(1); + expect(logs[0].message).toBe('Task queued'); + }); + + it('passes triggerType and triggerLabel when provided', async () => { + prismaMock.execution.create.mockResolvedValue({ id: 'exec-1' } as any); + + await SystemTaskRunner.create('IntegrityCheck', 'Scheduler', 'daily-job'); + + expect(prismaMock.execution.create).toHaveBeenCalledWith( + expect.objectContaining({ + data: expect.objectContaining({ + triggerType: 'Scheduler', + triggerLabel: 'daily-job', + }), + }) + ); + }); + + it('exposes the execution id via the id getter', async () => { + prismaMock.execution.create.mockResolvedValue({ id: 'exec-abc' } as any); + const runner = await SystemTaskRunner.create('IntegrityCheck'); + expect(runner.id).toBe('exec-abc'); + }); + }); + + describe('start()', () => { + it('transitions execution status to Running', async () => { + const runner = await makeRunner(); + prismaMock.execution.updateMany.mockResolvedValue({ count: 1 }); + + await runner.start(); + + expect(prismaMock.execution.updateMany).toHaveBeenCalledWith( + expect.objectContaining({ + where: { id: 'exec-1', status: 'Pending' }, + data: expect.objectContaining({ status: 'Running' }), + }) + ); + }); + + it('throws when execution has already been claimed (concurrent call)', async () => { + const runner = await makeRunner(); + prismaMock.execution.updateMany.mockResolvedValue({ count: 0 }); + + await expect(runner.start()).rejects.toThrow('Execution already claimed by a concurrent call'); + }); + }); + + describe('finish()', () => { + it('updates execution status to Success', async () => { + const runner = await makeRunner(); + + await runner.finish('Success'); + + expect(prismaMock.execution.update).toHaveBeenCalledWith( + expect.objectContaining({ + where: { id: 'exec-1' }, + data: expect.objectContaining({ status: 'Success' }), + }) + ); + }); + + it('updates execution status to Failed', async () => { + const runner = await makeRunner(); + + await runner.finish('Failed'); + + expect(prismaMock.execution.update).toHaveBeenCalledWith( + expect.objectContaining({ + data: expect.objectContaining({ status: 'Failed' }), + }) + ); + }); + + it('sets endedAt on finish', async () => { + const runner = await makeRunner(); + + await runner.finish('Success'); + + const call = prismaMock.execution.update.mock.calls.at(-1)![0]; + expect(call.data.endedAt).toBeDefined(); + }); + }); + + describe('setStage()', () => { + it('updates currentStage and sets progress to the stage range minimum', async () => { + const runner = await makeRunner(); + + runner.setStage('Running'); + + // Force flush to check the update + await runner.flushLogs(true); + + const call = prismaMock.execution.update.mock.calls.at(-1)![0]; + const meta = JSON.parse(call.data.metadata); + expect(meta.stage).toBe('Running'); + expect(meta.progress).toBe(10); + }); + + it('emits a completion log for the previous stage when switching stages', async () => { + const runner = await makeRunner(); + runner.setStage('Running'); // start Running + runner.setStage('Completed'); // switch away from Running - adds completion log + + await runner.finish('Success'); + + const call = prismaMock.execution.update.mock.calls.at(-1)![0]; + const logs = JSON.parse(call.data.logs); + const completionEntry = logs.find( + (l: { message: string }) => l.message.includes('Running completed') + ); + expect(completionEntry).toBeDefined(); + }); + + it('does not emit completion log when stage stays the same', async () => { + const runner = await makeRunner(); + runner.setStage('Running'); + runner.setStage('Running'); // no change + + await runner.flushLogs(true); + + const call = prismaMock.execution.update.mock.calls.at(-1)![0]; + const logs = JSON.parse(call.data.logs); + const completionEntries = logs.filter( + (l: { message: string }) => l.message.includes('Running completed') + ); + expect(completionEntries).toHaveLength(0); + }); + }); + + describe('updateStageProgress()', () => { + it('interpolates internalPercent within the stage range', async () => { + const runner = await makeRunner(); + runner.setStage('Running'); // range [10, 90] + runner.updateStageProgress(50); // midpoint → 10 + (90-10)*0.5 = 50 + + // finish() drains all pending void flushes before its own final DB write + await runner.finish('Success'); + + const call = prismaMock.execution.update.mock.calls.at(-1)![0]; + const meta = JSON.parse(call.data.metadata); + expect(meta.progress).toBe(50); + }); + + it('clamps internalPercent to 0-100 range', async () => { + const runner = await makeRunner(); + runner.setStage('Running'); // range [10, 90] + runner.updateStageProgress(150); // clamped to 100 → max of range = 90 + + await runner.finish('Success'); + + const call = prismaMock.execution.update.mock.calls.at(-1)![0]; + const meta = JSON.parse(call.data.metadata); + expect(meta.progress).toBe(90); + }); + }); + + describe('logEntry()', () => { + it('adds a log entry with the current stage', async () => { + const runner = await makeRunner(); + runner.setStage('Running'); + runner.logEntry('Processing file', 'info'); + + // finish() ensures all pending void flushes complete before writing final state + await runner.finish('Success'); + + const call = prismaMock.execution.update.mock.calls.at(-1)![0]; + const logs = JSON.parse(call.data.logs); + const entry = logs.find((l: { message: string }) => l.message === 'Processing file'); + expect(entry).toBeDefined(); + expect(entry.stage).toBe('Running'); + expect(entry.level).toBe('info'); + }); + + it('includes optional details in the log entry', async () => { + const runner = await makeRunner(); + runner.logEntry('Command output', 'info', 'command', 'line1\nline2'); + + await runner.flushLogs(true); + + const call = prismaMock.execution.update.mock.calls.at(-1)![0]; + const logs = JSON.parse(call.data.logs); + const entry = logs.find((l: { message: string }) => l.message === 'Command output'); + expect(entry.details).toBe('line1\nline2'); + }); + }); + + describe('flushLogs()', () => { + it('persists current logs and metadata to the database', async () => { + const runner = await makeRunner(); + runner.logEntry('Test log'); + + await runner.flushLogs(true); + + expect(prismaMock.execution.update).toHaveBeenCalledWith( + expect.objectContaining({ + where: { id: 'exec-1' }, + data: expect.objectContaining({ + logs: expect.stringContaining('Test log'), + metadata: expect.any(String), + }), + }) + ); + }); + + it('skips flush when called within 1 second and force is false', async () => { + const runner = await makeRunner(); + + await runner.flushLogs(true); // forced flush, sets lastLogFlush to now + const callCount = prismaMock.execution.update.mock.calls.length; + + await runner.flushLogs(false); // within 1s, should skip + + expect(prismaMock.execution.update.mock.calls.length).toBe(callCount); + }); + }); +}); diff --git a/tests/unit/services/integrity-service.test.ts b/tests/unit/services/integrity-service.test.ts index bda1f289..9eacb914 100644 --- a/tests/unit/services/integrity-service.test.ts +++ b/tests/unit/services/integrity-service.test.ts @@ -8,7 +8,11 @@ vi.mock('@/lib/prisma', () => ({ default: { adapterConfig: { findMany: vi.fn() }, job: { findMany: vi.fn() }, - systemSetting: { findUnique: vi.fn().mockResolvedValue(null) }, + systemSetting: { + findUnique: vi.fn().mockImplementation(({ where }: { where: { key: string } }) => + Promise.resolve(where.key === 'integrity.scanMode' ? { value: 'destinations' } : null) + ), + }, }, })); @@ -244,4 +248,202 @@ describe('IntegrityService', () => { expect(result.totalFiles).toBe(2); expect(result.passed).toBe(2); }); + + describe('Jobs scan mode (gatherFilesFromJobs)', () => { + function makeJobWithDest(jobName: string, destConfigId: string, overrides: Record = {}) { + return { + id: `job-${jobName}`, + name: jobName, + enabled: true, + skipVerification: false, + destinations: [ + { + configId: destConfigId, + config: { + id: destConfigId, + adapterId: 'local', + name: `Storage for ${jobName}`, + config: '{}', + primaryCredentialId: null, + sshCredentialId: null, + metadata: null, + }, + }, + ], + ...overrides, + }; + } + + beforeEach(() => { + // Switch to jobs mode: no scanMode setting + vi.mocked(prisma.systemSetting.findUnique).mockResolvedValue(null); + }); + + it('scans job-linked files when scanMode is not set', async () => { + const adapter = makeStorageAdapter({ + list: vi.fn().mockResolvedValue([ + { name: 'backup.sql', path: 'ProdJob/backup.sql' }, + ]), + }); + (prisma.job.findMany as ReturnType).mockResolvedValue([ + makeJobWithDest('ProdJob', 'dest-1'), + ]); + (registry.get as ReturnType).mockReturnValue(adapter); + vi.mocked(verificationService.verifyFile).mockResolvedValue({ + status: 'passed', + verifiedAt: new Date().toISOString(), + }); + + const result = await integrityService.runFullIntegrityCheck(); + + expect(result.totalFiles).toBe(1); + expect(result.passed).toBe(1); + }); + + it('skips jobs with skipVerification flag', async () => { + const adapter = makeStorageAdapter({ + list: vi.fn().mockResolvedValue([{ name: 'backup.sql', path: 'SkipJob/backup.sql' }]), + }); + (prisma.job.findMany as ReturnType).mockResolvedValue([ + makeJobWithDest('SkipJob', 'dest-1', { skipVerification: true }), + ]); + (registry.get as ReturnType).mockReturnValue(adapter); + + const result = await integrityService.runFullIntegrityCheck(); + + expect(result.totalFiles).toBe(0); + }); + + it('filters out files that do not start with the job name prefix', async () => { + const adapter = makeStorageAdapter({ + list: vi.fn().mockResolvedValue([ + { name: 'mine.sql', path: 'ProdJob/mine.sql' }, + { name: 'other.sql', path: 'OtherJob/other.sql' }, + ]), + }); + (prisma.job.findMany as ReturnType).mockResolvedValue([ + makeJobWithDest('ProdJob', 'dest-1'), + ]); + (registry.get as ReturnType).mockReturnValue(adapter); + vi.mocked(verificationService.verifyFile).mockResolvedValue({ + status: 'passed', + verifiedAt: new Date().toISOString(), + }); + + const result = await integrityService.runFullIntegrityCheck(); + + expect(result.totalFiles).toBe(1); + }); + + it('excludes .meta.json sidecar files from verification', async () => { + const adapter = makeStorageAdapter({ + list: vi.fn().mockResolvedValue([ + { name: 'backup.sql', path: 'Job/backup.sql' }, + { name: 'backup.sql.meta.json', path: 'Job/backup.sql.meta.json' }, + ]), + }); + (prisma.job.findMany as ReturnType).mockResolvedValue([ + makeJobWithDest('Job', 'dest-1'), + ]); + (registry.get as ReturnType).mockReturnValue(adapter); + vi.mocked(verificationService.verifyFile).mockResolvedValue({ + status: 'passed', + verifiedAt: new Date().toISOString(), + }); + + const result = await integrityService.runFullIntegrityCheck(); + + expect(result.totalFiles).toBe(1); + }); + + it('deduplicates files when two jobs share the same destination', async () => { + const adapter = makeStorageAdapter({ + list: vi.fn().mockResolvedValue([ + { name: 'job1.sql', path: 'Job1/job1.sql' }, + { name: 'job2.sql', path: 'Job2/job2.sql' }, + ]), + }); + const sharedDestId = 'shared-dest'; + (prisma.job.findMany as ReturnType).mockResolvedValue([ + makeJobWithDest('Job1', sharedDestId), + makeJobWithDest('Job2', sharedDestId), + ]); + (registry.get as ReturnType).mockReturnValue(adapter); + vi.mocked(verificationService.verifyFile).mockResolvedValue({ + status: 'passed', + verifiedAt: new Date().toISOString(), + }); + + const result = await integrityService.runFullIntegrityCheck(); + + // list() called once despite two jobs sharing the destination + expect(adapter.list).toHaveBeenCalledTimes(1); + expect(result.totalFiles).toBe(2); + }); + + it('skips destinations with skipVerification set in metadata', async () => { + const adapter = makeStorageAdapter({ + list: vi.fn().mockResolvedValue([{ name: 'backup.sql', path: 'Job/backup.sql' }]), + }); + const job = { + ...makeJobWithDest('Job', 'dest-1'), + destinations: [ + { + configId: 'dest-1', + config: { + id: 'dest-1', + adapterId: 'local', + name: 'Skipped Storage', + config: '{}', + primaryCredentialId: null, + sshCredentialId: null, + metadata: JSON.stringify({ skipVerification: true }), + }, + }, + ], + }; + (prisma.job.findMany as ReturnType).mockResolvedValue([job]); + (registry.get as ReturnType).mockReturnValue(adapter); + + const result = await integrityService.runFullIntegrityCheck(); + + expect(result.totalFiles).toBe(0); + }); + + it('applies maxAgeDays filter in jobs mode', async () => { + const oldDate = new Date(Date.now() - 10 * 86_400_000).toISOString(); + const recentDate = new Date(Date.now() - 1 * 86_400_000).toISOString(); + const adapter = makeStorageAdapter({ + list: vi.fn().mockResolvedValue([ + { name: 'old.sql', path: 'Job/old.sql', lastModified: oldDate, size: 100 }, + { name: 'new.sql', path: 'Job/new.sql', lastModified: recentDate, size: 100 }, + ]), + }); + (prisma.job.findMany as ReturnType).mockResolvedValue([ + makeJobWithDest('Job', 'dest-1'), + ]); + (registry.get as ReturnType).mockReturnValue(adapter); + // maxAgeDays = 5 + vi.mocked(prisma.systemSetting.findUnique).mockImplementation(({ where }: { where: { key: string } }) => { + if (where.key === 'integrity.maxAgeDays') return Promise.resolve({ value: '5' } as any); + return Promise.resolve(null); + }); + vi.mocked(verificationService.verifyFile).mockResolvedValue({ + status: 'passed', + verifiedAt: new Date().toISOString(), + }); + + const result = await integrityService.runFullIntegrityCheck(); + + expect(result.totalFiles).toBe(1); + }); + + it('returns zero results when no jobs exist', async () => { + (prisma.job.findMany as ReturnType).mockResolvedValue([]); + + const result = await integrityService.runFullIntegrityCheck(); + + expect(result.totalFiles).toBe(0); + }); + }); }); diff --git a/tests/unit/services/job-service.test.ts b/tests/unit/services/job-service.test.ts index 363630cf..d7f8dcdd 100644 --- a/tests/unit/services/job-service.test.ts +++ b/tests/unit/services/job-service.test.ts @@ -59,6 +59,7 @@ describe('JobService', () => { compression: "NONE", pgCompression: "", notificationEvents: "ALWAYS", + skipVerification: false, notifications: { connect: [{ id: 'notif-1' }] }, diff --git a/tests/unit/services/verification-service.test.ts b/tests/unit/services/verification-service.test.ts new file mode 100644 index 00000000..d2fd4590 --- /dev/null +++ b/tests/unit/services/verification-service.test.ts @@ -0,0 +1,378 @@ +import { describe, it, expect, vi, beforeEach } from 'vitest'; +import type { BackupMetadata } from '@/lib/core/interfaces'; + +vi.mock('@/lib/prisma', () => ({ + default: { + adapterConfig: { findUnique: vi.fn() }, + }, +})); + +vi.mock('@/lib/core/registry', () => ({ + registry: { get: vi.fn() }, +})); + +vi.mock('@/lib/adapters', () => ({ + registerAdapters: vi.fn(), +})); + +vi.mock('@/lib/adapters/config-resolver', () => ({ + resolveAdapterConfig: vi.fn().mockResolvedValue({ resolved: true }), +})); + +vi.mock('@/lib/crypto/checksum', () => ({ + calculateFileChecksum: vi.fn(), +})); + +vi.mock('@/lib/temp-dir', () => ({ + getTempDir: vi.fn().mockReturnValue('/tmp'), +})); + +vi.mock('fs/promises', () => { + const readFile = vi.fn().mockResolvedValue('{}'); + const unlink = vi.fn().mockResolvedValue(undefined); + const writeFile = vi.fn().mockResolvedValue(undefined); + return { + default: { readFile, unlink, writeFile }, + readFile, + unlink, + writeFile, + }; +}); + +vi.mock('@/lib/logging/logger', () => ({ + logger: { + child: vi.fn().mockReturnValue({ + info: vi.fn(), + warn: vi.fn(), + error: vi.fn(), + debug: vi.fn(), + }), + }, +})); + +vi.mock('@/lib/logging/errors', () => ({ + wrapError: vi.fn((e) => e), +})); + +vi.mock('@/services/storage/storage-service', () => ({ + storageService: { + updateStorageListCacheEntry: vi.fn().mockResolvedValue(undefined), + }, +})); + +import prisma from '@/lib/prisma'; +import { registry } from '@/lib/core/registry'; +import { calculateFileChecksum } from '@/lib/crypto/checksum'; +import { verificationService } from '@/services/storage/verification-service'; +import fsPromises from 'fs/promises'; + +function makeAdapterConfig(overrides: Record = {}) { + return { + id: 'config-1', + adapterId: 'local', + name: 'Local Storage', + config: '{}', + primaryCredentialId: null, + sshCredentialId: null, + metadata: null, + ...overrides, + }; +} + +function makeAdapter(overrides: Record = {}) { + return { + download: vi.fn().mockResolvedValue(false), + upload: vi.fn().mockResolvedValue(undefined), + read: vi.fn().mockResolvedValue(null), + list: vi.fn().mockResolvedValue([]), + delete: vi.fn().mockResolvedValue(undefined), + ...overrides, + }; +} + +function makeMetadata(overrides: Partial = {}): BackupMetadata { + return { + version: 1, + jobId: 'job-1', + jobName: 'Test Job', + sourceName: 'Test Source', + sourceType: 'mysql', + databases: ['testdb'], + timestamp: '2024-01-01T00:00:00.000Z', + originalFileName: 'backup.sql', + sourceId: 'src-1', + checksum: 'sha256:abc123', + ...overrides, + }; +} + +describe('VerificationService', () => { + beforeEach(() => { + vi.clearAllMocks(); + vi.mocked(calculateFileChecksum).mockResolvedValue('sha256:abc123'); + }); + + it('throws when adapter config is not found', async () => { + (prisma.adapterConfig.findUnique as ReturnType).mockResolvedValue(null); + + await expect( + verificationService.verifyFile('missing-id', 'backup.sql', 'manual') + ).rejects.toThrow('Storage configuration not found'); + }); + + it('throws when adapter is not in registry', async () => { + (prisma.adapterConfig.findUnique as ReturnType).mockResolvedValue(makeAdapterConfig()); + (registry.get as ReturnType).mockReturnValue(null); + + await expect( + verificationService.verifyFile('config-1', 'backup.sql', 'manual') + ).rejects.toThrow("Adapter 'local' not found"); + }); + + it('returns no_metadata when adapter.read returns null and download returns false', async () => { + const adapter = makeAdapter({ read: vi.fn().mockResolvedValue(null) }); + (prisma.adapterConfig.findUnique as ReturnType).mockResolvedValue(makeAdapterConfig()); + (registry.get as ReturnType).mockReturnValue(adapter); + + const result = await verificationService.verifyFile('config-1', 'backup.sql', 'manual'); + + expect(result.status).toBe('no_metadata'); + }); + + it('returns no_metadata when adapter has no read method and download returns false', async () => { + const adapter = makeAdapter({ read: undefined }); + (prisma.adapterConfig.findUnique as ReturnType).mockResolvedValue(makeAdapterConfig()); + (registry.get as ReturnType).mockReturnValue(adapter); + + const result = await verificationService.verifyFile('config-1', 'backup.sql', 'manual'); + + expect(result.status).toBe('no_metadata'); + expect(adapter.download).toHaveBeenCalledWith( + expect.anything(), + 'backup.sql.meta.json', + expect.stringContaining('/tmp') + ); + }); + + it('returns no_checksum when metadata has no checksum fields', async () => { + const metadata = makeMetadata({ checksum: undefined, checksumMd5: undefined }); + const adapter = makeAdapter({ read: vi.fn().mockResolvedValue(JSON.stringify(metadata)) }); + (prisma.adapterConfig.findUnique as ReturnType).mockResolvedValue(makeAdapterConfig()); + (registry.get as ReturnType).mockReturnValue(adapter); + + const result = await verificationService.verifyFile('config-1', 'backup.sql', 'manual'); + + expect(result.status).toBe('no_checksum'); + }); + + it('returns skipped when skipIfPassed is true and metadata.verification.passed is true', async () => { + const metadata = makeMetadata({ + verification: { verifiedAt: '2024-01-01T00:00:00.000Z', passed: true, trigger: 'manual' }, + }); + const adapter = makeAdapter({ read: vi.fn().mockResolvedValue(JSON.stringify(metadata)) }); + (prisma.adapterConfig.findUnique as ReturnType).mockResolvedValue(makeAdapterConfig()); + (registry.get as ReturnType).mockReturnValue(adapter); + + const result = await verificationService.verifyFile('config-1', 'backup.sql', 'manual', { skipIfPassed: true }); + + expect(result.status).toBe('skipped'); + }); + + it('does not skip when skipIfPassed is true but verification has not passed', async () => { + const metadata = makeMetadata({ + verification: { verifiedAt: '2024-01-01T00:00:00.000Z', passed: false, trigger: 'manual' }, + }); + const adapter = makeAdapter({ + read: vi.fn().mockResolvedValue(JSON.stringify(metadata)), + download: vi.fn().mockResolvedValue(true), + }); + (prisma.adapterConfig.findUnique as ReturnType).mockResolvedValue(makeAdapterConfig()); + (registry.get as ReturnType).mockReturnValue(adapter); + + const result = await verificationService.verifyFile('config-1', 'backup.sql', 'manual', { skipIfPassed: true }); + + expect(result.status).not.toBe('skipped'); + }); + + it('returns passed via native verifyChecksum', async () => { + const metadata = makeMetadata(); + const adapter = makeAdapter({ + read: vi.fn().mockResolvedValue(JSON.stringify(metadata)), + verifyChecksum: vi.fn().mockResolvedValue('passed'), + }); + (prisma.adapterConfig.findUnique as ReturnType).mockResolvedValue(makeAdapterConfig()); + (registry.get as ReturnType).mockReturnValue(adapter); + + const result = await verificationService.verifyFile('config-1', 'backup.sql', 'manual'); + + expect(result.status).toBe('passed'); + expect(adapter.verifyChecksum).toHaveBeenCalledWith( + expect.anything(), + 'backup.sql', + expect.objectContaining({ sha256: 'sha256:abc123' }) + ); + }); + + it('returns failed via native verifyChecksum', async () => { + const metadata = makeMetadata(); + const adapter = makeAdapter({ + read: vi.fn().mockResolvedValue(JSON.stringify(metadata)), + verifyChecksum: vi.fn().mockResolvedValue('failed'), + }); + (prisma.adapterConfig.findUnique as ReturnType).mockResolvedValue(makeAdapterConfig()); + (registry.get as ReturnType).mockReturnValue(adapter); + + const result = await verificationService.verifyFile('config-1', 'backup.sql', 'manual'); + + expect(result.status).toBe('failed'); + }); + + it('falls back to download when native verifyChecksum returns unsupported', async () => { + const metadata = makeMetadata(); + const adapter = makeAdapter({ + read: vi.fn().mockResolvedValue(JSON.stringify(metadata)), + verifyChecksum: vi.fn().mockResolvedValue('unsupported'), + download: vi.fn().mockResolvedValue(true), + }); + (prisma.adapterConfig.findUnique as ReturnType).mockResolvedValue(makeAdapterConfig()); + (registry.get as ReturnType).mockReturnValue(adapter); + vi.mocked(calculateFileChecksum).mockResolvedValue('sha256:abc123'); + + const result = await verificationService.verifyFile('config-1', 'backup.sql', 'manual'); + + expect(result.status).toBe('passed'); + expect(adapter.download).toHaveBeenCalledWith( + expect.anything(), + 'backup.sql', + expect.stringContaining('/tmp') + ); + }); + + it('returns passed when download checksum matches', async () => { + const metadata = makeMetadata({ checksum: 'sha256:expected' }); + const adapter = makeAdapter({ + read: vi.fn().mockResolvedValue(JSON.stringify(metadata)), + download: vi.fn().mockResolvedValue(true), + }); + (prisma.adapterConfig.findUnique as ReturnType).mockResolvedValue(makeAdapterConfig()); + (registry.get as ReturnType).mockReturnValue(adapter); + vi.mocked(calculateFileChecksum).mockResolvedValue('sha256:expected'); + + const result = await verificationService.verifyFile('config-1', 'backup.sql', 'manual'); + + expect(result.status).toBe('passed'); + expect(result.expectedChecksum).toBe('sha256:expected'); + expect(result.actualChecksum).toBeUndefined(); + }); + + it('returns failed when download checksum does not match', async () => { + const metadata = makeMetadata({ checksum: 'sha256:expected' }); + const adapter = makeAdapter({ + read: vi.fn().mockResolvedValue(JSON.stringify(metadata)), + download: vi.fn().mockResolvedValue(true), + }); + (prisma.adapterConfig.findUnique as ReturnType).mockResolvedValue(makeAdapterConfig()); + (registry.get as ReturnType).mockReturnValue(adapter); + vi.mocked(calculateFileChecksum).mockResolvedValue('sha256:tampered'); + + const result = await verificationService.verifyFile('config-1', 'backup.sql', 'manual'); + + expect(result.status).toBe('failed'); + expect(result.expectedChecksum).toBe('sha256:expected'); + expect(result.actualChecksum).toBe('sha256:tampered'); + }); + + it('returns download_error when adapter.download returns false for the backup file', async () => { + const metadata = makeMetadata(); + const adapter = makeAdapter({ + read: vi.fn().mockResolvedValue(JSON.stringify(metadata)), + download: vi.fn().mockResolvedValue(false), + }); + (prisma.adapterConfig.findUnique as ReturnType).mockResolvedValue(makeAdapterConfig()); + (registry.get as ReturnType).mockReturnValue(adapter); + + const result = await verificationService.verifyFile('config-1', 'backup.sql', 'manual'); + + expect(result.status).toBe('download_error'); + }); + + it('returns download_error when adapter.download throws', async () => { + const metadata = makeMetadata(); + const adapter = makeAdapter({ + read: vi.fn().mockResolvedValue(JSON.stringify(metadata)), + download: vi.fn().mockRejectedValue(new Error('Network timeout')), + }); + (prisma.adapterConfig.findUnique as ReturnType).mockResolvedValue(makeAdapterConfig()); + (registry.get as ReturnType).mockReturnValue(adapter); + + const result = await verificationService.verifyFile('config-1', 'backup.sql', 'manual'); + + expect(result.status).toBe('download_error'); + }); + + it('reads metadata via adapter.download when adapter has no read method', async () => { + const metadata = makeMetadata(); + const adapter = makeAdapter({ + read: undefined, + download: vi.fn().mockResolvedValue(true), + }); + (prisma.adapterConfig.findUnique as ReturnType).mockResolvedValue(makeAdapterConfig()); + (registry.get as ReturnType).mockReturnValue(adapter); + vi.mocked(fsPromises.readFile as ReturnType).mockResolvedValueOnce(JSON.stringify(metadata)); + + await verificationService.verifyFile('config-1', 'backup.sql', 'manual'); + + expect(adapter.download).toHaveBeenCalledWith( + expect.anything(), + 'backup.sql.meta.json', + expect.stringContaining('/tmp') + ); + }); + + it('calls adapter.upload to persist verification result after native check', async () => { + const metadata = makeMetadata(); + const adapter = makeAdapter({ + read: vi.fn().mockResolvedValue(JSON.stringify(metadata)), + verifyChecksum: vi.fn().mockResolvedValue('passed'), + }); + (prisma.adapterConfig.findUnique as ReturnType).mockResolvedValue(makeAdapterConfig()); + (registry.get as ReturnType).mockReturnValue(adapter); + + await verificationService.verifyFile('config-1', 'backup.sql', 'manual'); + + expect(adapter.upload).toHaveBeenCalledWith( + expect.anything(), + expect.stringContaining('/tmp'), + 'backup.sql.meta.json' + ); + }); + + it('calls adapter.upload to persist verification result after download-based check', async () => { + const metadata = makeMetadata(); + const adapter = makeAdapter({ + read: vi.fn().mockResolvedValue(JSON.stringify(metadata)), + download: vi.fn().mockResolvedValue(true), + }); + (prisma.adapterConfig.findUnique as ReturnType).mockResolvedValue(makeAdapterConfig()); + (registry.get as ReturnType).mockReturnValue(adapter); + vi.mocked(calculateFileChecksum).mockResolvedValue('sha256:abc123'); + + await verificationService.verifyFile('config-1', 'backup.sql', 'manual'); + + expect(adapter.upload).toHaveBeenCalledWith( + expect.anything(), + expect.stringContaining('/tmp'), + 'backup.sql.meta.json' + ); + }); + + it('includes verifiedAt in every result', async () => { + (prisma.adapterConfig.findUnique as ReturnType).mockResolvedValue(makeAdapterConfig()); + (registry.get as ReturnType).mockReturnValue(makeAdapter({ read: vi.fn().mockResolvedValue(null) })); + + const result = await verificationService.verifyFile('config-1', 'backup.sql', 'manual'); + + expect(result.verifiedAt).toBeDefined(); + expect(new Date(result.verifiedAt).getTime()).not.toBeNaN(); + }); +}); From 704ee3062f838ca47da99ff3effd87343c045715 Mon Sep 17 00:00:00 2001 From: Manu Date: Sun, 14 Jun 2026 13:56:52 +0200 Subject: [PATCH 20/24] Avoid stale OAuth token state; remove import Refactor OAuth button components to avoid stale/incorrect token state when credentialId or authorized props change: introduce a tokenCheck object, derive tokenState from it, and add an active cleanup flag to prevent state updates after unmount or race conditions. Preserve fail-open behavior on fetch errors. Also remove an unused `path` import from integrity-service.ts. --- src/components/adapter/dropbox-oauth-button.tsx | 15 ++++++++++----- .../adapter/google-drive-oauth-button.tsx | 15 ++++++++++----- src/components/adapter/onedrive-oauth-button.tsx | 15 ++++++++++----- src/services/backup/integrity-service.ts | 1 - 4 files changed, 30 insertions(+), 16 deletions(-) diff --git a/src/components/adapter/dropbox-oauth-button.tsx b/src/components/adapter/dropbox-oauth-button.tsx index d37fd9fb..40d3aa45 100644 --- a/src/components/adapter/dropbox-oauth-button.tsx +++ b/src/components/adapter/dropbox-oauth-button.tsx @@ -21,22 +21,27 @@ interface DropboxOAuthButtonProps { */ export function DropboxOAuthButton({ credentialId, authorized, onAuthorized }: DropboxOAuthButtonProps) { const [isLoading, setIsLoading] = useState(false); - const [tokenState, setTokenState] = useState<"checking" | "valid" | "expired" | null>(null); + const [tokenCheck, setTokenCheck] = useState<{ id: string; auth: boolean; result: "valid" | "expired" } | null>(null); + + const tokenState: "checking" | "valid" | "expired" | null = + !authorized || !credentialId ? null + : tokenCheck?.id === credentialId && tokenCheck?.auth === authorized ? tokenCheck.result + : "checking"; useEffect(() => { if (!authorized || !credentialId) { - setTokenState(null); return; } - setTokenState("checking"); + let active = true; fetch("/api/adapters/dropbox/validate-token", { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ credentialId }), }) .then((r) => r.json()) - .then((data) => setTokenState(data.valid ? "valid" : "expired")) - .catch(() => setTokenState("valid")); // Fail open - don't block UX on network error + .then((data) => { if (active) setTokenCheck({ id: credentialId, auth: !!authorized, result: data.valid ? "valid" : "expired" }); }) + .catch(() => { if (active) setTokenCheck({ id: credentialId, auth: !!authorized, result: "valid" }); }); + return () => { active = false; }; }, [authorized, credentialId]); if (!credentialId) { diff --git a/src/components/adapter/google-drive-oauth-button.tsx b/src/components/adapter/google-drive-oauth-button.tsx index 3843927f..b172b2c7 100644 --- a/src/components/adapter/google-drive-oauth-button.tsx +++ b/src/components/adapter/google-drive-oauth-button.tsx @@ -21,22 +21,27 @@ interface GoogleDriveOAuthButtonProps { */ export function GoogleDriveOAuthButton({ credentialId, authorized, onAuthorized }: GoogleDriveOAuthButtonProps) { const [isLoading, setIsLoading] = useState(false); - const [tokenState, setTokenState] = useState<"checking" | "valid" | "expired" | null>(null); + const [tokenCheck, setTokenCheck] = useState<{ id: string; auth: boolean; result: "valid" | "expired" } | null>(null); + + const tokenState: "checking" | "valid" | "expired" | null = + !authorized || !credentialId ? null + : tokenCheck?.id === credentialId && tokenCheck?.auth === authorized ? tokenCheck.result + : "checking"; useEffect(() => { if (!authorized || !credentialId) { - setTokenState(null); return; } - setTokenState("checking"); + let active = true; fetch("/api/adapters/google-drive/validate-token", { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ credentialId }), }) .then((r) => r.json()) - .then((data) => setTokenState(data.valid ? "valid" : "expired")) - .catch(() => setTokenState("valid")); // Fail open - don't block UX on network error + .then((data) => { if (active) setTokenCheck({ id: credentialId, auth: !!authorized, result: data.valid ? "valid" : "expired" }); }) + .catch(() => { if (active) setTokenCheck({ id: credentialId, auth: !!authorized, result: "valid" }); }); + return () => { active = false; }; }, [authorized, credentialId]); if (!credentialId) { diff --git a/src/components/adapter/onedrive-oauth-button.tsx b/src/components/adapter/onedrive-oauth-button.tsx index 5e2aee43..c9144c90 100644 --- a/src/components/adapter/onedrive-oauth-button.tsx +++ b/src/components/adapter/onedrive-oauth-button.tsx @@ -21,22 +21,27 @@ interface OneDriveOAuthButtonProps { */ export function OneDriveOAuthButton({ credentialId, authorized, onAuthorized }: OneDriveOAuthButtonProps) { const [isLoading, setIsLoading] = useState(false); - const [tokenState, setTokenState] = useState<"checking" | "valid" | "expired" | null>(null); + const [tokenCheck, setTokenCheck] = useState<{ id: string; auth: boolean; result: "valid" | "expired" } | null>(null); + + const tokenState: "checking" | "valid" | "expired" | null = + !authorized || !credentialId ? null + : tokenCheck?.id === credentialId && tokenCheck?.auth === authorized ? tokenCheck.result + : "checking"; useEffect(() => { if (!authorized || !credentialId) { - setTokenState(null); return; } - setTokenState("checking"); + let active = true; fetch("/api/adapters/onedrive/validate-token", { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ credentialId }), }) .then((r) => r.json()) - .then((data) => setTokenState(data.valid ? "valid" : "expired")) - .catch(() => setTokenState("valid")); // Fail open - don't block UX on network error + .then((data) => { if (active) setTokenCheck({ id: credentialId, auth: !!authorized, result: data.valid ? "valid" : "expired" }); }) + .catch(() => { if (active) setTokenCheck({ id: credentialId, auth: !!authorized, result: "valid" }); }); + return () => { active = false; }; }, [authorized, credentialId]); if (!credentialId) { diff --git a/src/services/backup/integrity-service.ts b/src/services/backup/integrity-service.ts index 69fa2a43..25235f50 100644 --- a/src/services/backup/integrity-service.ts +++ b/src/services/backup/integrity-service.ts @@ -1,5 +1,4 @@ import prisma from "@/lib/prisma"; -import path from "path"; import { registry } from "@/lib/core/registry"; import { registerAdapters } from "@/lib/adapters"; import { StorageAdapter } from "@/lib/core/interfaces"; From c239df14ee044ff2df3b360fa3c22a67b858bc86 Mon Sep 17 00:00:00 2001 From: Manu Date: Sun, 14 Jun 2026 14:17:16 +0200 Subject: [PATCH 21/24] Fix TypeScript test typing and add storage mocks Update unit tests to satisfy TypeScript types and add missing storage-service mocks. Cast call.data.metadata/logs to string before JSON.parse in SystemTaskRunner tests, add vi.mock stubs for storageService (append/update/remove) in upload and retention tests, refine prisma.systemSetting.findUnique mock typing and remove an any cast, and add verifyChecksum field to the adapter mock in verification-service tests. --- tests/unit/lib/runner/system-task-runner.test.ts | 14 +++++++------- tests/unit/runner/steps/03-upload.test.ts | 7 +++++++ tests/unit/runner/steps/05-retention.test.ts | 8 ++++++++ tests/unit/services/integrity-service.test.ts | 4 ++-- tests/unit/services/verification-service.test.ts | 1 + 5 files changed, 25 insertions(+), 9 deletions(-) diff --git a/tests/unit/lib/runner/system-task-runner.test.ts b/tests/unit/lib/runner/system-task-runner.test.ts index 46bad46d..5809190e 100644 --- a/tests/unit/lib/runner/system-task-runner.test.ts +++ b/tests/unit/lib/runner/system-task-runner.test.ts @@ -153,7 +153,7 @@ describe('SystemTaskRunner', () => { await runner.flushLogs(true); const call = prismaMock.execution.update.mock.calls.at(-1)![0]; - const meta = JSON.parse(call.data.metadata); + const meta = JSON.parse(call.data.metadata as string); expect(meta.stage).toBe('Running'); expect(meta.progress).toBe(10); }); @@ -166,7 +166,7 @@ describe('SystemTaskRunner', () => { await runner.finish('Success'); const call = prismaMock.execution.update.mock.calls.at(-1)![0]; - const logs = JSON.parse(call.data.logs); + const logs = JSON.parse(call.data.logs as string); const completionEntry = logs.find( (l: { message: string }) => l.message.includes('Running completed') ); @@ -181,7 +181,7 @@ describe('SystemTaskRunner', () => { await runner.flushLogs(true); const call = prismaMock.execution.update.mock.calls.at(-1)![0]; - const logs = JSON.parse(call.data.logs); + const logs = JSON.parse(call.data.logs as string); const completionEntries = logs.filter( (l: { message: string }) => l.message.includes('Running completed') ); @@ -199,7 +199,7 @@ describe('SystemTaskRunner', () => { await runner.finish('Success'); const call = prismaMock.execution.update.mock.calls.at(-1)![0]; - const meta = JSON.parse(call.data.metadata); + const meta = JSON.parse(call.data.metadata as string); expect(meta.progress).toBe(50); }); @@ -211,7 +211,7 @@ describe('SystemTaskRunner', () => { await runner.finish('Success'); const call = prismaMock.execution.update.mock.calls.at(-1)![0]; - const meta = JSON.parse(call.data.metadata); + const meta = JSON.parse(call.data.metadata as string); expect(meta.progress).toBe(90); }); }); @@ -226,7 +226,7 @@ describe('SystemTaskRunner', () => { await runner.finish('Success'); const call = prismaMock.execution.update.mock.calls.at(-1)![0]; - const logs = JSON.parse(call.data.logs); + const logs = JSON.parse(call.data.logs as string); const entry = logs.find((l: { message: string }) => l.message === 'Processing file'); expect(entry).toBeDefined(); expect(entry.stage).toBe('Running'); @@ -240,7 +240,7 @@ describe('SystemTaskRunner', () => { await runner.flushLogs(true); const call = prismaMock.execution.update.mock.calls.at(-1)![0]; - const logs = JSON.parse(call.data.logs); + const logs = JSON.parse(call.data.logs as string); const entry = logs.find((l: { message: string }) => l.message === 'Command output'); expect(entry.details).toBe('line1\nline2'); }); diff --git a/tests/unit/runner/steps/03-upload.test.ts b/tests/unit/runner/steps/03-upload.test.ts index c1193372..d6098880 100644 --- a/tests/unit/runner/steps/03-upload.test.ts +++ b/tests/unit/runner/steps/03-upload.test.ts @@ -98,6 +98,13 @@ vi.mock('@/services/storage/verification-service', () => ({ }, })); +vi.mock('@/services/storage/storage-service', () => ({ + storageService: { + appendStorageListCacheEntry: vi.fn().mockResolvedValue(undefined), + updateStorageListCacheEntry: vi.fn().mockResolvedValue(undefined), + }, +})); + // --- Helpers --- function makeDestination(overrides: Partial = {}): DestinationContext { diff --git a/tests/unit/runner/steps/05-retention.test.ts b/tests/unit/runner/steps/05-retention.test.ts index 26fd5321..cc45db50 100644 --- a/tests/unit/runner/steps/05-retention.test.ts +++ b/tests/unit/runner/steps/05-retention.test.ts @@ -22,6 +22,14 @@ vi.mock('@/services/dashboard-service', () => ({ refreshStorageStatsCache: vi.fn().mockResolvedValue(undefined), })); +vi.mock('@/services/storage/storage-service', () => ({ + storageService: { + appendStorageListCacheEntry: vi.fn().mockResolvedValue(undefined), + updateStorageListCacheEntry: vi.fn().mockResolvedValue(undefined), + removeStorageListCacheEntry: vi.fn().mockResolvedValue(undefined), + }, +})); + vi.mock('@/lib/prisma', () => ({ default: { systemSetting: { findUnique: vi.fn().mockResolvedValue(null) }, diff --git a/tests/unit/services/integrity-service.test.ts b/tests/unit/services/integrity-service.test.ts index 9eacb914..224477e5 100644 --- a/tests/unit/services/integrity-service.test.ts +++ b/tests/unit/services/integrity-service.test.ts @@ -424,8 +424,8 @@ describe('IntegrityService', () => { ]); (registry.get as ReturnType).mockReturnValue(adapter); // maxAgeDays = 5 - vi.mocked(prisma.systemSetting.findUnique).mockImplementation(({ where }: { where: { key: string } }) => { - if (where.key === 'integrity.maxAgeDays') return Promise.resolve({ value: '5' } as any); + (prisma.systemSetting.findUnique as ReturnType).mockImplementation(({ where }: { where: { key?: string } }) => { + if (where.key === 'integrity.maxAgeDays') return Promise.resolve({ value: '5' }); return Promise.resolve(null); }); vi.mocked(verificationService.verifyFile).mockResolvedValue({ diff --git a/tests/unit/services/verification-service.test.ts b/tests/unit/services/verification-service.test.ts index d2fd4590..d1e90711 100644 --- a/tests/unit/services/verification-service.test.ts +++ b/tests/unit/services/verification-service.test.ts @@ -86,6 +86,7 @@ function makeAdapter(overrides: Record = {}) { read: vi.fn().mockResolvedValue(null), list: vi.fn().mockResolvedValue([]), delete: vi.fn().mockResolvedValue(undefined), + verifyChecksum: undefined as ReturnType | undefined, ...overrides, }; } From 5b710410c6c83a090936f49350cb2a6a6d17e8b7 Mon Sep 17 00:00:00 2001 From: Manu Date: Sun, 14 Jun 2026 14:27:44 +0200 Subject: [PATCH 22/24] Database Explorer: search & sortable columns Add client-side search, column sorting, and refresh support to the Database Explorer table list. Introduces a search input with clear button, clickable column headers to sort by Name/Type/Rows/Size (with visual sort icons), and a refresh button that shows loading state. Uses useMemo for filtered/sorted results and preserves existing empty/missing-size handling. Also updates the changelog to document the new Explorer table list search and sorting features. --- docs/changelog.md | 1 + .../explorer/database-table-list.tsx | 220 ++++++++++++++---- 2 files changed, 176 insertions(+), 45 deletions(-) diff --git a/docs/changelog.md b/docs/changelog.md index 801cbcae..456dbde8 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -28,6 +28,7 @@ All notable changes to DBackup are documented here. - **integrity**: Manual system task runs auto-redirect to the live execution history when "Auto-redirect on job start" is enabled. - **ui**: All data tables (Storage, Jobs, Sources, Destinations, Notifications) now support horizontal scrolling when columns overflow. - **explorer**: Database version history now detects and visually distinguishes downgrades - the change log shows an orange downward arrow for downgrades vs. green upward arrow for upgrades, and downgrade notifications are sent with a distinct "Downgrade" label and warning color. +- **explorer**: Database Explorer table list now supports searching by name and sorting by Name, Type, Rows, or Size via clickable column headers. ### 🧪 Tests diff --git a/src/components/dashboard/explorer/database-table-list.tsx b/src/components/dashboard/explorer/database-table-list.tsx index 3d2a586a..423e0540 100644 --- a/src/components/dashboard/explorer/database-table-list.tsx +++ b/src/components/dashboard/explorer/database-table-list.tsx @@ -1,11 +1,24 @@ "use client"; -import { useState, useCallback, useEffect } from "react"; +import { useState, useCallback, useEffect, useMemo } from "react"; import { Card, CardContent, CardDescription, CardHeader, CardTitle } from "@/components/ui/card"; import { Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from "@/components/ui/table"; +import { Input } from "@/components/ui/input"; +import { Button } from "@/components/ui/button"; import { Skeleton } from "@/components/ui/skeleton"; import { Badge } from "@/components/ui/badge"; -import { AlertTriangle, TableIcon, Database, Rows3 } from "lucide-react"; +import { + AlertTriangle, + TableIcon, + Database, + Rows3, + Search, + ArrowUpDown, + ArrowUp, + ArrowDown, + X, + RefreshCw, +} from "lucide-react"; import { toast } from "sonner"; import { formatBytes } from "@/lib/utils"; @@ -22,6 +35,8 @@ interface DatabaseTableListProps { onTableClick: (tableName: string) => void; } +type SortCol = "name" | "type" | "rows" | "size"; + const TYPE_BADGE: Record = { table: { label: "Table", variant: "secondary" }, view: { label: "View", variant: "outline" }, @@ -33,6 +48,9 @@ export function DatabaseTableList({ sourceId, database, onTableClick }: Database const [tables, setTables] = useState([]); const [isLoading, setIsLoading] = useState(false); const [error, setError] = useState(null); + const [search, setSearch] = useState(""); + const [sortCol, setSortCol] = useState(null); + const [sortDir, setSortDir] = useState<"asc" | "desc">("asc"); const fetchTables = useCallback(async () => { setIsLoading(true); @@ -68,6 +86,44 @@ export function DatabaseTableList({ sourceId, database, onTableClick }: Database const hasSize = tables.some(t => t.sizeInBytes != null); const totalRows = tables.reduce((s, t) => s + (t.rowCount ?? 0), 0); + const displayed = useMemo(() => { + let result = tables.filter(t => + t.name.toLowerCase().includes(search.toLowerCase()) + ); + if (sortCol) { + result = [...result].sort((a, b) => { + let cmp = 0; + if (sortCol === "name") { + cmp = a.name.localeCompare(b.name); + } else if (sortCol === "type") { + cmp = (a.type ?? "").localeCompare(b.type ?? ""); + } else if (sortCol === "rows") { + cmp = (a.rowCount ?? -1) - (b.rowCount ?? -1); + } else if (sortCol === "size") { + cmp = (a.sizeInBytes ?? -1) - (b.sizeInBytes ?? -1); + } + return sortDir === "asc" ? cmp : -cmp; + }); + } + return result; + }, [tables, search, sortCol, sortDir]); + + function toggleSort(col: SortCol) { + if (sortCol === col) { + setSortDir(d => (d === "asc" ? "desc" : "asc")); + } else { + setSortCol(col); + setSortDir("asc"); + } + } + + function SortIcon({ col }: { col: SortCol }) { + if (sortCol !== col) return ; + return sortDir === "asc" + ? + : ; + } + return ( @@ -83,6 +139,15 @@ export function DatabaseTableList({ sourceId, database, onTableClick }: Database : `${tables.length} object${tables.length !== 1 ? "s" : ""}${totalRows > 0 ? ` · ~${totalRows.toLocaleString()} rows` : ""}`}
+
@@ -110,53 +175,118 @@ export function DatabaseTableList({ sourceId, database, onTableClick }: Database

No tables found in this database.

) : ( -
-
- - - Name - Type - Rows - {hasSize && Size} - - - - {tables.map(table => { - const typeMeta = TYPE_BADGE[table.type ?? "table"] ?? TYPE_BADGE.table; - return ( - onTableClick(table.name)} + <> +
+
+ + setSearch(e.target.value)} + className="pl-8 h-8 text-sm" + /> +
+ {search && ( + + )} +
+
+
+ + + toggleSort("name")} > - - - - {table.name} - - - - - {typeMeta.label} - - - - - - {table.rowCount != null ? table.rowCount.toLocaleString() : "-"} + + Name + + + toggleSort("type")} + > + + Type + + + toggleSort("rows")} + > + + Rows + + + {hasSize && ( + toggleSort("size")} + > + + Size + + )} + + + + {displayed.length === 0 ? ( + + + No tables match your search. - {hasSize && ( - - {table.sizeInBytes != null ? formatBytes(table.sizeInBytes) : "-"} - - )} - ); - })} - -
-
+ ) : ( + displayed.map(table => { + const typeMeta = TYPE_BADGE[table.type ?? "table"] ?? TYPE_BADGE.table; + return ( + onTableClick(table.name)} + > + + + + {table.name} + + + + + {typeMeta.label} + + + + + + {table.rowCount != null ? table.rowCount.toLocaleString() : "-"} + + + {hasSize && ( + + {table.sizeInBytes != null ? formatBytes(table.sizeInBytes) : "-"} + + )} + + ); + }) + )} + + +
+ )} From 2208735d387b17b788020cdb35c5459eaafbae80 Mon Sep 17 00:00:00 2001 From: Manu Date: Sun, 14 Jun 2026 14:34:11 +0200 Subject: [PATCH 23/24] Release v2.7.0: update versions & changelog Bump project and docs to v2.7.0 for the v2.7.0 release. Updated version fields in package.json and docs/package.json, updated OpenAPI metadata in api-docs/openapi.yaml and public/openapi.yaml, and added the v2.7.0 release entry (released June 14, 2026) to docs/changelog.md, including updated Docker image tags. --- api-docs/openapi.yaml | 2 +- docs/changelog.md | 8 ++++---- docs/package.json | 2 +- package.json | 2 +- public/openapi.yaml | 2 +- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/api-docs/openapi.yaml b/api-docs/openapi.yaml index 3ae76fd9..df88ef14 100644 --- a/api-docs/openapi.yaml +++ b/api-docs/openapi.yaml @@ -1,7 +1,7 @@ openapi: 3.1.0 info: title: DBackup API - version: 2.6.0 + version: 2.7.0 description: | REST API for DBackup - a self-hosted database backup automation platform with encryption, compression, and smart retention. diff --git a/docs/changelog.md b/docs/changelog.md index 456dbde8..c7fb9ca3 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -2,8 +2,8 @@ All notable changes to DBackup are documented here. -## vNEXT -*Release: In Progress* +## v2.7.0 - Backup Integrity Verification, Storage Explorer Caching, and Multiple Improvements +*Released: June 14, 2026* ### ✨ Features @@ -38,8 +38,8 @@ All notable changes to DBackup are documented here. ### 🐳 Docker -- **Image**: `skyfay/dbackup:vNEXT` -- **Also tagged as**: `latest`, `vNEXT` +- **Image**: `skyfay/dbackup:v2.7.0` +- **Also tagged as**: `latest`, `v2` - **CI Image**: `skyfay/dbackup:ci` - **Platforms**: linux/amd64, linux/arm64 diff --git a/docs/package.json b/docs/package.json index d2f3906c..a5d4179e 100644 --- a/docs/package.json +++ b/docs/package.json @@ -1,6 +1,6 @@ { "name": "dbackup-docs", - "version": "2.6.0", + "version": "2.7.0", "private": true, "scripts": { "dev": "vitepress dev", diff --git a/package.json b/package.json index 7c55c9a6..cff016dc 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "dbackup", - "version": "2.6.0", + "version": "2.7.0", "private": true, "scripts": { "dev": "prisma migrate deploy && prisma generate && next dev", diff --git a/public/openapi.yaml b/public/openapi.yaml index ced1abba..36cfc1d1 100644 --- a/public/openapi.yaml +++ b/public/openapi.yaml @@ -1,7 +1,7 @@ openapi: 3.1.0 info: title: DBackup API - version: 2.6.0 + version: 2.7.0 description: | REST API for DBackup - a self-hosted database backup automation platform with encryption, compression, and smart retention. From 116152804827e2f999f6128c429b062193385848 Mon Sep 17 00:00:00 2001 From: Manu Date: Sun, 14 Jun 2026 14:41:00 +0200 Subject: [PATCH 24/24] Mock storage-service in multi-destination test Add a vi.mock for '@/services/storage/storage-service' in tests/unit/runner/multi-destination-upload.test.ts to stub storageService.appendStorageListCacheEntry (mockResolvedValue undefined). This prevents real storage cache modifications during the unit test run. --- tests/unit/runner/multi-destination-upload.test.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/unit/runner/multi-destination-upload.test.ts b/tests/unit/runner/multi-destination-upload.test.ts index 83d9a3d9..b4d1a74d 100644 --- a/tests/unit/runner/multi-destination-upload.test.ts +++ b/tests/unit/runner/multi-destination-upload.test.ts @@ -28,6 +28,11 @@ vi.mock('@/services/storage/verification-service', () => ({ verifyFile: vi.fn().mockResolvedValue({ status: 'passed', verifiedAt: new Date().toISOString() }), }, })); +vi.mock('@/services/storage/storage-service', () => ({ + storageService: { + appendStorageListCacheEntry: vi.fn().mockResolvedValue(undefined), + }, +})); vi.mock('@/lib/temp-dir', () => ({ getTempDir: vi.fn().mockReturnValue('/tmp'), }));