diff --git a/.changeset/android-media-upload.md b/.changeset/android-media-upload.md new file mode 100644 index 0000000000..69a316d2b4 --- /dev/null +++ b/.changeset/android-media-upload.md @@ -0,0 +1,5 @@ +--- +default: patch +--- + +stream android media uploads to fix OOM and show progress and diverse input improvement. also purge orphaned upload temp files on startup. diff --git a/package.json b/package.json index 342b04f884..deaa0cc1d6 100644 --- a/package.json +++ b/package.json @@ -39,6 +39,7 @@ "@choochmeque/tauri-plugin-notifications-api": "0.5.0-rc.11", "@fontsource-variable/nunito": "5.2.7", "@fontsource/space-mono": "5.2.9", + "@noble/hashes": "^2.2.0", "@phosphor-icons/react": "^2.1.10", "@sableclient/twemoji-font": "^1.0.4", "@sentry/react": "^10.63.0", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 2856921370..c53fb51dfe 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -32,6 +32,9 @@ importers: '@fontsource/space-mono': specifier: 5.2.9 version: 5.2.9 + '@noble/hashes': + specifier: ^2.2.0 + version: 2.2.0 '@phosphor-icons/react': specifier: ^2.1.10 version: 2.1.10(react-dom@18.3.1(react@18.3.1))(react@18.3.1) @@ -314,7 +317,7 @@ importers: version: 0.7.1 jsdom: specifier: ^29.1.1 - version: 29.1.1 + version: 29.1.1(@noble/hashes@2.2.0) knip: specifier: 6.23.0 version: 6.23.0 @@ -350,7 +353,7 @@ importers: version: 1.6.0(@swc/helpers@0.5.23)(rollup@2.80.0)(vite@7.3.5(@types/node@24.13.2)(jiti@2.7.0)(lightningcss@1.32.0)(terser@5.48.0)(yaml@2.9.0)) vitest: specifier: ^4.1.9 - version: 4.1.9(@types/node@24.13.2)(@vitest/coverage-v8@4.1.9)(@vitest/ui@4.1.9)(jsdom@29.1.1)(vite@7.3.5(@types/node@24.13.2)(jiti@2.7.0)(lightningcss@1.32.0)(terser@5.48.0)(yaml@2.9.0)) + version: 4.1.9(@types/node@24.13.2)(@vitest/coverage-v8@4.1.9)(@vitest/ui@4.1.9)(jsdom@29.1.1(@noble/hashes@2.2.0))(vite@7.3.5(@types/node@24.13.2)(jiti@2.7.0)(lightningcss@1.32.0)(terser@5.48.0)(yaml@2.9.0)) wrangler: specifier: ^4.106.0 version: 4.106.0 @@ -1651,6 +1654,10 @@ packages: '@emnapi/core': ^1.7.1 '@emnapi/runtime': ^1.7.1 + '@noble/hashes@2.2.0': + resolution: {integrity: sha512-IYqDGiTXab6FniAgnSdZwgWbomxpy9FtYvLKs7wCUs2a8RkITG+DFGO1DM9cr+E3/RgADRpFjrKVaJ1z6sjtEg==} + engines: {node: '>= 20.19.0'} + '@oxc-parser/binding-android-arm-eabi@0.137.0': resolution: {integrity: sha512-KDs+0VPdEmasOkpuJHW9V5WCF+cvYdMQv2Jd+aJXt+cxIx12NToRQRbXaRwUEDsZw+/jMk81Ve8ZFbjUkJTOwA==} engines: {node: ^20.19.0 || >=22.12.0} @@ -6434,7 +6441,9 @@ snapshots: '@esbuild/win32-x64@0.28.1': optional: true - '@exodus/bytes@1.15.1': {} + '@exodus/bytes@1.15.1(@noble/hashes@2.2.0)': + optionalDependencies: + '@noble/hashes': 2.2.0 '@fontsource-variable/nunito@5.2.7': {} @@ -6652,6 +6661,8 @@ snapshots: '@tybys/wasm-util': 0.10.2 optional: true + '@noble/hashes@2.2.0': {} + '@oxc-parser/binding-android-arm-eabi@0.137.0': optional: true @@ -7744,7 +7755,7 @@ snapshots: obug: 2.1.3 std-env: 4.1.0 tinyrainbow: 3.1.0 - vitest: 4.1.9(@types/node@24.13.2)(@vitest/coverage-v8@4.1.9)(@vitest/ui@4.1.9)(jsdom@29.1.1)(vite@7.3.5(@types/node@24.13.2)(jiti@2.7.0)(lightningcss@1.32.0)(terser@5.48.0)(yaml@2.9.0)) + vitest: 4.1.9(@types/node@24.13.2)(@vitest/coverage-v8@4.1.9)(@vitest/ui@4.1.9)(jsdom@29.1.1(@noble/hashes@2.2.0))(vite@7.3.5(@types/node@24.13.2)(jiti@2.7.0)(lightningcss@1.32.0)(terser@5.48.0)(yaml@2.9.0)) '@vitest/expect@4.1.9': dependencies: @@ -7790,7 +7801,7 @@ snapshots: sirv: 3.0.2 tinyglobby: 0.2.17 tinyrainbow: 3.1.0 - vitest: 4.1.9(@types/node@24.13.2)(@vitest/coverage-v8@4.1.9)(@vitest/ui@4.1.9)(jsdom@29.1.1)(vite@7.3.5(@types/node@24.13.2)(jiti@2.7.0)(lightningcss@1.32.0)(terser@5.48.0)(yaml@2.9.0)) + vitest: 4.1.9(@types/node@24.13.2)(@vitest/coverage-v8@4.1.9)(@vitest/ui@4.1.9)(jsdom@29.1.1(@noble/hashes@2.2.0))(vite@7.3.5(@types/node@24.13.2)(jiti@2.7.0)(lightningcss@1.32.0)(terser@5.48.0)(yaml@2.9.0)) '@vitest/utils@4.1.9': dependencies: @@ -8055,10 +8066,10 @@ snapshots: csstype@3.2.3: {} - data-urls@7.0.0: + data-urls@7.0.0(@noble/hashes@2.2.0): dependencies: whatwg-mimetype: 5.0.0 - whatwg-url: 16.0.1 + whatwg-url: 16.0.1(@noble/hashes@2.2.0) transitivePeerDependencies: - '@noble/hashes' @@ -8540,9 +8551,9 @@ snapshots: domhandler: 6.0.1 htmlparser2: 12.0.0 - html-encoding-sniffer@6.0.0: + html-encoding-sniffer@6.0.0(@noble/hashes@2.2.0): dependencies: - '@exodus/bytes': 1.15.1 + '@exodus/bytes': 1.15.1(@noble/hashes@2.2.0) transitivePeerDependencies: - '@noble/hashes' @@ -8795,17 +8806,17 @@ snapshots: dependencies: argparse: 2.0.1 - jsdom@29.1.1: + jsdom@29.1.1(@noble/hashes@2.2.0): dependencies: '@asamuzakjp/css-color': 5.1.11 '@asamuzakjp/dom-selector': 7.1.1 '@bramus/specificity': 2.4.2 '@csstools/css-syntax-patches-for-csstree': 1.1.5(css-tree@3.2.1) - '@exodus/bytes': 1.15.1 + '@exodus/bytes': 1.15.1(@noble/hashes@2.2.0) css-tree: 3.2.1 - data-urls: 7.0.0 + data-urls: 7.0.0(@noble/hashes@2.2.0) decimal.js: 10.6.0 - html-encoding-sniffer: 6.0.0 + html-encoding-sniffer: 6.0.0(@noble/hashes@2.2.0) is-potential-custom-element-name: 1.0.1 lru-cache: 11.5.1 parse5: 8.0.1 @@ -8816,7 +8827,7 @@ snapshots: w3c-xmlserializer: 5.0.0 webidl-conversions: 8.0.1 whatwg-mimetype: 5.0.0 - whatwg-url: 16.0.1 + whatwg-url: 16.0.1(@noble/hashes@2.2.0) xml-name-validator: 5.0.0 transitivePeerDependencies: - '@noble/hashes' @@ -10060,7 +10071,7 @@ snapshots: terser: 5.48.0 yaml: 2.9.0 - vitest@4.1.9(@types/node@24.13.2)(@vitest/coverage-v8@4.1.9)(@vitest/ui@4.1.9)(jsdom@29.1.1)(vite@7.3.5(@types/node@24.13.2)(jiti@2.7.0)(lightningcss@1.32.0)(terser@5.48.0)(yaml@2.9.0)): + vitest@4.1.9(@types/node@24.13.2)(@vitest/coverage-v8@4.1.9)(@vitest/ui@4.1.9)(jsdom@29.1.1(@noble/hashes@2.2.0))(vite@7.3.5(@types/node@24.13.2)(jiti@2.7.0)(lightningcss@1.32.0)(terser@5.48.0)(yaml@2.9.0)): dependencies: '@vitest/expect': 4.1.9 '@vitest/mocker': 4.1.9(vite@7.3.5(@types/node@24.13.2)(jiti@2.7.0)(lightningcss@1.32.0)(terser@5.48.0)(yaml@2.9.0)) @@ -10086,7 +10097,7 @@ snapshots: '@types/node': 24.13.2 '@vitest/coverage-v8': 4.1.9(vitest@4.1.9) '@vitest/ui': 4.1.9(vitest@4.1.9) - jsdom: 29.1.1 + jsdom: 29.1.1(@noble/hashes@2.2.0) transitivePeerDependencies: - msw @@ -10106,9 +10117,9 @@ snapshots: whatwg-mimetype@5.0.0: {} - whatwg-url@16.0.1: + whatwg-url@16.0.1(@noble/hashes@2.2.0): dependencies: - '@exodus/bytes': 1.15.1 + '@exodus/bytes': 1.15.1(@noble/hashes@2.2.0) tr46: 6.0.0 webidl-conversions: 8.0.1 transitivePeerDependencies: diff --git a/src-tauri/Cargo.lock b/src-tauri/Cargo.lock index 0fa65b93e3..2c32b2e738 100644 --- a/src-tauri/Cargo.lock +++ b/src-tauri/Cargo.lock @@ -4970,6 +4970,7 @@ dependencies = [ "cookie_store", "encoding_rs", "futures-core", + "futures-util", "h2 0.4.14", "http 1.4.0", "http-body 1.0.1", @@ -4991,12 +4992,14 @@ dependencies = [ "sync_wrapper 1.0.2", "tokio", "tokio-rustls", + "tokio-util", "tower 0.5.3", "tower-http 0.6.8", "tower-service", "url", "wasm-bindgen", "wasm-bindgen-futures", + "wasm-streams 0.4.2", "web-sys", "webpki-roots", ] @@ -5036,7 +5039,7 @@ dependencies = [ "url", "wasm-bindgen", "wasm-bindgen-futures", - "wasm-streams", + "wasm-streams 0.5.0", "web-sys", ] @@ -5218,14 +5221,18 @@ checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" name = "sable" version = "0.1.0" dependencies = [ + "async-stream", + "base64 0.22.1", "cef", "enigo", + "futures-util", "gtk", "jni 0.21.1", "libloading 0.8.9", "log", "objc2", "percent-encoding", + "reqwest 0.12.28", "serde", "serde_json", "sha2", @@ -5250,6 +5257,7 @@ dependencies = [ "tauri-runtime-cef", "tauri-typegen", "tokio", + "tokio-util", "ts-rs", "webkit2gtk", "windows", @@ -7806,6 +7814,19 @@ dependencies = [ "wasmparser", ] +[[package]] +name = "wasm-streams" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15053d8d85c7eccdbefef60f06769760a563c7f0a9d6902a13d35c7800b0ad65" +dependencies = [ + "futures-util", + "js-sys", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + [[package]] name = "wasm-streams" version = "0.5.0" diff --git a/src-tauri/Cargo.toml b/src-tauri/Cargo.toml index c8f9488393..101e8d646a 100644 --- a/src-tauri/Cargo.toml +++ b/src-tauri/Cargo.toml @@ -22,10 +22,15 @@ tauri-typegen = "0.5" serde_json = "1.0" serde = { version = "1.0", features = ["derive"] } log = "0.4" -tokio = { version = "1", features = ["time", "sync"] } +tokio = { version = "1", features = ["time", "sync", "fs", "io-util"] } ts-rs = "12.0.0" sha2 = "0.10" percent-encoding = "2" +reqwest = { version = "0.12", default-features = false, features = ["stream"] } +async-stream = "0.3" +base64 = "0.22" +tokio-util = { version = "0.7", features = ["codec"] } +futures-util = "0.3" tauri = { version = "2.11.5", default-features = false, features = [ "tray-icon", diff --git a/src-tauri/src/lib.rs b/src-tauri/src/lib.rs index 24207fc97d..9aa3058a28 100644 --- a/src-tauri/src/lib.rs +++ b/src-tauri/src/lib.rs @@ -268,6 +268,8 @@ pub fn run() { network::media_protocol::respond, ) .setup(|app| { + network::native_upload::cleanup_uploads(app.handle()); + #[cfg(any(target_os = "linux", all(debug_assertions, windows)))] { use tauri_plugin_deep_link::DeepLinkExt; @@ -323,6 +325,9 @@ pub fn run() { .invoke_handler(tauri::generate_handler![ network::loopback_http::abort_loopback_fetch, network::loopback_http::loopback_fetch, + network::native_upload::native_upload, + network::native_upload::upload_write_chunk, + network::native_upload::abort_native_upload, network::media_protocol::set_media_session, network::media_protocol::clear_media_session, #[cfg(target_os = "android")] diff --git a/src-tauri/src/network/mod.rs b/src-tauri/src/network/mod.rs index 1f6c79b863..ae26b7f556 100644 --- a/src-tauri/src/network/mod.rs +++ b/src-tauri/src/network/mod.rs @@ -1,2 +1,3 @@ pub mod loopback_http; pub mod media_protocol; +pub mod native_upload; diff --git a/src-tauri/src/network/native_upload.rs b/src-tauri/src/network/native_upload.rs new file mode 100644 index 0000000000..9e92e7487d --- /dev/null +++ b/src-tauri/src/network/native_upload.rs @@ -0,0 +1,224 @@ +use std::{ + collections::HashMap, + path::PathBuf, + sync::{LazyLock, Mutex}, + time::Duration, +}; + +use base64::Engine; +use futures_util::StreamExt; +use serde::Serialize; +use tauri::{ipc::Channel, AppHandle, Manager, Runtime}; +use tauri_plugin_http::reqwest::{ + header::{AUTHORIZATION, CONTENT_LENGTH, CONTENT_TYPE}, + Body, Client, ClientBuilder, Method, Url, +}; +use tokio::{ + fs::{File, OpenOptions}, + io::AsyncWriteExt, + sync::watch, +}; +use tokio_util::codec::{BytesCodec, FramedRead}; + +const UPLOAD_SUBDIR: &str = "native-uploads"; + +static UPLOAD_ABORT_SENDERS: LazyLock>>> = + LazyLock::new(|| Mutex::new(HashMap::new())); + +static UPLOAD_CLIENT: LazyLock = LazyLock::new(|| { + ClientBuilder::new() + .no_proxy() + .connect_timeout(Duration::from_secs(15)) + .build() + .expect("failed to build native upload client") +}); + +#[derive(Debug, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct NativeUploadResponse { + status: u16, + body: String, +} + +#[derive(Clone, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct ProgressPayload { + loaded: u64, + total: u64, +} + +fn upload_temp_path(app: &AppHandle, request_id: &str) -> Result { + if request_id.is_empty() + || !request_id + .chars() + .all(|c| c.is_ascii_alphanumeric() || c == '-' || c == '_') + { + return Err("invalid upload id".into()); + } + let dir = app + .path() + .app_cache_dir() + .map_err(|err| err.to_string())? + .join(UPLOAD_SUBDIR); + Ok(dir.join(request_id)) +} + +/// Drop leftover upload temp files from a session that ended without running +/// the normal cleanup (crash, OS kill, ...). No upload can be in flight yet at +/// startup, so removing the whole subdir is safe; it is recreated on demand by +/// `upload_write_chunk`. +pub fn cleanup_uploads(app: &AppHandle) { + let Ok(dir) = app.path().app_cache_dir() else { + return; + }; + let _ = std::fs::remove_dir_all(dir.join(UPLOAD_SUBDIR)); +} + +fn register_abort_sender(request_id: &str) -> watch::Receiver { + let (sender, receiver) = watch::channel(false); + UPLOAD_ABORT_SENDERS + .lock() + .expect("upload abort senders poisoned") + .insert(request_id.to_owned(), sender); + receiver +} + +fn remove_abort_sender(request_id: &str) { + UPLOAD_ABORT_SENDERS + .lock() + .expect("upload abort senders poisoned") + .remove(request_id); +} + +async fn wait_for_abort_signal(receiver: &mut watch::Receiver) { + if *receiver.borrow() { + return; + } + while receiver.changed().await.is_ok() { + if *receiver.borrow() { + return; + } + } + std::future::pending::<()>().await; +} + +#[tauri::command] +pub async fn upload_write_chunk( + app: AppHandle, + request_id: String, + chunk: String, +) -> Result<(), String> { + let path = upload_temp_path(&app, &request_id)?; + if let Some(parent) = path.parent() { + tokio::fs::create_dir_all(parent) + .await + .map_err(|err| err.to_string())?; + } + let bytes = base64::engine::general_purpose::STANDARD + .decode(chunk.as_bytes()) + .map_err(|err| err.to_string())?; + let mut file = OpenOptions::new() + .create(true) + .append(true) + .open(&path) + .await + .map_err(|err| err.to_string())?; + file.write_all(&bytes) + .await + .map_err(|err| err.to_string())?; + Ok(()) +} + +#[tauri::command] +pub async fn abort_native_upload(app: AppHandle, request_id: String) { + if let Some(sender) = UPLOAD_ABORT_SENDERS + .lock() + .expect("upload abort senders poisoned") + .remove(&request_id) + { + let _ = sender.send(true); + } + if let Ok(path) = upload_temp_path(&app, &request_id) { + let _ = tokio::fs::remove_file(path).await; + } +} + +#[tauri::command] +pub async fn native_upload( + app: AppHandle, + request_id: String, + url: String, + content_type: String, + authorization: Option, + on_progress: Channel, +) -> Result { + let path = upload_temp_path(&app, &request_id)?; + let result = run_upload( + &path, + url, + content_type, + authorization, + &request_id, + on_progress, + ) + .await; + let _ = tokio::fs::remove_file(&path).await; + remove_abort_sender(&request_id); + result +} + +async fn run_upload( + path: &PathBuf, + url: String, + content_type: String, + authorization: Option, + request_id: &str, + on_progress: Channel, +) -> Result { + let url = Url::parse(url.trim()).map_err(|err| err.to_string())?; + if url.scheme() != "http" && url.scheme() != "https" { + return Err("native_upload only allows http(s) URLs".into()); + } + + let file = File::open(path) + .await + .map_err(|_| "no upload data for this request".to_string())?; + let total = file.metadata().await.map_err(|err| err.to_string())?.len(); + + let progress_stream = async_stream::stream! { + let mut reader = FramedRead::new(file, BytesCodec::new()); + let mut loaded: u64 = 0; + while let Some(item) = reader.next().await { + match item { + Ok(chunk) => { + loaded += chunk.len() as u64; + let _ = on_progress.send(ProgressPayload { loaded, total }); + yield Ok::<_, std::io::Error>(chunk.freeze()); + } + Err(err) => yield Err(err), + } + } + }; + + let mut builder = UPLOAD_CLIENT + .request(Method::POST, url) + .header(CONTENT_TYPE, content_type) + .header(CONTENT_LENGTH, total) + .body(Body::wrap_stream(progress_stream)); + if let Some(auth) = authorization { + builder = builder.header(AUTHORIZATION, auth); + } + + let mut abort_receiver = register_abort_sender(request_id); + let response = tokio::select! { + response = builder.send() => response.map_err(|err| err.to_string())?, + _ = wait_for_abort_signal(&mut abort_receiver) => return Err("Upload aborted".into()), + }; + let status = response.status().as_u16(); + let body = tokio::select! { + body = response.text() => body.map_err(|err| err.to_string())?, + _ = wait_for_abort_signal(&mut abort_receiver) => return Err("Upload aborted".into()), + }; + + Ok(NativeUploadResponse { status, body }) +} diff --git a/src/app/components/editor/Editor.tsx b/src/app/components/editor/Editor.tsx index 076cae68e8..a9ff50793a 100644 --- a/src/app/components/editor/Editor.tsx +++ b/src/app/components/editor/Editor.tsx @@ -1,4 +1,9 @@ -import type { ClipboardEventHandler, KeyboardEventHandler, ReactNode } from 'react'; +import type { + ClipboardEventHandler, + KeyboardEventHandler, + MutableRefObject, + ReactNode, +} from 'react'; import { forwardRef, useCallback, useEffect, useLayoutEffect, useRef, useState } from 'react'; import { Box, Scroll, Text } from 'folds'; import type { Descendant, Editor } from 'slate'; @@ -73,6 +78,8 @@ type CustomEditorProps = { onPaste?: ClipboardEventHandler; className?: string; variant?: 'Surface' | 'SurfaceVariant' | 'Background'; + enterKeyHint?: 'enter' | 'send'; + suppressBlurRefocusRef?: MutableRefObject; }; export const CustomEditor = forwardRef( ( @@ -93,6 +100,8 @@ export const CustomEditor = forwardRef( onPaste, className, variant = 'SurfaceVariant', + enterKeyHint, + suppressBlurRefocusRef, }, ref ) => { @@ -443,9 +452,12 @@ export const CustomEditor = forwardRef( onPaste={onPaste} // Defer to OS capitalization setting (respects iOS sentence-case toggle). autoCapitalize="sentences" + autoCorrect="on" + enterKeyHint={enterKeyHint} // keeps focus after pressing send, but yields to another editor. onBlur={(evt) => { if (!mobileOrTablet()) return; + if (suppressBlurRefocusRef?.current) return; const next = evt.relatedTarget as HTMLElement | null; if (next && next !== editableRef.current && next.isContentEditable) return; ReactEditor.focus(editor); diff --git a/src/app/components/upload-board/UploadBoard.tsx b/src/app/components/upload-board/UploadBoard.tsx index 6875a50900..f0a26d386f 100644 --- a/src/app/components/upload-board/UploadBoard.tsx +++ b/src/app/components/upload-board/UploadBoard.tsx @@ -1,11 +1,11 @@ import type { MutableRefObject, ReactNode } from 'react'; -import { useImperativeHandle, useRef } from 'react'; +import { useEffect, useImperativeHandle, useRef } from 'react'; import { Badge, Box, Chip, Header, Spinner, Text, as, percent } from 'folds'; import { CaretRight, CaretUp, X, sizedIcon } from '$components/icons/phosphor'; import classNames from 'classnames'; import { useAtomValue } from 'jotai'; -import type { TUploadFamilyObserverAtom, Upload, UploadSuccess } from '$state/upload'; +import type { TUploadFamilyObserverAtom, Upload } from '$state/upload'; import { UploadStatus } from '$state/upload'; import * as css from './UploadBoard.css'; @@ -34,7 +34,8 @@ type UploadBoardHeaderProps = { onToggle: () => void; uploadFamilyObserverAtom: TUploadFamilyObserverAtom; onCancel: (uploads: Upload[]) => void; - onSend: (uploads: UploadSuccess[]) => Promise; + onSend: (uploads: Upload[]) => Promise; + onBusyChange?: (busy: boolean) => void; imperativeHandlerRef: MutableRefObject; }; @@ -44,6 +45,7 @@ export function UploadBoardHeader({ uploadFamilyObserverAtom, onCancel, onSend, + onBusyChange, imperativeHandlerRef, }: UploadBoardHeaderProps) { const sendingRef = useRef(false); @@ -51,6 +53,11 @@ export function UploadBoardHeader({ const isSuccess = uploads.every((upload) => upload.status === UploadStatus.Success); const isError = uploads.some((upload) => upload.status === UploadStatus.Error); + const busy = uploads.length > 0 && !isSuccess && !isError; + useEffect(() => { + onBusyChange?.(busy); + return () => onBusyChange?.(false); + }, [busy, onBusyChange]); const progress = uploads.reduce( (acc, upload) => { acc.total += upload.file.size; @@ -68,8 +75,16 @@ export function UploadBoardHeader({ const handleSend = async () => { if (sendingRef.current) return; sendingRef.current = true; - await onSend(uploads.filter((upload) => upload.status === UploadStatus.Success)); - sendingRef.current = false; + try { + await onSend( + uploads.filter( + (upload) => + upload.status === UploadStatus.Success || upload.status === UploadStatus.Loading + ) + ); + } finally { + sendingRef.current = false; + } }; useImperativeHandle(imperativeHandlerRef, () => ({ diff --git a/src/app/components/upload-card/UploadCard.tsx b/src/app/components/upload-card/UploadCard.tsx index 9360a26546..d20b0ce269 100644 --- a/src/app/components/upload-card/UploadCard.tsx +++ b/src/app/components/upload-card/UploadCard.tsx @@ -26,7 +26,7 @@ export const UploadCard = forwardRef< > {before} - + {children} {after} diff --git a/src/app/components/upload-card/UploadCardRenderer.tsx b/src/app/components/upload-card/UploadCardRenderer.tsx index c00058fb5f..9625b5f837 100644 --- a/src/app/components/upload-card/UploadCardRenderer.tsx +++ b/src/app/components/upload-card/UploadCardRenderer.tsx @@ -390,7 +390,7 @@ export function UploadCardRenderer({ const [isDescribed, setIsDescribed] = useState(false); - if (upload.status === UploadStatus.Idle && !fileSizeExceeded) { + if (upload.status === UploadStatus.Idle && !fileSizeExceeded && !fileItem.encrypting) { startUpload(); } @@ -560,7 +560,7 @@ export function UploadCardRenderer({ } > - + {file.name} {upload.status === UploadStatus.Success && diff --git a/src/app/features/room/RoomInput.tsx b/src/app/features/room/RoomInput.tsx index db3c24b7e0..5452c8e92d 100644 --- a/src/app/features/room/RoomInput.tsx +++ b/src/app/features/room/RoomInput.tsx @@ -29,6 +29,7 @@ import { OverlayCenter, PopOut, Scroll, + Spinner, Text, toRem, } from 'folds'; @@ -339,7 +340,11 @@ export const RoomInput = forwardRef( const [replyDraft, setReplyDraft] = useAtom(roomIdToReplyDraftAtomFamily(draftKey)); const [uploadBoard, setUploadBoard] = useState(true); + const [uploadSending, setUploadSending] = useState(false); + const [uploadBusy, setUploadBusy] = useState(false); const [selectedFiles, setSelectedFiles] = useAtom(roomIdToUploadItemsAtomFamily(draftKey)); + const isEncrypting = selectedFiles.some((f) => f.encrypting); + const sendBusy = uploadSending || isEncrypting || uploadBusy; const uploadFamilyObserverAtom = createUploadFamilyObserverAtom( roomUploadAtomFamily, selectedFiles.map((f) => f.file) @@ -347,6 +352,13 @@ export const RoomInput = forwardRef( const uploadBoardHandlers = useRef(); const longPressTimer = useRef | null>(null); const isLongPress = useRef(false); + const suppressBlurRefocusRef = useRef(false); + const suppressEditorRefocus = useCallback(() => { + suppressBlurRefocusRef.current = true; + requestAnimationFrame(() => { + suppressBlurRefocusRef.current = false; + }); + }, []); const imagePackRooms: Room[] = useImagePackRooms(roomId, roomToParents); @@ -387,39 +399,46 @@ export const RoomInput = forwardRef( async (files: File[], audioMeta?: { waveform: number[]; audioDuration: number }) => { setUploadBoard(true); const safeFiles = files.map(safeFile); - const fileItems: TUploadItem[] = []; + const makeMetadata = () => ({ + markedAsSpoiler: false, + waveform: audioMeta?.waveform, + audioDuration: audioMeta?.audioDuration, + }); if (room.hasEncryptionStateEvent()) { - const encryptFiles = fulfilledPromiseSettledResult( - await Promise.allSettled(safeFiles.map((f) => encryptFile(f))) - ); - encryptFiles.forEach((ef) => - fileItems.push({ - ...ef, - metadata: { - markedAsSpoiler: false, - waveform: audioMeta?.waveform, - audioDuration: audioMeta?.audioDuration, - }, - }) - ); - } else { - safeFiles.forEach((f) => - fileItems.push({ - file: f, - originalFile: f, - encInfo: undefined, - metadata: { - markedAsSpoiler: false, - waveform: audioMeta?.waveform, - audioDuration: audioMeta?.audioDuration, - }, - }) - ); + const placeholders: TUploadItem[] = safeFiles.map((f) => ({ + file: f, + originalFile: f, + encInfo: undefined, + encrypting: true, + metadata: makeMetadata(), + })); + setSelectedFiles({ type: 'PUT', item: placeholders }); + placeholders.forEach((placeholder) => { + encryptFile(placeholder.originalFile) + .then((ef) => + setSelectedFiles({ + type: 'REPLACE', + item: placeholder, + replacement: { ...ef, encrypting: false, metadata: placeholder.metadata }, + }) + ) + .catch((encryptError: unknown) => { + log.warn('Failed to encrypt file for upload:', encryptError); + setSelectedFiles({ type: 'DELETE', item: placeholder }); + }); + }); + return; } + setSelectedFiles({ type: 'PUT', - item: fileItems, + item: safeFiles.map((f) => ({ + file: f, + originalFile: f, + encInfo: undefined, + metadata: makeMetadata(), + })), }); }, [setSelectedFiles, room] @@ -708,7 +727,7 @@ export const RoomInput = forwardRef( return getFileMsgContent(fileItem, upload.mxc); }; - const handleSendUpload = async (uploads: UploadSuccess[]) => { + const handleSendUpload = async (uploads: Upload[]) => { const plainText = toPlainText(editor.children).trim(); const caption = plainText.length > 0 ? plainText : undefined; let customHtml = trimCustomHtml( @@ -720,11 +739,26 @@ export const RoomInput = forwardRef( const formattedCaption = caption && !customHtmlEqualsPlainText(customHtml, plainText) ? customHtml : undefined; - if (uploads.length == 1 && sendIndividualAttachmentAsCaption) { - const upload = uploads[0]; + const resolved = fulfilledPromiseSettledResult( + await Promise.allSettled( + uploads.map(async (upload): Promise => { + if (upload.status === UploadStatus.Success) return upload; + if (upload.status === UploadStatus.Loading) { + const response = await upload.promise; + if (!response.content_uri) throw new Error('Upload failed'); + return { status: UploadStatus.Success, file: upload.file, mxc: response.content_uri }; + } + throw new Error('Upload not ready'); + }) + ) + ); + if (resolved.length === 0) return; + + if (resolved.length == 1 && sendIndividualAttachmentAsCaption) { + const upload = resolved[0]; if (!upload) throw new Error('Broken upload'); let content = await uploadToContent(upload); - handleCancelUpload(uploads); + handleCancelUpload(resolved); content.body = caption ?? ''; content.formatted_body = undefined; @@ -737,13 +771,13 @@ export const RoomInput = forwardRef( await handleSendContents([content]); return; } - if (uploads.length >= 2 && enableMediaGalleries) { - const itemsPromises = uploads.map(async (upload) => { + if (resolved.length >= 2 && enableMediaGalleries) { + const itemsPromises = resolved.map(async (upload) => { const fileItem = selectedFiles.find((f) => f.file === upload.file); if (!fileItem) throw new Error('Broken upload'); return getGalleryItemContent(mx, fileItem, upload.mxc); }); - handleCancelUpload(uploads); + handleCancelUpload(resolved); const items = fulfilledPromiseSettledResult(await Promise.allSettled(itemsPromises)); if (items.length === 0) return; @@ -753,8 +787,8 @@ export const RoomInput = forwardRef( await handleSendContents([galleryContent]); return; } - const contentsPromises = uploads.map(uploadToContent); - handleCancelUpload(uploads); + const contentsPromises = resolved.map(uploadToContent); + handleCancelUpload(resolved); const contents = fulfilledPromiseSettledResult(await Promise.allSettled(contentsPromises)); await handleSendContents(contents); @@ -800,6 +834,7 @@ export const RoomInput = forwardRef( ); const submit = useCallback(async () => { + if (selectedFiles.some((f) => f.encrypting)) return; uploadBoardHandlers.current?.handleSend(); if ( (selectedFiles.length >= 2 && enableMediaGalleries) || @@ -1486,6 +1521,8 @@ export const RoomInput = forwardRef( editor={editor} key={inputKey} placeholder="Send a message..." + enterKeyHint={enterForNewline ? 'enter' : 'send'} + suppressBlurRefocusRef={suppressBlurRefocusRef} onKeyDown={handleKeyDown} onKeyUp={handleKeyUp} onPaste={handlePaste} @@ -1500,7 +1537,15 @@ export const RoomInput = forwardRef( open={uploadBoard} onToggle={() => setUploadBoard(!uploadBoard)} uploadFamilyObserverAtom={uploadFamilyObserverAtom} - onSend={handleSendUpload} + onSend={async (uploads) => { + setUploadSending(true); + try { + await handleSendUpload(uploads); + } finally { + setUploadSending(false); + } + }} + onBusyChange={setUploadBusy} imperativeHandlerRef={uploadBoardHandlers} onCancel={handleCancelUpload} /> @@ -1713,6 +1758,7 @@ export const RoomInput = forwardRef( ? pickFile('*') : setAddMenuAnchor(evt.currentTarget.getBoundingClientRect()) } + onPointerDown={suppressEditorRefocus} variant="SurfaceVariant" size="300" radii="300" @@ -1785,97 +1831,130 @@ export const RoomInput = forwardRef( - {() => ( - { - setEmojiBoardTab((t) => { - if (t) { - if (!mobileOrTablet()) ReactEditor.focus(editor); - return undefined; - } - return t; - }); - }} - /> - } - > - {showGifPicker && ( - setEmojiBoardTab(EmojiBoardTab.Gif)} - variant="SurfaceVariant" - size="300" - radii="300" - style={{ backgroundColor: 'transparent' }} - > - {composerIcon(GifIcon, { - weight: emojiBoardTab === EmojiBoardTab.Gif ? 'fill' : 'regular', - })} - - )} - {!hideStickerBtn && ( + {() => { + const emojiBoard = ( + { + setEmojiBoardTab((t) => { + if (t) { + if (!mobileOrTablet()) ReactEditor.focus(editor); + return undefined; + } + return t; + }); + }} + /> + ); + const triggers = ( + <> + {showGifPicker && ( + setEmojiBoardTab(EmojiBoardTab.Gif)} + onPointerDown={suppressEditorRefocus} + variant="SurfaceVariant" + size="300" + radii="300" + style={{ backgroundColor: 'transparent' }} + > + {composerIcon(GifIcon, { + weight: emojiBoardTab === EmojiBoardTab.Gif ? 'fill' : 'regular', + })} + + )} + {!hideStickerBtn && ( + setEmojiBoardTab(EmojiBoardTab.Sticker)} + onPointerDown={suppressEditorRefocus} + variant="SurfaceVariant" + size="300" + radii="300" + style={{ backgroundColor: 'transparent' }} + title="open sticker picker" + aria-label="Open sticker picker" + > + {composerIcon(Sticker, { + weight: emojiBoardTab === EmojiBoardTab.Sticker ? 'fill' : 'regular', + })} + + )} setEmojiBoardTab(EmojiBoardTab.Sticker)} + ref={emojiBtnRef} + aria-pressed={ + hideStickerBtn + ? emojiBoardTab === EmojiBoardTab.Emoji || + emojiBoardTab === EmojiBoardTab.Gif + : emojiBoardTab === EmojiBoardTab.Emoji + } + onClick={() => setEmojiBoardTab(EmojiBoardTab.Emoji)} + onPointerDown={suppressEditorRefocus} variant="SurfaceVariant" size="300" radii="300" style={{ backgroundColor: 'transparent' }} - title="open sticker picker" - aria-label="Open sticker picker" + title="open emoji picker" + aria-label="Open emoji picker" > - {composerIcon(Sticker, { - weight: emojiBoardTab === EmojiBoardTab.Sticker ? 'fill' : 'regular', + {composerIcon(Smiley, { + weight: hideStickerBtn + ? emojiBoardTab + ? 'fill' + : 'regular' + : emojiBoardTab === EmojiBoardTab.Emoji + ? 'fill' + : 'regular', })} - )} - + ); + if (mobileOrTablet()) { + return ( + <> + {triggers} + }> +
+ {emojiBoard} +
+ + + ); + } + return ( + setEmojiBoardTab(EmojiBoardTab.Emoji)} - variant="SurfaceVariant" - size="300" - radii="300" - style={{ backgroundColor: 'transparent' }} - title="open emoji picker" - aria-label="Open emoji picker" + content={emojiBoard} > - {composerIcon(Smiley, { - weight: hideStickerBtn - ? emojiBoardTab - ? 'fill' - : 'regular' - : emojiBoardTab === EmojiBoardTab.Emoji - ? 'fill' - : 'regular', - })} - - - )} + {triggers} + + ); + }} ( { if (isLongPress.current) { @@ -1959,7 +2039,13 @@ export const RoomInput = forwardRef( radii="0" className={delayedEventsSupported ? css.SplitSendButton : undefined} > - {scheduledTime ? composerIcon(Clock) : composerIcon(PaperPlaneTilt)} + {sendBusy ? ( + + ) : scheduledTime ? ( + composerIcon(Clock) + ) : ( + composerIcon(PaperPlaneTilt) + )} {delayedEventsSupported && !mobileOrTablet() && ( ( } }, [saveState, save]); + const suppressBlurRefocusRef = useRef(false); + const suppressEditorRefocus = useCallback(() => { + suppressBlurRefocusRef.current = true; + requestAnimationFrame(() => { + suppressBlurRefocusRef.current = false; + }); + }, []); + const handleKeyDown: KeyboardEventHandler = useCallback( (evt) => { if ( @@ -527,6 +546,7 @@ export const MessageEditor = as<'div', MessageEditorProps>( ( ( > Save - + Cancel - {(anchor: RectCords | undefined, setAnchor) => ( - { - setAnchor((v) => { - if (v) { - if (!mobileOrTablet()) ReactEditor.focus(editor); - return undefined; - } - return v; - }); - }} - /> - } - > + {(anchor: RectCords | undefined, setAnchor) => { + const emojiBoard = ( + { + setAnchor((v) => { + if (v) { + if (!mobileOrTablet()) ReactEditor.focus(editor); + return undefined; + } + return v; + }); + }} + /> + ); + const trigger = ( ( evt.currentTarget.getBoundingClientRect() )) as MouseEventHandler } + onPointerDown={suppressEditorRefocus} variant="SurfaceVariant" size="300" radii="300" @@ -600,8 +623,40 @@ export const MessageEditor = as<'div', MessageEditorProps>( weight: anchor !== undefined ? 'fill' : 'regular', })} - - )} + ); + if (mobileOrTablet()) { + return ( + <> + {trigger} + }> +
+ {emojiBoard} +
+ + + ); + } + return ( + + {trigger} + + ); + }} diff --git a/src/app/features/room/message/Reactions.tsx b/src/app/features/room/message/Reactions.tsx index e0ef83fb70..0578bb0b0b 100644 --- a/src/app/features/room/message/Reactions.tsx +++ b/src/app/features/room/message/Reactions.tsx @@ -22,6 +22,7 @@ import { useMatrixClient } from '$hooks/useMatrixClient'; import { factoryEventSentBy } from '$utils/matrix'; import { Reaction, ReactionTooltipMsg } from '$components/message'; import { EmojiBoard } from '$components/emoji-board'; +import { mobileOrTablet } from '$utils/user-agent'; import { sizedIcon, Smiley } from '$components/icons/phosphor'; import { useRelations } from '$hooks/useRelations'; import { stopPropagation } from '$utils/keyboard'; @@ -122,17 +123,15 @@ export const Reactions = as<'div', ReactionsProps>( ); })} - {canSendReaction && reactions.length > 0 && ( - 0 && + (() => { + const emojiBoard = ( { onReactionToggle(mEventId, key); setEmojiBoardAnchor(undefined); @@ -143,32 +142,64 @@ export const Reactions = as<'div', ReactionsProps>( }} requestClose={() => setEmojiBoardAnchor(undefined)} /> + ); + const trigger = ( + + Add Reaction + + } + > + {(targetRef) => ( + + {sizedIcon(Smiley, '100', { filled: !!emojiBoardAnchor })} + + )} + + ); + if (mobileOrTablet()) { + return ( + <> + {trigger} + }> +
+ {emojiBoard} +
+ + + ); } - > - - Add Reaction - - } - > - {(targetRef) => ( - - {sizedIcon(Smiley, '100', { filled: !!emojiBoardAnchor })} - - )} - - - )} + return ( + + {trigger} + + ); + })()} {reactions.length > 0 && ( { diff --git a/src/app/features/room/msgContent.ts b/src/app/features/room/msgContent.ts index 2246b36d16..ff47bc58ee 100644 --- a/src/app/features/room/msgContent.ts +++ b/src/app/features/room/msgContent.ts @@ -22,7 +22,7 @@ import { import { mimeTypeToExt } from '$utils/mimeTypes'; import type { TUploadItem } from '$state/room/roomInputDrafts'; import type { GifData } from '$components/emoji-board/types'; -import { encodeBlurHash } from '$utils/blurHash'; +import { encodeBlurHashAsync } from '$utils/blurHash'; import { scaleYDimension } from '$utils/common'; import { createLogger } from '$utils/debug'; import { @@ -73,7 +73,11 @@ export const getImageMsgContent = async ( [MATRIX_UNSTABLE_SPOILER_PROPERTY_NAME]: metadata.markedAsSpoiler, }; if (imgEl) { - const blurHash = encodeBlurHash(imgEl, 512, scaleYDimension(imgEl.width, 512, imgEl.height)); + const blurHash = await encodeBlurHashAsync( + imgEl, + 512, + scaleYDimension(imgEl.width, 512, imgEl.height) + ); content.info = { ...getImageInfo(imgEl, file), @@ -122,11 +126,12 @@ export const getVideoMsgContent = async ( ) ); if (thumbContent && thumbContent.thumbnail_info) { - thumbContent.thumbnail_info[MATRIX_UNSTABLE_BLUR_HASH_PROPERTY_NAME] = encodeBlurHash( - videoEl, - 512, - scaleYDimension(videoEl.videoWidth, 512, videoEl.videoHeight) - ); + thumbContent.thumbnail_info[MATRIX_UNSTABLE_BLUR_HASH_PROPERTY_NAME] = + await encodeBlurHashAsync( + videoEl, + 512, + scaleYDimension(videoEl.videoWidth, 512, videoEl.videoHeight) + ); } if (thumbError) log.warn('Failed to generate video thumbnail:', thumbError); content.info = { @@ -288,7 +293,11 @@ export const getGifMsgContent = async ( } if (imgEl) { - const blurHash = encodeBlurHash(imgEl, 512, scaleYDimension(imgEl.width, 512, imgEl.height)); + const blurHash = await encodeBlurHashAsync( + imgEl, + 512, + scaleYDimension(imgEl.width, 512, imgEl.height) + ); if (blurHash) { content.info[MATRIX_UNSTABLE_BLUR_HASH_PROPERTY_NAME] = blurHash; } diff --git a/src/app/generated/tauri/commands.ts b/src/app/generated/tauri/commands.ts index 5dbe7aa786..27a99e2066 100644 --- a/src/app/generated/tauri/commands.ts +++ b/src/app/generated/tauri/commands.ts @@ -1,19 +1,23 @@ /** * Auto-generated TypeScript bindings for Tauri commands * Generated by tauri-typegen v0.5.0 - * Generated at: 2026-07-20T16:38:06.342501881+00:00 + * Generated at: 2026-07-20T19:29:46.118402061+00:00 * Generator: none * * Do not edit manually - regenerate using: cargo tauri-typegen generate */ -import { invoke } from '@tauri-apps/api/core'; +import { invoke, Channel } from '@tauri-apps/api/core'; import * as types from './types'; export async function abortLoopbackFetch(params: types.AbortLoopbackFetchParams): Promise { return invoke('abort_loopback_fetch', params); } +export async function abortNativeUpload(params: types.AbortNativeUploadParams): Promise { + return invoke('abort_native_upload', params); +} + export async function clearMediaSession(): Promise { return invoke('clear_media_session'); } @@ -34,6 +38,10 @@ export async function loopbackFetch(params: types.LoopbackFetchParams): Promise< return invoke('loopback_fetch', params); } +export async function nativeUpload(params: types.NativeUploadParams): Promise { + return invoke('native_upload', params); +} + export async function saveDownload(params: types.SaveDownloadParams): Promise { return invoke('save_download', params); } @@ -69,3 +77,7 @@ export async function stopWindowTracking(): Promise { export async function syncDesktopSettings(params: types.SyncDesktopSettingsParams): Promise { return invoke('sync_desktop_settings', params); } + +export async function uploadWriteChunk(params: types.UploadWriteChunkParams): Promise { + return invoke('upload_write_chunk', params); +} diff --git a/src/app/generated/tauri/index.ts b/src/app/generated/tauri/index.ts index 8b14e93b19..5b327d1d8e 100644 --- a/src/app/generated/tauri/index.ts +++ b/src/app/generated/tauri/index.ts @@ -1,7 +1,7 @@ /** * Auto-generated TypeScript bindings for Tauri commands * Generated by tauri-typegen v0.5.0 - * Generated at: 2026-07-20T16:38:06.342846421+00:00 + * Generated at: 2026-07-20T19:29:46.118855830+00:00 * Generator: none * * Do not edit manually - regenerate using: cargo tauri-typegen generate diff --git a/src/app/generated/tauri/types.ts b/src/app/generated/tauri/types.ts index 0ad332e0c1..36d349b021 100644 --- a/src/app/generated/tauri/types.ts +++ b/src/app/generated/tauri/types.ts @@ -1,12 +1,15 @@ /** * Auto-generated TypeScript bindings for Tauri commands * Generated by tauri-typegen v0.5.0 - * Generated at: 2026-07-20T16:38:06.341992373+00:00 + * Generated at: 2026-07-20T19:29:46.117694182+00:00 * Generator: none * * Do not edit manually - regenerate using: cargo tauri-typegen generate */ + +import type { Channel } from '@tauri-apps/api/core'; + export interface DesktopRuntimeState { trayAvailable: boolean; } @@ -32,6 +35,16 @@ export interface LoopbackFetchResponse { body: number[]; } +export interface NativeUploadResponse { + status: number; + body: string; +} + +export interface ProgressPayload { + loaded: number; + total: number; +} + export interface WindowTarget { window_class?: string | null; exe_name?: string | null; @@ -42,11 +55,25 @@ export interface AbortLoopbackFetchParams { [key: string]: unknown; } +export interface AbortNativeUploadParams { + requestId: string; + [key: string]: unknown; +} + export interface LoopbackFetchParams { request: LoopbackFetchRequest; [key: string]: unknown; } +export interface NativeUploadParams { + requestId: string; + url: string; + contentType: string; + authorization?: string | null; + onProgress: Channel; + [key: string]: unknown; +} + export interface SaveDownloadParams { filename: string; bytes: number[]; @@ -84,3 +111,9 @@ export interface SyncDesktopSettingsParams { [key: string]: unknown; } +export interface UploadWriteChunkParams { + requestId: string; + chunk: string; + [key: string]: unknown; +} + diff --git a/src/app/state/room/roomInputDrafts.ts b/src/app/state/room/roomInputDrafts.ts index e79abbe0f6..8e90abcc45 100644 --- a/src/app/state/room/roomInputDrafts.ts +++ b/src/app/state/room/roomInputDrafts.ts @@ -18,6 +18,7 @@ export type TUploadItem = { originalFile: TUploadContent; metadata: TUploadMetadata; encInfo: EncryptedAttachmentInfo | undefined; + encrypting?: boolean; body?: string; format?: string; formatted_body?: string; diff --git a/src/app/utils/attachmentCrypto.test.ts b/src/app/utils/attachmentCrypto.test.ts new file mode 100644 index 0000000000..1a7ba4545d --- /dev/null +++ b/src/app/utils/attachmentCrypto.test.ts @@ -0,0 +1,42 @@ +// @vitest-environment node +import { describe, it, expect, beforeAll } from 'vitest'; +import { decryptAttachment } from 'browser-encrypt-attachment'; +import { encryptAttachmentStreaming } from './attachmentCrypto'; + +beforeAll(() => { + (globalThis as unknown as { window: unknown }).window = { + crypto: globalThis.crypto, + atob: (s: string) => Buffer.from(s, 'base64').toString('binary'), + btoa: (s: string) => Buffer.from(s, 'binary').toString('base64'), + }; +}); + +const roundTrip = async (size: number) => { + const original = new Uint8Array(size); + for (let i = 0; i < size; i += 1) original[i] = (i * 131 + 7) & 0xff; + const { blob, info } = await encryptAttachmentStreaming(new Blob([original])); + const cipher = await blob.arrayBuffer(); + const decrypted = new Uint8Array(await decryptAttachment(cipher, info)); + return { original, decrypted, info, cipherLength: cipher.byteLength }; +}; + +describe('encryptAttachmentStreaming', () => { + it('is byte-compatible with browser-encrypt-attachment across chunk boundaries', async () => { + const size = 4 * 1024 * 1024 * 2 + 12_345; // >2 chunks, non-block-aligned tail + const { original, decrypted, info, cipherLength } = await roundTrip(size); + expect(info.v).toBe('v2'); + expect(cipherLength).toBe(size); + expect(decrypted.length).toBe(size); + expect(Buffer.compare(Buffer.from(decrypted), Buffer.from(original))).toBe(0); + }); + + it('round-trips a small single-chunk file', async () => { + const { original, decrypted } = await roundTrip(137); + expect(Buffer.compare(Buffer.from(decrypted), Buffer.from(original))).toBe(0); + }); + + it('handles an exact chunk-size boundary', async () => { + const { original, decrypted } = await roundTrip(4 * 1024 * 1024); + expect(Buffer.compare(Buffer.from(decrypted), Buffer.from(original))).toBe(0); + }); +}); diff --git a/src/app/utils/attachmentCrypto.ts b/src/app/utils/attachmentCrypto.ts new file mode 100644 index 0000000000..dae7d55ccb --- /dev/null +++ b/src/app/utils/attachmentCrypto.ts @@ -0,0 +1,57 @@ +import { sha256 } from '@noble/hashes/sha2.js'; +import type { EncryptedAttachmentInfo } from 'browser-encrypt-attachment'; + +const CHUNK_SIZE = 4 * 1024 * 1024; + +const encodeUnpaddedBase64 = (bytes: Uint8Array): string => + btoa(String.fromCharCode(...bytes)).replace(/=+$/, ''); + +const counterForBlock = (nonce: Uint8Array, blockIndex: number): Uint8Array => { + const counter = new Uint8Array(16); + counter.set(nonce, 0); + const view = new DataView(counter.buffer); + view.setUint32(8, Math.floor(blockIndex / 2 ** 32)); + view.setUint32(12, blockIndex >>> 0); + return counter; +}; + +export const encryptAttachmentStreaming = async ( + file: Blob +): Promise<{ blob: Blob; info: EncryptedAttachmentInfo }> => { + const key = await crypto.subtle.generateKey({ name: 'AES-CTR', length: 256 }, true, [ + 'encrypt', + 'decrypt', + ]); + const exportedKey = (await crypto.subtle.exportKey('jwk', key)) as EncryptedAttachmentInfo['key']; + + const iv = new Uint8Array(16); + crypto.getRandomValues(iv.subarray(0, 8)); + const nonce = iv.subarray(0, 8); + + const hasher = sha256.create(); + const parts: BlobPart[] = []; + + for (let offset = 0; offset < file.size; offset += CHUNK_SIZE) { + // eslint-disable-next-line no-await-in-loop -- stream chunks to cap peak memory + const plainChunk = await file.slice(offset, offset + CHUNK_SIZE).arrayBuffer(); + const counter = counterForBlock(nonce, offset / 16); + // eslint-disable-next-line no-await-in-loop -- stream chunks to cap peak memory + const cipherChunk = await crypto.subtle.encrypt( + { name: 'AES-CTR', counter, length: 64 }, + key, + plainChunk + ); + const bytes = new Uint8Array(cipherChunk); + hasher.update(bytes); + parts.push(new Blob([bytes])); + } + + const info: EncryptedAttachmentInfo = { + v: 'v2', + key: exportedKey, + iv: encodeUnpaddedBase64(iv), + hashes: { sha256: encodeUnpaddedBase64(hasher.digest()) }, + }; + + return { blob: new Blob(parts, { type: file.type }), info }; +}; diff --git a/src/app/utils/blurHash.ts b/src/app/utils/blurHash.ts index 3fce2662ed..ca0ec4bd62 100644 --- a/src/app/utils/blurHash.ts +++ b/src/app/utils/blurHash.ts @@ -1,4 +1,5 @@ import { encode, isBlurhashValid } from 'blurhash'; +import { blurHashWorkerSupported, encodeBlurHashInWorker } from '$utils/mediaWorker'; export const encodeBlurHash = ( img: HTMLImageElement | HTMLVideoElement, @@ -23,6 +24,27 @@ export const encodeBlurHash = ( } }; +export const encodeBlurHashAsync = async ( + img: HTMLImageElement | HTMLVideoElement, + width: number, + height: number +): Promise => { + if (blurHashWorkerSupported()) { + let bitmap: ImageBitmap | undefined; + try { + bitmap = await createImageBitmap(img, { + resizeWidth: width, + resizeHeight: height, + resizeQuality: 'low', + }); + return await encodeBlurHashInWorker(bitmap, width, height); + } catch { + bitmap?.close(); + } + } + return encodeBlurHash(img, width, height); +}; + export const validBlurHash = (hash?: string): string | undefined => { if (typeof hash === 'string') { const validity = isBlurhashValid(hash); diff --git a/src/app/utils/matrix.ts b/src/app/utils/matrix.ts index c97e083eff..91f05e3716 100644 --- a/src/app/utils/matrix.ts +++ b/src/app/utils/matrix.ts @@ -1,6 +1,6 @@ import type { EncryptedAttachmentInfo } from 'browser-encrypt-attachment'; -import { decryptAttachment, encryptAttachment } from 'browser-encrypt-attachment'; -import { convertFileSrc, isTauri } from '@tauri-apps/api/core'; +import { decryptAttachment } from 'browser-encrypt-attachment'; +import { Channel, convertFileSrc, invoke, isTauri } from '@tauri-apps/api/core'; import type { AccountDataEvents, EventTimelineSet, @@ -23,7 +23,8 @@ import to from 'await-to-js'; import type { IImageInfo, IThumbnailContent, IVideoInfo } from '$types/matrix/common'; import * as Sentry from '@sentry/react'; -import { fetch } from '$utils/fetch'; +import { encryptBlobInWorker } from '$utils/mediaWorker'; +import { encryptAttachmentStreaming } from '$utils/attachmentCrypto'; import { getEventReactions, getStateEvent } from './room'; import { getReactionContent } from './messageReaction'; import { matchMxId, validMxId } from './mxIdHelper'; @@ -125,14 +126,19 @@ export const encryptFile = async ( file: File; originalFile: T; }> => { - const dataBuffer = await file.arrayBuffer(); - const encryptedAttachment = await encryptAttachment(dataBuffer); + let blob: Blob; + let info: EncryptedAttachmentInfo; + try { + ({ blob, info } = await encryptBlobInWorker(file)); + } catch { + ({ blob, info } = await encryptAttachmentStreaming(file)); + } const fileName = getUploadFileName(file); - const encFile = new File([encryptedAttachment.data], fileName, { + const encFile = new File([blob], fileName, { type: file.type, }); return { - encInfo: encryptedAttachment.info, + encInfo: info, file: encFile, originalFile: file, }; @@ -165,6 +171,19 @@ export type UploadContentOpts = { * and gets blocked, so uploads to the homeserver fail. Route the upload through * our Tauri-aware `fetch` instead, keeping the SDK path (with progress) on web. */ +const UPLOAD_CHUNK_SIZE = 2 * 1024 * 1024; + +const blobToBase64 = (blob: Blob): Promise => + new Promise((resolve, reject) => { + const reader = new FileReader(); + reader.addEventListener('load', () => { + const result = reader.result as string; + resolve(result.slice(result.indexOf(',') + 1)); + }); + reader.addEventListener('error', () => reject(reader.error)); + reader.readAsDataURL(blob); + }); + const tauriUploadAbortControllers = new WeakMap, AbortController>(); type UploadFileType = TUploadContent | Blob | XMLHttpRequestBodyInit; @@ -190,33 +209,76 @@ export const uploadContentToServer = ( url.searchParams.set('filename', fileName); } - const headers: Record = { 'Content-Type': contentType }; const accessToken = mx.getAccessToken(); - if (accessToken) { - headers.Authorization = `Bearer ${accessToken}`; - } + const requestId = + globalThis.crypto?.randomUUID?.() ?? + `upload-${Date.now()}-${Math.random().toString(16).slice(2)}`; const promise = (async (): Promise => { - const response = await fetch(url.toString(), { - method: 'POST', - headers, - body: file, - signal: abortController.signal, - }); - if (!response.ok) { - const body = await response.text().catch(() => ''); - let parsed: { errcode?: string; error?: string } = {}; - try { - parsed = JSON.parse(body); - } catch { - // Non-JSON error body; fall back to the status text. + const blob = file instanceof Blob ? file : new Blob([file as BlobPart]); + const total = blob.size; + + const throwIfAborted = () => { + if (abortController.signal.aborted) { + throw new DOMException('The operation was aborted', 'AbortError'); } - throw new MatrixError({ - errcode: parsed.errcode, - error: parsed.error ?? `Upload failed with status ${response.status}`, + }; + const onAbort = () => { + void invoke('abort_native_upload', { requestId }); + }; + abortController.signal.addEventListener('abort', onAbort, { once: true }); + + const half = Math.floor(total / 2); + const onProgress = new Channel<{ loaded: number; total: number }>(); + if (opts.progressHandler) { + // eslint-disable-next-line unicorn/prefer-add-event-listener -- Channel only exposes onmessage + onProgress.onmessage = (payload) => { + const sent = payload.total ? payload.loaded / payload.total : 0; + opts.progressHandler?.({ loaded: half + Math.floor(sent * (total - half)), total }); + }; + } + + const writeChunk = async (start: number) => { + const chunk = await blobToBase64(blob.slice(start, start + UPLOAD_CHUNK_SIZE)); + await invoke('upload_write_chunk', { requestId, chunk }); + }; + + try { + for (let offset = 0; offset < total; offset += UPLOAD_CHUNK_SIZE) { + throwIfAborted(); + const end = Math.min(offset + UPLOAD_CHUNK_SIZE, total); + // eslint-disable-next-line no-await-in-loop -- sequential chunks bound webview memory + await writeChunk(offset); + opts.progressHandler?.({ loaded: Math.floor(end / 2), total }); + } + throwIfAborted(); + + const result = await invoke<{ status: number; body: string }>('native_upload', { + requestId, + url: url.toString(), + contentType, + authorization: accessToken ? `Bearer ${accessToken}` : null, + onProgress, }); + if (result.status < 200 || result.status >= 300) { + let parsed: { errcode?: string; error?: string } = {}; + try { + parsed = JSON.parse(result.body); + } catch { + // Non-JSON error body; fall back to the status code. + } + throw new MatrixError({ + errcode: parsed.errcode, + error: parsed.error ?? `Upload failed with status ${result.status}`, + }); + } + return JSON.parse(result.body) as UploadResponse; + } catch (err) { + void invoke('abort_native_upload', { requestId }); + throw err; + } finally { + abortController.signal.removeEventListener('abort', onAbort); } - return (await response.json()) as UploadResponse; })(); tauriUploadAbortControllers.set(promise, abortController); diff --git a/src/app/utils/mediaWorker.ts b/src/app/utils/mediaWorker.ts new file mode 100644 index 0000000000..b5a7b183ef --- /dev/null +++ b/src/app/utils/mediaWorker.ts @@ -0,0 +1,72 @@ +import type { EncryptedAttachmentInfo } from 'browser-encrypt-attachment'; + +type Pending = { resolve: (value: unknown) => void; reject: (reason?: unknown) => void }; + +let worker: Worker | undefined; +let disabled = false; +let seq = 0; +const pending = new Map(); + +const rejectAll = (reason: Error): void => { + for (const entry of pending.values()) entry.reject(reason); + pending.clear(); +}; + +const getWorker = (): Worker | undefined => { + if (disabled) return undefined; + if (worker) return worker; + if (typeof Worker === 'undefined') { + disabled = true; + return undefined; + } + try { + worker = new Worker(new URL('../workers/media.worker.ts', import.meta.url), { type: 'module' }); + } catch { + disabled = true; + return undefined; + } + worker.addEventListener('message', (event: MessageEvent) => { + const { id, error, ...rest } = event.data as { id: number; error?: string }; + const entry = pending.get(id); + if (!entry) return; + pending.delete(id); + if (error) entry.reject(new Error(error)); + else entry.resolve(rest); + }); + worker.addEventListener('error', () => { + rejectAll(new Error('media worker crashed')); + worker?.terminate(); + worker = undefined; + disabled = true; + }); + return worker; +}; + +const call = (message: object, transfer: Transferable[] = []): Promise => { + const active = getWorker(); + if (!active) return Promise.reject(new Error('media worker unavailable')); + const id = (seq += 1); + return new Promise((resolve, reject) => { + pending.set(id, { resolve: resolve as (value: unknown) => void, reject }); + active.postMessage({ id, ...message }, transfer); + }); +}; + +export const blurHashWorkerSupported = (): boolean => + typeof Worker !== 'undefined' && + typeof OffscreenCanvas !== 'undefined' && + typeof createImageBitmap === 'function'; + +export const encryptBlobInWorker = ( + file: Blob +): Promise<{ blob: Blob; info: EncryptedAttachmentInfo }> => + call<{ blob: Blob; info: EncryptedAttachmentInfo }>({ type: 'encrypt', file }); + +export const encodeBlurHashInWorker = ( + bitmap: ImageBitmap, + width: number, + height: number +): Promise => + call<{ hash: string | undefined }>({ type: 'blurhash', bitmap, width, height }, [bitmap]).then( + (result) => result.hash + ); diff --git a/src/app/workers/media.worker.ts b/src/app/workers/media.worker.ts new file mode 100644 index 0000000000..ffcf84cfec --- /dev/null +++ b/src/app/workers/media.worker.ts @@ -0,0 +1,49 @@ +import { encode } from 'blurhash'; +import { encryptAttachmentStreaming } from '$utils/attachmentCrypto'; + +type EncryptRequest = { id: number; type: 'encrypt'; file: Blob }; +type BlurHashRequest = { + id: number; + type: 'blurhash'; + bitmap: ImageBitmap; + width: number; + height: number; +}; +type Request = EncryptRequest | BlurHashRequest; + +const post = (message: unknown, transfer: Transferable[] = []): void => { + (self as unknown as { postMessage: (m: unknown, t: Transferable[]) => void }).postMessage( + message, + transfer + ); +}; + +const handleEncrypt = async (req: EncryptRequest): Promise => { + const { blob, info } = await encryptAttachmentStreaming(req.file); + post({ id: req.id, blob, info }); +}; + +const handleBlurHash = (req: BlurHashRequest): void => { + const { bitmap, width, height } = req; + try { + const canvas = new OffscreenCanvas(width, height); + const ctx = canvas.getContext('2d'); + if (!ctx) { + post({ id: req.id, hash: undefined }); + return; + } + ctx.drawImage(bitmap, 0, 0, width, height); + const imageData = ctx.getImageData(0, 0, width, height); + post({ id: req.id, hash: encode(imageData.data, width, height, 4, 4) }); + } finally { + bitmap.close(); + } +}; + +self.addEventListener('message', (event: MessageEvent): void => { + const req = event.data; + const run = req.type === 'encrypt' ? handleEncrypt(req) : Promise.resolve(handleBlurHash(req)); + run.catch((error: unknown) => { + post({ id: req.id, error: error instanceof Error ? error.message : String(error) }); + }); +});