0.15.0 - Avatar Batching

Author: ShayBox

Cargo.lock

@@ -1,6 +1,6 @@
 [package]
 name = "vrc-log"
-version = "0.14.2"
+version = "0.15.0"
 edition = "2024"
 license = "MIT"
 readme = "README.md"
@@ -21,31 +21,33 @@ chrono = "0.4"
 colored = "3"
 crossterm = { version = "0.29", optional = true }
 derive-config = { version = "2", features = ["dirs", "toml"] }
-discord-presence = { version = "2", optional = true }
-downcast-rs = "2.0.2"
-flume = "0.12.0"
+discord-presence = { version = "3", optional = true }
+flume = "0.12"
 futures = "0.3"
-inquire = "0.8"
-itertools = "0.14.0"
+inquire = "0.9"
+itertools = "0.14"
 lazy-regex = "3"
 notify = "8"
 parking_lot = "0.12"
-reqwest = { version = "0.12", default-features = false, features = [
-    "json",
-], optional = true }
-rusqlite = "0.37.0"
+reqwest = { version = "0.13", features = ["json", "query"], optional = true }
+rusqlite = "0.38"
 serde = { version = "1", features = ["derive"] }
 serde_json = "1"
 strum = { version = "0.27", features = ["derive"] }
 terminal-link = "0.1"
 time = { version = "0.3", features = ["macros"] }
 tokio = { version = "1", features = ["full"] }
-tokio-rusqlite-new = { version = "0.11", features = [
-    "bundled",
-], optional = true }
+tokio-rusqlite-new = { version = "0.12", features = ["bundled"], optional = true }
 tracing = "0.1"
 tracing-subscriber = { version = "=0.3.19", features = ["env-filter", "time"] }
 
+[dev-dependencies]
+criterion = "0.8"
+reqwest = "0.13"
+tempfile = "3"
+tokio = { version = "1", features = ["full"] }
+uuid = { version = "1", features = ["v4"] }
+
 [target.'cfg(windows)'.dependencies]
 windows = { version = "0.62", features = [
     "Win32_Foundation",
@@ -62,7 +64,6 @@ default = [
     "vrcdb",
     "vrcwb",
     "title",
-    "rustls-tls",
 ]
 
 # VRChat Avatar Database Providers
@@ -76,9 +77,6 @@ vrcwb = ["dep:reqwest", "discord"]
 discord = ["dep:discord-presence", "dep:cached"]
 title = ["dep:crossterm"]
 
-native-tls = ["reqwest/native-tls"]
-rustls-tls = ["reqwest/rustls-tls"]
-
 [[bench]]
 name = "cache_bench"
 harness = false
@@ -87,7 +85,6 @@ harness = false
 name = "cache_disk_bench"
 harness = false
 
-
 # https://github.com/johnthagen/min-sized-rust
 [profile.release]
 strip = true      # Automatically strip symbols from the binary.
@@ -102,11 +99,3 @@ nursery = { level = "warn", priority = -1 }
 cargo = { level = "warn", priority = -1 }
 multiple_crate_versions = "allow"
 similar_names = "allow"
-
-[dev-dependencies]
-criterion = "0.8.1"
-httptest = "0.16.3"
-reqwest = "0.12"
-tempfile = "3.23.0"
-tokio = { version = "1", features = ["full"] }
-uuid = { version = "1.19.0", features = ["v4"] }

Cargo.toml

@@ -10,15 +10,16 @@ use crossterm::{execute, terminal::SetTitle};
 use derive_config::{ConfigError, DeriveTomlConfig};
 use notify::PollWatcher;
 use terminal_link::Link;
-use time::{UtcOffset, macros::format_description};
+use time::{macros::format_description, UtcOffset};
 use tokio::signal;
 use tracing::level_filters::LevelFilter;
-use tracing_subscriber::{EnvFilter, fmt::time::OffsetTime};
+use tracing_subscriber::{fmt::time::OffsetTime, EnvFilter};
 use vrc_log::{
-    CARGO_PKG_HOMEPAGE, provider,
-    provider::{ProviderKind, avtrdb::AvtrDBActor, prelude::*},
+    provider,
+    provider::{avtrdb::AvtrDBActor, prelude::*, ProviderKind},
     settings::Settings,
     vrchat::{VRCHAT_AMP_PATH, VRCHAT_LOW_PATH},
+    CARGO_PKG_HOMEPAGE,
 };
 
 /* Watchers will stop working if they get dropped. */

src/main.rs

@@ -1,56 +1,70 @@
-use std::sync::Arc;
-
-use crate::print_colorized;
-use crate::provider::Provider;
+use std::{collections::HashMap, sync::Arc};
 
 #[cfg(feature = "cache")]
 use crate::cache;
+use crate::{print_colorized, provider::Provider};
 
 #[cfg(feature = "cache")]
 pub async fn process_with_cache<I: IntoIterator<Item = String>>(
     providers: Vec<Arc<Box<dyn Provider>>>,
     cache: &cache::Cache,
     avatar_ids: I,
 ) -> anyhow::Result<()> {
-    let checked_ids = cache.check_all_ids(avatar_ids).await?;
+    let checked_ids = cache
+        .check_all_ids(avatar_ids)
+        .await?
+        .into_iter()
+        .collect::<Vec<_>>();
 
     let (tx, rx) = flume::unbounded();
+    let checked_ids = Arc::new(checked_ids);
+    let mut base_bits = HashMap::new();
 
-    for (id, provider_bits) in checked_ids {
-        print_colorized(&id);
+    for (id, provider_bits) in checked_ids.iter() {
+        print_colorized(id);
+        base_bits.insert(id.clone(), *provider_bits);
+    }
 
-        for p in providers
-            .iter()
-            .filter(|provider| provider_bits & provider.kind() as u32 == 0)
-        {
-            let p = p.clone();
-            let id_clone = id.clone();
-            let tx_clone = tx.clone();
-            tokio::spawn(async move {
-                let kind = p.kind();
-                match p.send_avatar_id(&id_clone).await {
+    for provider in &providers {
+        let provider = provider.clone();
+        let tx_clone = tx.clone();
+        let checked_ids = checked_ids.clone();
+        tokio::spawn(async move {
+            let kind = provider.kind();
+            let kind_bit = kind as u32;
+            for (id, provider_bits) in checked_ids.iter() {
+                if provider_bits & kind_bit != 0 {
+                    continue;
+                }
+                match provider.send_avatar_id(id).await {
                     Ok(success) => {
                         if success {
                             info!("^ Successfully Submitted to {kind}");
-                            let _ = tx_clone
-                                .send_async((id_clone, provider_bits | kind as u32))
-                                .await;
+                            let _ = tx_clone.send_async((id.clone(), kind_bit)).await;
                         }
                     }
                     Err(err) => {
                         error!("^ Failed to submit to {kind}: {err}");
                     }
                 }
-            });
-        }
+            }
+        });
     }
 
     drop(tx);
 
-    let mut buffer = Vec::new();
+    let mut updated_bits = HashMap::new();
 
-    while let Ok(msg) = rx.recv_async().await {
-        buffer.push(msg);
+    while let Ok((id, kind_bit)) = rx.recv_async().await {
+        let entry = updated_bits.entry(id).or_insert(0);
+        *entry |= kind_bit;
+    }
+
+    let mut buffer = Vec::new();
+    for (id, bits) in base_bits {
+        if let Some(add_bits) = updated_bits.get(&id) {
+            buffer.push((id, bits | add_bits));
+        }
     }
 
     cache.store_avatar_ids_with_providers(buffer).await
@@ -90,19 +104,20 @@ pub async fn process_without_cache<I: IntoIterator<Item = String>>(
 
 #[cfg(test)]
 mod tests {
-    use crate::provider::ProviderKind;
+    use std::sync::Arc;
 
-    use super::*;
     use anyhow::Result;
     use async_trait::async_trait;
-    use std::sync::Arc;
     use strum::IntoEnumIterator;
     use tokio::sync::Mutex;
 
+    use super::*;
+    use crate::provider::ProviderKind;
+
     #[derive(Clone)]
     struct MockProvider {
-        kind: ProviderKind,
-        sent: Arc<Mutex<Vec<String>>>,
+        kind:    ProviderKind,
+        sent:    Arc<Mutex<Vec<String>>>,
         succeed: bool,
     }
 

src/process.rs

@@ -1,6 +1,6 @@
 use std::time::Duration;
 
-use anyhow::{Result, bail};
+use anyhow::{bail, Result};
 use colored::{Color, Colorize};
 use flume::{Receiver, Sender};
 use reqwest::{Client, StatusCode, Url};
@@ -10,9 +10,9 @@ use terminal_link::Link;
 use tokio::time::Instant;
 
 use crate::{
-    USER_AGENT,
     provider::{Provider, ProviderKind},
     settings::Settings,
+    USER_AGENT,
 };
 
 const INGEST_BASE_URL: &str = "https://api.avtrdb.com/v3/";
@@ -31,13 +31,13 @@ pub struct AvtrDB {
 }
 
 pub struct AvtrDBActor<'s> {
-    settings: &'s Settings,
-    client: Client,
-    buffer: Vec<String>,
-    base_url: String,
-    channel: Receiver<String>,
+    settings:       &'s Settings,
+    client:         Client,
+    buffer:         Vec<String>,
+    base_url:       String,
+    channel:        Receiver<String>,
     flush_interval: Duration,
-    last_flush: Instant,
+    last_flush:     Instant,
 }
 
 impl<'s> AvtrDBActor<'s> {
@@ -177,8 +177,8 @@ impl AvtrDB {
 #[allow(dead_code)]
 struct IngestResponse {
     avatars_enqueued: u64,
-    invalid_ids: u64,
-    ticket: String,
+    invalid_ids:      u64,
+    ticket:           String,
 }
 
 #[async_trait::async_trait]

src/provider/avtrdb.rs

@@ -24,16 +24,16 @@ pub enum ProviderKind {
     #[cfg(feature = "nsvr")]
     #[strum(to_string = "NSVR - NekoSune Community")]
     #[serde(alias = "VRCDS")]
-    NSVR = 1 << 1,
+    NSVR   = 1 << 1,
     #[cfg(feature = "paw")]
     #[strum(to_string = "PAW - Puppy's Avatar World")]
-    PAW = 1 << 2,
+    PAW    = 1 << 2,
     #[cfg(feature = "vrcdb")]
     #[strum(to_string = "VRCDB - Avatar Search")]
-    VRCDB = 1 << 3,
+    VRCDB  = 1 << 3,
     #[cfg(feature = "vrcwb")]
     #[strum(to_string = "VRCWB - World Balancer")]
-    VRCWB = 1 << 4,
+    VRCWB  = 1 << 4,
 }
 
 #[async_trait]

src/provider/mod.rs

@@ -1,21 +1,21 @@
 use std::time::Duration;
 
-use anyhow::{Result, bail};
+use anyhow::{bail, Result};
 use async_trait::async_trait;
 use reqwest::{Client, StatusCode};
 use serde_json::json;
 
 use crate::{
-    USER_AGENT,
     provider::{Provider, ProviderKind},
     settings::Settings,
+    USER_AGENT,
 };
 
 const URL: &str = "https://avtr.nekosunevr.co.uk/v1/vrchat/avatars/store/putavatarExternal";
 
 pub struct NSVR<'s> {
     settings: &'s Settings,
-    client: Client,
+    client:   Client,
 }
 
 impl<'s> NSVR<'s> {

src/provider/nsvr.rs

@@ -1,15 +1,15 @@
 use std::time::Duration;
 
-use anyhow::{Result, bail};
+use anyhow::{bail, Result};
 use async_trait::async_trait;
 use reqwest::{Client, StatusCode};
 use serde::Deserialize;
 use serde_json::Value;
 
 use crate::{
-    USER_AGENT,
     provider::{Provider, ProviderKind},
     settings::Settings,
+    USER_AGENT,
 };
 
 const URL: &str = "https://paw-api.amelia.fun/update";
@@ -31,9 +31,9 @@ impl Paw {
 #[allow(dead_code)]
 struct PawResponse {
     success: bool,
-    code: u16,
-    result: Option<Value>,
-    avatar: Option<Value>,
+    code:    u16,
+    result:  Option<Value>,
+    avatar:  Option<Value>,
 }
 
 #[async_trait]

src/provider/paw.rs

@@ -1,21 +1,21 @@
 use std::time::Duration;
 
-use anyhow::{Result, bail};
+use anyhow::{bail, Result};
 use async_trait::async_trait;
 use reqwest::{Client, StatusCode};
 use serde_json::json;
 
 use crate::{
-    USER_AGENT,
     provider::{Provider, ProviderKind},
     settings::Settings,
+    USER_AGENT,
 };
 
 const URL: &str = "https://search.bs002.de/api/Avatar/putavatar";
 
 pub struct VrcDB<'s> {
     settings: &'s Settings,
-    client: Client,
+    client:   Client,
 }
 
 impl<'s> VrcDB<'s> {