feat(oracles): integrate frigate ephemeral scanning

Author: sdmg15

Cargo.lock

@@ -10,7 +10,8 @@ use bdk_sp::{
         self,
         address::NetworkUnchecked,
         bip32,
-        consensus::Decodable,
+        consensus::{deserialize, Decodable},
+        hashes::Hash,
         hex::{DisplayHex, FromHex},
         key::Secp256k1,
         script::PushBytesBuf,
@@ -34,6 +35,7 @@ use bdk_sp_oracles::{
         TrustedPeer, UnboundedReceiver, Warning,
     },
     filters::kyoto::{FilterEvent, FilterSubscriber},
+    frigate::{FrigateClient, History, SubscribeRequest, UnsubscribeRequest, DUMMY_COINBASE},
     tweaks::blindbit::{BlindbitSubscriber, TweakEvent},
 };
 use bdk_sp_wallet::{
@@ -161,6 +163,16 @@ pub enum Commands {
         #[clap(long)]
         hash: Option<BlockHash>,
     },
+
+    ScanFrigate {
+        #[clap(flatten)]
+        rpc_args: RpcArgs,
+        #[clap(long)]
+        height: Option<u32>,
+        #[clap(long)]
+        hash: Option<BlockHash>,
+    },
+
     Create {
         /// Network
         #[clap(long, short, default_value = "signet")]
@@ -567,6 +579,167 @@ async fn main() -> anyhow::Result<()> {
                 );
             }
         }
+        Commands::ScanFrigate {
+            rpc_args,
+            height,
+            hash,
+        } => {
+            // The implementation done here differs from what is mentioned in the section
+            // https://github.com/sparrowwallet/frigate/tree/master?tab=readme-ov-file#blockchainsilentpaymentssubscribe
+            // This implementation is doing a one time scanning only. So instead of calling
+            // `blockchain.scripthash.subscribe` on each script from the wallet, we just subscribe
+            // and read the scanning result from the stream. On each result received we update the
+            // wallet state and once scanning progress reaches 1.0 (100%) we stop.
+            let sync_point = if let (Some(height), Some(hash)) = (height, hash) {
+                HeaderCheckpoint::new(height, hash)
+            } else if wallet.birthday.height <= wallet.chain().tip().height() {
+                let height = wallet.chain().tip().height();
+                let hash = wallet.chain().tip().hash();
+                HeaderCheckpoint::new(height, hash)
+            } else {
+                let checkpoint = wallet
+                    .chain()
+                    .get(wallet.birthday.height)
+                    .expect("should be something");
+                let height = checkpoint.height();
+                let hash = checkpoint.hash();
+                HeaderCheckpoint::new(height, hash)
+            };
+
+            let mut client = FrigateClient::connect(&rpc_args.url)
+                .await
+                .unwrap()
+                .with_timeout(tokio::time::Duration::from_secs(60));
+
+            let labels = wallet
+                .indexer()
+                .index()
+                .num_to_label
+                .clone()
+                .into_keys()
+                .collect::<Vec<u32>>();
+            let labels = if !labels.is_empty() {
+                Some(labels)
+            } else {
+                None
+            };
+
+            let subscribe_params = SubscribeRequest {
+                scan_priv_key: *wallet.indexer().scan_sk(),
+                spend_pub_key: *wallet.indexer().spend_pk(),
+                start_height: Some(sync_point.height),
+                labels,
+            };
+
+            // Attempt to subscribe; any timeout will trigger unsubscribe automatically.
+            match client.subscribe_with_timeout(&subscribe_params).await {
+                Ok(Some((histories, progress))) => {
+                    tracing::info!(
+                        "Initial subscription result: {} histories, progress {}",
+                        histories.len(),
+                        progress
+                    );
+                }
+                Ok(None) => {
+                    tracing::info!("Subscription acknowledged, awaiting notifications");
+                }
+                Err(e) => {
+                    tracing::error!("Subscribe failed: {}", e);
+                    return Err(e.into());
+                }
+            }
+
+            tracing::info!("Starting frigate scanning loop...");
+            loop {
+                match client.read_from_stream(4096).await {
+                    Ok(subscribe_result) => {
+                        if subscribe_result["params"].is_object() {
+                            let histories: Vec<History> = serde_json::from_value(
+                                subscribe_result["params"]["history"].clone(),
+                            )?;
+                            let progress = subscribe_result["params"]["progress"]
+                                .as_f64()
+                                .unwrap_or(0.0) as f32;
+
+                            let mut secrets_by_height: HashMap<u32, HashMap<Txid, PublicKey>> =
+                                HashMap::new();
+
+                            tracing::debug!("Received history {:#?}", histories);
+
+                            histories.iter().for_each(|h| {
+                                secrets_by_height
+                                    .entry(h.height)
+                                    .and_modify(|v| {
+                                        v.insert(h.tx_hash, h.tweak_key);
+                                    })
+                                    .or_insert(HashMap::from([(h.tx_hash, h.tweak_key)]));
+                            });
+
+                            // Filter when the height is 0, because that would mean mempool transaction
+                            for secret in secrets_by_height.into_iter().filter(|v| v.0 > 0) {
+                                // Since frigate doesn't provide a blockchain.getblock we will mimick that here
+                                // By constructing a block from the block header and the list of transactions
+                                // received from the scan request
+                                let mut raw_blk = client.get_block_header(secret.0).await.unwrap();
+                                raw_blk.push_str("00");
+
+                                // Push dummy coinbase
+                                let coinbase: Transaction =
+                                    deserialize(&Vec::<u8>::from_hex(DUMMY_COINBASE).unwrap())
+                                        .unwrap();
+                                let mut block: Block =
+                                    deserialize(&Vec::<u8>::from_hex(&raw_blk).unwrap()).unwrap();
+
+                                let mut blockhash = BlockHash::all_zeros();
+
+                                let mut txs: Vec<Transaction> = vec![coinbase];
+                                for key in secret.1.keys() {
+                                    let tx_result =
+                                        client.get_transaction(key.to_string()).await.unwrap();
+                                    let tx: Transaction =
+                                        deserialize(&Vec::<u8>::from_hex(&tx_result.1).unwrap())
+                                            .unwrap();
+                                    txs.push(tx);
+
+                                    blockhash = BlockHash::from_str(&tx_result.0).unwrap();
+                                }
+
+                                block.txdata = txs;
+                                tracing::debug!("Final block {:?}", block);
+                                wallet.apply_block_relevant(&block, secret.1, secret.0);
+
+                                tracing::debug!("Checkpoint hash {blockhash:?}");
+                                let checkpoint = wallet.chain().tip().insert(BlockId {
+                                    height: secret.0,
+                                    hash: blockhash,
+                                });
+                                wallet.update_chain(checkpoint);
+                            }
+
+                            tracing::info!("Progress {progress}");
+                            // Check the progress
+                            if progress >= 1.0 {
+                                tracing::info!("Scanning completed");
+                                break;
+                            }
+                        }
+                    }
+                    Err(e) if e.to_string().contains("timed out") => {
+                        tracing::warn!("read_from_stream timeout, exiting scan");
+                        let unsubscribe_request = UnsubscribeRequest {
+                            scan_privkey: *wallet.indexer().scan_sk(),
+                            spend_pubkey: *wallet.indexer().spend_pk(),
+                        };
+                        let _ = client.unsubscribe(&unsubscribe_request).await;
+                        break;
+                    }
+                    Err(e) => {
+                        tracing::error!("read_from_stream error: {}", e);
+                        return Err(e.into());
+                    }
+                }
+            }
+        }
         Commands::Balance => {
             fn print_balances<'a>(
                 title_str: &'a str,

cli/v2/src/main.rs

@@ -0,0 +1,51 @@
+#!/usr/bin/env bash
+
+########################### STAGE 1: setup ####################################
+
+# 1. Install dependencies locally and setup regtest environment
+just non_nix_init
+# 2. Check bitcoind is running on regtest
+just cli getblockchaininfo
+# 3. Check bdk-cli wallet was created correctly
+just regtest-bdk balance
+# 4. Check sp-cli wallet was created correctly
+just regtest-sp balance
+# 5. Synchronize bdk-cli wallet
+just regtest-bdk sync
+
+###################### STAGE 2: fund bdk-cli wallet ###########################
+
+# 6. Get a new address from bdk-cli wallet
+REGTEST_ADDRESS=$(just regtest-bdk unused_address | jq -r '.address' | tr -d '\n')
+# 7. Mine a few more blocks to fund the wallet
+just mine 1 $REGTEST_ADDRESS
+# 8. Mine some of them to the internal wallet to confirm the bdk-cli balance
+just mine 101
+# 9. Synchronize bdk-cli wallet
+just regtest-bdk sync
+# 10. Check balance
+just regtest-bdk balance
+
+################ STAGE 3: create a silent payment output ######################
+
+# 11. Get a silent payment code from sp-cli2 wallet
+SP_CODE=$(just regtest-sp code | jq -r '.silent_payment_code' | tr -d '\n')
+# 12. Create a transaction spending bdk-cli wallet UTXOs to a the previous silent payment code
+RAW_TX=$(just regtest-bdk create_sp_tx --to-sp $SP_CODE:10000 --fee_rate 5 | jq -r '.raw_tx' | tr -d '\n')
+TXID=$(just regtest-bdk broadcast --tx $RAW_TX | jq -r '.txid' | tr -d '\n')
+# 14. Mine a new block
+just mine 1
+# 15. Once the new transaction has been mined, synchronize bdk-cli wallet again
+just regtest-bdk sync
+
+# ################## STAGE 4: find a silent payment output ######################
+
+# 16. Now synchronize sp-cli2 wallet using frigate ephemeral scanning
+FRIGATE_HOST="127.0.0.1:57001"
+just regtest-sp scan-frigate --url $FRIGATE_HOST
+# 17. Check balance on sp-cli2 wallet
+just regtest-sp balance
+# 18. Check balance on bdk-cli wallet
+just regtest-bdk balance
+
+# At this point we will able to see SP outputs paid to out wallet!

doc/tabconf7/frigate_playbook.sh

@@ -123,7 +123,7 @@ build TAG="1.0.0" VERSION="29.0" RELEASE="29.0": machine
     RUN mkdir -p /build/frigate
     RUN mkdir -p /frigate
     WORKDIR /frigate
-    RUN git clone --recursive --branch 1.1.0 --depth 1 https://github.com/sparrowwallet/frigate.git .
+    RUN git clone --recursive --branch 1.3.2 --depth 1 https://github.com/sparrowwallet/frigate.git .
     RUN ./gradlew jpackage
     RUN cp -r ./build/jpackage/frigate /build/frigate
     RUN rm -rf /frigate

doc/tabconf7/justfile

@@ -13,9 +13,10 @@ redb = "2.4.0"
 rayon = "1.11.0"
 reqwest = { version = "0.12.23", features = ["json", "rustls-tls", "http2", "charset"], default-features = false }
 serde = { version = "1.0.219", features = ["serde_derive"] }
-serde_json = "1.0.142"
+serde_json = { version = "1.0.142", features = ["raw_value"]}
 url = "2.5.4"
 tracing = "0.1.41"
+jsonrpc = "=0.18.0"
 
 [lints]
 workspace = true