Accept a single rescan timestamp for all descriptors

Bitcoin Core scans since the earliest timestamp provided within the
`importmulti` batch, so specifying different timestamps isn't really
very useufl.

This is still supported via the lower-level Rust API, but not exposed
through the CLI/FF Interfaces, which are now simplified.

Author: shesek

CHANGELOG.md

@@ -38,6 +38,8 @@ Breaking CLI changes:
 
 - Renamed `--http-server-addr` to `--http-addr` and `--electrum-rpc-addr` to `--electrum-addr`
 
+- The CLI now accepts a single `--rescan-since` timestamp instead of a separate one for each descriptor/xpub.
+
 ## 0.1.5 - 2020-10-05
 
 - Reproducible builds using Docker (#51)

README.md

@@ -136,9 +136,10 @@ You can set multiple `--xpub`s to track. This also supports ypubs and zpubs.
 
 You can also track output script descriptors using `--descriptor`. For example, `--descriptor 'wpkh(<xpub>/0/*)'`.
 
-Rescanning can be controlled with `--xpub <xpub>@<rescan>`. You can specify `<rescan>` with the wallet birthday formatted
-as `yyyy-mm-dd` to scan from that date onwards only, or use `now` to disable rescanning and watch for new transactions only (for newly created wallets).
-*Setting this can significantly speed up scanning and is highly recommended.*
+To speed up rescanning for historical transactions, you can provide the wallet creation date with `--rescan-since <timestmap>`.
+The timestamp can be a `YYYY-MM-DD` formatted string, or 'now' to disable rescanning and watch for new
+transactions only (for newly created wallets).
+*Setting this is highly recommended.*
 
 By default, the Electrum server will be bound on port `50001`/`60001`/`60401` (according to the network)
 and the HTTP server will be bound on port `3060`. This can be controlled with `--electrum-addr`
@@ -166,7 +167,8 @@ For example:
 NETWORK=regtest
 GAP_LIMIT=20
 XPUBS='<xpub1>;<xpub2>'
-DESCRIPTORS='pkh(<xpub>/0/*)@2020-01-01'
+DESCRIPTORS='pkh(<xpub>/0/*)'
+RESCAN_SINCE=2020-01-01
 ```
 
 Setting the environment variables directly is also supported.
@@ -187,7 +189,7 @@ This removes several large dependencies and disables the `track-spends` database
 
 You can use bwt with pruning, but:
 
-1. You will have to provide a rescan date (via `--xpub <xpub>@<rescan>`) that is within the range of non-pruned blocks, or use `none` to disable rescanning entirely.
+1. You will have to provide a rescan date (via `--rescan-since`) that is within the range of non-pruned blocks, or use `none` to disable rescanning entirely.
 
 2. Electrum needs to be run with `--skipmerklecheck` to tolerate missing SPV proofs for transactions in pruned blocks.
 

contrib/electrum-plugin/bwt.py

@@ -58,6 +58,7 @@ def start(self):
             '--network', get_network_name(),
             '--bitcoind-url', self.bitcoind_url,
             '--bitcoind-dir', self.bitcoind_dir,
+            '--rescan-since', self.rescan_since,
             '--electrum-addr', '127.0.0.1:%d' % self.rpc_port,
             '--electrum-skip-merkle',
             '--no-startup-banner',
@@ -74,7 +75,7 @@ def start(self):
 
         for wallet in self.wallets:
             for xpub in wallet.get_master_public_keys():
-                args.extend([ '--xpub', '%s@%s' % (xpub, self.rescan_since) ])
+                args.extend([ '--xpub', xpub ])
 
         for i in range(self.verbose):
             args.append('-v')

contrib/nodejs-bwt-daemon/README.md

@@ -32,8 +32,8 @@ location, on the default ports and with cookie auth enabled, this should Just Wo
 ```js
 import BwtDaemon from 'bwt-daemon'
 
-let bwtd = await BwtDaemon({
-  xpubs: [ [ 'xpub66...', 'now' ] ],
+const bwtd = await BwtDaemon({
+  xpubs: [ 'xpub66...' ],
   electrum: true,
 })
 
@@ -43,17 +43,19 @@ console.log('bwt electrum server ready on', bwtd.electrum_addr)
 With some more advanced options:
 
 ```js
-let bwtd = await BwtDaemon({
+const bwtd = await BwtDaemon({
   // Network and Bitcoin Core RPC settings
   network: 'regtest',
   bitcoind_dir: '/home/satoshi/.bitcoin',
   bitcoind_url: 'http://127.0.0.1:9008/',
   bitcoind_wallet: 'bwt',
 
-  // Descriptors or xpubs to track as an array of (desc_or_xpub, rescan_since) tuples
-  // Use 'now' to look for new transactions only, or the unix timestamp to begin rescanning from.
-  descriptors: [ [ 'wpkh(tpub61.../0/*)', 'now' ] ],
-  xpubs: [ [ 'tpub66...', 'now' ] ],
+  // Descriptors and xpubs to track
+  descriptors: [ 'wpkh(tpub61.../0/*)' ],
+  xpubs: [ 'tpub66...' ],
+
+  // Rescan since timestamp. Accepts unix timestamps, date strings, Date objects, or 'now' to look for new transactions only
+  rescan_since: '2020-01-01',
 
   // Enable HTTP and Electrum servers
   http: true,

contrib/nodejs-bwt-daemon/example.js

@@ -7,7 +7,7 @@ const BwtDaemon = require('bwt-daemon')
     network: 'regtest',
     bitcoind_dir: '/tmp/bd1',
     bitcoind_wallet: 'bwt',
-    descriptors: [ [ my_desc, 'now' ] ],
+    descriptors: [ my_desc ],
     electrum: true,
     http: true,
     verbose: 2,

contrib/nodejs-bwt-daemon/index.js

@@ -40,6 +40,10 @@ function init(options) {
       delete options.progress
     }
 
+    if (options.rescan_since) {
+      options.rescan_since = parse_timestamp(options.rescan_since)
+    }
+
     // Convenience shortcuts
     if (options.electrum) {
       options.electrum_addr || (options.electrum_addr = '127.0.0.1:0')
@@ -97,3 +101,19 @@ class BwtDaemon {
 }
 
 module.exports = init.BwtDaemon = init
+
+// Utility
+
+function parse_timestamp(ts) {
+  // Pass 'now' as is
+  if (ts == 'now') return ts
+  // Date objects
+  if (ts.getTime) return ts.getTime()/1000|0
+  // Unix timestamp
+  if (!isNaN(ts)) return +ts
+  // Date string (e.g. YYYY-MM-DD)
+  const dt = new Date(ts)
+  if (!isNaN(dt.getTime())) return dt.getTime()/1000|0
+
+  throw new Error(`Invalid rescan since value: ${ts}`)
+}

examples/use-from-rust.rs

@@ -10,7 +10,8 @@ fn main() -> Result<()> {
         bitcoind_dir: Some("/home/satoshi/.bitcoin".into()),
         bitcoind_wallet: Some("bwt".into()),
         electrum_addr: Some("127.0.0.1:0".parse().unwrap()),
-        descriptors: vec![(my_desc.parse().unwrap(), RescanSince::Timestamp(0))],
+        descriptors: vec![my_desc.parse().unwrap()],
+        rescan_since: RescanSince::Now,
         verbose: 2,
         ..Default::default()
     };

src/app.rs

@@ -40,6 +40,7 @@ impl App {
             &config.descriptors[..],
             &config.xpubs[..],
             &config.bare_xpubs[..],
+            config.rescan_since,
             config.network,
             config.gap_limit,
             config.initial_import_size,

src/config.rs

@@ -127,37 +127,61 @@ pub struct Config {
     #[cfg_attr(feature = "cli", structopt(
         short = "d",
         long = "descriptor",
-        help = "Descriptors to track (scans for history from the genesis by default, use <desc>@<yyyy-mm-dd> or <desc>@<unix-epoch> to specify a rescan timestmap, or <desc>@none to disable rescan)",
+        help = "Add a descriptor to track",
         parse(try_from_str = parse_desc),
         env, hide_env_values(true),
         use_delimiter(true), value_delimiter(";"),
         display_order(20)
     ))]
     #[serde(default)]
-    pub descriptors: Vec<(ExtendedDescriptor, RescanSince)>,
+    pub descriptors: Vec<ExtendedDescriptor>,
 
-    #[cfg_attr(feature = "cli", structopt(
-        short = "x",
-        long = "xpub",
-        help = "xpubs to track (represented as two separate descriptors for the internal/external chains, supports <xpub>@<rescan-time>)",
-        parse(try_from_str = parse_xpub),
-        env, hide_env_values(true),
-        use_delimiter(true), value_delimiter(";"),
-        display_order(21)
-    ))]
+    #[cfg_attr(
+        feature = "cli",
+        structopt(
+            short = "x",
+            long = "xpub",
+            help = "Add an extended public key to track (with separate internal/external chains)",
+            env,
+            hide_env_values(true),
+            use_delimiter(true),
+            value_delimiter(";"),
+            display_order(21)
+        )
+    )]
     #[serde(default)]
-    pub xpubs: Vec<(XyzPubKey, RescanSince)>,
+    pub xpubs: Vec<XyzPubKey>,
 
-    #[cfg_attr(feature = "cli", structopt(
-        short = "X",
-        long = "bare-xpub",
-        help = "Bare xpubs to track (like --xpub, but does not derive separate internal/external chains)",
-        parse(try_from_str = parse_xpub),
-        env, hide_env_values(true), use_delimiter(true),
-        display_order(22)
-    ))]
+    #[cfg_attr(
+        feature = "cli",
+        structopt(
+            short = "X",
+            long = "bare-xpub",
+            help = "Add a bare extended public key to track (without separate internal/external chains)",
+            env,
+            hide_env_values(true),
+            use_delimiter(true),
+            display_order(22)
+        )
+    )]
     #[serde(default)]
-    pub bare_xpubs: Vec<(XyzPubKey, RescanSince)>,
+    pub bare_xpubs: Vec<XyzPubKey>,
+
+    #[cfg_attr(
+        feature = "cli",
+        structopt(
+            short = "R",
+            long,
+            help = "Start date for wallet history rescan. Accepts YYYY-MM-DD formatted strings, unix timestamps, or 'now' to watch for new transactions only",
+            parse(try_from_str = parse_rescan),
+            default_value = "0",
+            env,
+            hide_env_values(true),
+            display_order(29)
+        )
+    )]
+    #[serde(default = "default_rescan_since")]
+    pub rescan_since: RescanSince,
 
     #[cfg_attr(
         feature = "cli",
@@ -452,34 +476,23 @@ fn apply_log_env(mut builder: LogBuilder) -> LogBuilder {
 }
 
 #[cfg(feature = "cli")]
-fn parse_desc(s: &str) -> Result<(ExtendedDescriptor, RescanSince)> {
+fn parse_desc(s: &str) -> Result<ExtendedDescriptor> {
     use crate::util::descriptor::DescriptorChecksum;
-    let mut parts = s.trim().splitn(2, '@');
-    let desc = ExtendedDescriptor::parse_with_checksum(parts.next().req()?)?;
-    let rescan = parse_rescan(parts.next())?;
-    Ok((desc, rescan))
+    Ok(ExtendedDescriptor::parse_with_checksum(s)?)
 }
 
 #[cfg(feature = "cli")]
-fn parse_xpub(s: &str) -> Result<(XyzPubKey, RescanSince)> {
-    let mut parts = s.trim().splitn(2, '@');
-    let xpub = parts.next().req()?.parse()?;
-    let rescan = parse_rescan(parts.next())?;
-    Ok((xpub, rescan))
-}
-
-#[cfg(feature = "cli")]
-fn parse_rescan(s: Option<&str>) -> Result<RescanSince> {
+fn parse_rescan(s: &str) -> Result<RescanSince> {
     use crate::error::Context;
     Ok(match s {
-        None | Some("all") => RescanSince::Timestamp(0),
-        Some("now") | Some("none") => RescanSince::Now,
-        Some(s) => {
+        "all" | "genesis" => RescanSince::Timestamp(0),
+        "now" | "none" => RescanSince::Now,
+        s => {
             // try as a unix timestamp first, then as a datetime string
             RescanSince::Timestamp(
                 s.parse::<u64>()
                     .or_else(|_| parse_yyyymmdd(s))
-                    .context("invalid rescan value")?,
+                    .context("invalid rescan-since value")?,
             )
         }
     })
@@ -566,6 +579,7 @@ defaultable!(Config,
   )
   @custom(
     network=Network::Bitcoin,
+    rescan_since=RescanSince::Timestamp(0),
     gap_limit=20,
     initial_import_size=350,
     poll_interval=time::Duration::from_secs(5),
@@ -577,6 +591,9 @@ defaultable!(Config,
 fn default_network() -> Network {
     Network::Bitcoin
 }
+fn default_rescan_since() -> RescanSince {
+    RescanSince::Timestamp(0)
+}
 fn default_gap_limit() -> u32 {
     20
 }

src/wallet.rs

@@ -35,46 +35,47 @@ impl WalletWatcher {
     }
 
     pub fn from_config(
-        descs: &[(ExtendedDescriptor, RescanSince)],
-        xpubs: &[(XyzPubKey, RescanSince)],
-        bare_xpubs: &[(XyzPubKey, RescanSince)],
+        descs: &[ExtendedDescriptor],
+        xpubs: &[XyzPubKey],
+        bare_xpubs: &[XyzPubKey],
+        rescan_since: RescanSince,
         network: Network,
         gap_limit: u32,
         initial_import_size: u32,
     ) -> Result<Self> {
         let mut wallets = vec![];
-        for (desc, rescan) in descs {
+        for desc in descs {
             wallets.push(
                 Wallet::from_descriptor(
                     desc.clone(),
                     network,
                     gap_limit,
                     initial_import_size,
-                    *rescan,
+                    rescan_since,
                 )
                 .with_context(|| format!("invalid descriptor {}", desc))?,
             );
         }
-        for (xpub, rescan) in xpubs {
+        for xpub in xpubs {
             wallets.append(
                 &mut Wallet::from_xpub(
                     xpub.clone(),
                     network,
                     gap_limit,
                     initial_import_size,
-                    *rescan,
+                    rescan_since,
                 )
                 .with_context(|| format!("invalid xpub {}", xpub))?,
             );
         }
-        for (xpub, rescan) in bare_xpubs {
+        for xpub in bare_xpubs {
             wallets.push(
                 Wallet::from_bare_xpub(
                     xpub.clone(),
                     network,
                     gap_limit,
                     initial_import_size,
-                    *rescan,
+                    rescan_since,
                 )
                 .with_context(|| format!("invalid xpub {}", xpub))?,
             );