Add google bot IP "firewall"

[minor]

Author: joecorall

CLAUDE.md

@@ -62,9 +62,10 @@ services:
             traefik.http.middlewares.captcha-protect.plugin.captcha-protect.captchaProvider: turnstile
             traefik.http.middlewares.captcha-protect.plugin.captcha-protect.siteKey: ${TURNSTILE_SITE_KEY}
             traefik.http.middlewares.captcha-protect.plugin.captcha-protect.secretKey: ${TURNSTILE_SECRET_KEY}
-            traefik.http.middlewares.captcha-protect.plugin.captcha-protect.goodBots: apple.com,archive.org,commoncrawl.org,duckduckgo.com,facebook.com,google.com,googlebot.com,googleusercontent.com,instagram.com,kagibot.org,linkedin.com,msn.com,openalex.org,twitter.com,x.com
+            traefik.http.middlewares.captcha-protect.plugin.captcha-protect.goodBots: apple.com,archive.org,commoncrawl.org,duckduckgo.com,facebook.com,google.com,instagram.com,kagibot.org,linkedin.com,msn.com,openalex.org,twitter.com,x.com
             traefik.http.middlewares.captcha-protect.plugin.captcha-protect.persistentStateFile: /tmp/state.json
             traefik.http.middlewares.captcha-protect.plugin.captcha-protect.enableStateReconciliation: "false"
+            traefik.http.middlewares.captcha-protect.plugin.captcha-protect.enableGooglebotIPCheck: "true"
             traefik.http.middlewares.captcha-protect.plugin.captcha-protect.periodSeconds: 30
             traefik.http.middlewares.captcha-protect.plugin.captcha-protect.failureThreshold: 3
         networks:
@@ -82,7 +83,7 @@ services:
             --providers.docker=true
             --providers.docker.network=default
             --experimental.plugins.captcha-protect.modulename=github.com/libops/captcha-protect
-            --experimental.plugins.captcha-protect.version=v1.11.1
+            --experimental.plugins.captcha-protect.version=v1.12.0
         volumes:
             - /var/run/docker.sock:/var/run/docker.sock:z
             - /CHANGEME/TO/A/HOST/PATH/FOR/STATE/FILE:/tmp/state.json:rw
@@ -128,6 +129,7 @@ services:
 | `enableStatsPage`       | `string`                | `"false"`                | Allows `exemptIps` to access `/captcha-protect/stats` to monitor the rate limiter.                                                                                                               |
 | `logLevel`              | `string`                | `"INFO"`                 | Log level for the middleware. Options: `ERROR`, `WARNING`, `INFO`, or `DEBUG`.                                                                                                                   |
 | `persistentStateFile`   | `string`                | `""`                     | File path to persist rate limiter state across Traefik restarts. In Docker, mount this file from the host.                                                                                       |
+| `enableGooglebotIPCheck`    | `string`            | `"false"`                | Treat google bot IPs are good bots                                                                                                                                                               |
 | `enableStateReconciliation` | `string`            | `"false"`                | When `"true"`, reads and merges disk state before each save to prevent multiple instances from overwriting data. Adds extra I/O overhead. Only enable for multi-instance deployments sharing state. **Performance warning**: Not recommended for sites with >1M unique visitors due to reconciliation overhead (5-8s per cycle at scale). |
 
 ### Circuit Breaker (failover if a captcha provider is unavailable)
@@ -152,14 +154,17 @@ The circuit breaker provides automatic failover when the primary captcha provide
 
 ### Good Bots
 
-To avoid having this middleware impact your SEO score, it's recommended to provide a value for `goodBots`. By default, no bots will be allowed to crawl your protected routes beyond the rate limit unless their second level domain (e.g. `google.com`) is configured as a good bot.
+To avoid having this middleware impact your SEO score, it's recommended to provide a value for `goodBots`. By default, no bots will be allowed to crawl your protected routes beyond the rate limit unless their second level domain (e.g. `bing.com`) is configured as a good bot.
 
 A good default value for `goodBots` would be:
 
 ```
-goodBots: apple.com,archive.org,duckduckgo.com,facebook.com,google.com,googlebot.com,googleusercontent.com,instagram.com,kagibot.org,linkedin.com,msn.com,openalex.org,twitter.com,x.com
+enableGooglebotIPCheck: "true"
+goodBots: apple.com,archive.org,duckduckgo.com,facebook.com,google.com,instagram.com,kagibot.org,linkedin.com,msn.com,openalex.org,twitter.com,x.com
 ```
 
+Since google publishes their bot IPs, we can also leverage their API to let google based on IP. This can be enabled with enableGooglebotIPCheck: "true"
+
 **However** if you set the config parameter `protectParameters="true"`, even good bots won't be allowed to crawl protected routes if a URL parameter is on the request (e.g. `/foo?bar=baz`). This `protectParameters` feature is meant to help protect faceted search pages.
 
 

README.md

@@ -19,6 +19,7 @@ services:
             traefik.http.middlewares.captcha-protect.plugin.captcha-protect.ipForwardedHeader: "X-Forwarded-For"
             traefik.http.middlewares.captcha-protect.plugin.captcha-protect.logLevel: "DEBUG"
             traefik.http.middlewares.captcha-protect.plugin.captcha-protect.goodBots: ""
+            traefik.http.middlewares.captcha-protect.plugin.captcha-protect.enableGooglebotIPCheck: "true"
             traefik.http.middlewares.captcha-protect.plugin.captcha-protect.protectRoutes: "/"
             traefik.http.middlewares.captcha-protect.plugin.captcha-protect.persistentStateFile: "/tmp/state.json"
             traefik.http.middlewares.captcha-protect.plugin.captcha-protect.enableStateReconciliation: "true"
@@ -48,6 +49,7 @@ services:
             traefik.http.middlewares.captcha-protect.plugin.captcha-protect.ipForwardedHeader: "X-Forwarded-For"
             traefik.http.middlewares.captcha-protect.plugin.captcha-protect.logLevel: "DEBUG"
             traefik.http.middlewares.captcha-protect.plugin.captcha-protect.goodBots: ""
+            traefik.http.middlewares.captcha-protect.plugin.captcha-protect.enableGooglebotIPCheck: "true"
             traefik.http.middlewares.captcha-protect.plugin.captcha-protect.protectRoutes: "/"
             traefik.http.middlewares.captcha-protect.plugin.captcha-protect.persistentStateFile: "/tmp/state.json"
             traefik.http.middlewares.captcha-protect.plugin.captcha-protect.enableStateReconciliation: "true"

ci/docker-compose.yml

@@ -0,0 +1,54 @@
+package state
+
+import (
+	"log/slog"
+	"net"
+	"sync"
+)
+
+// GooglebotIPs holds the list of Googlebot IP ranges, providing a thread-safe way to check if an IP is a Googlebot.
+type GooglebotIPs struct {
+	cidrs []*net.IPNet
+	mu    sync.RWMutex
+}
+
+// NewGooglebotIPs creates and initializes a new GooglebotIPs object.
+func NewGooglebotIPs() *GooglebotIPs {
+	return &GooglebotIPs{
+		cidrs: make([]*net.IPNet, 0),
+	}
+}
+
+// Update parses a slice of CIDR strings and replaces the existing IP ranges with the new ones.
+// It logs an error for any CIDR string that fails to parse.
+func (g *GooglebotIPs) Update(cidrs []string, log *slog.Logger) {
+	g.mu.Lock()
+	defer g.mu.Unlock()
+
+	g.cidrs = make([]*net.IPNet, 0, len(cidrs))
+
+	for _, s := range cidrs {
+		_, network, err := net.ParseCIDR(s)
+		if err != nil {
+			log.Error("error parsing CIDR", "cidr", s, "err", err)
+
+			continue
+		}
+
+		g.cidrs = append(g.cidrs, network)
+	}
+}
+
+// Contains checks if the given IP address is within any of the stored Googlebot IP ranges.
+func (g *GooglebotIPs) Contains(ip net.IP) bool {
+	g.mu.RLock()
+	defer g.mu.RUnlock()
+
+	for _, network := range g.cidrs {
+		if network.Contains(ip) {
+			return true
+		}
+	}
+
+	return false
+}