Test: Add tests for exponential backoff (prebid#2970)

Author: marki1an

src/test/groovy/org/prebid/server/functional/testcontainers/scaffolding/VendorList.groovy

@@ -2,6 +2,7 @@ package org.prebid.server.functional.testcontainers.scaffolding
 
 import org.mockserver.matchers.TimeToLive
 import org.mockserver.matchers.Times
+import org.mockserver.model.Delay
 import org.mockserver.model.HttpRequest
 import org.mockserver.model.HttpResponse
 import org.testcontainers.containers.MockServerContainer
@@ -17,7 +18,7 @@ import static org.prebid.server.functional.util.privacy.TcfConsent.TcfPolicyVers
 
 class VendorList extends NetworkScaffolding {
 
-    private static final String VENDOR_LIST_ENDPOINT = "/{TCF_POLICY}/vendor-list.json"
+    private static final String VENDOR_LIST_ENDPOINT = "/v{TCF_POLICY}/vendor-list.json"
 
     VendorList(MockServerContainer mockServerContainer) {
         super(mockServerContainer, VENDOR_LIST_ENDPOINT)
@@ -39,16 +40,19 @@ class VendorList extends NetworkScaffolding {
     }
 
     void setResponse(TcfPolicyVersion tcfPolicyVersion = TCF_POLICY_V2,
+                     Delay delay = null,
                      Map<Integer, Vendor> vendors = [(GENERIC_VENDOR_ID): Vendor.getDefaultVendor(GENERIC_VENDOR_ID)]) {
-        def prepareEndpoint = endpoint.replace("{TCF_POLICY}", "v" + tcfPolicyVersion.vendorListVersion)
+        def prepareEndpoint = endpoint.replace("{TCF_POLICY}", tcfPolicyVersion.vendorListVersion.toString())
         def prepareEncodeResponseBody = encode(defaultVendorListResponse.tap {
             it.tcfPolicyVersion = tcfPolicyVersion.vendorListVersion
             it.vendors = vendors
         })
 
         mockServerClient.when(request().withPath(prepareEndpoint), Times.unlimited(), TimeToLive.unlimited(), -10)
-                .respond { request -> request.withPath(endpoint)
-                            ? response().withStatusCode(OK_200.code()).withBody(prepareEncodeResponseBody)
-                            : HttpResponse.notFoundResponse()}
+                .respond { request ->
+                    request.withPath(endpoint)
+                            ? response().withStatusCode(OK_200.code()).withDelay(delay).withBody(prepareEncodeResponseBody)
+                            : HttpResponse.notFoundResponse()
+                }
     }
 }

src/test/groovy/org/prebid/server/functional/tests/privacy/GdprAmpSpec.groovy

@@ -1,5 +1,6 @@
 package org.prebid.server.functional.tests.privacy
 
+import org.mockserver.model.Delay
 import org.prebid.server.functional.model.ChannelType
 import org.prebid.server.functional.model.config.AccountConfig
 import org.prebid.server.functional.model.config.AccountGdprConfig
@@ -327,7 +328,7 @@ class GdprAmpSpec extends PrivacyBaseSpec {
         privacyPbsService.sendAmpRequest(ampRequest)
 
         then: "Used vendor list have proper specification version of GVL"
-        def properVendorListPath = "/app/prebid-server/data/vendorlist-v${tcfPolicyVersion.vendorListVersion}/${tcfPolicyVersion.vendorListVersion}.json"
+        def properVendorListPath = VENDOR_LIST_PATH.replace("{VendorVersion}", tcfPolicyVersion.vendorListVersion.toString())
         PBSUtils.waitUntil { privacyPbsService.isFileExist(properVendorListPath) }
         def vendorList = privacyPbsService.getValueFromContainer(properVendorListPath, VendorListConsent.class)
         assert vendorList.tcfPolicyVersion == tcfPolicyVersion.vendorListVersion
@@ -369,4 +370,64 @@ class GdprAmpSpec extends PrivacyBaseSpec {
         assert response.ext?.warnings[PREBID]*.message ==
                 ["Parsing consent string: ${tcfConsent} failed. TCF policy version ${invalidTcfPolicyVersion} is not supported" as String]
     }
+
+    def "PBS amp should emit the same error without a second GVL list request if a retry is too soon for the exponential-backoff"() {
+        given: "Test start time"
+        def startTime = Instant.now()
+
+        and: "Prepare tcf consent string"
+        def tcfConsent = new TcfConsent.Builder()
+                .setPurposesLITransparency(BASIC_ADS)
+                .setTcfPolicyVersion(tcfPolicyVersion.value)
+                .setVendorListVersion(tcfPolicyVersion.vendorListVersion)
+                .setVendorLegitimateInterest([GENERIC_VENDOR_ID])
+                .build()
+
+        and: "AMP request"
+        def ampRequest = getGdprAmpRequest(tcfConsent)
+
+        and: "Default stored request"
+        def ampStoredRequest = BidRequest.defaultBidRequest
+
+        and: "Stored request in DB"
+        def storedRequest = StoredRequest.getStoredRequest(ampRequest, ampStoredRequest)
+        storedRequestDao.save(storedRequest)
+
+        and: "Reset valid vendor list response"
+        vendorListResponse.reset()
+
+        and: "Set vendor list response with delay"
+        vendorListResponse.setResponse(tcfPolicyVersion, Delay.seconds(EXPONENTIAL_BACKOFF_MAX_DELAY + 3))
+
+        when: "PBS processes amp request"
+        privacyPbsService.sendAmpRequest(ampRequest)
+
+        then: "PBS shouldn't fetch vendor list"
+        def vendorListPath = VENDOR_LIST_PATH.replace("{VendorVersion}", tcfPolicyVersion.vendorListVersion.toString())
+        assert !privacyPbsService.isFileExist(vendorListPath)
+
+        and: "Logs should contain proper vendor list version"
+        def logs = privacyPbsService.getLogsByTime(startTime)
+        def tcfError = "TCF 2 vendor list for version v${tcfPolicyVersion.vendorListVersion}.${tcfPolicyVersion.vendorListVersion} not found, started downloading."
+        assert getLogsByText(logs, tcfError)
+
+        and: "Second start for fetch second round of logs"
+        def secondStartTime = Instant.now()
+
+        when: "PBS processes amp request"
+        privacyPbsService.sendAmpRequest(ampRequest)
+
+        then: "PBS shouldn't fetch vendor list"
+        assert !privacyPbsService.isFileExist(vendorListPath)
+
+        and: "Logs should contain proper vendor list version"
+        def logsSecond = privacyPbsService.getLogsByTime(secondStartTime)
+        assert getLogsByText(logsSecond, tcfError)
+
+        and: "Reset vendor list response"
+        vendorListResponse.reset()
+
+        where:
+        tcfPolicyVersion << [TCF_POLICY_V2, TCF_POLICY_V3]
+    }
 }

src/test/groovy/org/prebid/server/functional/tests/privacy/GdprAuctionSpec.groovy

@@ -1,5 +1,6 @@
 package org.prebid.server.functional.tests.privacy
 
+import org.mockserver.model.Delay
 import org.prebid.server.functional.model.ChannelType
 import org.prebid.server.functional.model.config.AccountGdprConfig
 import org.prebid.server.functional.model.request.auction.DistributionChannel
@@ -275,7 +276,7 @@ class GdprAuctionSpec extends PrivacyBaseSpec {
         privacyPbsService.sendAuctionRequest(bidRequest)
 
         then: "Used vendor list have proper specification version of GVL"
-        def properVendorListPath = "/app/prebid-server/data/vendorlist-v${tcfPolicyVersion.vendorListVersion}/${tcfPolicyVersion.vendorListVersion}.json"
+        def properVendorListPath = VENDOR_LIST_PATH.replace("{VendorVersion}", tcfPolicyVersion.vendorListVersion.toString())
         PBSUtils.waitUntil { privacyPbsService.isFileExist(properVendorListPath) }
         def vendorList = privacyPbsService.getValueFromContainer(properVendorListPath, VendorListConsent.class)
         assert vendorList.tcfPolicyVersion == tcfPolicyVersion.vendorListVersion
@@ -313,4 +314,57 @@ class GdprAuctionSpec extends PrivacyBaseSpec {
         assert response.ext?.warnings[ErrorType.PREBID]*.message ==
                 ["Parsing consent string: ${tcfConsent} failed. TCF policy version ${invalidTcfPolicyVersion} is not supported" as String]
     }
+
+    def "PBS auction should emit the same error without a second GVL list request if a retry is too soon for the exponential-backoff"() {
+        given: "Test start time"
+        def startTime = Instant.now()
+
+        and: "Tcf consent setup"
+        def tcfConsent = new TcfConsent.Builder()
+                .setPurposesLITransparency(BASIC_ADS)
+                .setTcfPolicyVersion(tcfPolicyVersion.value)
+                .setVendorListVersion(tcfPolicyVersion.vendorListVersion)
+                .setVendorLegitimateInterest([GENERIC_VENDOR_ID])
+                .build()
+
+        and: "Bid request"
+        def bidRequest = getGdprBidRequest(tcfConsent)
+
+        and: "Reset valid vendor list response"
+        vendorListResponse.reset()
+
+        and: "Set vendor list response with delay"
+        vendorListResponse.setResponse(tcfPolicyVersion, Delay.seconds(EXPONENTIAL_BACKOFF_MAX_DELAY + 3))
+
+        when: "PBS processes auction request"
+        privacyPbsService.sendAuctionRequest(bidRequest)
+
+        then: "Used vendor list have proper specification version of GVL"
+        def properVendorListPath = VENDOR_LIST_PATH.replace("{VendorVersion}", tcfPolicyVersion.vendorListVersion.toString())
+        assert !privacyPbsService.isFileExist(properVendorListPath)
+
+        and: "Logs should contain proper vendor list version"
+        def logs = privacyPbsService.getLogsByTime(startTime)
+        def tcfError = "TCF 2 vendor list for version v${tcfPolicyVersion.vendorListVersion}.${tcfPolicyVersion.vendorListVersion} not found, started downloading."
+        assert getLogsByText(logs, tcfError)
+
+        and: "Second start for fetch second round of logs"
+        def secondStartTime = Instant.now()
+
+        when: "PBS processes amp request"
+        privacyPbsService.sendAuctionRequest(bidRequest)
+
+        then: "PBS shouldn't fetch vendor list"
+        assert !privacyPbsService.isFileExist(properVendorListPath)
+
+        and: "Logs should contain proper vendor list version"
+        def logsSecond = privacyPbsService.getLogsByTime(secondStartTime)
+        assert getLogsByText(logsSecond, tcfError)
+
+        and: "Reset vendor list response"
+        vendorListResponse.reset()
+
+        where:
+        tcfPolicyVersion << [TCF_POLICY_V2, TCF_POLICY_V3]
+    }
 }

src/test/groovy/org/prebid/server/functional/tests/privacy/PrivacyBaseSpec.groovy

@@ -66,8 +66,8 @@ abstract class PrivacyBaseSpec extends BaseSpec {
                                                                 "gdpr.vendorlist.v3.http-endpoint-template": "$networkServiceContainer.rootUri/v3/vendor-list.json".toString()]
     protected static final Map<String, String> SETTING_CONFIG = ["settings.enforce-valid-account": 'true']
     protected static final Map<String, String> GENERIC_VENDOR_CONFIG = ["adapters.generic.meta-info.vendor-id": GENERIC_VENDOR_ID as String,
-                                                                      "gdpr.host-vendor-id"                 : GENERIC_VENDOR_ID as String,
-                                                                      "adapters.generic.ccpa-enforced"      : "true"]
+                                                                        "gdpr.host-vendor-id"                 : GENERIC_VENDOR_ID as String,
+                                                                        "adapters.generic.ccpa-enforced"      : "true"]
 
     @Shared
     protected static final int PURPOSES_ONLY_GVL_VERSION = PBSUtils.getRandomNumber(0, 4095)
@@ -78,17 +78,29 @@ abstract class PrivacyBaseSpec extends BaseSpec {
     @Shared
     protected static final int PURPOSES_AND_LEG_INT_PURPOSES_GVL_VERSION = PBSUtils.getRandomNumberWithExclusion([PURPOSES_ONLY_GVL_VERSION, LEG_INT_PURPOSES_ONLY_GVL_VERSION, LEG_INT_AND_FLEXIBLE_PURPOSES_GVL_VERSION], 0, 4095)
 
+    protected static final int EXPONENTIAL_BACKOFF_MAX_DELAY = 1
+
+    private static final Map<String, String> RETRY_POLICY_EXPONENTIAL_CONFIG = [
+            "gdpr.vendorlist.v2.retry-policy.exponential-backoff.delay-millis"    : 1 as String,
+            "gdpr.vendorlist.v2.retry-policy.exponential-backoff.max-delay-millis": EXPONENTIAL_BACKOFF_MAX_DELAY as String,
+            "gdpr.vendorlist.v2.retry-policy.exponential-backoff.factor"          : Long.MAX_VALUE as String,
+            "gdpr.vendorlist.v3.retry-policy.exponential-backoff.delay-millis"    : 1 as String,
+            "gdpr.vendorlist.v3.retry-policy.exponential-backoff.max-delay-millis": EXPONENTIAL_BACKOFF_MAX_DELAY as String,
+            "gdpr.vendorlist.v3.retry-policy.exponential-backoff.factor"          : Long.MAX_VALUE as String]
+
     private static final PbsPgConfig pgConfig = new PbsPgConfig(networkServiceContainer)
 
-    protected static final Map<String, String> PBS_CONFIG = OPENX_CONFIG + OPENX_COOKIE_SYNC_CONFIG +
-            GENERIC_COOKIE_SYNC_CONFIG + pgConfig.properties + GDPR_VENDOR_LIST_CONFIG + SETTING_CONFIG + GENERIC_VENDOR_CONFIG
+    protected static final String VENDOR_LIST_PATH = "/app/prebid-server/data/vendorlist-v{VendorVersion}/{VendorVersion}.json"
     protected static final String VALID_VALUE_FOR_GPC_HEADER = "1"
     protected static final GppConsent SIMPLE_GPC_DISALLOW_LOGIC = new UspNatV1Consent.Builder().setGpc(true).build()
     protected static final VendorList vendorListResponse = new VendorList(networkServiceContainer)
 
     @Shared
     protected final PrebidServerService privacyPbsService = pbsServiceFactory.getService(GDPR_VENDOR_LIST_CONFIG +
-            GENERIC_COOKIE_SYNC_CONFIG + GENERIC_VENDOR_CONFIG)
+            GENERIC_COOKIE_SYNC_CONFIG + GENERIC_VENDOR_CONFIG + RETRY_POLICY_EXPONENTIAL_CONFIG)
+
+    protected static final Map<String, String> PBS_CONFIG = OPENX_CONFIG + OPENX_COOKIE_SYNC_CONFIG +
+            GENERIC_COOKIE_SYNC_CONFIG + pgConfig.properties + GDPR_VENDOR_LIST_CONFIG + SETTING_CONFIG + GENERIC_VENDOR_CONFIG
 
     @Shared
     protected final PrebidServerService activityPbsService = pbsServiceFactory.getService(PBS_CONFIG)