Implement rate limit protection

rubengees · rubengees · commit e9a40cf33c16 · 2020-08-15T19:29:17.000+02:00
diff --git a/README.md b/README.md
@@ -35,7 +35,7 @@ All requests are done through the `ProxerApi` class. You initialize an instance
 The most simple initialization looks like this:
 
 ```java
-ProxerApi api = new ProxerConnection.Builder("yourApiKey").build();
+ProxerApi api = new ProxerApi.Builder("yourApiKey").build();
 ```
 
 You can customize the `ProxerApi` in the following ways:
@@ -204,6 +204,27 @@ ProxerApi api = new ProxerApi.Builder("yourApiKey")
 
 > The token is only stored in memory with the default `LoginTokenManager`. This means, that you lose the login information when the application terminates. You may want to persist it into a `File` or `SharedPreferences` on `Android`.
 
+### Rate limiting
+
+The api has rate limiting in place with individual limits for each api class. If more requests than the limit are performed,
+a `ProxerException` (with the `ServerErrorType.CAPTCHA`) is thrown and the user has to solve a captcha. The link to the captcha page can be obtained with
+`ProxerUrls.captchaWeb()`.
+
+##### Rate limit protection
+
+Recent versions have a mechanism for protecting against rate limiting implemented. It can be enabled by calling the method:
+
+```java
+ProxerApi api = new ProxerApi.Builder("yourApiKey")
+        .enableRateLimitProtection()
+        .build();
+```
+
+This cancels requests which would trigger the rate limit and throw a `ProxerException` (with the `ServerErrorType.RATE_LIMIT`).
+The user does not need to solve the captcha but simply wait then.
+
+> This does not completely captcha errors. If multiple users call from the same network, the rate limit can still be hit and thus the error needs to be handled properly. 
+
 ### Utils
 
 This library offers two utility classes: `ProxerUrls` and `ProxerUtils`.
diff --git a/library/src/main/kotlin/me/proxer/library/ProxerApi.kt b/library/src/main/kotlin/me/proxer/library/ProxerApi.kt
@@ -39,6 +39,7 @@ import me.proxer.library.internal.interceptor.HeaderInterceptor
 import me.proxer.library.internal.interceptor.HttpsEnforcingInterceptor
 import me.proxer.library.internal.interceptor.LoginTokenInterceptor
 import me.proxer.library.internal.interceptor.OneShotInterceptor
+import me.proxer.library.internal.interceptor.RateLimitInterceptor
 import me.proxer.library.util.ProxerUrls
 import okhttp3.CertificatePinner
 import okhttp3.OkHttpClient
@@ -181,6 +182,7 @@ class ProxerApi private constructor(retrofit: Retrofit) {
         private var moshi: Moshi? = null
         private var client: OkHttpClient? = null
         private var retrofit: Retrofit? = null
+        private var enableRateLimitProtection: Boolean = false
 
         /**
          * Sets a custom login token manager.
@@ -220,35 +222,45 @@ class ProxerApi private constructor(retrofit: Retrofit) {
          */
         fun retrofit(retrofit: Retrofit) = this.apply { this.retrofit = retrofit }
 
+        /**
+         * Enables the rate limit protection to avoid users having to fill out captchas.
+         */
+        fun enableRateLimitProtection() = this.apply { this.enableRateLimitProtection = true }
+
         /**
          * Finally builds the [ProxerApi] with the provided adjustments.
          */
         fun build(): ProxerApi {
-            return ProxerApi(buildRetrofit())
+            val moshi = buildMoshi()
+
+            return ProxerApi(buildRetrofit(moshi))
         }
 
-        private fun buildRetrofit(): Retrofit {
+        private fun buildRetrofit(moshi: Moshi): Retrofit {
             return (retrofit?.newBuilder() ?: Retrofit.Builder())
                 .baseUrl(ProxerUrls.apiBase)
-                .client(buildClient())
+                .client(buildClient(moshi))
                 .addCallAdapterFactory(ProxerResponseCallAdapterFactory())
-                .addConverterFactory(MoshiConverterFactory.create(buildMoshi()))
+                .addConverterFactory(MoshiConverterFactory.create(moshi))
                 .addConverterFactory(EnumRetrofitConverterFactory())
                 .build()
         }
 
-        private fun buildClient(): OkHttpClient {
+        private fun buildClient(moshi: Moshi): OkHttpClient {
             return (client?.newBuilder() ?: OkHttpClient.Builder())
                 .apply {
                     interceptors().apply {
                         addAll(
-                            0, listOf(
+                            0,
+                            listOf(
                                 HeaderInterceptor(apiKey, buildUserAgent()),
                                 LoginTokenInterceptor(buildLoginTokenManager()),
                                 HttpsEnforcingInterceptor(),
                                 OneShotInterceptor()
                             )
                         )
+
+                        if (enableRateLimitProtection) add(0, RateLimitInterceptor(moshi))
                     }
 
                     certificatePinner(constructCertificatePinner())
diff --git a/library/src/main/kotlin/me/proxer/library/ProxerException.kt b/library/src/main/kotlin/me/proxer/library/ProxerException.kt
@@ -157,6 +157,7 @@ class ProxerException : Exception {
         COMMENT_ALREADY_EXISTS(3078),
 
         UNKNOWN(10_000),
+        RATE_LIMIT(99_998),
         INTERNAL(99_999);
 
         companion object {
diff --git a/library/src/main/kotlin/me/proxer/library/internal/interceptor/LoginTokenInterceptor.kt b/library/src/main/kotlin/me/proxer/library/internal/interceptor/LoginTokenInterceptor.kt
@@ -7,6 +7,8 @@ import me.proxer.library.ProxerException.ServerErrorType
 import me.proxer.library.util.ProxerUrls
 import okhttp3.Interceptor
 import okhttp3.Response
+import java.util.concurrent.locks.ReentrantLock
+import kotlin.concurrent.withLock
 
 /**
  * @author Ruben Gees
@@ -31,7 +33,7 @@ internal class LoginTokenInterceptor(private val loginTokenManager: LoginTokenMa
             .build()
     }
 
-    private val lock = Any()
+    private val lock = ReentrantLock()
 
     override fun intercept(chain: Interceptor.Chain): Response {
         val oldRequest = chain.request()
@@ -40,7 +42,7 @@ internal class LoginTokenInterceptor(private val loginTokenManager: LoginTokenMa
             val newRequestBuilder = oldRequest.newBuilder()
 
             if (oldRequest.url != LOGIN_URL) {
-                val loginToken = synchronized(lock) {
+                val loginToken = lock.withLock {
                     loginTokenManager.provide()
                 }
 
@@ -59,7 +61,7 @@ internal class LoginTokenInterceptor(private val loginTokenManager: LoginTokenMa
         if (response.isSuccessful) {
             val (errorCode, token) = peekResponse(response)
 
-            synchronized(lock) {
+            lock.withLock {
                 val errorType = errorCode?.let { ServerErrorType.fromErrorCode(it) }
 
                 if (errorType != null) {
diff --git a/library/src/main/kotlin/me/proxer/library/internal/interceptor/RateLimitInterceptor.kt b/library/src/main/kotlin/me/proxer/library/internal/interceptor/RateLimitInterceptor.kt
@@ -0,0 +1,115 @@
+package me.proxer.library.internal.interceptor
+
+import com.squareup.moshi.Moshi
+import com.squareup.moshi.Types
+import me.proxer.library.ProxerException.ServerErrorType.RATE_LIMIT
+import me.proxer.library.internal.ProxerResponse
+import me.proxer.library.util.ProxerUrls
+import okhttp3.Interceptor
+import okhttp3.MediaType.Companion.toMediaTypeOrNull
+import okhttp3.Protocol
+import okhttp3.Request
+import okhttp3.Response
+import okhttp3.ResponseBody.Companion.toResponseBody
+import java.util.Date
+import java.util.concurrent.locks.ReentrantReadWriteLock
+import kotlin.concurrent.read
+import kotlin.concurrent.write
+
+internal class RateLimitInterceptor(private val moshi: Moshi) : Interceptor {
+
+    private companion object {
+        private const val WAIT_THRESHOLD = 2_000
+
+        private val errorMediaType = "application/json".toMediaTypeOrNull()
+        private val responseType = Types.newParameterizedType(ProxerResponse::class.java, Unit::class.java)
+
+        private val limits = mapOf(
+            "anime" to Limit(20, 360_000),
+            "apps" to Limit(100, 30_000), // Not enforced by api.
+            "chat" to Limit(30, 30_000),
+            "comment" to Limit(100, 30_000), // Not enforced by api.
+            "files" to Limit(30, 100_000),
+            "forum" to Limit(100, 30_000), // Not enforced by api.
+            "info" to Limit(60, 360_000),
+            "list" to Limit(60, 60_000),
+            "manga" to Limit(10, 300_000),
+            "media" to Limit(100, 30_000), // Not enforced by api.
+            "messenger" to Limit(100, 50_000),
+            "misc" to Limit(100, 30_000), // Not enforced by api.
+            "notifications" to Limit(100, 30_000), // Not enforced by api.
+            "ucp" to Limit(40, 60_000),
+            "user" to Limit(40, 360_000),
+            "users" to Limit(40, 360_000),
+            "wiki" to Limit(100, 30_000) // Not enforced by api.
+        )
+    }
+
+    private var state = limits.mapValues { State(0, null) }
+
+    private val lock = ReentrantReadWriteLock()
+
+    @Suppress("ReturnCount")
+    override fun intercept(chain: Interceptor.Chain): Response {
+        val oldRequest = chain.request()
+
+        if (oldRequest.url.host != ProxerUrls.apiBase.host) {
+            return chain.proceed(oldRequest)
+        }
+
+        val apiClass = oldRequest.url.pathSegments.getOrNull(2)
+        val limit = limits[apiClass]
+
+        if (apiClass == null || limit == null) {
+            return chain.proceed(oldRequest)
+        }
+
+        return intercept(apiClass, limit, chain, oldRequest)
+    }
+
+    @Suppress("ReturnCount")
+    private fun intercept(
+        apiClass: String,
+        limit: Limit,
+        chain: Interceptor.Chain,
+        oldRequest: Request
+    ): Response {
+        val (requests, firstRequest) = lock.read { state.getValue(apiClass) }
+        val timeDifferenceMillis = if (firstRequest != null) Date().time - firstRequest.time else null
+
+        if (timeDifferenceMillis == null || timeDifferenceMillis > limit.millis) {
+            lock.write { state = state.update(apiClass, 1, Date()) }
+
+            return chain.proceed(oldRequest)
+        } else if (requests + 2 >= limit.maxRequests) {
+            return if ((limit.millis - timeDifferenceMillis) < WAIT_THRESHOLD) {
+                Thread.sleep(timeDifferenceMillis)
+
+                intercept(chain)
+            } else {
+                val errorBody = moshi
+                    .adapter<ProxerResponse<Unit?>>(responseType)
+                    .toJson(ProxerResponse(true, "Rate limit", RATE_LIMIT.code, null))
+
+                Response.Builder()
+                    .body(errorBody.toResponseBody(errorMediaType))
+                    .protocol(Protocol.HTTP_1_1)
+                    .request(oldRequest)
+                    .code(200)
+                    .message("")
+                    .build()
+            }
+        } else {
+            lock.write { state = state.update(apiClass, requests + 1, firstRequest) }
+
+            return chain.proceed(oldRequest)
+        }
+    }
+
+    private fun Map<String, State>.update(apiClass: String, requests: Int, firstRequest: Date?): Map<String, State> {
+        return toMutableMap().apply { put(apiClass, State(requests, firstRequest)) }
+    }
+
+    private data class Limit(val maxRequests: Int, val millis: Int)
+    private data class State(val requests: Int, val firstRequest: Date?)
+}
diff --git a/library/src/test/kotlin/me/proxer/library/internal/interceptor/RateLimitInterceptorTest.kt b/library/src/test/kotlin/me/proxer/library/internal/interceptor/RateLimitInterceptorTest.kt
@@ -0,0 +1,72 @@
+package me.proxer.library.internal.interceptor
+
+import me.proxer.library.ProxerApi
+import me.proxer.library.ProxerException
+import me.proxer.library.ProxerException.ServerErrorType
+import me.proxer.library.ProxerTest
+import me.proxer.library.enums.Language
+import me.proxer.library.runRequest
+import org.amshove.kluent.invoking
+import org.amshove.kluent.shouldBe
+import org.amshove.kluent.shouldThrow
+import org.junit.jupiter.api.Disabled
+import org.junit.jupiter.api.Test
+
+class RateLimitInterceptorTest : ProxerTest() {
+
+    private val rateLimitApi = ProxerApi.Builder("mock-key")
+        .enableRateLimitProtection()
+        .client(client)
+        .build()
+
+    @Test
+    fun testSingleRequest() {
+        server.runRequest("chapter.json") {
+            rateLimitApi.manga.chapter("123", 123, Language.ENGLISH).build().execute()
+        }
+    }
+
+    @Test
+    fun testRateLimitError() {
+        // Limit 10.
+        repeat(8) {
+            server.runRequest("chapter.json") {
+                rateLimitApi.manga.chapter("123", 123, Language.ENGLISH).build().execute()
+            }
+        }
+
+        val result = invoking {
+            server.runRequest("chapter.json") {
+                rateLimitApi.manga.chapter("123", 123, Language.ENGLISH).build().execute()
+            }
+        } shouldThrow ProxerException::class
+
+        result.exception.errorType shouldBe ProxerException.ErrorType.SERVER
+        result.exception.serverErrorType shouldBe ServerErrorType.RATE_LIMIT
+    }
+
+    @Test
+    @Disabled
+    fun testResetStateAfterTimeLimit() {
+        // Limit 30.
+        repeat(28) {
+            server.runRequest("chat_messages.json") {
+                rateLimitApi.chat.messages("123").build().execute()
+            }
+        }
+
+        val result = invoking {
+            server.runRequest("chat_messages.json") {
+                rateLimitApi.chat.messages("123").build().execute()
+            }
+        } shouldThrow ProxerException::class
+
+        result.exception.serverErrorType shouldBe ServerErrorType.RATE_LIMIT
+
+        Thread.sleep(30_000)
+
+        server.runRequest("chat_messages.json") {
+            rateLimitApi.chat.messages("123").build().execute()
+        }
+    }
+}
