Skip to content

Commit 3faf604

Browse files
krzema12krzema12
authored
chore(server): count calls to GitHub's API as metric (#2168)
Part of #2160. Thanks to this, we'll be able to assess the load on GitHub, and better understand if some server's instabilities are related to e.g. being throttled by GitHub.
1 parent 9698245 commit 3faf604

8 files changed

Lines changed: 73 additions & 21 deletions

File tree

‎jit-binding-server/src/main/kotlin/io/github/typesafegithub/workflows/jitbindingserver/Main.kt‎

Lines changed: 16 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,8 @@ import io.ktor.server.engine.embeddedServer
2121
import io.ktor.server.netty.Netty
2222
import io.ktor.server.response.respondText
2323
import io.ktor.server.routing.routing
24+
import io.micrometer.core.instrument.MeterRegistry
25+
import io.micrometer.core.instrument.Tag
2426
import io.micrometer.prometheusmetrics.PrometheusConfig
2527
import io.micrometer.prometheusmetrics.PrometheusMeterRegistry
2628
import java.time.Duration
@@ -65,7 +67,12 @@ fun main() {
6567

6668
fun Application.appModule(
6769
buildVersionArtifacts: suspend (ActionCoords, HttpClient) -> VersionArtifacts?,
68-
buildPackageArtifacts: suspend (ActionCoords, String, (Collection<ActionCoords>) -> Unit) -> Map<String, String>,
70+
buildPackageArtifacts: suspend (
71+
ActionCoords,
72+
String,
73+
(Collection<ActionCoords>) -> Unit,
74+
MeterRegistry,
75+
) -> Map<String, String>,
6976
getGithubAuthToken: () -> String,
7077
) {
7178
val httpClient =
@@ -75,6 +82,7 @@ fun Application.appModule(
7582
val counter =
7683
prometheusRegistry.counter(
7784
"calls_to_github",
85+
listOf(Tag.of("type", "static")),
7886
)
7987
counter.increment()
8088
}
@@ -106,7 +114,12 @@ private fun buildBindingsCache(
106114
@Suppress("ktlint:standard:function-signature") // Conflict with detekt.
107115
private fun buildMetadataCache(
108116
bindingsCache: LoadingCache<ActionCoords, CachedVersionArtifact>,
109-
buildPackageArtifacts: suspend (ActionCoords, String, (Collection<ActionCoords>) -> Unit) -> Map<String, String>,
117+
buildPackageArtifacts: suspend (
118+
ActionCoords,
119+
String,
120+
(Collection<ActionCoords>) -> Unit,
121+
MeterRegistry,
122+
) -> Map<String, String>,
110123
getGithubAuthToken: () -> String,
111124
): LoadingCache<ActionCoords, CachedMetadataArtifact> =
112125
Caffeine
@@ -118,6 +131,7 @@ private fun buildMetadataCache(
118131
it,
119132
getGithubAuthToken(),
120133
{ coords -> prefetchBindingArtifacts(coords, bindingsCache) },
134+
prometheusRegistry,
121135
)
122136
}
123137

‎jit-binding-server/src/test/kotlin/io/github/typesafegithub/workflows/jitbindingserver/ArtifactRoutesTest.kt‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@ class ArtifactRoutesTest :
3030
)
3131
},
3232
// Irrelevant for these tests.
33-
buildPackageArtifacts = { _, _, _ -> emptyMap() },
33+
buildPackageArtifacts = { _, _, _, _ -> emptyMap() },
3434
getGithubAuthToken = { "" },
3535
)
3636
}
@@ -51,7 +51,7 @@ class ArtifactRoutesTest :
5151
appModule(
5252
buildVersionArtifacts = { _, _ -> null },
5353
// Irrelevant for these tests.
54-
buildPackageArtifacts = { _, _, _ -> emptyMap() },
54+
buildPackageArtifacts = { _, _, _, _ -> emptyMap() },
5555
getGithubAuthToken = { "" },
5656
)
5757
}
@@ -71,7 +71,7 @@ class ArtifactRoutesTest :
7171
appModule(
7272
buildVersionArtifacts = { _, _ -> error("An internal error occurred!") },
7373
// Irrelevant for these tests.
74-
buildPackageArtifacts = { _, _, _ -> emptyMap() },
74+
buildPackageArtifacts = { _, _, _, _ -> emptyMap() },
7575
getGithubAuthToken = { "" },
7676
)
7777
}
@@ -98,7 +98,7 @@ class ArtifactRoutesTest :
9898
appModule(
9999
buildVersionArtifacts = mockBuildVersionArtifacts,
100100
// Irrelevant for these tests.
101-
buildPackageArtifacts = { _, _, _ -> emptyMap() },
101+
buildPackageArtifacts = { _, _, _, _ -> emptyMap() },
102102
getGithubAuthToken = { "" },
103103
)
104104
}

‎jit-binding-server/src/test/kotlin/io/github/typesafegithub/workflows/jitbindingserver/MetadataRoutesTest.kt‎

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ import io.ktor.client.request.get
88
import io.ktor.client.statement.bodyAsText
99
import io.ktor.http.HttpStatusCode
1010
import io.ktor.server.testing.testApplication
11+
import io.micrometer.core.instrument.MeterRegistry
1112
import io.mockk.every
1213
import io.mockk.mockk
1314
import io.mockk.verify
@@ -20,7 +21,7 @@ class MetadataRoutesTest :
2021
// Given
2122
application {
2223
appModule(
23-
buildPackageArtifacts = { _, _, _ ->
24+
buildPackageArtifacts = { _, _, _, _ ->
2425
mapOf("maven-metadata.xml" to "Some XML contents")
2526
},
2627
getGithubAuthToken = { "some-token" },
@@ -48,7 +49,7 @@ class MetadataRoutesTest :
4849
// Given
4950
application {
5051
appModule(
51-
buildPackageArtifacts = { _, _, _ ->
52+
buildPackageArtifacts = { _, _, _, _ ->
5253
emptyMap()
5354
},
5455
getGithubAuthToken = { "some-token" },
@@ -75,7 +76,7 @@ class MetadataRoutesTest :
7576
// Given
7677
application {
7778
appModule(
78-
buildPackageArtifacts = { _, _, _ ->
79+
buildPackageArtifacts = { _, _, _, _ ->
7980
error("An internal error occurred!")
8081
},
8182
getGithubAuthToken = { "some-token" },
@@ -106,9 +107,10 @@ class MetadataRoutesTest :
106107
ActionCoords,
107108
String,
108109
(Collection<ActionCoords>) -> Unit,
110+
MeterRegistry?,
109111
) -> Map<String, String>,
110112
>()
111-
every { mockBuildPackageArtifacts(any(), any(), any()) } throws
113+
every { mockBuildPackageArtifacts(any(), any(), any(), any()) } throws
112114
Exception("An internal error occurred!") andThen
113115
mapOf("maven-metadata.xml" to "Some XML contents")
114116
application {
@@ -135,7 +137,7 @@ class MetadataRoutesTest :
135137
// Then
136138
response2.status shouldBe HttpStatusCode.OK
137139

138-
verify(exactly = 2) { mockBuildPackageArtifacts(any(), any(), any()) }
140+
verify(exactly = 2) { mockBuildPackageArtifacts(any(), any(), any(), any()) }
139141
}
140142
}
141143
}

‎maven-binding-builder/src/main/kotlin/io/github/typesafegithub/workflows/mavenbinding/MavenMetadataBuilding.kt‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,21 +7,24 @@ import io.github.typesafegithub.workflows.actionbindinggenerator.domain.ActionCo
77
import io.github.typesafegithub.workflows.actionbindinggenerator.domain.SignificantVersion.FULL
88
import io.github.typesafegithub.workflows.shared.internal.fetchAvailableVersions
99
import io.github.typesafegithub.workflows.shared.internal.model.Version
10+
import io.micrometer.core.instrument.MeterRegistry
1011
import java.time.format.DateTimeFormatter
1112

1213
private val logger = logger { }
1314

1415
internal suspend fun ActionCoords.buildMavenMetadataFile(
1516
githubAuthToken: String,
17+
meterRegistry: MeterRegistry? = null,
1618
fetchAvailableVersions: suspend (
1719
owner: String,
1820
name: String,
1921
githubAuthToken: String?,
22+
meterRegistry: MeterRegistry?,
2023
) -> Either<String, List<Version>> = ::fetchAvailableVersions,
2124
prefetchBindingArtifacts: (Collection<ActionCoords>) -> Unit = {},
2225
): String? {
2326
val availableVersions =
24-
fetchAvailableVersions(owner, name, githubAuthToken)
27+
fetchAvailableVersions(owner, name, githubAuthToken, meterRegistry)
2528
.getOrElse {
2629
logger.error { it }
2730
emptyList()

‎maven-binding-builder/src/main/kotlin/io/github/typesafegithub/workflows/mavenbinding/PackageArtifactsBuilding.kt‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,19 @@
11
package io.github.typesafegithub.workflows.mavenbinding
22

33
import io.github.typesafegithub.workflows.actionbindinggenerator.domain.ActionCoords
4+
import io.micrometer.core.instrument.MeterRegistry
45

56
suspend fun buildPackageArtifacts(
67
actionCoords: ActionCoords,
78
githubAuthToken: String,
89
prefetchBindingArtifacts: (Collection<ActionCoords>) -> Unit,
10+
meterRegistry: MeterRegistry,
911
): Map<String, String> {
1012
val mavenMetadata =
1113
actionCoords.buildMavenMetadataFile(
1214
githubAuthToken = githubAuthToken,
1315
prefetchBindingArtifacts = prefetchBindingArtifacts,
16+
meterRegistry = meterRegistry,
1417
) ?: return emptyMap()
1518
return mapOf(
1619
"maven-metadata.xml" to mavenMetadata,

‎maven-binding-builder/src/test/kotlin/io/github/typesafegithub/workflows/mavenbinding/MavenMetadataBuildingTest.kt‎

Lines changed: 9 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ import io.github.typesafegithub.workflows.shared.internal.model.Version
99
import io.kotest.core.spec.style.FunSpec
1010
import io.kotest.matchers.nulls.shouldBeNull
1111
import io.kotest.matchers.shouldBe
12+
import io.micrometer.core.instrument.MeterRegistry
1213
import java.time.ZonedDateTime
1314

1415
class MavenMetadataBuildingTest :
@@ -26,7 +27,8 @@ class MavenMetadataBuildingTest :
2627
String,
2728
String,
2829
String?,
29-
) -> Either<String, List<Version>> = { _, _, _ ->
30+
MeterRegistry?,
31+
) -> Either<String, List<Version>> = { _, _, _, _ ->
3032
listOf(
3133
Version(version = "v3-beta", dateProvider = { ZonedDateTime.parse("2024-07-01T00:00:00Z") }),
3234
Version(version = "v2", dateProvider = { ZonedDateTime.parse("2024-05-01T00:00:00Z") }),
@@ -70,7 +72,8 @@ class MavenMetadataBuildingTest :
7072
String,
7173
String,
7274
String?,
73-
) -> Either<String, List<Version>> = { _, _, _ ->
75+
MeterRegistry?,
76+
) -> Either<String, List<Version>> = { _, _, _, _ ->
7477
listOf(
7578
Version(version = "v1.1", dateProvider = { ZonedDateTime.parse("2024-03-07T00:00:00Z") }),
7679
Version(version = "v1.1.0", dateProvider = { ZonedDateTime.parse("2024-03-07T00:00:00Z") }),
@@ -95,7 +98,8 @@ class MavenMetadataBuildingTest :
9598
String,
9699
String,
97100
String?,
98-
) -> Either<String, List<Version>> = { _, _, _ ->
101+
MeterRegistry?,
102+
) -> Either<String, List<Version>> = { _, _, _, _ ->
99103
emptyList<Version>().right()
100104
}
101105

@@ -115,7 +119,8 @@ class MavenMetadataBuildingTest :
115119
String,
116120
String,
117121
String?,
118-
) -> Either<String, List<Version>> = { owner, name, _ ->
122+
MeterRegistry?,
123+
) -> Either<String, List<Version>> = { owner, name, _, _ ->
119124
listOf(
120125
Version(version = "v3-beta", dateProvider = { ZonedDateTime.parse("2024-07-01T00:00:00Z") }),
121126
Version(version = "v2", dateProvider = { ZonedDateTime.parse("2024-05-01T00:00:00Z") }),

‎shared-internal/build.gradle.kts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ dependencies {
1414
// we cannot use a BOM due to limitation in kotlin scripting when resolving the transitive KMM variant dependencies
1515
// note: see https://youtrack.jetbrains.com/issue/KT-67618
1616
api("io.ktor:ktor-client-core:3.3.3")
17+
api("io.micrometer:micrometer-core:1.15.5")
1718
implementation("io.ktor:ktor-client-cio:3.3.3")
1819
implementation("io.ktor:ktor-client-content-negotiation:3.3.3")
1920
implementation("io.ktor:ktor-client-logging:3.3.3")

‎shared-internal/src/main/kotlin/io/github/typesafegithub/workflows/shared/internal/GithubApi.kt‎

Lines changed: 29 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -7,15 +7,18 @@ import io.github.oshai.kotlinlogging.KotlinLogging.logger
77
import io.github.typesafegithub.workflows.shared.internal.model.Version
88
import io.ktor.client.HttpClient
99
import io.ktor.client.call.body
10+
import io.ktor.client.plugins.HttpSend
1011
import io.ktor.client.plugins.contentnegotiation.ContentNegotiation
1112
import io.ktor.client.plugins.logging.LogLevel.ALL
1213
import io.ktor.client.plugins.logging.Logger
1314
import io.ktor.client.plugins.logging.Logging
15+
import io.ktor.client.plugins.plugin
1416
import io.ktor.client.request.bearerAuth
1517
import io.ktor.client.request.get
1618
import io.ktor.client.statement.bodyAsText
1719
import io.ktor.http.isSuccess
1820
import io.ktor.serialization.kotlinx.json.json
21+
import io.micrometer.core.instrument.MeterRegistry
1922
import kotlinx.serialization.Serializable
2023
import kotlinx.serialization.json.Json
2124
import java.time.ZonedDateTime
@@ -26,25 +29,29 @@ suspend fun fetchAvailableVersions(
2629
owner: String,
2730
name: String,
2831
githubAuthToken: String?,
32+
meterRegistry: MeterRegistry? = null,
2933
githubEndpoint: String = "https://api.github.com",
3034
): Either<String, List<Version>> =
3135
either {
32-
buildHttpClient().use { httpClient ->
36+
buildHttpClient(meterRegistry = meterRegistry).use { httpClient ->
3337
return listOf(
3438
apiTagsUrl(githubEndpoint = githubEndpoint, owner = owner, name = name),
3539
apiBranchesUrl(githubEndpoint = githubEndpoint, owner = owner, name = name),
3640
).flatMap { url -> fetchGithubRefs(url, githubAuthToken, httpClient).bind() }
37-
.versions(githubAuthToken)
41+
.versions(githubAuthToken, meterRegistry = meterRegistry)
3842
}
3943
}
4044

41-
private fun List<GithubRef>.versions(githubAuthToken: String?): Either<String, List<Version>> =
45+
private fun List<GithubRef>.versions(
46+
githubAuthToken: String?,
47+
meterRegistry: MeterRegistry?,
48+
): Either<String, List<Version>> =
4249
either {
4350
this@versions.map { githubRef ->
4451
val version = githubRef.ref.substringAfterLast("/")
4552
Version(version) {
4653
val response =
47-
buildHttpClient().use { httpClient ->
54+
buildHttpClient(meterRegistry = meterRegistry).use { httpClient ->
4855
httpClient
4956
.get(urlString = githubRef.`object`.url) {
5057
if (githubAuthToken != null) {
@@ -122,7 +129,7 @@ private data class Person(
122129
val date: String,
123130
)
124131

125-
private fun buildHttpClient() =
132+
private fun buildHttpClient(meterRegistry: MeterRegistry?) =
126133
HttpClient {
127134
val klogger = logger
128135
install(Logging) {
@@ -141,4 +148,21 @@ private fun buildHttpClient() =
141148
},
142149
)
143150
}
151+
}.apply {
152+
if (meterRegistry != null) {
153+
plugin(HttpSend).intercept { request ->
154+
if (request.url.host == "api.github.com") {
155+
val counter =
156+
meterRegistry.counter(
157+
"calls_to_github",
158+
listOf(
159+
io.micrometer.core.instrument.Tag
160+
.of("type", "api"),
161+
),
162+
)
163+
counter.increment()
164+
}
165+
execute(request)
166+
}
167+
}
144168
}

0 commit comments

Comments
 (0)