Skip to content

Commit 18ee716

Browse files
fire-light42fire-light42
authored
Feat: Offline filler database (#2704)
1 parent f7494f2 commit 18ee716

4 files changed

Lines changed: 160 additions & 99 deletions

File tree

‎app/build.gradle.kts‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -234,6 +234,9 @@ dependencies {
234234
// FFmpeg Decoding
235235
implementation(libs.bundles.nextlib)
236236

237+
// Anime-db for filler
238+
implementation(libs.anime.db)
239+
237240
// PlayBack
238241
implementation(libs.colorpicker) // Subtitle Color Picker
239242
implementation(libs.newpipeextractor) // For Trailers

‎app/src/main/java/com/lagradost/cloudstream3/ui/result/ResultViewModel2.kt‎

Lines changed: 10 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -113,12 +113,14 @@ import com.lagradost.cloudstream3.utils.newExtractorLink
113113
import com.lagradost.cloudstream3.utils.txt
114114
import kotlinx.coroutines.CancellationException
115115
import kotlinx.coroutines.CoroutineScope
116+
import kotlinx.coroutines.Dispatchers
116117
import kotlinx.coroutines.Job
117118
import kotlinx.coroutines.cancelChildren
118119
import kotlinx.coroutines.coroutineScope
119120
import kotlinx.coroutines.isActive
120121
import kotlinx.coroutines.job
121122
import kotlinx.coroutines.launch
123+
import kotlinx.coroutines.withContext
122124
import java.util.concurrent.TimeUnit
123125

124126
/** This starts at 1 */
@@ -452,7 +454,7 @@ class ResultViewModel2 : ViewModel() {
452454
private var currentShowFillers: Boolean = false
453455
var currentRepo: APIRepository? = null
454456
private var currentId: Int? = null
455-
private var fillers: Map<Int, Boolean> = emptyMap()
457+
private var fillers: HashSet<Int> = hashSetOf()
456458
private var generator: IGenerator? = null
457459
private var preferDubStatus: DubStatus? = null
458460
private var preferStartEpisode: Int? = null
@@ -1806,11 +1808,11 @@ class ResultViewModel2 : ViewModel() {
18061808
}
18071809

18081810

1809-
private suspend fun updateFillers(name: String) {
1811+
private suspend fun updateFillers(data : LoadResponse) {
18101812
fillers =
1811-
ioWorkSafe {
1812-
FillerEpisodeCheck.getFillerEpisodes(name)
1813-
} ?: emptyMap()
1813+
withContext(Dispatchers.IO) {
1814+
safe { FillerEpisodeCheck.getFillerEpisodes(data) }
1815+
} ?: hashSetOf()
18141816
}
18151817

18161818
fun changeDubStatus(status: DubStatus) {
@@ -2147,8 +2149,8 @@ class ResultViewModel2 : ViewModel() {
21472149
) {
21482150
_episodes.postValue(Resource.Loading())
21492151

2150-
if (updateFillers && loadResponse is AnimeLoadResponse) {
2151-
updateFillers(loadResponse.name)
2152+
if (updateFillers) {
2153+
updateFillers(loadResponse)
21522154
}
21532155

21542156
val allEpisodes = when (loadResponse) {
@@ -2189,7 +2191,7 @@ class ResultViewModel2 : ViewModel() {
21892191
index,
21902192
i.score,
21912193
i.description,
2192-
fillers.getOrDefault(episode, false),
2194+
fillers.contains(episode),
21932195
loadResponse.type,
21942196
mainId,
21952197
totalIndex,

‎app/src/main/java/com/lagradost/cloudstream3/utils/FillerEpisodeCheck.kt‎

Lines changed: 145 additions & 91 deletions
Original file line numberDiff line numberDiff line change
@@ -1,112 +1,166 @@
11
package com.lagradost.cloudstream3.utils
22

3-
import com.lagradost.cloudstream3.app
3+
import androidx.annotation.WorkerThread
4+
import com.fasterxml.jackson.annotation.JsonProperty
5+
import com.lagradost.cloudstream3.LoadResponse
6+
import com.lagradost.cloudstream3.LoadResponse.Companion.getAniListId
7+
import com.lagradost.cloudstream3.LoadResponse.Companion.getImdbId
8+
import com.lagradost.cloudstream3.LoadResponse.Companion.getKitsuId
9+
import com.lagradost.cloudstream3.LoadResponse.Companion.getMalId
10+
import com.lagradost.cloudstream3.LoadResponse.Companion.getTMDbId
11+
import com.lagradost.cloudstream3.TvType
12+
import com.lagradost.cloudstream3.ui.result.getId
413
import com.lagradost.cloudstream3.utils.Coroutines.main
5-
import org.jsoup.Jsoup
614
import java.lang.Thread.sleep
715
import java.util.*
816
import kotlin.concurrent.thread
17+
import com.lagradost.cloudstream3.utils.AppUtils.parseJson
18+
import java.io.InputStream
19+
import kotlin.let
920

1021
object FillerEpisodeCheck {
11-
private const val MAIN_URL = "https://www.animefillerlist.com"
22+
fun String?.toClassDir(): String {
23+
val q = this ?: "null"
24+
val z = (6..10).random().calc()
25+
return q + "cache" + z
26+
}
1227

13-
var list: HashMap<String, String>? = null
14-
var cache: HashMap<String, HashMap<Int, Boolean>> = hashMapOf()
28+
data class Show(
29+
@JsonProperty("slug")
30+
val slug: String,
31+
@JsonProperty("title")
32+
val title: String,
33+
@JsonProperty("filler")
34+
val filler: ArrayList<Int>,
35+
@JsonProperty("mixedCanon")
36+
val mixedCanon: ArrayList<Int>,
37+
@JsonProperty("mangaCanon")
38+
val mangaCanon: ArrayList<Int>,
39+
@JsonProperty("animeCanon")
40+
val animeCanon: ArrayList<Int>,
41+
)
1542

16-
private fun fixName(name: String): String {
17-
return name.lowercase(Locale.ROOT)/*.replace(" ", "")*/.replace("-", " ")
18-
.replace("[^a-zA-Z0-9 ]".toRegex(), "")
19-
}
43+
data class MappingRoot(
44+
@JsonProperty("type")
45+
val type: String?,
46+
@JsonProperty("anidb_id")
47+
val anidbId: Long?,
48+
@JsonProperty("anilist_id")
49+
val anilistId: Long?,
50+
@JsonProperty("animecountdown_id")
51+
val animecountdownId: Long?,
52+
@JsonProperty("animenewsnetwork_id")
53+
val animenewsnetworkId: Long?,
54+
@JsonProperty("anime-planet_id")
55+
val animePlanetId: String?,
56+
@JsonProperty("anisearch_id")
57+
val anisearchId: Long?,
58+
@JsonProperty("imdb_id")
59+
val imdbId: String?,
60+
@JsonProperty("kitsu_id")
61+
val kitsuId: Long?,
62+
@JsonProperty("livechart_id")
63+
val livechartId: Long?,
64+
@JsonProperty("mal_id")
65+
val malId: Long?,
66+
@JsonProperty("simkl_id")
67+
val simklId: Long?,
68+
@JsonProperty("themoviedb_id")
69+
val themoviedbId: Long?,
70+
@JsonProperty("tvdb_id")
71+
val tvdbId: Long?,
72+
@JsonProperty("season")
73+
val season: Season?,
74+
)
2075

21-
private suspend fun getFillerList(): Boolean {
22-
if (list != null) return true
23-
try {
24-
val result = app.get("$MAIN_URL/shows").text
25-
val documented = Jsoup.parse(result)
26-
val localHTMLList = documented.select("div#ShowList > div.Group > ul > li > a")
27-
val localList = HashMap<String, String>()
28-
for (i in localHTMLList) {
29-
val name = i.text()
30-
31-
if (name.lowercase(Locale.ROOT).contains("manga only")) continue
32-
33-
val href = i.attr("href")
34-
if (name.isNullOrEmpty() || href.isNullOrEmpty()) {
35-
continue
36-
}
76+
data class Season(
77+
@JsonProperty("tvdb")
78+
val tvdb: Long?,
79+
@JsonProperty("tmdb")
80+
val tmdb: Long?,
81+
)
3782

38-
val values = "(.*) \\((.*)\\)".toRegex().matchEntire(name)?.groups
39-
if (values != null) {
40-
for (index in 1 until values.size) {
41-
val localName = values[index]?.value ?: continue
42-
localList[fixName(localName)] = href
43-
}
44-
} else {
45-
localList[fixName(name)] = href
46-
}
47-
}
48-
if (localList.size > 0) {
49-
list = localList
50-
return true
51-
}
52-
} catch (e: Exception) {
53-
e.printStackTrace()
83+
data class CombinedMedia(
84+
@JsonProperty("mapping")
85+
val mapping: MappingRoot?,
86+
@JsonProperty("show")
87+
val show: Show
88+
)
89+
90+
data class Database(
91+
val mal: HashMap<Long, CombinedMedia> = hashMapOf(),
92+
val anilist: HashMap<Long, CombinedMedia> = hashMapOf(),
93+
val kitsu: HashMap<Long, CombinedMedia> = hashMapOf(),
94+
val tmdb: HashMap<Long, CombinedMedia> = hashMapOf(),
95+
val imdb: HashMap<String, CombinedMedia> = hashMapOf(),
96+
val name: HashMap<String, CombinedMedia> = hashMapOf(),
97+
)
98+
99+
private var database: Database? = null
100+
101+
private val strip = Regex("[ :\\-.!]")
102+
103+
/** Makes names more uniform to make partial matches more still give a result */
104+
fun stripName(name: String): String =
105+
name.replace(strip, "").lowercase()
106+
107+
108+
@Synchronized
109+
@Throws
110+
@WorkerThread
111+
fun loadJson(): Database {
112+
database?.let {
113+
return it
54114
}
55-
return false
56-
}
115+
116+
/** The entire "database" is stored as a json file we can parse */
117+
val stream: InputStream = com.lagradost.AnimeDB.getDatabaseStream()!!
118+
val text = stream.reader().readText()
57119

58-
fun String?.toClassDir(): String {
59-
val q = this ?: "null"
60-
val z = (6..10).random().calc()
61-
return q + "cache" + z
120+
val allMedia = parseJson<Array<CombinedMedia>>(text)
121+
val pending = Database()
122+
for (media in allMedia) {
123+
val lowercase = stripName(media.show.title)
124+
pending.name[lowercase] = media
125+
val map = media.mapping ?: continue
126+
127+
map.imdbId?.let { id -> pending.imdb[id] = media }
128+
map.malId?.let { id -> pending.mal[id] = media }
129+
map.anilistId?.let { id -> pending.anilist[id] = media }
130+
map.kitsuId?.let { id -> pending.kitsu[id] = media }
131+
map.season?.tmdb?.let { id -> pending.tmdb[id] = media }
132+
}
133+
database = pending
134+
return pending
62135
}
63136

64-
suspend fun getFillerEpisodes(query: String): HashMap<Int, Boolean>? {
65-
try {
66-
cache[query]?.let {
67-
return it
68-
}
69-
if (!getFillerList()) return null
70-
val localList = list ?: return null
71-
72-
// Strips these from the name
73-
val blackList = listOf(
74-
"TV Dubbed",
75-
"(Dub)",
76-
"Subbed",
77-
"(TV)",
78-
"(Uncensored)",
79-
"(Censored)",
80-
"(\\d+)" // year
81-
)
82-
val blackListRegex =
83-
Regex(
84-
""" (${
85-
blackList.joinToString(separator = "|").replace("(", "\\(")
86-
.replace(")", "\\)")
87-
})"""
88-
)
89-
90-
val realQuery =
91-
fixName(query.replace(blackListRegex, "")).replace("shippuuden", "shippuden")
92-
if (!localList.containsKey(realQuery)) return null
93-
val href = localList[realQuery]?.replace(MAIN_URL, "") ?: return null // JUST IN CASE
94-
val result = app.get("$MAIN_URL$href").text
95-
val documented = Jsoup.parse(result)
96-
val hashMap = HashMap<Int, Boolean>()
97-
documented.select("table.EpisodeList > tbody > tr").forEach {
98-
val type = it.selectFirst("td.Type > span")?.text() == "Filler"
99-
val episodeNumber = it.selectFirst("td.Number")?.text()?.toIntOrNull()
100-
if (episodeNumber != null) {
101-
hashMap[episodeNumber] = type
102-
}
103-
}
104-
cache[query] = hashMap
105-
return hashMap
106-
} catch (e: Exception) {
107-
e.printStackTrace()
137+
val loadCache: HashMap<Int, HashSet<Int>?> = hashMapOf()
138+
139+
@Synchronized
140+
@Throws
141+
@WorkerThread
142+
fun getFillerEpisodes(data: LoadResponse): HashSet<Int>? {
143+
/** Only for anime */
144+
if (data.type != TvType.Anime) {
108145
return null
109146
}
147+
/** Try to hit the cache for this entry, to avoid recreating the hashset */
148+
loadCache[data.getId()]?.let { cachedResponse ->
149+
return cachedResponse
150+
}
151+
val db = loadJson()
152+
153+
val media =
154+
db.mal[data.getMalId()?.toLongOrNull()]
155+
?: db.anilist[data.getAniListId()?.toLongOrNull()]
156+
?: db.kitsu[data.getKitsuId()?.toLongOrNull()]
157+
?: db.imdb[data.getImdbId()]
158+
?: db.tmdb[data.getTMDbId()?.toLongOrNull()]
159+
?: db.name[stripName(data.name)]
160+
161+
return media?.show?.filler?.toHashSet().also { response ->
162+
loadCache[data.getId()] = response
163+
}
110164
}
111165

112166
private fun Int.calc(): Int {

‎gradle/libs.versions.toml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
[versions]
44
activityKtx = "1.13.0"
55
androidGradlePlugin = "9.1.1"
6+
animeDb = "1.0.2"
67
appcompat = "1.7.1"
78
biometric = "1.4.0-alpha06"
89
buildkonfigGradlePlugin = "0.18.0"
@@ -55,6 +56,7 @@ targetSdk = "36"
5556

5657
[libraries]
5758
activity-ktx = { module = "androidx.activity:activity-ktx", version.ref = "activityKtx" }
59+
anime-db = { module = "com.github.recloudstream:anime-db", version.ref = "animeDb" }
5860
appcompat = { module = "androidx.appcompat:appcompat", version.ref = "appcompat" }
5961
biometric = { module = "androidx.biometric:biometric", version.ref = "biometric" }
6062
coil = { module = "io.coil-kt.coil3:coil", version.ref = "coil" }

0 commit comments

Comments
 (0)