Kemono & Coomer | Fixed Latest, Added Filters, Increased Post Limit (#5383)

* Fixed Latest, Added Filters, Increased Post Limit

* Fix Lint Errors

- There's no "Coomer" multisrc.

* Add Favourites Filter

- For #547

* Added `KemonoFavouritesDto`

- also a `fav` var in KemonoCreatorDTo

* Apply suggestion

* Apply suggestion

* Apply Suggestions

- Apply AwkwardPeak's suggestions

* Fix Lint Errors

lib-multisrc/kemono/build.gradle.kts

@@ -2,4 +2,4 @@ plugins {
     id("lib-multisrc")
 }
 
-baseVersionCode = 12
+baseVersionCode = 13

lib-multisrc/kemono/src/eu/kanade/tachiyomi/multisrc/kemono/Kemono.kt

@@ -4,29 +4,26 @@ import android.app.Application
 import androidx.preference.ListPreference
 import androidx.preference.PreferenceScreen
 import androidx.preference.SwitchPreferenceCompat
+import eu.kanade.tachiyomi.multisrc.kemono.KemonoCreatorDto.Companion.serviceName
 import eu.kanade.tachiyomi.network.GET
 import eu.kanade.tachiyomi.network.interceptor.rateLimit
 import eu.kanade.tachiyomi.source.ConfigurableSource
+import eu.kanade.tachiyomi.source.model.Filter
 import eu.kanade.tachiyomi.source.model.FilterList
 import eu.kanade.tachiyomi.source.model.MangasPage
 import eu.kanade.tachiyomi.source.model.Page
 import eu.kanade.tachiyomi.source.model.SChapter
 import eu.kanade.tachiyomi.source.model.SManga
 import eu.kanade.tachiyomi.source.online.HttpSource
-import eu.kanade.tachiyomi.util.asJsoup
 import kotlinx.serialization.json.Json
 import kotlinx.serialization.json.decodeFromStream
-import okhttp3.Call
-import okhttp3.Callback
 import okhttp3.Request
 import okhttp3.Response
-import okio.blackholeSink
-import org.jsoup.select.Evaluator
 import rx.Observable
 import uy.kohesive.injekt.Injekt
 import uy.kohesive.injekt.api.get
 import uy.kohesive.injekt.injectLazy
-import java.io.IOException
+import java.lang.Thread.sleep
 import java.util.TimeZone
 import kotlin.math.min
 
@@ -51,6 +48,8 @@ open class Kemono(
 
     private val imgCdnUrl = baseUrl.replace("//", "//img.")
 
+    private var mangasCache: List<KemonoCreatorDto> = emptyList()
+
     private fun String.formatAvatarUrl(): String = removePrefix("https://").replaceBefore('/', imgCdnUrl)
 
     override fun popularMangaRequest(page: Int) = throw UnsupportedOperationException()
@@ -63,82 +62,123 @@ open class Kemono(
 
     override fun fetchPopularManga(page: Int): Observable<MangasPage> {
         return Observable.fromCallable {
-            fetchNewDesignListing(page, "/artists", compareByDescending { it.favorited })
+            searchMangas(page, sortBy = "pop" to "desc")
         }
     }
 
     override fun fetchLatestUpdates(page: Int): Observable<MangasPage> {
         return Observable.fromCallable {
-            fetchNewDesignListing(page, "/artists/updated", compareByDescending { it.updatedDate })
-        }
-    }
-
-    private fun fetchNewDesignListing(
-        page: Int,
-        path: String,
-        comparator: Comparator<KemonoCreatorDto>,
-    ): MangasPage {
-        val baseUrl = baseUrl
-        return if (page == 1) {
-            val document = client.newCall(GET(baseUrl + path, headers)).execute().asJsoup()
-            val cardList = document.selectFirst(Evaluator.Class("card-list__items"))!!
-            val creators = cardList.children().map {
-                SManga.create().apply {
-                    url = it.attr("href")
-                    title = it.selectFirst(Evaluator.Class("user-card__name"))!!.ownText()
-                    author = it.selectFirst(Evaluator.Class("user-card__service"))!!.ownText()
-                    thumbnail_url = it.selectFirst(Evaluator.Tag("img"))!!.absUrl("src").formatAvatarUrl()
-                    description = PROMPT
-                    initialized = true
-                }
-            }.filterUnsupported()
-            MangasPage(creators, true).also { cacheCreators() }
-        } else {
-            fetchCreatorsPage(page) { it.apply { sortWith(comparator) } }
+            searchMangas(page, sortBy = "lat" to "desc")
         }
     }
 
     override fun fetchSearchManga(page: Int, query: String, filters: FilterList): Observable<MangasPage> = Observable.fromCallable {
-        if (query.isBlank()) throw Exception("Query is empty")
-        fetchCreatorsPage(page) { all ->
-            val result = all.filterTo(ArrayList()) { it.name.contains(query, ignoreCase = true) }
-            if (result.isEmpty()) return@fetchCreatorsPage emptyList()
-            if (result[0].favorited != -1) {
-                result.sortByDescending { it.favorited }
-            } else {
-                result.sortByDescending { it.updatedDate }
+        searchMangas(page, query, filters)
+    }
+
+    private fun searchMangas(page: Int = 1, title: String = "", filters: FilterList? = null, sortBy: Pair<String, String> = "" to ""): MangasPage {
+        var sort = sortBy
+        val typeIncluded: MutableList<String> = mutableListOf()
+        val typeExcluded: MutableList<String> = mutableListOf()
+        var fav: Boolean? = null
+        filters?.forEach { filter ->
+            when (filter) {
+                is SortFilter -> {
+                    sort = filter.getValue() to if (filter.state!!.ascending) "asc" else "desc"
+                }
+                is TypeFilter -> {
+                    filter.state.filter { state -> state.isIncluded() }.forEach { tri ->
+                        typeIncluded.add(tri.value)
+                    }
+
+                    filter.state.filter { state -> state.isExcluded() }.forEach { tri ->
+                        typeExcluded.add(tri.value)
+                    }
+                }
+                is FavouritesFilter -> {
+                    fav = when (filter.state[0].state) {
+                        0 -> null
+                        1 -> true
+                        else -> false
+                    }
+                }
+                else -> {}
             }
-            result
         }
-    }
 
-    private fun fetchCreatorsPage(
-        page: Int,
-        block: (ArrayList<KemonoCreatorDto>) -> List<KemonoCreatorDto>,
-    ): MangasPage {
-        val imgCdnUrl = this.imgCdnUrl
-        val response = client.newCall(GET("$baseUrl/$apiPath/creators", headers)).execute()
-        val allCreators = block(response.parseAs())
-        val count = allCreators.size
-        val fromIndex = (page - 1) * NEW_PAGE_SIZE
-        val toIndex = min(count, fromIndex + NEW_PAGE_SIZE)
-        val creators = allCreators.subList(fromIndex, toIndex)
-            .map { it.toSManga(imgCdnUrl) }
-            .filterUnsupported()
-        return MangasPage(creators, toIndex < count)
-    }
+        var mangas = mangasCache
+        if (page == 1) {
+            var favourites: List<KemonoFavouritesDto> = emptyList()
+            if (fav != null) {
+                val favores = client.newCall(GET("$baseUrl/$apiPath/account/favorites", headers)).execute()
 
-    private fun cacheCreators() {
-        val callback = object : Callback {
-            override fun onResponse(call: Call, response: Response) =
-                response.body.source().run {
-                    readAll(blackholeSink())
-                    close()
+                if (favores.code == 401) throw Exception("You are not Logged In")
+                favourites = favores.parseAs<List<KemonoFavouritesDto>>().filterNot { it.service.lowercase() == "discord" }
+            }
+
+            val response = client.newCall(GET("$baseUrl/$apiPath/creators", headers)).execute()
+            val allCreators = response.parseAs<List<KemonoCreatorDto>>().filterNot { it.service.lowercase() == "discord" }
+            mangas = allCreators.filter {
+                val includeType = typeIncluded.isEmpty() || typeIncluded.contains(it.service.serviceName().lowercase())
+                val excludeType = typeExcluded.isNotEmpty() && typeExcluded.contains(it.service.serviceName().lowercase())
+
+                val regularSearch = it.name.contains(title, true)
+
+                val isFavourited = when (fav) {
+                    true -> favourites.any { f -> f.id == it.id.also { _ -> it.fav = f.faved_seq } }
+                    false -> favourites.none { f -> f.id == it.id }
+                    else -> true
                 }
 
-            override fun onFailure(call: Call, e: IOException) = Unit
+                includeType && !excludeType && isFavourited &&
+                    regularSearch
+            }.also { mangasCache = mangas }
         }
-        client.newCall(GET("$baseUrl/$apiPath/creators", headers)).enqueue(callback)
+
+        val sorted = when (sort.first) {
+            "pop" -> {
+                if (sort.second == "desc") {
+                    mangas.sortedByDescending { it.favorited }
+                } else {
+                    mangas.sortedBy { it.favorited }
+                }
+            }
+            "tit" -> {
+                if (sort.second == "desc") {
+                    mangas.sortedByDescending { it.name }
+                } else {
+                    mangas.sortedBy { it.name }
+                }
+            }
+            "new" -> {
+                if (sort.second == "desc") {
+                    mangas.sortedByDescending { it.id }
+                } else {
+                    mangas.sortedBy { it.id }
+                }
+            }
+            "fav" -> {
+                if (fav != true) throw Exception("Please check 'Favourites Only' Filter")
+                if (sort.second == "desc") {
+                    mangas.sortedByDescending { it.fav }
+                } else {
+                    mangas.sortedBy { it.fav }
+                }
+            }
+            else -> {
+                if (sort.second == "desc") {
+                    mangas.sortedByDescending { it.updatedDate }
+                } else {
+                    mangas.sortedBy { it.updatedDate }
+                }
+            }
+        }
+        val maxIndex = mangas.size
+        val fromIndex = (page - 1) * PAGE_CREATORS_LIMIT
+        val toIndex = min(maxIndex, fromIndex + PAGE_CREATORS_LIMIT)
+
+        val final = sorted.subList(fromIndex, toIndex).map { it.toSManga(imgCdnUrl) }
+        return MangasPage(final, toIndex != maxIndex)
     }
 
     override fun searchMangaRequest(page: Int, query: String, filters: FilterList) = throw UnsupportedOperationException()
@@ -156,28 +196,31 @@ open class Kemono(
             "Pixiv Fanbox", "Fantia" -> TimeZone.getTimeZone("GMT+09:00")
             else -> TimeZone.getTimeZone("GMT")
         }
-        val maxPosts = preferences.getString(POST_PAGES_PREF, POST_PAGES_DEFAULT)!!
-            .toInt().coerceAtMost(POST_PAGES_MAX) * POST_PAGE_SIZE
+        val prefMaxPost = preferences.getString(POST_PAGES_PREF, POST_PAGES_DEFAULT)!!
+            .toInt().coerceAtMost(POST_PAGES_MAX) * PAGE_POST_LIMIT
         var offset = 0
         var hasNextPage = true
         val result = ArrayList<SChapter>()
-        while (offset < maxPosts && hasNextPage) {
-            val request = GET("$baseUrl/$apiPath${manga.url}?limit=$POST_PAGE_SIZE&o=$offset", headers)
+        while (offset < prefMaxPost && hasNextPage) {
+            val request = GET("$baseUrl/$apiPath${manga.url}?o=$offset", headers)
             val page: List<KemonoPostDto> = retry(request).parseAs()
             page.forEach { post -> if (post.images.isNotEmpty()) result.add(post.toSChapter()) }
-            offset += POST_PAGE_SIZE
-            hasNextPage = page.size == POST_PAGE_SIZE
+            offset += PAGE_POST_LIMIT
+            hasNextPage = page.size == PAGE_POST_LIMIT
         }
         result
     }
 
     private fun retry(request: Request): Response {
         var code = 0
-        repeat(3) {
+        repeat(5) {
             val response = client.newCall(request).execute()
             if (response.isSuccessful) return response
             response.close()
             code = response.code
+            if (code == 429) {
+                sleep(10000)
+            }
         }
         throw Exception("HTTP error $code")
     }
@@ -217,10 +260,8 @@ open class Kemono(
             key = POST_PAGES_PREF
             title = "Maximum posts to load"
             summary = "Loading more posts costs more time and network traffic.\nCurrently: %s"
-            entryValues = (1..POST_PAGES_MAX).map { it.toString() }.toTypedArray()
-            entries = (1..POST_PAGES_MAX).map {
-                if (it == 1) "1 page ($POST_PAGE_SIZE posts)" else "$it pages (${it * POST_PAGE_SIZE} posts)"
-            }.toTypedArray()
+            entryValues = Array(POST_PAGES_MAX) { (it + 1).toString() }
+            entries = Array(POST_PAGES_MAX) { "${(it + 1)} pages (${(it + 1) * PAGE_POST_LIMIT} posts)" }
             setDefaultValue(POST_PAGES_DEFAULT)
         }.let { screen.addPreference(it) }
 
@@ -232,16 +273,55 @@ open class Kemono(
         }.let(screen::addPreference)
     }
 
+    // Filters
+
+    override fun getFilterList(): FilterList =
+        FilterList(
+            SortFilter(
+                "Sort by",
+                Filter.Sort.Selection(0, false),
+                getSortsList,
+            ),
+            TypeFilter("Types", getTypes),
+            FavouritesFilter(),
+        )
+
+    open val getTypes: List<String> = emptyList()
+
+    open val getSortsList: List<Pair<String, String>> = listOf(
+        Pair("Popularity", "pop"),
+        Pair("Date Indexed", "new"),
+        Pair("Date Updated", "lat"),
+        Pair("Alphabetical Order", "tit"),
+        Pair("Service", "serv"),
+        Pair("Date Favourited", "fav"),
+    )
+
+    internal open class TypeFilter(name: String, vals: List<String>) :
+        Filter.Group<TriFilter>(
+            name,
+            vals.map { TriFilter(it, it.lowercase()) },
+        )
+
+    internal class FavouritesFilter() :
+        Filter.Group<TriFilter>(
+            "Favourites",
+            listOf(TriFilter("Favourites Only", "fav")),
+        )
+    internal open class TriFilter(name: String, val value: String) : Filter.TriState(name)
+
+    internal open class SortFilter(name: String, selection: Selection, private val vals: List<Pair<String, String>>) :
+        Filter.Sort(name, vals.map { it.first }.toTypedArray(), selection) {
+        fun getValue() = vals[state!!.index].second
+    }
     companion object {
-        private const val NEW_PAGE_SIZE = 50
+        private const val PAGE_POST_LIMIT = 50
+        private const val PAGE_CREATORS_LIMIT = 50
         const val PROMPT = "You can change how many posts to load in the extension preferences."
 
-        private const val POST_PAGE_SIZE = 50
         private const val POST_PAGES_PREF = "POST_PAGES"
         private const val POST_PAGES_DEFAULT = "1"
-        private const val POST_PAGES_MAX = 50
-
-        private fun List<SManga>.filterUnsupported() = filterNot { it.author == "Discord" }
+        private const val POST_PAGES_MAX = 75
 
         // private const val BASE_URL_PREF = "BASE_URL"
         private const val USE_LOW_RES_IMG = "USE_LOW_RES_IMG"

lib-multisrc/kemono/src/eu/kanade/tachiyomi/multisrc/kemono/KemonoDto.kt

@@ -7,15 +7,23 @@ import kotlinx.serialization.json.JsonPrimitive
 import kotlinx.serialization.json.double
 import java.text.SimpleDateFormat
 import java.util.Locale
+@Serializable
+class KemonoFavouritesDto(
+    val id: String,
+    val name: String,
+    val service: String,
+    val faved_seq: Long,
+)
 
 @Serializable
 class KemonoCreatorDto(
-    private val id: String,
+    val id: String,
     val name: String,
-    private val service: String,
+    val service: String,
     private val updated: JsonPrimitive,
     val favorited: Int = -1,
 ) {
+    var fav: Long = 0
     val updatedDate get() = when {
         updated.isString -> dateFormat.parse(updated.content)?.time ?: 0
         else -> (updated.double * 1000).toLong()

src/all/coomer/src/eu/kanade/tachiyomi/extension/all/coomer/Coomer.kt

@@ -2,4 +2,10 @@ package eu.kanade.tachiyomi.extension.all.coomer
 
 import eu.kanade.tachiyomi.multisrc.kemono.Kemono
 
-class Coomer : Kemono("Coomer", "https://coomer.su", "all")
+class Coomer : Kemono("Coomer", "https://coomer.su", "all") {
+    override val getTypes = listOf(
+        "OnlyFans",
+        "Fansly",
+        "CandFans",
+    )
+}

src/all/kemono/src/eu/kanade/tachiyomi/extension/all/kemono/Kemono.kt

@@ -2,4 +2,15 @@ package eu.kanade.tachiyomi.extension.all.kemono
 
 import eu.kanade.tachiyomi.multisrc.kemono.Kemono
 
-class Kemono : Kemono("Kemono", "https://kemono.su", "all")
+class Kemono : Kemono("Kemono", "https://kemono.su", "all") {
+    override val getTypes = listOf(
+        "Patreon",
+        "Pixiv Fanbox",
+        "Discord",
+        "Fantia",
+        "Afdian",
+        "Boosty",
+        "Gumroad",
+        "SubscribeStar",
+    )
+}