Add Weekly Young Magazine (Yanmaga) (#1318)

src/ja/yanmaga/build.gradle

@@ -0,0 +1,12 @@
+ext {
+    extName = "Weekly Young Magazine"
+    extClass = ".YanmagaFactory"
+    extVersionCode = 1
+    isNsfw = true
+}
+
+apply from: "$rootDir/common.gradle"
+
+dependencies {
+    implementation(project(":lib:speedbinb"))
+}

src/ja/yanmaga/src/eu/kanade/tachiyomi/extension/ja/yanmaga/ParseInsertAdjacentHTML.kt

@@ -0,0 +1,29 @@
+package eu.kanade.tachiyomi.extension.ja.yanmaga
+
+import app.cash.quickjs.QuickJs
+
+private val INSERT_ADJACENT_HTML_REGEX = Regex(
+    """\s*\.\s*insertAdjacentHTML\s*\(\s*['"](beforebegin|afterbegin|beforeend|afterend)['"]\s*,\s*""",
+)
+
+/**
+ * Get the inserted content from a script containing a bunch of insertAdjacentHTML calls.
+ */
+internal fun parseInsertAdjacentHtmlScript(script: String, targetName: String = "target"): List<String> =
+    QuickJs.create().use { qjs ->
+        val cleanedScript = script.split("\n")
+            .filterNot {
+                it.contains("var $targetName") || it.contains("$targetName.classList")
+            }
+            .joinToString("\n")
+            .replace(INSERT_ADJACENT_HTML_REGEX, ".push(")
+        val result = qjs.evaluate(
+            """
+                const $targetName = [];
+                $cleanedScript
+                $targetName
+            """.trimIndent(),
+        )
+
+        (result as Array<*>).map { it as String }
+    }

src/ja/yanmaga/src/eu/kanade/tachiyomi/extension/ja/yanmaga/Yanmaga.kt

@@ -0,0 +1,169 @@
+package eu.kanade.tachiyomi.extension.ja.yanmaga
+
+import eu.kanade.tachiyomi.lib.speedbinb.SpeedBinbInterceptor
+import eu.kanade.tachiyomi.lib.speedbinb.SpeedBinbReader
+import eu.kanade.tachiyomi.network.GET
+import eu.kanade.tachiyomi.source.model.FilterList
+import eu.kanade.tachiyomi.source.model.Page
+import eu.kanade.tachiyomi.source.model.SChapter
+import eu.kanade.tachiyomi.source.model.SManga
+import eu.kanade.tachiyomi.source.online.ParsedHttpSource
+import eu.kanade.tachiyomi.util.asJsoup
+import kotlinx.serialization.json.Json
+import okhttp3.HttpUrl.Companion.toHttpUrl
+import okhttp3.Request
+import okhttp3.Response
+import org.jsoup.Jsoup
+import org.jsoup.nodes.Document
+import org.jsoup.nodes.Element
+import uy.kohesive.injekt.Injekt
+import uy.kohesive.injekt.api.get
+import java.text.SimpleDateFormat
+import java.util.Locale
+
+abstract class Yanmaga(
+    private val searchCategoryClass: String,
+    private val highQualityImages: Boolean = false,
+    private val dateFormat: SimpleDateFormat = SimpleDateFormat("yyyy/MM/dd", Locale.ROOT),
+) : ParsedHttpSource() {
+
+    override val baseUrl = "https://yanmaga.jp"
+
+    override val lang = "ja"
+
+    protected val json = Injekt.get<Json>()
+
+    override val client = network.client.newBuilder()
+        .addInterceptor(SpeedBinbInterceptor(json))
+        .build()
+
+    override fun headersBuilder() = super.headersBuilder()
+        .add("Referer", "$baseUrl/")
+
+    override fun searchMangaRequest(page: Int, query: String, filters: FilterList): Request {
+        val url = baseUrl.toHttpUrl().newBuilder().apply {
+            addPathSegment("search")
+            addQueryParameter("q", query)
+            addQueryParameter("kind", "human")
+
+            if (page > 1) {
+                addQueryParameter("page", page.toString())
+            }
+
+            addQueryParameter("search-submit", "")
+        }.build()
+
+        return GET(url, headers)
+    }
+
+    override fun searchMangaSelector() = "ul.search-list > li.search-item:has(.$searchCategoryClass)"
+
+    override fun searchMangaFromElement(element: Element) = SManga.create().apply {
+        setUrlWithoutDomain(element.selectFirst("a")!!.attr("href"))
+        title = element.selectFirst(".search-item-title")!!.text()
+        thumbnail_url = element.selectFirst(".search-item-thumbnail-image img")?.absUrl("src")
+    }
+
+    override fun searchMangaNextPageSelector() = "ul.pagination > li.page-item > a.page-next"
+
+    // Longer chapter lists are fetched through AJAX, the response being a JavaScript script
+    // that inserts raw HTML into the DOM. Horror.
+    override fun chapterListParse(response: Response): List<SChapter> {
+        val document = response.asJsoup()
+
+        if (document.selectFirst(".js-episode") == null) {
+            return document.select(chapterListSelector())
+                .map { chapterFromElement(it) }
+                .filter { it.url.isNotEmpty() }
+        }
+
+        val chapterUrl = response.request.url.toString()
+        val firstChapterList = document
+            .select("ul.mod-episode-list:first-of-type > li.mod-episode-item")
+            .map { chapterFromElement(it) }
+        val lastChapterList = document
+            .select("ul.mod-episode-list:last-of-type > li.mod-episode-item")
+            .map { chapterFromElement(it) }
+        val totalChapterCount = document
+            .selectFirst("#contents")
+            ?.attr("data-count")
+            ?.toInt()
+            ?: return firstChapterList + lastChapterList
+        val chapterMoreButton = document.selectFirst(".mod-episode-more-button[data-offset][data-path]")
+            ?: return firstChapterList + lastChapterList
+        val chapterOffset = chapterMoreButton.attr("data-offset").toInt()
+        val chapterAjaxUrl = chapterMoreButton.attr("abs:data-path").toHttpUrl()
+        val chaptersPerPage = document
+            .selectFirst("script:containsData(gon.episode_more)")
+            ?.data()
+            ?.substringAfter("gon.episode_more=")
+            ?.substringBefore(";")
+            ?.toInt()
+            ?: 150
+        val headers = headers.newBuilder()
+            .set("Referer", chapterUrl)
+            .set("X-CSRF-Token", document.selectFirst("meta[name=csrf-token]")!!.attr("content"))
+            .set("X-Requested-With", "XMLHttpRequest")
+            .build()
+
+        return buildList(totalChapterCount) {
+            addAll(firstChapterList)
+
+            for (i in chapterOffset until totalChapterCount - lastChapterList.size step chaptersPerPage) {
+                val limit = totalChapterCount - lastChapterList.size - i
+                val url = chapterAjaxUrl.newBuilder().apply {
+                    addQueryParameter("offset", i.toString())
+
+                    if (limit < 150) {
+                        addQueryParameter("limit", limit.toString())
+                    }
+
+                    addQueryParameter("cb", System.currentTimeMillis().toString())
+                }.build()
+                val script = client.newCall(GET(url, headers)).execute().body.string()
+
+                parseInsertAdjacentHtmlScript(script)
+                    .map { chapterFromElement(Jsoup.parseBodyFragment(it, chapterUrl)) }
+                    .let { addAll(it) }
+            }
+
+            addAll(lastChapterList)
+        }
+            .filter { it.url.isNotEmpty() }
+    }
+
+    override fun chapterListSelector() = "ul.mod-episode-list > li.mod-episode-item"
+
+    override fun chapterFromElement(element: Element) = SChapter.create().apply {
+        // The first chapter sometimes is a fake one. However, this still count towards the total
+        // chapter count, so we can't filter this out yet.
+        url = ""
+        element.selectFirst("a.mod-episode-link")?.attr("href")?.let {
+            setUrlWithoutDomain(it)
+        }
+        name = element.selectFirst(".mod-episode-title")!!.text()
+        date_upload = try {
+            dateFormat.parse(element.selectFirst(".mod-episode-date")!!.text())!!.time
+        } catch (_: Exception) {
+            0L
+        }
+    }
+
+    private val reader by lazy { SpeedBinbReader(client, headers, json, highQualityImages) }
+
+    override fun pageListParse(document: Document): List<Page> {
+        if (document.selectFirst(".ga-rental-modal-sign-up") != null) {
+            // Please log in with WebView to read this story
+            throw Exception("このストーリーを読むには WebView でログイン")
+        }
+
+        if (document.selectFirst(".ga-modal-open") != null) {
+            // Rent this story with points in WebView
+            throw Exception("WebView でポイントを使用してこのストーリーをレンタル")
+        }
+
+        return reader.pageListParse(document)
+    }
+
+    override fun imageUrlParse(document: Document) = throw UnsupportedOperationException()
+}

src/ja/yanmaga/src/eu/kanade/tachiyomi/extension/ja/yanmaga/YanmagaComics.kt

@@ -0,0 +1,134 @@
+package eu.kanade.tachiyomi.extension.ja.yanmaga
+
+import eu.kanade.tachiyomi.network.GET
+import eu.kanade.tachiyomi.network.asObservableSuccess
+import eu.kanade.tachiyomi.source.model.MangasPage
+import eu.kanade.tachiyomi.source.model.SManga
+import eu.kanade.tachiyomi.util.asJsoup
+import okhttp3.Request
+import okhttp3.Response
+import org.jsoup.Jsoup
+import org.jsoup.nodes.Document
+import org.jsoup.nodes.Element
+import org.jsoup.select.Elements
+import rx.Observable
+
+class YanmagaComics : Yanmaga("search-item-category--comics") {
+
+    override val name = "ヤンマガ（マンガ）"
+
+    override val supportsLatest = true
+
+    private lateinit var directory: Elements
+
+    override fun fetchPopularManga(page: Int): Observable<MangasPage> {
+        return if (page == 1) {
+            client.newCall(popularMangaRequest(page))
+                .asObservableSuccess()
+                .map { popularMangaParse(it) }
+        } else {
+            Observable.just(parseDirectory(page))
+        }
+    }
+
+    override fun popularMangaRequest(page: Int) = GET("$baseUrl/comics", headers)
+
+    override fun popularMangaParse(response: Response): MangasPage {
+        val document = response.asJsoup()
+
+        directory = document.select(popularMangaSelector())
+        return parseDirectory(1)
+    }
+
+    private fun parseDirectory(page: Int): MangasPage {
+        val endRange = minOf(page * 24, directory.size)
+        val manga = directory.subList((page - 1) * 24, endRange).map { popularMangaFromElement(it) }
+        val hasNextPage = endRange < directory.lastIndex
+
+        return MangasPage(manga, hasNextPage)
+    }
+
+    override fun popularMangaSelector() = "a.ga-comics-book-item"
+
+    override fun popularMangaFromElement(element: Element) = SManga.create().apply {
+        setUrlWithoutDomain(element.attr("href"))
+        title = element.selectFirst(".mod-book-title")!!.text()
+        thumbnail_url = element.selectFirst(".mod-book-image img")?.absUrl("data-src")
+    }
+
+    override fun popularMangaNextPageSelector() = throw UnsupportedOperationException()
+
+    private var latestUpdatesCsrfToken: String? = null
+    private var latestUpdatesMoreUrl: String? = null
+    private var latestUpdatesCount: Int = 0
+
+    override fun latestUpdatesRequest(page: Int): Request {
+        val pageUrl = "$baseUrl/comics/series/newer"
+
+        if (page == 1) {
+            return GET(pageUrl, headers)
+        }
+
+        val offset = (page - 1) * LATEST_UPDATES_PER_PAGE
+        val headers = headers.newBuilder()
+            .set("Referer", pageUrl)
+            .set("X-CSRF-Token", latestUpdatesCsrfToken!!)
+            .set("X-Requested-With", "XMLHttpRequest")
+            .build()
+
+        return GET("${latestUpdatesMoreUrl!!}?offset=$offset", headers)
+    }
+
+    override fun latestUpdatesParse(response: Response): MangasPage {
+        val pageUrl = "$baseUrl/comics/series/newer"
+        val url = response.request.url
+
+        return if (url.pathSegments.last() == "newer") {
+            val document = response.asJsoup()
+
+            latestUpdatesCsrfToken = document.selectFirst("meta[name=csrf-token]")!!.attr("content")
+            document.selectFirst(".newer-older-episode-more-button[data-count][data-path]")!!.let {
+                latestUpdatesMoreUrl = it.attr("abs:data-path")
+                latestUpdatesCount = it.attr("data-count").toInt()
+            }
+
+            val manga = document.select(latestUpdatesSelector())
+                .map { latestUpdatesFromElement(it) }
+            val hasNextPage = latestUpdatesCount > LATEST_UPDATES_PER_PAGE
+
+            MangasPage(manga, hasNextPage)
+        } else {
+            val offset = url.queryParameter("offset")!!.toInt()
+            val manga = parseInsertAdjacentHtmlScript(response.body.string())
+                .map { latestUpdatesFromElement(Jsoup.parseBodyFragment(it, pageUrl)) }
+            val hasNextPage = offset + LATEST_UPDATES_PER_PAGE < latestUpdatesCount
+
+            MangasPage(manga, hasNextPage)
+        }
+    }
+
+    override fun latestUpdatesSelector() = "#comic-episodes-newer > div"
+
+    override fun latestUpdatesFromElement(element: Element) = SManga.create().apply {
+        setUrlWithoutDomain(element.selectFirst("a")!!.attr("href"))
+        title = element.selectFirst(".text-wrapper h2")!!.text()
+        thumbnail_url = element.selectFirst(".img-bg-wrapper")?.absUrl("data-bg")
+    }
+
+    override fun latestUpdatesNextPageSelector() = throw UnsupportedOperationException()
+
+    override fun mangaDetailsParse(document: Document) = SManga.create().apply {
+        title = document.selectFirst(".detailv2-outline-title")!!.text()
+        author = document.select(".detailv2-outline-author-item a").joinToString { it.text() }
+        description = document.selectFirst(".detailv2-description")?.text()
+        genre = document.select(".detailv2-tag .ga-tag").joinToString { it.text() }
+        thumbnail_url = document.selectFirst(".detailv2-thumbnail-image img")?.absUrl("src")
+        status = if (document.selectFirst(".detailv2-link-note") != null) {
+            SManga.ONGOING
+        } else {
+            SManga.COMPLETED
+        }
+    }
+}
+
+private const val LATEST_UPDATES_PER_PAGE = 12

src/ja/yanmaga/src/eu/kanade/tachiyomi/extension/ja/yanmaga/YanmagaFactory.kt

@@ -0,0 +1,10 @@
+package eu.kanade.tachiyomi.extension.ja.yanmaga
+
+import eu.kanade.tachiyomi.source.SourceFactory
+
+class YanmagaFactory : SourceFactory {
+    override fun createSources() = listOf(
+        YanmagaComics(),
+        YanmagaGravures(),
+    )
+}

src/ja/yanmaga/src/eu/kanade/tachiyomi/extension/ja/yanmaga/YanmagaGravures.kt

@@ -0,0 +1,72 @@
+package eu.kanade.tachiyomi.extension.ja.yanmaga
+
+import eu.kanade.tachiyomi.network.GET
+import eu.kanade.tachiyomi.source.model.SChapter
+import eu.kanade.tachiyomi.source.model.SManga
+import eu.kanade.tachiyomi.source.model.UpdateStrategy
+import org.jsoup.nodes.Document
+import org.jsoup.nodes.Element
+import rx.Observable
+
+class YanmagaGravures : Yanmaga("search-item-category--gravures", true) {
+
+    override val name = "ヤンマガ（グラビア）"
+
+    override val supportsLatest = false
+
+    override fun popularMangaRequest(page: Int) = GET("$baseUrl/gravures/series?page=$page", headers)
+
+    override fun popularMangaSelector() = "a.banner-link"
+
+    override fun popularMangaFromElement(element: Element) = SManga.create().apply {
+        setUrlWithoutDomain(element.attr("href"))
+        title = element.selectFirst(".text-wrapper h2")!!.text()
+        thumbnail_url = element.selectFirst(".img-bg-wrapper")?.absUrl("data-bg")
+    }
+
+    override fun popularMangaNextPageSelector() = "ul.pagination > li.page-item > a.page-next"
+
+    override fun latestUpdatesRequest(page: Int) = throw UnsupportedOperationException()
+
+    override fun latestUpdatesSelector() = throw UnsupportedOperationException()
+
+    override fun latestUpdatesFromElement(element: Element) = throw UnsupportedOperationException()
+
+    override fun latestUpdatesNextPageSelector() = throw UnsupportedOperationException()
+
+    // Search returns gravure books instead of series
+    override fun searchMangaFromElement(element: Element) = super.searchMangaFromElement(element)
+        .apply {
+            status = SManga.COMPLETED
+            update_strategy = UpdateStrategy.ONLY_FETCH_ONCE
+        }
+
+    override fun fetchMangaDetails(manga: SManga): Observable<SManga> {
+        return if (manga.url.contains("/series/")) {
+            super.fetchMangaDetails(manga)
+        } else {
+            Observable.just(manga)
+        }
+    }
+
+    override fun mangaDetailsParse(document: Document) = SManga.create().apply {
+        title = document.selectFirst(".detail-header-title")!!.text()
+        genre = document.select(".ga-tag").joinToString { it.text() }
+        thumbnail_url = document.selectFirst(".detail-header-image img")?.absUrl("src")
+    }
+
+    override fun fetchChapterList(manga: SManga): Observable<List<SChapter>> {
+        return if (manga.url.contains("/series/")) {
+            super.fetchChapterList(manga)
+        } else {
+            Observable.just(
+                listOf(
+                    SChapter.create().apply {
+                        url = manga.url
+                        name = "作品"
+                    },
+                ),
+            )
+        }
+    }
+}