diff --git a/build.gradle b/build.gradle index bd9a865..ec87641 100644 --- a/build.gradle +++ b/build.gradle @@ -31,6 +31,7 @@ dependencies { implementation group: "org.http4k", name: "http4k-core", version: "$http_4k_version" implementation group: "org.http4k", name: "http4k-format-jackson", version: "$http_4k_version" + implementation group: "com.fasterxml.jackson.datatype", name: "jackson-datatype-jsr310", version: "2.11.1" implementation group: "org.http4k", name: "http4k-client-apache", version: "$http_4k_version" implementation group: "org.http4k", name: "http4k-server-netty", version: "$http_4k_version" runtimeOnly group: "io.netty", name: "netty-tcnative-boringssl-static", version: "2.0.30.Final" @@ -42,8 +43,8 @@ dependencies { implementation group: "org.xerial", name: "sqlite-jdbc", version: "3.30.1" -// implementation "com.goterl.lazycode:lazysodium-java:4.2.6" -// implementation "net.java.dev.jna:jna:5.5.0" + implementation "com.goterl.lazycode:lazysodium-java:4.2.6" + implementation "net.java.dev.jna:jna:5.5.0" } java { diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index 7fb2880..fc72034 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,5 +1,5 @@ -#Wed May 27 21:24:59 CDT 2020 -distributionUrl=https\://services.gradle.org/distributions/gradle-6.5.1-bin.zip +#Thu Jul 02 11:52:16 CDT 2020 +distributionUrl=https\://services.gradle.org/distributions/gradle-6.5.1-all.zip distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists zipStorePath=wrapper/dists diff --git a/src/main/kotlin/mdnet/base/Constants.kt b/src/main/kotlin/mdnet/base/Constants.kt index e86d23c..c7ef950 100644 --- a/src/main/kotlin/mdnet/base/Constants.kt +++ b/src/main/kotlin/mdnet/base/Constants.kt @@ -18,9 +18,6 @@ along with this MangaDex@Home. If not, see . */ package mdnet.base -import com.fasterxml.jackson.core.JsonParser -import com.fasterxml.jackson.databind.ObjectMapper -import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper import java.time.Duration object Constants { @@ -28,7 +25,6 @@ object Constants { const val CLIENT_VERSION = "1.0.0" const val WEBUI_VERSION = "0.1.1" val MAX_AGE_CACHE: Duration = Duration.ofDays(14) - val JACKSON: ObjectMapper = jacksonObjectMapper().configure(JsonParser.Feature.ALLOW_COMMENTS, true) const val MAX_READ_TIME_SECONDS = 300 const val MAX_WRITE_TIME_SECONDS = 60 diff --git a/src/main/kotlin/mdnet/base/Main.kt b/src/main/kotlin/mdnet/base/Main.kt index 6d56fb8..fea408a 100644 --- a/src/main/kotlin/mdnet/base/Main.kt +++ b/src/main/kotlin/mdnet/base/Main.kt @@ -19,20 +19,24 @@ along with this MangaDex@Home. If not, see . package mdnet.base import ch.qos.logback.classic.LoggerContext +import com.fasterxml.jackson.core.JsonParser import com.fasterxml.jackson.core.JsonProcessingException +import com.fasterxml.jackson.databind.ObjectMapper +import com.fasterxml.jackson.databind.SerializationFeature import com.fasterxml.jackson.databind.exc.UnrecognizedPropertyException +import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper import com.fasterxml.jackson.module.kotlin.readValue import java.io.FileReader import java.io.FileWriter import java.io.IOException import java.util.regex.Pattern import kotlin.system.exitProcess -import mdnet.base.Constants.JACKSON import mdnet.base.settings.ClientSettings import org.slf4j.LoggerFactory object Main { private val LOGGER = LoggerFactory.getLogger(Main::class.java) + private val JACKSON: ObjectMapper = jacksonObjectMapper().enable(SerializationFeature.INDENT_OUTPUT).configure(JsonParser.Feature.ALLOW_COMMENTS, true) @JvmStatic fun main(args: Array) { diff --git a/src/main/kotlin/mdnet/base/MangaDexClient.kt b/src/main/kotlin/mdnet/base/MangaDexClient.kt index 79d7a37..84103c0 100644 --- a/src/main/kotlin/mdnet/base/MangaDexClient.kt +++ b/src/main/kotlin/mdnet/base/MangaDexClient.kt @@ -20,6 +20,8 @@ along with this MangaDex@Home. If not, see . package mdnet.base import ch.qos.logback.classic.LoggerContext +import com.fasterxml.jackson.databind.ObjectMapper +import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper import com.fasterxml.jackson.module.kotlin.readValue import java.io.File import java.io.IOException @@ -30,7 +32,6 @@ import java.util.concurrent.Executors import java.util.concurrent.TimeUnit import java.util.concurrent.atomic.AtomicBoolean import java.util.concurrent.atomic.AtomicReference -import mdnet.base.Constants.JACKSON import mdnet.base.Main.dieWithError import mdnet.base.data.Statistics import mdnet.base.server.getServer @@ -320,5 +321,6 @@ class MangaDexClient(private val clientSettings: ClientSettings) { companion object { private val LOGGER = LoggerFactory.getLogger(MangaDexClient::class.java) + private val JACKSON: ObjectMapper = jacksonObjectMapper() } } diff --git a/src/main/kotlin/mdnet/base/ServerHandler.kt b/src/main/kotlin/mdnet/base/ServerHandler.kt index d8e283d..4e3e560 100644 --- a/src/main/kotlin/mdnet/base/ServerHandler.kt +++ b/src/main/kotlin/mdnet/base/ServerHandler.kt @@ -31,6 +31,7 @@ import org.http4k.format.ConfigurableJackson import org.http4k.format.asConfigurable import org.http4k.format.withStandardMappings import org.slf4j.LoggerFactory + object ServerHandlerJackson : ConfigurableJackson( KotlinModule() .asConfigurable() @@ -86,7 +87,7 @@ class ServerHandler(private val settings: ClientSettings) { val response = client(request) return if (response.status.successful) { - SERVER_SETTINGS_LENS(response) + SERVER_SETTINGS_LENS(response).also { println(it) } } else { null } @@ -108,7 +109,7 @@ class ServerHandler(private val settings: ClientSettings) { } private fun getServerAddress(): String { - return if (settings.devSettings == null || !settings.devSettings.isDev) + return if (settings.devSettings?.isDev != true) SERVER_ADDRESS else SERVER_ADDRESS_DEV @@ -119,6 +120,6 @@ class ServerHandler(private val settings: ClientSettings) { private val STRING_ANY_MAP_LENS = Body.auto>().toLens() private val SERVER_SETTINGS_LENS = Body.auto().toLens() private const val SERVER_ADDRESS = "https://api.mangadex.network/" - private const val SERVER_ADDRESS_DEV = "https://mangadex-test.net/" + private const val SERVER_ADDRESS_DEV = "http://localhost:28080/" } } diff --git a/src/main/kotlin/mdnet/base/data/Token.kt b/src/main/kotlin/mdnet/base/data/Token.kt new file mode 100644 index 0000000..68ce10d --- /dev/null +++ b/src/main/kotlin/mdnet/base/data/Token.kt @@ -0,0 +1,8 @@ +package mdnet.base.data + +import com.fasterxml.jackson.databind.PropertyNamingStrategy +import com.fasterxml.jackson.databind.annotation.JsonNaming +import java.time.OffsetDateTime + +@JsonNaming(PropertyNamingStrategy.SnakeCaseStrategy::class) +data class Token(val expires: OffsetDateTime, val ip: String, val hash: String, val clientId: String) diff --git a/src/main/kotlin/mdnet/base/server/Application.kt b/src/main/kotlin/mdnet/base/server/Application.kt index f0a88a5..295c936 100644 --- a/src/main/kotlin/mdnet/base/server/Application.kt +++ b/src/main/kotlin/mdnet/base/server/Application.kt @@ -39,7 +39,7 @@ private val LOGGER = LoggerFactory.getLogger("Application") fun getServer(cache: DiskLruCache, serverSettings: ServerSettings, clientSettings: ClientSettings, statistics: AtomicReference, isHandled: AtomicBoolean): Http4kServer { val database = Database.connect("jdbc:sqlite:cache/data.db", "org.sqlite.JDBC") - val imageServer = ImageServer(cache, statistics, serverSettings.imageServer, database, isHandled) + val imageServer = ImageServer(cache, statistics, serverSettings, database, isHandled) return timeRequest() .then(catchAllHideDetails()) @@ -65,15 +65,24 @@ fun getServer(cache: DiskLruCache, serverSettings: ServerSettings, clientSetting fun timeRequest(): Filter { return Filter { next: HttpHandler -> { request: Request -> + val cleanedUri = request.uri.path.let { + if (it.startsWith("/data")) { + it + } else { + it.replaceBefore("/data", "/{token}") + } + } + + if (LOGGER.isInfoEnabled) { + LOGGER.info("Request for $cleanedUri received from ${request.source?.address}") + } + val start = System.currentTimeMillis() val response = next(request) val latency = System.currentTimeMillis() - start - if (LOGGER.isTraceEnabled && response.header("X-Uri") != null) { - val sanitizedUri = response.header("X-Uri") - if (LOGGER.isInfoEnabled) { - LOGGER.info("Request for $sanitizedUri completed in ${latency}ms") - } + if (LOGGER.isInfoEnabled) { + LOGGER.info("Request for $cleanedUri completed (TTFB) in ${latency}ms") } response.header("X-Time-Taken", latency.toString()) } diff --git a/src/main/kotlin/mdnet/base/server/ImageServer.kt b/src/main/kotlin/mdnet/base/server/ImageServer.kt index 7382fd2..f739722 100644 --- a/src/main/kotlin/mdnet/base/server/ImageServer.kt +++ b/src/main/kotlin/mdnet/base/server/ImageServer.kt @@ -19,12 +19,22 @@ along with this MangaDex@Home. If not, see . /* ktlint-disable no-wildcard-imports */ package mdnet.base.server +import com.fasterxml.jackson.databind.DeserializationFeature +import com.fasterxml.jackson.databind.ObjectMapper +import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule +import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper +import com.fasterxml.jackson.module.kotlin.readValue +import com.goterl.lazycode.lazysodium.LazySodiumJava +import com.goterl.lazycode.lazysodium.SodiumJava +import com.goterl.lazycode.lazysodium.exceptions.SodiumException import java.io.BufferedInputStream import java.io.BufferedOutputStream import java.io.File import java.io.InputStream import java.security.MessageDigest import java.time.Clock +import java.time.OffsetDateTime +import java.util.* import java.util.concurrent.Executors import java.util.concurrent.atomic.AtomicBoolean import java.util.concurrent.atomic.AtomicReference @@ -36,6 +46,8 @@ import mdnet.base.Constants import mdnet.base.data.ImageData import mdnet.base.data.ImageDatum import mdnet.base.data.Statistics +import mdnet.base.data.Token +import mdnet.base.settings.ServerSettings import mdnet.cache.CachingInputStream import mdnet.cache.DiskLruCache import org.apache.http.client.config.CookieSpecs @@ -54,7 +66,7 @@ import org.slf4j.LoggerFactory private const val THREADS_TO_ALLOCATE = 262144 // 2**18 -class ImageServer(private val cache: DiskLruCache, private val statistics: AtomicReference, private val upstreamUrl: String, private val database: Database, private val handled: AtomicBoolean) { +class ImageServer(private val cache: DiskLruCache, private val statistics: AtomicReference, private val serverSettings: ServerSettings, private val database: Database, private val handled: AtomicBoolean) { init { transaction(database) { SchemaUtils.create(ImageData) @@ -65,74 +77,102 @@ class ImageServer(private val cache: DiskLruCache, private val statistics: Atomi .disableConnectionState() .setDefaultRequestConfig( RequestConfig.custom() - .setCookieSpec(CookieSpecs.IGNORE_COOKIES) - .setConnectTimeout(3000) - .setSocketTimeout(3000) - .setConnectionRequestTimeout(3000) - .build()) + .setCookieSpec(CookieSpecs.IGNORE_COOKIES) + .setConnectTimeout(3000) + .setSocketTimeout(3000) + .setConnectionRequestTimeout(3000) + .build()) .setMaxConnTotal(THREADS_TO_ALLOCATE) .setMaxConnPerRoute(THREADS_TO_ALLOCATE) .build()) - fun handler(dataSaver: Boolean, tokenized: Boolean = false): HttpHandler = baseHandler().then { request -> - val chapterHash = Path.of("chapterHash")(request) - val fileName = Path.of("fileName")(request) + fun handler(dataSaver: Boolean, tokenized: Boolean = false): HttpHandler { + val sodium = LazySodiumJava(SodiumJava()) - val sanitizedUri = if (dataSaver) { - "/data-saver" - } else { - "/data" - } + "/$chapterHash/$fileName" + return baseHandler().then { request -> + val chapterHash = Path.of("chapterHash")(request) + val fileName = Path.of("fileName")(request) - if (LOGGER.isInfoEnabled) { - LOGGER.info("Request for $sanitizedUri received from ${request.source?.address}") - } - statistics.getAndUpdate { - it.copy(requestsServed = it.requestsServed + 1) - } + val sanitizedUri = if (dataSaver) { + "/data-saver" + } else { + "/data" + } + "/$chapterHash/$fileName" - val rc4Bytes = if (dataSaver) { - md5Bytes("saver$chapterHash.$fileName") - } else { - md5Bytes("$chapterHash.$fileName") - } - val imageId = printHexString(rc4Bytes) - - val snapshot = cache.getUnsafe(imageId.toCacheId()) - val imageDatum = synchronized(database) { - transaction(database) { - ImageDatum.findById(imageId) - } - } - - handled.set(true) - if (request.header("Referer")?.startsWith("https://mangadex.org") == false) { - snapshot?.close() - Response(Status.FORBIDDEN) - } else if (snapshot != null && imageDatum != null) { - request.handleCacheHit(sanitizedUri, getRc4(rc4Bytes), snapshot, imageDatum) - .header("X-Uri", sanitizedUri) - } else { - if (snapshot != null) { - snapshot.close() - if (LOGGER.isWarnEnabled) { - LOGGER.warn("Removing cache file for $sanitizedUri without corresponding DB entry") + if (tokenized || serverSettings.forceToken) { + val tokenArr = Base64.getUrlDecoder().decode(Path.of("token")(request)) + val token = JACKSON.readValue( + try { + sodium.cryptoBoxOpenEasyAfterNm( + tokenArr.sliceArray(24 until tokenArr.size), tokenArr.sliceArray(0 until 24), serverSettings.sharedKey + ) + } catch (_: SodiumException) { + if (LOGGER.isInfoEnabled) { + LOGGER.info("Request for $sanitizedUri rejected for invalid token") + } + return@then Response(Status.FORBIDDEN) + } + ) + if (OffsetDateTime.now().isAfter(token.expires)) { + if (LOGGER.isInfoEnabled) { + LOGGER.info("Request for $sanitizedUri rejected for expired token") + } + return@then Response(Status.GONE) } - cache.removeUnsafe(imageId.toCacheId()) - } - if (imageDatum != null) { - if (LOGGER.isWarnEnabled) { - LOGGER.warn("Deleting DB entry for $sanitizedUri without corresponding file") + + if (token.hash != chapterHash) { + if (LOGGER.isInfoEnabled) { + LOGGER.info("Request for $sanitizedUri rejected for inapplicable token") + } + return@then Response(Status.FORBIDDEN) } - synchronized(database) { - transaction(database) { - imageDatum.delete() + } + + statistics.getAndUpdate { + it.copy(requestsServed = it.requestsServed + 1) + } + + val rc4Bytes = if (dataSaver) { + md5Bytes("saver$chapterHash.$fileName") + } else { + md5Bytes("$chapterHash.$fileName") + } + val imageId = printHexString(rc4Bytes) + + val snapshot = cache.getUnsafe(imageId.toCacheId()) + val imageDatum = synchronized(database) { + transaction(database) { + ImageDatum.findById(imageId) + } + } + + handled.set(true) + if (request.header("Referer")?.startsWith("https://mangadex.org") == false) { + snapshot?.close() + Response(Status.FORBIDDEN) + } else if (snapshot != null && imageDatum != null) { + request.handleCacheHit(sanitizedUri, getRc4(rc4Bytes), snapshot, imageDatum) + } else { + if (snapshot != null) { + snapshot.close() + if (LOGGER.isWarnEnabled) { + LOGGER.warn("Removing cache file for $sanitizedUri without corresponding DB entry") + } + cache.removeUnsafe(imageId.toCacheId()) + } + if (imageDatum != null) { + if (LOGGER.isWarnEnabled) { + LOGGER.warn("Deleting DB entry for $sanitizedUri without corresponding file") + } + synchronized(database) { + transaction(database) { + imageDatum.delete() + } } } - } - request.handleCacheMiss(sanitizedUri, getRc4(rc4Bytes), imageId) - .header("X-Uri", sanitizedUri) + request.handleCacheMiss(sanitizedUri, getRc4(rc4Bytes), imageId) + } } } @@ -177,7 +217,7 @@ class ImageServer(private val cache: DiskLruCache, private val statistics: Atomi it.copy(cacheMisses = it.cacheMisses + 1) } - val mdResponse = client(Request(Method.GET, "$upstreamUrl$sanitizedUri")) + val mdResponse = client(Request(Method.GET, "${serverSettings.imageServer}$sanitizedUri")) if (mdResponse.status != Status.OK) { if (LOGGER.isTraceEnabled) { @@ -272,17 +312,20 @@ class ImageServer(private val cache: DiskLruCache, private val statistics: Atomi companion object { private val LOGGER = LoggerFactory.getLogger(ImageServer::class.java) + private val JACKSON: ObjectMapper = jacksonObjectMapper() + .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false) + .registerModule(JavaTimeModule()) private fun baseHandler(): Filter = CachingFilters.Response.MaxAge(Clock.systemUTC(), Constants.MAX_AGE_CACHE) .then(ServerFilters.Cors( - CorsPolicy( - origins = listOf("https://mangadex.org"), - headers = listOf("*"), - methods = Method.values().toList() - ) + CorsPolicy( + origins = listOf("https://mangadex.org"), + headers = listOf("*"), + methods = Method.values().toList() ) ) + ) .then(Filter { next: HttpHandler -> { request: Request -> val response = next(request) diff --git a/src/main/kotlin/mdnet/base/server/naclbox.kt b/src/main/kotlin/mdnet/base/server/naclbox.kt new file mode 100644 index 0000000..7b7a7da --- /dev/null +++ b/src/main/kotlin/mdnet/base/server/naclbox.kt @@ -0,0 +1,42 @@ +/* +Mangadex@Home +Copyright (c) 2020, MangaDex Network +This file is part of MangaDex@Home. + +MangaDex@Home is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +MangaDex@Home is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this MangaDex@Home. If not, see . + */ +/* ktlint-disable no-wildcard-imports */ +package mdnet.base.server + +import com.goterl.lazycode.lazysodium.LazySodiumJava +import com.goterl.lazycode.lazysodium.exceptions.SodiumException +import com.goterl.lazycode.lazysodium.interfaces.Box + +@Throws(SodiumException::class) +fun LazySodiumJava.cryptoBoxOpenEasyAfterNm(cipherBytes: ByteArray, nonce: ByteArray, sharedKey: ByteArray): String { + if (!Box.Checker.checkNonce(nonce.size)) { + throw SodiumException("Incorrect nonce length.") + } + + if (!Box.Checker.checkBeforeNmBytes(sharedKey.size)) { + throw SodiumException("Incorrect shared secret key length.") + } + + val message = ByteArray(cipherBytes.size - Box.MACBYTES) + val res: Boolean = cryptoBoxOpenEasyAfterNm(message, cipherBytes, cipherBytes.size.toLong(), nonce, sharedKey) + if (!res) { + throw SodiumException("Could not fully complete shared secret key decryption.") + } + return str(message) +} diff --git a/src/main/kotlin/mdnet/base/settings/ServerSettings.kt b/src/main/kotlin/mdnet/base/settings/ServerSettings.kt index 81057bf..338b879 100644 --- a/src/main/kotlin/mdnet/base/settings/ServerSettings.kt +++ b/src/main/kotlin/mdnet/base/settings/ServerSettings.kt @@ -27,10 +27,40 @@ data class ServerSettings( val imageServer: String, val latestBuild: Int, val url: String, + val sharedKey: ByteArray, val compromised: Boolean, val paused: Boolean, + val forceToken: Boolean = false, val tls: TlsCert? -) +) { + override fun equals(other: Any?): Boolean { + if (this === other) return true + if (javaClass != other?.javaClass) return false + + other as ServerSettings + + if (imageServer != other.imageServer) return false + if (latestBuild != other.latestBuild) return false + if (url != other.url) return false + if (!sharedKey.contentEquals(other.sharedKey)) return false + if (compromised != other.compromised) return false + if (paused != other.paused) return false + if (tls != other.tls) return false + + return true + } + + override fun hashCode(): Int { + var result = imageServer.hashCode() + result = 31 * result + latestBuild + result = 31 * result + url.hashCode() + result = 31 * result + sharedKey.contentHashCode() + result = 31 * result + compromised.hashCode() + result = 31 * result + paused.hashCode() + result = 31 * result + (tls?.hashCode() ?: 0) + return result + } +} @JsonNaming(PropertyNamingStrategy.SnakeCaseStrategy::class) data class TlsCert(