Validate tokens

build.gradle

@@ -31,6 +31,7 @@ dependencies {
 
   implementation group: "org.http4k", name: "http4k-core", version: "$http_4k_version"
   implementation group: "org.http4k", name: "http4k-format-jackson", version: "$http_4k_version"
+  implementation group: "com.fasterxml.jackson.datatype", name: "jackson-datatype-jsr310", version: "2.11.1"
   implementation group: "org.http4k", name: "http4k-client-apache", version: "$http_4k_version"
   implementation group: "org.http4k", name: "http4k-server-netty", version: "$http_4k_version"
   runtimeOnly group: "io.netty", name: "netty-tcnative-boringssl-static", version: "2.0.30.Final"
@@ -42,8 +43,8 @@ dependencies {
 
   implementation group: "org.xerial", name: "sqlite-jdbc", version: "3.30.1"
 
-//  implementation "com.goterl.lazycode:lazysodium-java:4.2.6"
+  implementation "com.goterl.lazycode:lazysodium-java:4.2.6"
-//  implementation "net.java.dev.jna:jna:5.5.0"
+  implementation "net.java.dev.jna:jna:5.5.0"
 }
 
 java {

gradle/wrapper/gradle-wrapper.properties

@@ -1,5 +1,5 @@
-#Wed May 27 21:24:59 CDT 2020
+#Thu Jul 02 11:52:16 CDT 2020
-distributionUrl=https\://services.gradle.org/distributions/gradle-6.5.1-bin.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-6.5.1-all.zip
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
 zipStorePath=wrapper/dists

src/main/kotlin/mdnet/base/Constants.kt

@@ -18,9 +18,6 @@ along with this MangaDex@Home.  If not, see <http://www.gnu.org/licenses/>.
  */
 package mdnet.base
 
-import com.fasterxml.jackson.core.JsonParser
-import com.fasterxml.jackson.databind.ObjectMapper
-import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper
 import java.time.Duration
 
 object Constants {
@@ -28,7 +25,6 @@ object Constants {
     const val CLIENT_VERSION = "1.0.0"
     const val WEBUI_VERSION = "0.1.1"
     val MAX_AGE_CACHE: Duration = Duration.ofDays(14)
-    val JACKSON: ObjectMapper = jacksonObjectMapper().configure(JsonParser.Feature.ALLOW_COMMENTS, true)
 
     const val MAX_READ_TIME_SECONDS = 300
     const val MAX_WRITE_TIME_SECONDS = 60

src/main/kotlin/mdnet/base/Main.kt

@@ -19,20 +19,24 @@ along with this MangaDex@Home.  If not, see <http://www.gnu.org/licenses/>.
 package mdnet.base
 
 import ch.qos.logback.classic.LoggerContext
+import com.fasterxml.jackson.core.JsonParser
 import com.fasterxml.jackson.core.JsonProcessingException
+import com.fasterxml.jackson.databind.ObjectMapper
+import com.fasterxml.jackson.databind.SerializationFeature
 import com.fasterxml.jackson.databind.exc.UnrecognizedPropertyException
+import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper
 import com.fasterxml.jackson.module.kotlin.readValue
 import java.io.FileReader
 import java.io.FileWriter
 import java.io.IOException
 import java.util.regex.Pattern
 import kotlin.system.exitProcess
-import mdnet.base.Constants.JACKSON
 import mdnet.base.settings.ClientSettings
 import org.slf4j.LoggerFactory
 
 object Main {
     private val LOGGER = LoggerFactory.getLogger(Main::class.java)
+    private val JACKSON: ObjectMapper = jacksonObjectMapper().enable(SerializationFeature.INDENT_OUTPUT).configure(JsonParser.Feature.ALLOW_COMMENTS, true)
 
     @JvmStatic
     fun main(args: Array<String>) {

src/main/kotlin/mdnet/base/MangaDexClient.kt

@@ -20,6 +20,8 @@ along with this MangaDex@Home.  If not, see <http://www.gnu.org/licenses/>.
 package mdnet.base
 
 import ch.qos.logback.classic.LoggerContext
+import com.fasterxml.jackson.databind.ObjectMapper
+import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper
 import com.fasterxml.jackson.module.kotlin.readValue
 import java.io.File
 import java.io.IOException
@@ -30,7 +32,6 @@ import java.util.concurrent.Executors
 import java.util.concurrent.TimeUnit
 import java.util.concurrent.atomic.AtomicBoolean
 import java.util.concurrent.atomic.AtomicReference
-import mdnet.base.Constants.JACKSON
 import mdnet.base.Main.dieWithError
 import mdnet.base.data.Statistics
 import mdnet.base.server.getServer
@@ -320,5 +321,6 @@ class MangaDexClient(private val clientSettings: ClientSettings) {
 
     companion object {
         private val LOGGER = LoggerFactory.getLogger(MangaDexClient::class.java)
+        private val JACKSON: ObjectMapper = jacksonObjectMapper()
     }
 }

src/main/kotlin/mdnet/base/ServerHandler.kt

@@ -31,6 +31,7 @@ import org.http4k.format.ConfigurableJackson
 import org.http4k.format.asConfigurable
 import org.http4k.format.withStandardMappings
 import org.slf4j.LoggerFactory
+
 object ServerHandlerJackson : ConfigurableJackson(
     KotlinModule()
     .asConfigurable()
@@ -86,7 +87,7 @@ class ServerHandler(private val settings: ClientSettings) {
         val response = client(request)
 
         return if (response.status.successful) {
-            SERVER_SETTINGS_LENS(response)
+            SERVER_SETTINGS_LENS(response).also { println(it) }
         } else {
             null
         }
@@ -108,7 +109,7 @@ class ServerHandler(private val settings: ClientSettings) {
     }
 
     private fun getServerAddress(): String {
-        return if (settings.devSettings == null || !settings.devSettings.isDev)
+        return if (settings.devSettings?.isDev != true)
             SERVER_ADDRESS
         else
             SERVER_ADDRESS_DEV
@@ -119,6 +120,6 @@ class ServerHandler(private val settings: ClientSettings) {
         private val STRING_ANY_MAP_LENS = Body.auto<Map<String, Any>>().toLens()
         private val SERVER_SETTINGS_LENS = Body.auto<ServerSettings>().toLens()
         private const val SERVER_ADDRESS = "https://api.mangadex.network/"
-        private const val SERVER_ADDRESS_DEV = "https://mangadex-test.net/"
+        private const val SERVER_ADDRESS_DEV = "http://localhost:28080/"
     }
 }

src/main/kotlin/mdnet/base/data/Token.kt

@@ -0,0 +1,8 @@
+package mdnet.base.data
+
+import com.fasterxml.jackson.databind.PropertyNamingStrategy
+import com.fasterxml.jackson.databind.annotation.JsonNaming
+import java.time.OffsetDateTime
+
+@JsonNaming(PropertyNamingStrategy.SnakeCaseStrategy::class)
+data class Token(val expires: OffsetDateTime, val ip: String, val hash: String, val clientId: String)

src/main/kotlin/mdnet/base/server/Application.kt

@@ -39,7 +39,7 @@ private val LOGGER = LoggerFactory.getLogger("Application")
 
 fun getServer(cache: DiskLruCache, serverSettings: ServerSettings, clientSettings: ClientSettings, statistics: AtomicReference<Statistics>, isHandled: AtomicBoolean): Http4kServer {
     val database = Database.connect("jdbc:sqlite:cache/data.db", "org.sqlite.JDBC")
-    val imageServer = ImageServer(cache, statistics, serverSettings.imageServer, database, isHandled)
+    val imageServer = ImageServer(cache, statistics, serverSettings, database, isHandled)
 
     return timeRequest()
             .then(catchAllHideDetails())
@@ -65,15 +65,24 @@ fun getServer(cache: DiskLruCache, serverSettings: ServerSettings, clientSetting
 fun timeRequest(): Filter {
     return Filter { next: HttpHandler ->
         { request: Request ->
+            val cleanedUri = request.uri.path.let {
+                if (it.startsWith("/data")) {
+                    it
+                } else {
+                    it.replaceBefore("/data", "/{token}")
+                }
+            }
+
+            if (LOGGER.isInfoEnabled) {
+                LOGGER.info("Request for $cleanedUri received from ${request.source?.address}")
+            }
+
             val start = System.currentTimeMillis()
             val response = next(request)
             val latency = System.currentTimeMillis() - start
 
-            if (LOGGER.isTraceEnabled && response.header("X-Uri") != null) {
+            if (LOGGER.isInfoEnabled) {
-                val sanitizedUri = response.header("X-Uri")
+                LOGGER.info("Request for $cleanedUri completed (TTFB) in ${latency}ms")
-                if (LOGGER.isInfoEnabled) {
-                    LOGGER.info("Request for $sanitizedUri completed in ${latency}ms")
-                }
             }
             response.header("X-Time-Taken", latency.toString())
         }

src/main/kotlin/mdnet/base/server/ImageServer.kt

@@ -19,12 +19,22 @@ along with this MangaDex@Home.  If not, see <http://www.gnu.org/licenses/>.
 /* ktlint-disable no-wildcard-imports */
 package mdnet.base.server
 
+import com.fasterxml.jackson.databind.DeserializationFeature
+import com.fasterxml.jackson.databind.ObjectMapper
+import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule
+import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper
+import com.fasterxml.jackson.module.kotlin.readValue
+import com.goterl.lazycode.lazysodium.LazySodiumJava
+import com.goterl.lazycode.lazysodium.SodiumJava
+import com.goterl.lazycode.lazysodium.exceptions.SodiumException
 import java.io.BufferedInputStream
 import java.io.BufferedOutputStream
 import java.io.File
 import java.io.InputStream
 import java.security.MessageDigest
 import java.time.Clock
+import java.time.OffsetDateTime
+import java.util.*
 import java.util.concurrent.Executors
 import java.util.concurrent.atomic.AtomicBoolean
 import java.util.concurrent.atomic.AtomicReference
@@ -36,6 +46,8 @@ import mdnet.base.Constants
 import mdnet.base.data.ImageData
 import mdnet.base.data.ImageDatum
 import mdnet.base.data.Statistics
+import mdnet.base.data.Token
+import mdnet.base.settings.ServerSettings
 import mdnet.cache.CachingInputStream
 import mdnet.cache.DiskLruCache
 import org.apache.http.client.config.CookieSpecs
@@ -54,7 +66,7 @@ import org.slf4j.LoggerFactory
 
 private const val THREADS_TO_ALLOCATE = 262144 // 2**18
 
-class ImageServer(private val cache: DiskLruCache, private val statistics: AtomicReference<Statistics>, private val upstreamUrl: String, private val database: Database, private val handled: AtomicBoolean) {
+class ImageServer(private val cache: DiskLruCache, private val statistics: AtomicReference<Statistics>, private val serverSettings: ServerSettings, private val database: Database, private val handled: AtomicBoolean) {
     init {
         transaction(database) {
             SchemaUtils.create(ImageData)
@@ -65,74 +77,102 @@ class ImageServer(private val cache: DiskLruCache, private val statistics: Atomi
         .disableConnectionState()
         .setDefaultRequestConfig(
             RequestConfig.custom()
-            .setCookieSpec(CookieSpecs.IGNORE_COOKIES)
+                .setCookieSpec(CookieSpecs.IGNORE_COOKIES)
-            .setConnectTimeout(3000)
+                .setConnectTimeout(3000)
-            .setSocketTimeout(3000)
+                .setSocketTimeout(3000)
-            .setConnectionRequestTimeout(3000)
+                .setConnectionRequestTimeout(3000)
-            .build())
+                .build())
         .setMaxConnTotal(THREADS_TO_ALLOCATE)
         .setMaxConnPerRoute(THREADS_TO_ALLOCATE)
         .build())
 
-    fun handler(dataSaver: Boolean, tokenized: Boolean = false): HttpHandler = baseHandler().then { request ->
+    fun handler(dataSaver: Boolean, tokenized: Boolean = false): HttpHandler {
-        val chapterHash = Path.of("chapterHash")(request)
+        val sodium = LazySodiumJava(SodiumJava())
-        val fileName = Path.of("fileName")(request)
 
-        val sanitizedUri = if (dataSaver) {
+        return baseHandler().then { request ->
-            "/data-saver"
+            val chapterHash = Path.of("chapterHash")(request)
-        } else {
+            val fileName = Path.of("fileName")(request)
-            "/data"
-        } + "/$chapterHash/$fileName"
 
-        if (LOGGER.isInfoEnabled) {
+            val sanitizedUri = if (dataSaver) {
-            LOGGER.info("Request for $sanitizedUri received from ${request.source?.address}")
+                "/data-saver"
-        }
+            } else {
-        statistics.getAndUpdate {
+                "/data"
-            it.copy(requestsServed = it.requestsServed + 1)
+            } + "/$chapterHash/$fileName"
-        }
 
-        val rc4Bytes = if (dataSaver) {
+            if (tokenized || serverSettings.forceToken) {
-            md5Bytes("saver$chapterHash.$fileName")
+                val tokenArr = Base64.getUrlDecoder().decode(Path.of("token")(request))
-        } else {
+                val token = JACKSON.readValue<Token>(
-            md5Bytes("$chapterHash.$fileName")
+                    try {
-        }
+                        sodium.cryptoBoxOpenEasyAfterNm(
-        val imageId = printHexString(rc4Bytes)
+                            tokenArr.sliceArray(24 until tokenArr.size), tokenArr.sliceArray(0 until 24), serverSettings.sharedKey
-
+                        )
-        val snapshot = cache.getUnsafe(imageId.toCacheId())
+                    } catch (_: SodiumException) {
-        val imageDatum = synchronized(database) {
+                        if (LOGGER.isInfoEnabled) {
-            transaction(database) {
+                            LOGGER.info("Request for $sanitizedUri rejected for invalid token")
-                ImageDatum.findById(imageId)
+                        }
-            }
+                        return@then Response(Status.FORBIDDEN)
-        }
+                    }
-
+                )
-        handled.set(true)
+                if (OffsetDateTime.now().isAfter(token.expires)) {
-        if (request.header("Referer")?.startsWith("https://mangadex.org") == false) {
+                    if (LOGGER.isInfoEnabled) {
-            snapshot?.close()
+                        LOGGER.info("Request for $sanitizedUri rejected for expired token")
-            Response(Status.FORBIDDEN)
+                    }
-        } else if (snapshot != null && imageDatum != null) {
+                    return@then Response(Status.GONE)
-            request.handleCacheHit(sanitizedUri, getRc4(rc4Bytes), snapshot, imageDatum)
-                .header("X-Uri", sanitizedUri)
-        } else {
-            if (snapshot != null) {
-                snapshot.close()
-                if (LOGGER.isWarnEnabled) {
-                    LOGGER.warn("Removing cache file for $sanitizedUri without corresponding DB entry")
                 }
-                cache.removeUnsafe(imageId.toCacheId())
+
-            }
+                if (token.hash != chapterHash) {
-            if (imageDatum != null) {
+                    if (LOGGER.isInfoEnabled) {
-                if (LOGGER.isWarnEnabled) {
+                        LOGGER.info("Request for $sanitizedUri rejected for inapplicable token")
-                    LOGGER.warn("Deleting DB entry for $sanitizedUri without corresponding file")
+                    }
+                    return@then Response(Status.FORBIDDEN)
                 }
-                synchronized(database) {
+            }
-                    transaction(database) {
+
-                        imageDatum.delete()
+            statistics.getAndUpdate {
+                it.copy(requestsServed = it.requestsServed + 1)
+            }
+
+            val rc4Bytes = if (dataSaver) {
+                md5Bytes("saver$chapterHash.$fileName")
+            } else {
+                md5Bytes("$chapterHash.$fileName")
+            }
+            val imageId = printHexString(rc4Bytes)
+
+            val snapshot = cache.getUnsafe(imageId.toCacheId())
+            val imageDatum = synchronized(database) {
+                transaction(database) {
+                    ImageDatum.findById(imageId)
+                }
+            }
+
+            handled.set(true)
+            if (request.header("Referer")?.startsWith("https://mangadex.org") == false) {
+                snapshot?.close()
+                Response(Status.FORBIDDEN)
+            } else if (snapshot != null && imageDatum != null) {
+                request.handleCacheHit(sanitizedUri, getRc4(rc4Bytes), snapshot, imageDatum)
+            } else {
+                if (snapshot != null) {
+                    snapshot.close()
+                    if (LOGGER.isWarnEnabled) {
+                        LOGGER.warn("Removing cache file for $sanitizedUri without corresponding DB entry")
+                    }
+                    cache.removeUnsafe(imageId.toCacheId())
+                }
+                if (imageDatum != null) {
+                    if (LOGGER.isWarnEnabled) {
+                        LOGGER.warn("Deleting DB entry for $sanitizedUri without corresponding file")
+                    }
+                    synchronized(database) {
+                        transaction(database) {
+                            imageDatum.delete()
+                        }
                     }
                 }
-            }
 
-            request.handleCacheMiss(sanitizedUri, getRc4(rc4Bytes), imageId)
+                request.handleCacheMiss(sanitizedUri, getRc4(rc4Bytes), imageId)
-                .header("X-Uri", sanitizedUri)
+            }
         }
     }
 
@@ -177,7 +217,7 @@ class ImageServer(private val cache: DiskLruCache, private val statistics: Atomi
             it.copy(cacheMisses = it.cacheMisses + 1)
         }
 
-        val mdResponse = client(Request(Method.GET, "$upstreamUrl$sanitizedUri"))
+        val mdResponse = client(Request(Method.GET, "${serverSettings.imageServer}$sanitizedUri"))
 
         if (mdResponse.status != Status.OK) {
             if (LOGGER.isTraceEnabled) {
@@ -272,17 +312,20 @@ class ImageServer(private val cache: DiskLruCache, private val statistics: Atomi
 
     companion object {
         private val LOGGER = LoggerFactory.getLogger(ImageServer::class.java)
+        private val JACKSON: ObjectMapper = jacksonObjectMapper()
+            .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false)
+            .registerModule(JavaTimeModule())
 
         private fun baseHandler(): Filter =
             CachingFilters.Response.MaxAge(Clock.systemUTC(), Constants.MAX_AGE_CACHE)
                 .then(ServerFilters.Cors(
-                        CorsPolicy(
+                    CorsPolicy(
-                            origins = listOf("https://mangadex.org"),
+                        origins = listOf("https://mangadex.org"),
-                            headers = listOf("*"),
+                        headers = listOf("*"),
-                            methods = Method.values().toList()
+                        methods = Method.values().toList()
-                        )
                     )
                 )
+                )
                 .then(Filter { next: HttpHandler ->
                     { request: Request ->
                         val response = next(request)

src/main/kotlin/mdnet/base/server/naclbox.kt

@@ -0,0 +1,42 @@
+/*
+Mangadex@Home
+Copyright (c) 2020, MangaDex Network
+This file is part of MangaDex@Home.
+
+MangaDex@Home is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+MangaDex@Home is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this MangaDex@Home.  If not, see <http://www.gnu.org/licenses/>.
+ */
+/* ktlint-disable no-wildcard-imports */
+package mdnet.base.server
+
+import com.goterl.lazycode.lazysodium.LazySodiumJava
+import com.goterl.lazycode.lazysodium.exceptions.SodiumException
+import com.goterl.lazycode.lazysodium.interfaces.Box
+
+@Throws(SodiumException::class)
+fun LazySodiumJava.cryptoBoxOpenEasyAfterNm(cipherBytes: ByteArray, nonce: ByteArray, sharedKey: ByteArray): String {
+    if (!Box.Checker.checkNonce(nonce.size)) {
+        throw SodiumException("Incorrect nonce length.")
+    }
+
+    if (!Box.Checker.checkBeforeNmBytes(sharedKey.size)) {
+        throw SodiumException("Incorrect shared secret key length.")
+    }
+
+    val message = ByteArray(cipherBytes.size - Box.MACBYTES)
+    val res: Boolean = cryptoBoxOpenEasyAfterNm(message, cipherBytes, cipherBytes.size.toLong(), nonce, sharedKey)
+    if (!res) {
+        throw SodiumException("Could not fully complete shared secret key decryption.")
+    }
+    return str(message)
+}

src/main/kotlin/mdnet/base/settings/ServerSettings.kt

@@ -27,10 +27,40 @@ data class ServerSettings(
     val imageServer: String,
     val latestBuild: Int,
     val url: String,
+    val sharedKey: ByteArray,
     val compromised: Boolean,
     val paused: Boolean,
+    val forceToken: Boolean = false,
     val tls: TlsCert?
-)
+) {
+    override fun equals(other: Any?): Boolean {
+        if (this === other) return true
+        if (javaClass != other?.javaClass) return false
+
+        other as ServerSettings
+
+        if (imageServer != other.imageServer) return false
+        if (latestBuild != other.latestBuild) return false
+        if (url != other.url) return false
+        if (!sharedKey.contentEquals(other.sharedKey)) return false
+        if (compromised != other.compromised) return false
+        if (paused != other.paused) return false
+        if (tls != other.tls) return false
+
+        return true
+    }
+
+    override fun hashCode(): Int {
+        var result = imageServer.hashCode()
+        result = 31 * result + latestBuild
+        result = 31 * result + url.hashCode()
+        result = 31 * result + sharedKey.contentHashCode()
+        result = 31 * result + compromised.hashCode()
+        result = 31 * result + paused.hashCode()
+        result = 31 * result + (tls?.hashCode() ?: 0)
+        return result
+    }
+}
 
 @JsonNaming(PropertyNamingStrategy.SnakeCaseStrategy::class)
 data class TlsCert(