Mirrors/mangadex_at_home
1
0
Fork 0
mirror of https://gitlab.com/mangadex-pub/mangadex_at_home.git synced 2024-11-16 16:12:32 +01:00
Code Issues Releases Activity

Validate tokens

Browse Source
This commit is contained in:
carbotaniuman 2020-07-02 16:24:12 -05:00
parent ac324a7dd2
commit 331f0f030d
11 changed files with 219 additions and 83 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
build.gradle
View File

@ -31,6 +31,7 @@ dependencies {
implementation group: "org.http4k", name: "http4k-core", version: "$http_4k_version"
implementation group: "org.http4k", name: "http4k-format-jackson", version: "$http_4k_version"
implementation group: "com.fasterxml.jackson.datatype", name: "jackson-datatype-jsr310", version: "2.11.1"
implementation group: "org.http4k", name: "http4k-client-apache", version: "$http_4k_version"
implementation group: "org.http4k", name: "http4k-server-netty", version: "$http_4k_version"
runtimeOnly group: "io.netty", name: "netty-tcnative-boringssl-static", version: "2.0.30.Final"
@ -42,8 +43,8 @@ dependencies {
implementation group: "org.xerial", name: "sqlite-jdbc", version: "3.30.1"
// implementation "com.goterl.lazycode:lazysodium-java:4.2.6"
// implementation "net.java.dev.jna:jna:5.5.0"
implementation "com.goterl.lazycode:lazysodium-java:4.2.6"
implementation "net.java.dev.jna:jna:5.5.0"
}
java {

4
gradle/wrapper/gradle-wrapper.properties vendored
View File

@ -1,5 +1,5 @@
#Wed May 27 21:24:59 CDT 2020
distributionUrl=https\://services.gradle.org/distributions/gradle-6.5.1-bin.zip
#Thu Jul 02 11:52:16 CDT 2020
distributionUrl=https\://services.gradle.org/distributions/gradle-6.5.1-all.zip
distributionBase=GRADLE_USER_HOME
distributionPath=wrapper/dists
zipStorePath=wrapper/dists

4
src/main/kotlin/mdnet/base/Constants.kt
View File

@ -18,9 +18,6 @@ along with this MangaDex@Home. If not, see <http://www.gnu.org/licenses/>.
*/
package mdnet.base
import com.fasterxml.jackson.core.JsonParser
import com.fasterxml.jackson.databind.ObjectMapper
import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper
import java.time.Duration
object Constants {
@ -28,7 +25,6 @@ object Constants {
const val CLIENT_VERSION = "1.0.0"
const val WEBUI_VERSION = "0.1.1"
val MAX_AGE_CACHE: Duration = Duration.ofDays(14)
val JACKSON: ObjectMapper = jacksonObjectMapper().configure(JsonParser.Feature.ALLOW_COMMENTS, true)
const val MAX_READ_TIME_SECONDS = 300
const val MAX_WRITE_TIME_SECONDS = 60

6
src/main/kotlin/mdnet/base/Main.kt
View File

@ -19,20 +19,24 @@ along with this MangaDex@Home. If not, see <http://www.gnu.org/licenses/>.
package mdnet.base
import ch.qos.logback.classic.LoggerContext
import com.fasterxml.jackson.core.JsonParser
import com.fasterxml.jackson.core.JsonProcessingException
import com.fasterxml.jackson.databind.ObjectMapper
import com.fasterxml.jackson.databind.SerializationFeature
import com.fasterxml.jackson.databind.exc.UnrecognizedPropertyException
import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper
import com.fasterxml.jackson.module.kotlin.readValue
import java.io.FileReader
import java.io.FileWriter
import java.io.IOException
import java.util.regex.Pattern
import kotlin.system.exitProcess
import mdnet.base.Constants.JACKSON
import mdnet.base.settings.ClientSettings
import org.slf4j.LoggerFactory
object Main {
private val LOGGER = LoggerFactory.getLogger(Main::class.java)
private val JACKSON: ObjectMapper = jacksonObjectMapper().enable(SerializationFeature.INDENT_OUTPUT).configure(JsonParser.Feature.ALLOW_COMMENTS, true)
@JvmStatic
fun main(args: Array<String>) {

4
src/main/kotlin/mdnet/base/MangaDexClient.kt
View File

@ -20,6 +20,8 @@ along with this MangaDex@Home. If not, see <http://www.gnu.org/licenses/>.
package mdnet.base
import ch.qos.logback.classic.LoggerContext
import com.fasterxml.jackson.databind.ObjectMapper
import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper
import com.fasterxml.jackson.module.kotlin.readValue
import java.io.File
import java.io.IOException
@ -30,7 +32,6 @@ import java.util.concurrent.Executors
import java.util.concurrent.TimeUnit
import java.util.concurrent.atomic.AtomicBoolean
import java.util.concurrent.atomic.AtomicReference
import mdnet.base.Constants.JACKSON
import mdnet.base.Main.dieWithError
import mdnet.base.data.Statistics
import mdnet.base.server.getServer
@ -320,5 +321,6 @@ class MangaDexClient(private val clientSettings: ClientSettings) {
companion object {
private val LOGGER = LoggerFactory.getLogger(MangaDexClient::class.java)
private val JACKSON: ObjectMapper = jacksonObjectMapper()
}
}

7
src/main/kotlin/mdnet/base/ServerHandler.kt
View File

@ -31,6 +31,7 @@ import org.http4k.format.ConfigurableJackson
import org.http4k.format.asConfigurable
import org.http4k.format.withStandardMappings
import org.slf4j.LoggerFactory
object ServerHandlerJackson : ConfigurableJackson(
KotlinModule()
.asConfigurable()
@ -86,7 +87,7 @@ class ServerHandler(private val settings: ClientSettings) {
val response = client(request)
return if (response.status.successful) {
SERVER_SETTINGS_LENS(response)
SERVER_SETTINGS_LENS(response).also { println(it) }
} else {
null
}
@ -108,7 +109,7 @@ class ServerHandler(private val settings: ClientSettings) {
}
private fun getServerAddress(): String {
return if (settings.devSettings == null || !settings.devSettings.isDev)
return if (settings.devSettings?.isDev != true)
SERVER_ADDRESS
else
SERVER_ADDRESS_DEV
@ -119,6 +120,6 @@ class ServerHandler(private val settings: ClientSettings) {
private val STRING_ANY_MAP_LENS = Body.auto<Map<String, Any>>().toLens()
private val SERVER_SETTINGS_LENS = Body.auto<ServerSettings>().toLens()
private const val SERVER_ADDRESS = "https://api.mangadex.network/"
private const val SERVER_ADDRESS_DEV = "https://mangadex-test.net/"
private const val SERVER_ADDRESS_DEV = "http://localhost:28080/"
}
}

8
src/main/kotlin/mdnet/base/data/Token.kt Normal file
View File

@ -0,0 +1,8 @@
package mdnet.base.data
import com.fasterxml.jackson.databind.PropertyNamingStrategy
import com.fasterxml.jackson.databind.annotation.JsonNaming
import java.time.OffsetDateTime
@JsonNaming(PropertyNamingStrategy.SnakeCaseStrategy::class)
data class Token(val expires: OffsetDateTime, val ip: String, val hash: String, val clientId: String)

21
src/main/kotlin/mdnet/base/server/Application.kt
View File

@ -39,7 +39,7 @@ private val LOGGER = LoggerFactory.getLogger("Application")
fun getServer(cache: DiskLruCache, serverSettings: ServerSettings, clientSettings: ClientSettings, statistics: AtomicReference<Statistics>, isHandled: AtomicBoolean): Http4kServer {
val database = Database.connect("jdbc:sqlite:cache/data.db", "org.sqlite.JDBC")
val imageServer = ImageServer(cache, statistics, serverSettings.imageServer, database, isHandled)
val imageServer = ImageServer(cache, statistics, serverSettings, database, isHandled)
return timeRequest()
.then(catchAllHideDetails())
@ -65,15 +65,24 @@ fun getServer(cache: DiskLruCache, serverSettings: ServerSettings, clientSetting
fun timeRequest(): Filter {
return Filter { next: HttpHandler ->
{ request: Request ->
val cleanedUri = request.uri.path.let {
if (it.startsWith("/data")) {
it
} else {
it.replaceBefore("/data", "/{token}")
}
}
if (LOGGER.isInfoEnabled) {
LOGGER.info("Request for $cleanedUri received from ${request.source?.address}")
}
val start = System.currentTimeMillis()
val response = next(request)
val latency = System.currentTimeMillis() - start
if (LOGGER.isTraceEnabled && response.header("X-Uri") != null) {
val sanitizedUri = response.header("X-Uri")
if (LOGGER.isInfoEnabled) {
LOGGER.info("Request for $sanitizedUri completed in ${latency}ms")
}
if (LOGGER.isInfoEnabled) {
LOGGER.info("Request for $cleanedUri completed (TTFB) in ${latency}ms")
}
response.header("X-Time-Taken", latency.toString())
}

169
src/main/kotlin/mdnet/base/server/ImageServer.kt
View File

@ -19,12 +19,22 @@ along with this MangaDex@Home. If not, see <http://www.gnu.org/licenses/>.
/* ktlint-disable no-wildcard-imports */
package mdnet.base.server
import com.fasterxml.jackson.databind.DeserializationFeature
import com.fasterxml.jackson.databind.ObjectMapper
import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule
import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper
import com.fasterxml.jackson.module.kotlin.readValue
import com.goterl.lazycode.lazysodium.LazySodiumJava
import com.goterl.lazycode.lazysodium.SodiumJava
import com.goterl.lazycode.lazysodium.exceptions.SodiumException
import java.io.BufferedInputStream
import java.io.BufferedOutputStream
import java.io.File
import java.io.InputStream
import java.security.MessageDigest
import java.time.Clock
import java.time.OffsetDateTime
import java.util.*
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicBoolean
import java.util.concurrent.atomic.AtomicReference
@ -36,6 +46,8 @@ import mdnet.base.Constants
import mdnet.base.data.ImageData
import mdnet.base.data.ImageDatum
import mdnet.base.data.Statistics
import mdnet.base.data.Token
import mdnet.base.settings.ServerSettings
import mdnet.cache.CachingInputStream
import mdnet.cache.DiskLruCache
import org.apache.http.client.config.CookieSpecs
@ -54,7 +66,7 @@ import org.slf4j.LoggerFactory
private const val THREADS_TO_ALLOCATE = 262144 // 2**18
class ImageServer(private val cache: DiskLruCache, private val statistics: AtomicReference<Statistics>, private val upstreamUrl: String, private val database: Database, private val handled: AtomicBoolean) {
class ImageServer(private val cache: DiskLruCache, private val statistics: AtomicReference<Statistics>, private val serverSettings: ServerSettings, private val database: Database, private val handled: AtomicBoolean) {
init {
transaction(database) {
SchemaUtils.create(ImageData)
@ -65,74 +77,102 @@ class ImageServer(private val cache: DiskLruCache, private val statistics: Atomi
.disableConnectionState()
.setDefaultRequestConfig(
RequestConfig.custom()
.setCookieSpec(CookieSpecs.IGNORE_COOKIES)
.setConnectTimeout(3000)
.setSocketTimeout(3000)
.setConnectionRequestTimeout(3000)
.build())
.setCookieSpec(CookieSpecs.IGNORE_COOKIES)
.setConnectTimeout(3000)
.setSocketTimeout(3000)
.setConnectionRequestTimeout(3000)
.build())
.setMaxConnTotal(THREADS_TO_ALLOCATE)
.setMaxConnPerRoute(THREADS_TO_ALLOCATE)
.build())
fun handler(dataSaver: Boolean, tokenized: Boolean = false): HttpHandler = baseHandler().then { request ->
val chapterHash = Path.of("chapterHash")(request)
val fileName = Path.of("fileName")(request)
fun handler(dataSaver: Boolean, tokenized: Boolean = false): HttpHandler {
val sodium = LazySodiumJava(SodiumJava())
val sanitizedUri = if (dataSaver) {
"/data-saver"
} else {
"/data"
} + "/$chapterHash/$fileName"
return baseHandler().then { request ->
val chapterHash = Path.of("chapterHash")(request)
val fileName = Path.of("fileName")(request)
if (LOGGER.isInfoEnabled) {
LOGGER.info("Request for $sanitizedUri received from ${request.source?.address}")
}
statistics.getAndUpdate {
it.copy(requestsServed = it.requestsServed + 1)
}
val sanitizedUri = if (dataSaver) {
"/data-saver"
} else {
"/data"
} + "/$chapterHash/$fileName"
val rc4Bytes = if (dataSaver) {
md5Bytes("saver$chapterHash.$fileName")
} else {
md5Bytes("$chapterHash.$fileName")
}
val imageId = printHexString(rc4Bytes)
val snapshot = cache.getUnsafe(imageId.toCacheId())
val imageDatum = synchronized(database) {
transaction(database) {
ImageDatum.findById(imageId)
}
}
handled.set(true)
if (request.header("Referer")?.startsWith("https://mangadex.org") == false) {
snapshot?.close()
Response(Status.FORBIDDEN)
} else if (snapshot != null && imageDatum != null) {
request.handleCacheHit(sanitizedUri, getRc4(rc4Bytes), snapshot, imageDatum)
.header("X-Uri", sanitizedUri)
} else {
if (snapshot != null) {
snapshot.close()
if (LOGGER.isWarnEnabled) {
LOGGER.warn("Removing cache file for $sanitizedUri without corresponding DB entry")
if (tokenized || serverSettings.forceToken) {
val tokenArr = Base64.getUrlDecoder().decode(Path.of("token")(request))
val token = JACKSON.readValue<Token>(
try {
sodium.cryptoBoxOpenEasyAfterNm(
tokenArr.sliceArray(24 until tokenArr.size), tokenArr.sliceArray(0 until 24), serverSettings.sharedKey
)
} catch (_: SodiumException) {
if (LOGGER.isInfoEnabled) {
LOGGER.info("Request for $sanitizedUri rejected for invalid token")
}
return@then Response(Status.FORBIDDEN)
}
)
if (OffsetDateTime.now().isAfter(token.expires)) {
if (LOGGER.isInfoEnabled) {
LOGGER.info("Request for $sanitizedUri rejected for expired token")
}
return@then Response(Status.GONE)
}
cache.removeUnsafe(imageId.toCacheId())
}
if (imageDatum != null) {
if (LOGGER.isWarnEnabled) {
LOGGER.warn("Deleting DB entry for $sanitizedUri without corresponding file")
if (token.hash != chapterHash) {
if (LOGGER.isInfoEnabled) {
LOGGER.info("Request for $sanitizedUri rejected for inapplicable token")
}
return@then Response(Status.FORBIDDEN)
}
synchronized(database) {
transaction(database) {
imageDatum.delete()
}
statistics.getAndUpdate {
it.copy(requestsServed = it.requestsServed + 1)
}
val rc4Bytes = if (dataSaver) {
md5Bytes("saver$chapterHash.$fileName")
} else {
md5Bytes("$chapterHash.$fileName")
}
val imageId = printHexString(rc4Bytes)
val snapshot = cache.getUnsafe(imageId.toCacheId())
val imageDatum = synchronized(database) {
transaction(database) {
ImageDatum.findById(imageId)
}
}
handled.set(true)
if (request.header("Referer")?.startsWith("https://mangadex.org") == false) {
snapshot?.close()
Response(Status.FORBIDDEN)
} else if (snapshot != null && imageDatum != null) {
request.handleCacheHit(sanitizedUri, getRc4(rc4Bytes), snapshot, imageDatum)
} else {
if (snapshot != null) {
snapshot.close()
if (LOGGER.isWarnEnabled) {
LOGGER.warn("Removing cache file for $sanitizedUri without corresponding DB entry")
}
cache.removeUnsafe(imageId.toCacheId())
}
if (imageDatum != null) {
if (LOGGER.isWarnEnabled) {
LOGGER.warn("Deleting DB entry for $sanitizedUri without corresponding file")
}
synchronized(database) {
transaction(database) {
imageDatum.delete()
}
}
}
}
request.handleCacheMiss(sanitizedUri, getRc4(rc4Bytes), imageId)
.header("X-Uri", sanitizedUri)
request.handleCacheMiss(sanitizedUri, getRc4(rc4Bytes), imageId)
}
}
}
@ -177,7 +217,7 @@ class ImageServer(private val cache: DiskLruCache, private val statistics: Atomi
it.copy(cacheMisses = it.cacheMisses + 1)
}
val mdResponse = client(Request(Method.GET, "$upstreamUrl$sanitizedUri"))
val mdResponse = client(Request(Method.GET, "${serverSettings.imageServer}$sanitizedUri"))
if (mdResponse.status != Status.OK) {
if (LOGGER.isTraceEnabled) {
@ -272,17 +312,20 @@ class ImageServer(private val cache: DiskLruCache, private val statistics: Atomi
companion object {
private val LOGGER = LoggerFactory.getLogger(ImageServer::class.java)
private val JACKSON: ObjectMapper = jacksonObjectMapper()
.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false)
.registerModule(JavaTimeModule())
private fun baseHandler(): Filter =
CachingFilters.Response.MaxAge(Clock.systemUTC(), Constants.MAX_AGE_CACHE)
.then(ServerFilters.Cors(
CorsPolicy(
origins = listOf("https://mangadex.org"),
headers = listOf("*"),
methods = Method.values().toList()
)
CorsPolicy(
origins = listOf("https://mangadex.org"),
headers = listOf("*"),
methods = Method.values().toList()
)
)
)
.then(Filter { next: HttpHandler ->
{ request: Request ->
val response = next(request)

42
src/main/kotlin/mdnet/base/server/naclbox.kt Normal file
View File

@ -0,0 +1,42 @@
/*
Mangadex@Home
Copyright (c) 2020, MangaDex Network
This file is part of MangaDex@Home.
MangaDex@Home is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
MangaDex@Home is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this MangaDex@Home. If not, see <http://www.gnu.org/licenses/>.
*/
/* ktlint-disable no-wildcard-imports */
package mdnet.base.server
import com.goterl.lazycode.lazysodium.LazySodiumJava
import com.goterl.lazycode.lazysodium.exceptions.SodiumException
import com.goterl.lazycode.lazysodium.interfaces.Box
@Throws(SodiumException::class)
fun LazySodiumJava.cryptoBoxOpenEasyAfterNm(cipherBytes: ByteArray, nonce: ByteArray, sharedKey: ByteArray): String {
if (!Box.Checker.checkNonce(nonce.size)) {
throw SodiumException("Incorrect nonce length.")
}
if (!Box.Checker.checkBeforeNmBytes(sharedKey.size)) {
throw SodiumException("Incorrect shared secret key length.")
}
val message = ByteArray(cipherBytes.size - Box.MACBYTES)
val res: Boolean = cryptoBoxOpenEasyAfterNm(message, cipherBytes, cipherBytes.size.toLong(), nonce, sharedKey)
if (!res) {
throw SodiumException("Could not fully complete shared secret key decryption.")
}
return str(message)
}

32
src/main/kotlin/mdnet/base/settings/ServerSettings.kt
View File

@ -27,10 +27,40 @@ data class ServerSettings(
val imageServer: String,
val latestBuild: Int,
val url: String,
val sharedKey: ByteArray,
val compromised: Boolean,
val paused: Boolean,
val forceToken: Boolean = false,
val tls: TlsCert?
)
) {
override fun equals(other: Any?): Boolean {
if (this === other) return true
if (javaClass != other?.javaClass) return false
other as ServerSettings
if (imageServer != other.imageServer) return false
if (latestBuild != other.latestBuild) return false
if (url != other.url) return false
if (!sharedKey.contentEquals(other.sharedKey)) return false
if (compromised != other.compromised) return false
if (paused != other.paused) return false
if (tls != other.tls) return false
return true
}
override fun hashCode(): Int {
var result = imageServer.hashCode()
result = 31 * result + latestBuild
result = 31 * result + url.hashCode()
result = 31 * result + sharedKey.contentHashCode()
result = 31 * result + compromised.hashCode()
result = 31 * result + paused.hashCode()
result = 31 * result + (tls?.hashCode() ?: 0)
return result
}
}
@JsonNaming(PropertyNamingStrategy.SnakeCaseStrategy::class)
data class TlsCert(