diff --git a/CHANGELOG.md b/CHANGELOG.md index e7cdf0e..de15bda 100755 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,15 +6,20 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] ### Added +- [2020-08-11] New CLI for specifying database location, cache folder, and settings [@carbotaniuman]. ### Changed --- [2020-08-11] Change logging defaults [@carbotaniuman]. +- [2020-08-11] Change logging defaults [@carbotaniuman]. ### Deprecated ### Removed ### Fixed +- [2020-08-11] Bugs relating to `settings.json` changes [@carbotaniuman]. +- [2020-08-11] Logs taking up an absurd amount of space [@carbotaniuman]. +- [2020-08-11] Random crashes for no reason [@carbotaniuman]. +- [2020-08-11] SQLException is noww properly handled [@carbotaniuman]. ### Security diff --git a/build.gradle b/build.gradle index 02cbcca..999f6e9 100644 --- a/build.gradle +++ b/build.gradle @@ -1,6 +1,7 @@ plugins { id "java" id "org.jetbrains.kotlin.jvm" version "1.3.72" + id "org.jetbrains.kotlin.kapt" version "1.3.72" id "application" id "com.github.johnrengelman.shadow" version "5.2.0" id "com.diffplug.gradle.spotless" version "4.4.0" @@ -45,6 +46,15 @@ dependencies { implementation "com.goterl.lazycode:lazysodium-java:4.3.0" implementation "net.java.dev.jna:jna:5.5.0" + + implementation "info.picocli:picocli:4.5.0" + kapt "info.picocli:picocli-codegen:4.5.0" +} + +kapt { + arguments { + arg("project", "${project.group}/${project.name}") + } } java { diff --git a/src/main/kotlin/mdnet/base/Main.kt b/src/main/kotlin/mdnet/base/Main.kt index 229730b..4b0ead3 100644 --- a/src/main/kotlin/mdnet/base/Main.kt +++ b/src/main/kotlin/mdnet/base/Main.kt @@ -19,15 +19,48 @@ along with this MangaDex@Home. If not, see . package mdnet.base import ch.qos.logback.classic.LoggerContext +import java.io.File import kotlin.system.exitProcess import mdnet.BuildInfo import org.slf4j.LoggerFactory +import picocli.CommandLine object Main { private val LOGGER = LoggerFactory.getLogger(Main::class.java) @JvmStatic fun main(args: Array) { + CommandLine(ClientArgs()).execute(*args) + } + + fun dieWithError(e: Throwable): Nothing { + LOGGER.error(e) { "Critical Error" } + (LoggerFactory.getILoggerFactory() as LoggerContext).stop() + exitProcess(1) + } + + fun dieWithError(error: String): Nothing { + LOGGER.error { "Critical Error: $error" } + + (LoggerFactory.getILoggerFactory() as LoggerContext).stop() + exitProcess(1) + } +} + +@CommandLine.Command(name = "java -jar ", usageHelpWidth = 120, version = ["Client Version ${BuildInfo.VERSION} (Build ${Constants.CLIENT_BUILD})"]) +data class ClientArgs( + @field:CommandLine.Option(names = ["-s", "--settings"], defaultValue = "settings.json", paramLabel = "", description = ["the settings file (default: \${DEFAULT-VALUE})"]) + var settingsFile: File = File("settings.json"), + @field:CommandLine.Option(names = ["-d", "--database"], defaultValue = "cache\${sys:file.separator}data.db", paramLabel = "", description = ["the database file (default: \${DEFAULT-VALUE})"]) + var databaseFile: File = File("cache${File.separator}data.db"), + @field:CommandLine.Option(names = ["-c", "--cache"], defaultValue = "cache", paramLabel = "", description = ["the cache folder (default: \${DEFAULT-VALUE})"]) + var cacheFolder: File = File("cache"), + @field:CommandLine.Option(names = ["-h", "--help"], usageHelp = true, description = ["show this help message and exit"]) + var helpRequested: Boolean = false, + @field:CommandLine.Option(names = ["-v", "--version"], versionHelp = true, description = ["show the version message and exit"]) + var versionRequested: Boolean = false +) : Runnable { + override fun run() { println( "Mangadex@Home Client Version ${BuildInfo.VERSION} (Build ${Constants.CLIENT_BUILD}) initializing" ) @@ -48,31 +81,11 @@ object Main { along with Mangadex@Home. If not, see . """.trimIndent()) - var file = "settings.json" - if (args.size == 1) { - file = args[0] - } else if (args.isNotEmpty()) { - dieWithError("Expected one argument: path to config file, or nothing") - } - - val client = MangaDexClient(file) + val client = MangaDexClient(settingsFile, databaseFile, cacheFolder) Runtime.getRuntime().addShutdownHook(Thread { client.shutdown() (LoggerFactory.getILoggerFactory() as LoggerContext).stop() }) client.runLoop() } - - fun dieWithError(e: Throwable): Nothing { - LOGGER.error(e) { "Critical Error" } - (LoggerFactory.getILoggerFactory() as LoggerContext).stop() - exitProcess(1) - } - - fun dieWithError(error: String): Nothing { - LOGGER.error { "Critical Error: $error" } - - (LoggerFactory.getILoggerFactory() as LoggerContext).stop() - exitProcess(1) - } } diff --git a/src/main/kotlin/mdnet/base/MangaDexClient.kt b/src/main/kotlin/mdnet/base/MangaDexClient.kt index 577630d..27aa096 100644 --- a/src/main/kotlin/mdnet/base/MangaDexClient.kt +++ b/src/main/kotlin/mdnet/base/MangaDexClient.kt @@ -38,14 +38,16 @@ import mdnet.base.settings.* import mdnet.cache.DiskLruCache import mdnet.cache.HeaderMismatchException import org.http4k.server.Http4kServer +import org.jetbrains.exposed.sql.Database import org.slf4j.LoggerFactory // Exception class to handle when Client Settings have invalid values class ClientSettingsException(message: String) : Exception(message) -class MangaDexClient(private val clientSettingsFile: String) { +class MangaDexClient(private val settingsFile: File, databaseFile: File, cacheFolder: File) { // just for scheduling one task, so single-threaded private val executor = Executors.newSingleThreadScheduledExecutor() + private val database: Database private val cache: DiskLruCache private var settings: ClientSettings @@ -65,9 +67,13 @@ class MangaDexClient(private val clientSettingsFile: String) { dieWithError(e) } + LOGGER.info { "Client settings loaded: $settings" } + + database = Database.connect("jdbc:sqlite:$databaseFile", "org.sqlite.JDBC") + try { cache = DiskLruCache.open( - File("cache"), 1, 1, + cacheFolder, 1, 1, (settings.maxCacheSizeInMebibytes * 1024 * 1024 * 0.8).toLong() /* MiB to bytes */ ) } catch (e: HeaderMismatchException) { @@ -88,6 +94,9 @@ class MangaDexClient(private val clientSettingsFile: String) { LOGGER.warn(e) { "Reload of ClientSettings failed" } } }, 1, 1, TimeUnit.MINUTES) + + startImageServer() + startWebUi() } // Precondition: settings must be filled with up-to-date settings and `imageServer` must not be null @@ -96,37 +105,53 @@ class MangaDexClient(private val clientSettingsFile: String) { val imageServer = requireNotNull(imageServer) if (webUi != null) throw AssertionError() + LOGGER.info { "WebUI starting" } webUi = getUiServer(webSettings, imageServer.statistics, imageServer.statsMap).also { it.start() } - LOGGER.info { "WebUI was successfully started" } + LOGGER.info { "WebUI started" } } } // Precondition: settings must be filled with up-to-date settings private fun startImageServer() { if (imageServer != null) throw AssertionError() - imageServer = ServerManager(settings.serverSettings, settings.devSettings, settings.maxCacheSizeInMebibytes, cache).also { + LOGGER.info { "Server manager starting" } + imageServer = ServerManager(settings.serverSettings, settings.devSettings, settings.maxCacheSizeInMebibytes, cache, database).also { it.start() } - LOGGER.info { "Server manager was successfully started" } + LOGGER.info { "Server manager started" } } private fun stopImageServer() { + LOGGER.info { "Server manager stopping" } requireNotNull(imageServer).shutdown() - LOGGER.info { "Server manager was successfully stopped" } + imageServer = null + LOGGER.info { "Server manager stopped" } } private fun stopWebUi() { + LOGGER.info { "WebUI stopping" } requireNotNull(webUi).stop() - LOGGER.info { "Server manager was successfully stopped" } + webUi = null + LOGGER.info { "WebUI stopped" } } fun shutdown() { LOGGER.info { "Mangadex@Home Client shutting down" } - stopWebUi() - stopImageServer() + if (webUi != null) { + stopWebUi() + } + if (imageServer != null) { + stopImageServer() + } LOGGER.info { "Mangadex@Home Client has shut down" } + + try { + cache.close() + } catch (e: IOException) { + LOGGER.error(e) { "Cache failed to close" } + } } /** @@ -142,6 +167,7 @@ class MangaDexClient(private val clientSettingsFile: String) { LOGGER.info { "Client settings unchanged" } return } + LOGGER.info { "New settings loaded: $newSettings" } cache.maxSize = (newSettings.maxCacheSizeInMebibytes * 1024 * 1024 * 0.8).toLong() @@ -215,7 +241,7 @@ class MangaDexClient(private val clientSettingsFile: String) { } private fun readClientSettings(): ClientSettings { - return JACKSON.readValue(FileReader(clientSettingsFile)).apply(::validateSettings) + return JACKSON.readValue(FileReader(settingsFile)).apply(::validateSettings) } companion object { diff --git a/src/main/kotlin/mdnet/base/ServerManager.kt b/src/main/kotlin/mdnet/base/ServerManager.kt index c054d8a..9b6a789 100644 --- a/src/main/kotlin/mdnet/base/ServerManager.kt +++ b/src/main/kotlin/mdnet/base/ServerManager.kt @@ -5,7 +5,6 @@ import com.fasterxml.jackson.databind.ObjectMapper import com.fasterxml.jackson.databind.SerializationFeature import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper import com.fasterxml.jackson.module.kotlin.readValue -import java.io.IOException import java.time.Instant import java.util.Collections import java.util.LinkedHashMap @@ -21,6 +20,7 @@ import mdnet.base.settings.RemoteSettings import mdnet.base.settings.ServerSettings import mdnet.cache.DiskLruCache import org.http4k.server.Http4kServer +import org.jetbrains.exposed.sql.Database import org.slf4j.LoggerFactory sealed class State @@ -33,7 +33,7 @@ data class GracefulStop(val lastRunning: Running, val counts: Int = 0, val nextS // server is currently running data class Running(val server: Http4kServer, val settings: RemoteSettings, val serverSettings: ServerSettings, val devSettings: DevSettings) : State() -class ServerManager(serverSettings: ServerSettings, devSettings: DevSettings, maxCacheSizeInMebibytes: Long, private val cache: DiskLruCache) { +class ServerManager(serverSettings: ServerSettings, devSettings: DevSettings, maxCacheSizeInMebibytes: Long, private val cache: DiskLruCache, private val database: Database) { // this must remain single-threaded because of how the state mechanism works private val executor = Executors.newSingleThreadScheduledExecutor() @@ -68,6 +68,7 @@ class ServerManager(serverSettings: ServerSettings, devSettings: DevSettings, ma } fun start() { + LOGGER.info { "Image server starting" } loginAndStartServer() statsMap[Instant.now()] = statistics.get() @@ -165,6 +166,8 @@ class ServerManager(serverSettings: ServerSettings, devSettings: DevSettings, ma LOGGER.warn(e) { "Graceful shutdown checker failed" } } }, 45, 45, TimeUnit.SECONDS) + + LOGGER.info { "Image server has started" } } private fun pingControl() { @@ -197,7 +200,7 @@ class ServerManager(serverSettings: ServerSettings, devSettings: DevSettings, ma val remoteSettings = serverHandler.loginToControl() ?: Main.dieWithError("Failed to get a login response from server - check API secret for validity") - val server = getServer(cache, remoteSettings, state.serverSettings, statistics, isHandled).start() + val server = getServer(cache, database, remoteSettings, state.serverSettings, statistics, isHandled).start() if (remoteSettings.latestBuild > Constants.CLIENT_BUILD) { LOGGER.warn { @@ -253,12 +256,6 @@ class ServerManager(serverSettings: ServerSettings, devSettings: DevSettings, ma }, 0, TimeUnit.SECONDS) latch.await() - try { - cache.close() - } catch (e: IOException) { - LOGGER.error(e) { "Cache failed to close" } - } - executor.shutdown() LOGGER.info { "Image server has shut down" } } diff --git a/src/main/kotlin/mdnet/base/netty/ApplicationNetty.kt b/src/main/kotlin/mdnet/base/netty/ApplicationNetty.kt index b2ccd35..a51c83c 100644 --- a/src/main/kotlin/mdnet/base/netty/ApplicationNetty.kt +++ b/src/main/kotlin/mdnet/base/netty/ApplicationNetty.kt @@ -57,9 +57,9 @@ import org.http4k.server.Http4kServer import org.http4k.server.ServerConfig import org.slf4j.LoggerFactory -private val LOGGER = LoggerFactory.getLogger("Application") +private val LOGGER = LoggerFactory.getLogger("AppNetty") -class Netty(private val tls: TlsCert, internal val serverSettings: ServerSettings, private val statistics: AtomicReference) : ServerConfig { +class Netty(private val tls: TlsCert, private val serverSettings: ServerSettings, private val statistics: AtomicReference) : ServerConfig { override fun toServer(httpHandler: HttpHandler): Http4kServer = object : Http4kServer { private val masterGroup = NioEventLoopGroup(serverSettings.threads) private val workerGroup = NioEventLoopGroup(serverSettings.threads) diff --git a/src/main/kotlin/mdnet/base/netty/Keys.kt b/src/main/kotlin/mdnet/base/netty/Keys.kt index a2a89c2..bf2f82b 100644 --- a/src/main/kotlin/mdnet/base/netty/Keys.kt +++ b/src/main/kotlin/mdnet/base/netty/Keys.kt @@ -32,7 +32,7 @@ private const val PKCS_1_PEM_FOOTER = "-----END RSA PRIVATE KEY-----" private const val PKCS_8_PEM_HEADER = "-----BEGIN PRIVATE KEY-----" private const val PKCS_8_PEM_FOOTER = "-----END PRIVATE KEY-----" -internal fun loadKey(keyDataString: String): PrivateKey? { +fun loadKey(keyDataString: String): PrivateKey? { if (keyDataString.contains(PKCS_1_PEM_HEADER)) { val fixedString = keyDataString.replace(PKCS_1_PEM_HEADER, "").replace( PKCS_1_PEM_FOOTER, "") diff --git a/src/main/kotlin/mdnet/base/netty/WebUiNetty.kt b/src/main/kotlin/mdnet/base/netty/WebUiNetty.kt index 1bf98f3..caa67d5 100644 --- a/src/main/kotlin/mdnet/base/netty/WebUiNetty.kt +++ b/src/main/kotlin/mdnet/base/netty/WebUiNetty.kt @@ -32,7 +32,6 @@ import io.netty.handler.codec.http.HttpServerCodec import io.netty.handler.codec.http.HttpServerKeepAliveHandler import io.netty.handler.stream.ChunkedWriteHandler import java.net.InetSocketAddress -import java.util.concurrent.TimeUnit import org.http4k.core.HttpHandler import org.http4k.server.Http4kChannelHandler import org.http4k.server.Http4kServer @@ -67,9 +66,9 @@ class WebUiNetty(private val hostname: String, private val port: Int) : ServerCo } override fun stop() = apply { - masterGroup.shutdownGracefully(5, 15, TimeUnit.SECONDS).sync() - workerGroup.shutdownGracefully(5, 15, TimeUnit.SECONDS).sync() - closeFuture.sync() + closeFuture.cancel(false) + workerGroup.shutdownGracefully() + masterGroup.shutdownGracefully() } override fun port(): Int = address.port diff --git a/src/main/kotlin/mdnet/base/server/ImageServer.kt b/src/main/kotlin/mdnet/base/server/ImageServer.kt index bed3b2d..e3a455b 100644 --- a/src/main/kotlin/mdnet/base/server/ImageServer.kt +++ b/src/main/kotlin/mdnet/base/server/ImageServer.kt @@ -66,6 +66,7 @@ import org.http4k.routing.bind import org.http4k.routing.routes import org.http4k.server.Http4kServer import org.http4k.server.asServer +import org.jetbrains.exposed.exceptions.ExposedSQLException import org.jetbrains.exposed.sql.Database import org.jetbrains.exposed.sql.SchemaUtils import org.jetbrains.exposed.sql.transactions.transaction @@ -250,13 +251,20 @@ class ImageServer( LOGGER.trace { "Request for $sanitizedUri is being cached and served" } if (imageDatum == null) { - synchronized(database) { - transaction(database) { - ImageDatum.new(imageId) { - this.contentType = contentType - this.lastModified = lastModified + try { + synchronized(database) { + transaction(database) { + ImageDatum.new(imageId) { + this.contentType = contentType + this.lastModified = lastModified + } } } + } catch (_: ExposedSQLException) { + // some other code got to the database first, fall back to just serving + editor.abort() + LOGGER.trace { "Request for $sanitizedUri is being served" } + respondWithImage(mdResponse.body.stream, contentLength, contentType, lastModified, false) } } @@ -331,8 +339,7 @@ class ImageServer( private fun String.isImageMimetype() = this.toLowerCase().startsWith("image/") -fun getServer(cache: DiskLruCache, remoteSettings: RemoteSettings, serverSettings: ServerSettings, statistics: AtomicReference, isHandled: AtomicBoolean): Http4kServer { - val database = Database.connect("jdbc:sqlite:cache/data.db", "org.sqlite.JDBC") +fun getServer(cache: DiskLruCache, database: Database, remoteSettings: RemoteSettings, serverSettings: ServerSettings, statistics: AtomicReference, isHandled: AtomicBoolean): Http4kServer { val client = ApacheClient(responseBodyMode = BodyMode.Stream, client = HttpClients.custom() .disableConnectionState() .setDefaultRequestConfig( diff --git a/src/main/kotlin/mdnet/base/server/naclbox.kt b/src/main/kotlin/mdnet/base/server/naclbox.kt index c81d071..991de4d 100644 --- a/src/main/kotlin/mdnet/base/server/naclbox.kt +++ b/src/main/kotlin/mdnet/base/server/naclbox.kt @@ -27,7 +27,7 @@ import javax.crypto.Cipher import javax.crypto.spec.SecretKeySpec @Throws(SodiumException::class) -internal fun LazySodiumJava.cryptoBoxOpenEasyAfterNm(cipherBytes: ByteArray, nonce: ByteArray, sharedKey: ByteArray): String { +fun LazySodiumJava.cryptoBoxOpenEasyAfterNm(cipherBytes: ByteArray, nonce: ByteArray, sharedKey: ByteArray): String { if (!Box.Checker.checkNonce(nonce.size)) { throw SodiumException("Incorrect nonce length.") } @@ -44,18 +44,18 @@ internal fun LazySodiumJava.cryptoBoxOpenEasyAfterNm(cipherBytes: ByteArray, non return str(message) } -internal fun getRc4(key: ByteArray): Cipher { +fun getRc4(key: ByteArray): Cipher { val rc4 = Cipher.getInstance("RC4") rc4.init(Cipher.ENCRYPT_MODE, SecretKeySpec(key, "RC4")) return rc4 } -internal fun md5Bytes(stringToHash: String): ByteArray { +fun md5Bytes(stringToHash: String): ByteArray { val digest = MessageDigest.getInstance("MD5") return digest.digest(stringToHash.toByteArray()) } -internal fun printHexString(bytes: ByteArray): String { +fun printHexString(bytes: ByteArray): String { val sb = StringBuilder() for (b in bytes) { sb.append(String.format("%02x", b)) diff --git a/src/main/kotlin/mdnet/base/settings/ClientSettings.kt b/src/main/kotlin/mdnet/base/settings/ClientSettings.kt index 7d8258f..699f2fd 100644 --- a/src/main/kotlin/mdnet/base/settings/ClientSettings.kt +++ b/src/main/kotlin/mdnet/base/settings/ClientSettings.kt @@ -24,13 +24,43 @@ import com.fasterxml.jackson.databind.annotation.JsonNaming import dev.afanasev.sekret.Secret @JsonNaming(PropertyNamingStrategy.SnakeCaseStrategy::class) -data class ClientSettings( +class ClientSettings( val maxCacheSizeInMebibytes: Long = 20480, - @JsonUnwrapped - val serverSettings: ServerSettings = ServerSettings(), val webSettings: WebSettings? = null, val devSettings: DevSettings = DevSettings(isDev = false) -) +) { + // FIXME: jackson doesn't work with data classes and JsonUnwrapped + // fix this in 2.0 when we can break the settings file + // and remove the `@JsonUnwrapped` + @field:JsonUnwrapped + lateinit var serverSettings: ServerSettings + + override fun equals(other: Any?): Boolean { + if (this === other) return true + if (javaClass != other?.javaClass) return false + + other as ClientSettings + + if (maxCacheSizeInMebibytes != other.maxCacheSizeInMebibytes) return false + if (webSettings != other.webSettings) return false + if (devSettings != other.devSettings) return false + if (serverSettings != other.serverSettings) return false + + return true + } + + override fun hashCode(): Int { + var result = maxCacheSizeInMebibytes.hashCode() + result = 31 * result + (webSettings?.hashCode() ?: 0) + result = 31 * result + devSettings.hashCode() + result = 31 * result + serverSettings.hashCode() + return result + } + + override fun toString(): String { + return "ClientSettings(maxCacheSizeInMebibytes=$maxCacheSizeInMebibytes, webSettings=$webSettings, devSettings=$devSettings, serverSettings=$serverSettings)" + } +} @JsonNaming(PropertyNamingStrategy.SnakeCaseStrategy::class) data class ServerSettings( diff --git a/src/main/resources/logback.xml b/src/main/resources/logback.xml index 04a9f21..dd4fccc 100644 --- a/src/main/resources/logback.xml +++ b/src/main/resources/logback.xml @@ -5,12 +5,12 @@ log/latest.log - - log/logFile.%d{yyyy-MM-dd_HH}.log + + log/logFile.%d{yyyy-MM-dd_HH}.%i.log 12 100MB 1GB - + --> %d{YYYY-MM-dd HH:mm:ss} %-5level %logger{36} - %msg%n @@ -37,6 +37,6 @@ -