diff --git a/CHANGELOG.md b/CHANGELOG.md index a6bb66c..93a7cf5 100755 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [2.0.0-rc6] - 2021-01-27 ### Fixed - [2021-01-27] Upped max threadpool size [@carbotaniuman]. +- [2021-01-27] Switch to OkHttp and hopefully fix a class of bugs [@carbotaniuman]. +- [2021-01-27] Add ability for nodes to specify external ip [@carbotaniuman]. ## [2.0.0-rc5] - 2021-01-27 ### Changed diff --git a/build.gradle b/build.gradle index edba11c..11836ee 100644 --- a/build.gradle +++ b/build.gradle @@ -36,7 +36,7 @@ dependencies { implementation group: "org.http4k", name: "http4k-format-jackson", version: "$http_4k_version" implementation group: "com.fasterxml.jackson.dataformat", name: "jackson-dataformat-yaml", version: "2.12.1" implementation group: "com.fasterxml.jackson.datatype", name: "jackson-datatype-jsr310", version: "2.12.1" - implementation group: "org.http4k", name: "http4k-client-apache", version: "$http_4k_version" + implementation group: "org.http4k", name: "http4k-client-okhttp", version: "$http_4k_version" implementation group: "org.http4k", name: "http4k-metrics-micrometer", version: "$http_4k_version" implementation group: "org.http4k", name: "http4k-server-netty", version: "$http_4k_version" implementation group: "io.netty", name: "netty-transport-native-epoll", version: "4.1.58.Final", classifier: "linux-x86_64" diff --git a/src/main/kotlin/mdnet/BackendApi.kt b/src/main/kotlin/mdnet/BackendApi.kt index 6d30b02..35e73f3 100644 --- a/src/main/kotlin/mdnet/BackendApi.kt +++ b/src/main/kotlin/mdnet/BackendApi.kt @@ -23,13 +23,10 @@ import com.fasterxml.jackson.module.kotlin.KotlinModule import mdnet.ServerHandlerJackson.auto import mdnet.logging.info import mdnet.settings.ClientSettings +import mdnet.settings.LogoutRequest import mdnet.settings.RemoteSettings -import org.apache.hc.client5.http.impl.DefaultSchemePortResolver -import org.apache.hc.client5.http.impl.classic.HttpClients -import org.apache.hc.client5.http.impl.routing.DefaultRoutePlanner -import org.apache.hc.core5.http.HttpHost -import org.apache.hc.core5.http.protocol.HttpContext -import org.http4k.client.ApacheClient +import mdnet.settings.SettingsRequest +import org.http4k.client.OkHttp import org.http4k.core.Body import org.http4k.core.Method import org.http4k.core.Request @@ -37,7 +34,7 @@ import org.http4k.format.ConfigurableJackson import org.http4k.format.asConfigurable import org.http4k.format.withStandardMappings import org.slf4j.LoggerFactory -import java.net.InetAddress +import java.lang.RuntimeException object ServerHandlerJackson : ConfigurableJackson( KotlinModule() @@ -49,57 +46,44 @@ object ServerHandlerJackson : ConfigurableJackson( class BackendApi(private val settings: ClientSettings) { private val serverAddress = settings.devSettings.devUrl ?: SERVER_ADDRESS - private val client = ApacheClient( - client = HttpClients.custom() - .setRoutePlanner( - object : DefaultRoutePlanner(DefaultSchemePortResolver()) { - override fun determineLocalAddress(firstHop: HttpHost?, context: HttpContext?): InetAddress { - return InetAddress.getByName(settings.serverSettings.hostname) - } - } - ) - .build() - ) + private val client = OkHttp() fun logoutFromControl(): Boolean { + val serverSettings = settings.serverSettings + LOGGER.info { "Disconnecting from the control server" } - val params = mapOf( - "secret" to settings.serverSettings.secret + + val request = LOGOUT_REQUEST_LENS( + LogoutRequest(serverSettings.secret), + Request(Method.POST, serverAddress + "stop") ) - val request = STRING_ANY_MAP_LENS(params, Request(Method.POST, serverAddress + "stop")) val response = client(request) return response.status.successful } - private fun getPingParams(tlsCreatedAt: String? = null): Map { + private fun getPingParams(tlsCreatedAt: String? = null): SettingsRequest { val serverSettings = settings.serverSettings - return mapOf( - "secret" to serverSettings.secret, - "port" to let { - if (serverSettings.externalPort != 0) { - serverSettings.externalPort - } else { - serverSettings.port - } - }, - "disk_space" to settings.maxCacheSizeInMebibytes * 1024 * 1024, - "network_speed" to serverSettings.externalMaxKilobitsPerSecond * 1000 / 8, - "build_version" to Constants.CLIENT_BUILD - ).let { - if (tlsCreatedAt != null) { - it.plus("tls_created_at" to tlsCreatedAt) + return SettingsRequest( + secret = serverSettings.secret, + port = if (serverSettings.externalPort != 0) { + serverSettings.externalPort } else { - it - } - } + serverSettings.port + }, + buildVersion = Constants.CLIENT_BUILD, + diskSpace = settings.maxCacheSizeInMebibytes * 1024 * 1024, + networkSpeed = serverSettings.externalMaxKilobitsPerSecond * 1000 / 8, + ipAddress = serverSettings.externalIp, + tlsCreatedAt = tlsCreatedAt, + ) } - fun loginToControl(): RemoteSettings? { + fun loginToControl(): RemoteSettings { LOGGER.info { "Connecting to the control server" } - val request = STRING_ANY_MAP_LENS( + val request = SETTINGS_REQUEST_LENS( getPingParams(null), Request( Method.POST, @@ -111,14 +95,14 @@ class BackendApi(private val settings: ClientSettings) { return if (response.status.successful) { SERVER_SETTINGS_LENS(response) } else { - null + throw RuntimeException(response.bodyString()) } } fun pingControl(old: RemoteSettings): RemoteSettings? { LOGGER.info { "Pinging the control server" } - val request = STRING_ANY_MAP_LENS( + val request = SETTINGS_REQUEST_LENS( getPingParams(old.tls!!.createdAt), Request( Method.POST, @@ -136,7 +120,8 @@ class BackendApi(private val settings: ClientSettings) { companion object { private val LOGGER = LoggerFactory.getLogger(BackendApi::class.java) - private val STRING_ANY_MAP_LENS = Body.auto>().toLens() + private val SETTINGS_REQUEST_LENS = Body.auto().toLens() + private val LOGOUT_REQUEST_LENS = Body.auto().toLens() private val SERVER_SETTINGS_LENS = Body.auto().toLens() private const val SERVER_ADDRESS = "https://api.mangadex.network/" } diff --git a/src/main/kotlin/mdnet/ServerManager.kt b/src/main/kotlin/mdnet/ServerManager.kt index 4faef83..1a749e9 100644 --- a/src/main/kotlin/mdnet/ServerManager.kt +++ b/src/main/kotlin/mdnet/ServerManager.kt @@ -209,7 +209,6 @@ class ServerManager( this.state as Uninitialized val remoteSettings = backendApi.loginToControl() - ?: throw RuntimeException("Failed to get a login response from server") LOGGER.info { "Server settings received: $remoteSettings" } warnBasedOnSettings(remoteSettings) diff --git a/src/main/kotlin/mdnet/metrics/GeoIpMetricsFilter.kt b/src/main/kotlin/mdnet/metrics/GeoIpMetricsFilter.kt index a72324c..09ef17d 100644 --- a/src/main/kotlin/mdnet/metrics/GeoIpMetricsFilter.kt +++ b/src/main/kotlin/mdnet/metrics/GeoIpMetricsFilter.kt @@ -26,6 +26,7 @@ import mdnet.logging.debug import mdnet.logging.warn import org.apache.commons.compress.archivers.tar.TarArchiveInputStream import org.apache.commons.io.IOUtils +import org.http4k.client.OkHttp import org.http4k.core.Filter import org.http4k.core.HttpHandler import org.http4k.core.Method @@ -90,7 +91,6 @@ class GeoIpMetricsFilterBuilder( private val enableGeoIp: Boolean, private val license: String, private val registry: PrometheusMeterRegistry, - private val client: HttpHandler ) { fun build(): GeoIpMetricsFilter { return if (enableGeoIp) { @@ -108,6 +108,8 @@ class GeoIpMetricsFilterBuilder( val databaseFile = Files.createTempFile(databaseFileDir, "geoip2_country", ".mmdb") val geoIpDatabaseUri = GEOIP2_COUNTRY_URI_FORMAT.format(license) + + val client = OkHttp() val response = client(Request(Method.GET, geoIpDatabaseUri)) if (response.status != Status.OK) { throw IllegalStateException("Couldn't download GeoIP 2 database (http status: ${response.status})") diff --git a/src/main/kotlin/mdnet/server/ImageServer.kt b/src/main/kotlin/mdnet/server/ImageServer.kt index d4d95e7..ce0a8df 100644 --- a/src/main/kotlin/mdnet/server/ImageServer.kt +++ b/src/main/kotlin/mdnet/server/ImageServer.kt @@ -44,12 +44,8 @@ import mdnet.security.TweetNaclFast import mdnet.settings.MetricsSettings import mdnet.settings.RemoteSettings import mdnet.settings.ServerSettings -import org.apache.hc.client5.http.config.RequestConfig -import org.apache.hc.client5.http.cookie.StandardCookieSpec -import org.apache.hc.client5.http.impl.classic.HttpClients -import org.apache.hc.client5.http.impl.io.PoolingHttpClientConnectionManagerBuilder -import org.apache.hc.core5.util.Timeout -import org.http4k.client.ApacheClient +import okhttp3.OkHttpClient +import org.http4k.client.OkHttp import org.http4k.core.* import org.http4k.filter.CachingFilters import org.http4k.filter.ClientFilters @@ -66,6 +62,7 @@ import java.io.BufferedInputStream import java.io.BufferedOutputStream import java.io.InputStream import java.time.Clock +import java.time.Duration import java.time.OffsetDateTime import java.util.* import java.util.concurrent.Executors @@ -235,31 +232,20 @@ fun getServer( statistics: Statistics, registry: PrometheusMeterRegistry, ): Http4kServer { - val apache = ApacheClient( - responseBodyMode = BodyMode.Stream, - client = HttpClients.custom() - .disableConnectionState() - .setDefaultRequestConfig( - RequestConfig.custom() - .setCookieSpec(StandardCookieSpec.IGNORE) - .setConnectTimeout(Timeout.ofSeconds(2)) - .setResponseTimeout(Timeout.ofSeconds(2)) - .setConnectionRequestTimeout(Timeout.ofSeconds(1)) - .build() - ) - .setConnectionManager( - PoolingHttpClientConnectionManagerBuilder.create() - .setMaxConnTotal(500) - .setMaxConnPerRoute(500) - .build() - ) + val okHttpClient = OkHttp( + bodyMode = BodyMode.Stream, + client = OkHttpClient.Builder() + .followRedirects(false) + .connectTimeout(Duration.ofSeconds(2)) + .readTimeout(Duration.ofSeconds(5)) + .writeTimeout(Duration.ofSeconds(5)) .build() ) val client = ClientFilters.SetBaseUriFrom(remoteSettings.imageServer) .then(ClientFilters.MicrometerMetrics.RequestTimer(registry)) - .then(apache) + .then(okHttpClient) val imageServer = ImageServer( storage = storage, @@ -311,7 +297,7 @@ fun getServer( ).withFilter( ServerFilters.MicrometerMetrics.RequestTimer(registry, labeler = PostTransactionLabeler()) ).withFilter( - GeoIpMetricsFilterBuilder(metricsSettings.enableGeoip, metricsSettings.geoipLicenseKey, registry, apache).build() + GeoIpMetricsFilterBuilder(metricsSettings.enableGeoip, metricsSettings.geoipLicenseKey, registry).build() ) ) .asServer(Netty(remoteSettings.tls!!, serverSettings, statistics)) diff --git a/src/main/kotlin/mdnet/settings/ClientSettings.kt b/src/main/kotlin/mdnet/settings/ClientSettings.kt index ffb7701..1e9ae4b 100644 --- a/src/main/kotlin/mdnet/settings/ClientSettings.kt +++ b/src/main/kotlin/mdnet/settings/ClientSettings.kt @@ -35,6 +35,7 @@ data class ClientSettings( data class ServerSettings( @field:Secret val secret: String, val externalPort: Int = 0, + val externalIp: String? = null, val gracefulShutdownWaitSeconds: Int = 60, val hostname: String = "0.0.0.0", val maxKilobitsPerSecond: Long = 0, diff --git a/src/main/kotlin/mdnet/settings/SettingsRequest.kt b/src/main/kotlin/mdnet/settings/SettingsRequest.kt new file mode 100644 index 0000000..9a88046 --- /dev/null +++ b/src/main/kotlin/mdnet/settings/SettingsRequest.kt @@ -0,0 +1,39 @@ +/* +Mangadex@Home +Copyright (c) 2020, MangaDex Network +This file is part of MangaDex@Home. + +MangaDex@Home is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +MangaDex@Home is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this MangaDex@Home. If not, see . +*/ +package mdnet.settings + +import com.fasterxml.jackson.databind.PropertyNamingStrategies +import com.fasterxml.jackson.databind.annotation.JsonNaming +import dev.afanasev.sekret.Secret + +@JsonNaming(PropertyNamingStrategies.SnakeCaseStrategy::class) +data class SettingsRequest( + @field:Secret val secret: String, + val ipAddress: String?, + val port: Int, + val diskSpace: Long, + val networkSpeed: Long, + val buildVersion: Int, + val tlsCreatedAt: String?, +) + +@JsonNaming(PropertyNamingStrategies.SnakeCaseStrategy::class) +data class LogoutRequest( + @field:Secret val secret: String, +)