mirror of
https://gitlab.com/mangadex-pub/mangadex_at_home.git
synced 2024-11-16 16:12:32 +01:00
Merge branch 'fix-tls-and-crashes' into 'master'
Fix tls and crashes See merge request mangadex-pub/mangadex_at_home!99
This commit is contained in:
carbotaniuman
commit
85dce402df
10
CHANGELOG.md
10
CHANGELOG.md
@ -17,6 +17,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
||||
|
||||
### Security
|
||||
|
||||
## [2.0.2] - 2022-02-16
|
||||
### Removed
|
||||
- [2022-02-16] Remove TLS 1.0 and 1.1 support [@carbotaniuman].
|
||||
|
||||
### Fixed
|
||||
- [2022-02-16] Fix uncatched exceptions killing threads and not being logged [@carbotaniuman].
|
||||
|
||||
## [2.0.1] - 2021-05-27
|
||||
### Added
|
||||
- [2021-05-27] Added SNI check to prevent people from simply scanning nodes [@carbotaniuman].
|
||||
@ -396,7 +403,8 @@ This release contains many breaking changes! Of note are the changes to the cach
|
||||
### Fixed
|
||||
- [2020-06-11] Tweaked logging configuration to reduce log file sizes by [@carbotaniuman].
|
||||
|
||||
[Unreleased]: https://gitlab.com/mangadex/mangadex_at_home/-/compare/2.0.0...HEAD
|
||||
[Unreleased]: https://gitlab.com/mangadex/mangadex_at_home/-/compare/2.0.2...HEAD
|
||||
[2.0.2]: https://gitlab.com/mangadex/mangadex_at_home/-/compare/2.0.1...2.0.2
|
||||
[2.0.1]: https://gitlab.com/mangadex/mangadex_at_home/-/compare/2.0.0...2.0.1
|
||||
[2.0.0]: https://gitlab.com/mangadex/mangadex_at_home/-/compare/2.0.0-rc14...2.0.0
|
||||
[2.0.0-rc14]: https://gitlab.com/mangadex/mangadex_at_home/-/compare/2.0.0-rc13...2.0.0-rc14
|
||||
|
||||
@ -24,7 +24,6 @@ import io.micrometer.prometheus.PrometheusConfig
|
||||
import io.micrometer.prometheus.PrometheusMeterRegistry
|
||||
import mdnet.cache.ImageStorage
|
||||
import mdnet.data.Statistics
|
||||
import mdnet.logging.error
|
||||
import mdnet.logging.info
|
||||
import mdnet.logging.warn
|
||||
import mdnet.metrics.DefaultMicrometerMetrics
|
||||
@ -188,7 +187,7 @@ class ServerManager(
|
||||
}
|
||||
}
|
||||
} catch (e: Exception) {
|
||||
LOGGER.error(e) { "Main loop failed" }
|
||||
LOGGER.warn(e) { "Main loop failed" }
|
||||
}
|
||||
},
|
||||
5, 5, TimeUnit.SECONDS
|
||||
|
||||
46
src/main/kotlin/mdnet/cache/ImageStorage.kt
vendored
46
src/main/kotlin/mdnet/cache/ImageStorage.kt
vendored
@ -25,6 +25,7 @@ import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper
|
||||
import com.fasterxml.jackson.module.kotlin.readValue
|
||||
import mdnet.logging.info
|
||||
import mdnet.logging.trace
|
||||
import mdnet.logging.warn
|
||||
import org.apache.commons.io.file.PathUtils
|
||||
import org.ktorm.database.Database
|
||||
import org.ktorm.dsl.*
|
||||
@ -58,7 +59,7 @@ class ImageStorage(
|
||||
private val cacheDirectory: Path,
|
||||
private val database: Database,
|
||||
autoPrune: Boolean = true
|
||||
) {
|
||||
) : AutoCloseable {
|
||||
private val tempCacheDirectory = cacheDirectory.resolve("tmp")
|
||||
|
||||
private val evictor: ScheduledExecutorService = Executors.newScheduledThreadPool(2)
|
||||
@ -89,36 +90,43 @@ class ImageStorage(
|
||||
|
||||
evictor.scheduleWithFixedDelay(
|
||||
{
|
||||
val toUpdate = HashSet<String>()
|
||||
queue.drainTo(toUpdate)
|
||||
val now = Instant.now()
|
||||
try {
|
||||
val toUpdate = HashSet<String>()
|
||||
queue.drainTo(toUpdate)
|
||||
val now = Instant.now()
|
||||
|
||||
LOGGER.info { "Updating LRU times for ${toUpdate.size} entries" }
|
||||
synchronized(database) {
|
||||
database.batchUpdate(DbImage) {
|
||||
for (id in toUpdate) {
|
||||
item {
|
||||
set(DbImage.accessed, now)
|
||||
where {
|
||||
DbImage.id eq id
|
||||
LOGGER.info { "Updating LRU times for ${toUpdate.size} entries" }
|
||||
synchronized(database) {
|
||||
database.batchUpdate(DbImage) {
|
||||
for (id in toUpdate) {
|
||||
item {
|
||||
set(DbImage.accessed, now)
|
||||
where {
|
||||
DbImage.id eq id
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
calculateSize()
|
||||
} catch (e: Exception) {
|
||||
LOGGER.warn(e) { "Error updating LRU $this" }
|
||||
}
|
||||
calculateSize()
|
||||
},
|
||||
1, 1, TimeUnit.MINUTES
|
||||
30, 30, TimeUnit.SECONDS
|
||||
)
|
||||
|
||||
// evict LRU cache every 3 minutes
|
||||
if (autoPrune) {
|
||||
evictor.scheduleWithFixedDelay(
|
||||
{
|
||||
calculateSize()
|
||||
pruneImages()
|
||||
try {
|
||||
calculateSize()
|
||||
pruneImages()
|
||||
} catch (e: Exception) {
|
||||
LOGGER.warn(e) { "Error pruning images" }
|
||||
}
|
||||
},
|
||||
0, 3, TimeUnit.MINUTES
|
||||
0, 1, TimeUnit.MINUTES
|
||||
)
|
||||
}
|
||||
}
|
||||
@ -255,7 +263,7 @@ class ImageStorage(
|
||||
}
|
||||
}
|
||||
|
||||
fun close() {
|
||||
override fun close() {
|
||||
evictor.shutdown()
|
||||
evictor.awaitTermination(10, TimeUnit.SECONDS)
|
||||
}
|
||||
|
||||
@ -175,7 +175,7 @@ class Netty(
|
||||
val certs = getX509Certs(tls.certificate)
|
||||
val sslContext = SslContextBuilder
|
||||
.forServer(getPrivateKey(tls.privateKey), certs)
|
||||
.protocols("TLSv1.3", "TLSv1.2", "TLSv1.1", "TLSv1")
|
||||
.protocols("TLSv1.3", "TLSv1.2")
|
||||
.build()
|
||||
|
||||
val bootstrap = ServerBootstrap()
|
||||
|
||||
28
src/test/kotlin/mdnet/cache/ImageStorageTest.kt
vendored
28
src/test/kotlin/mdnet/cache/ImageStorageTest.kt
vendored
@ -39,11 +39,13 @@ class ImageStorageTest : FreeSpec() {
|
||||
override fun isolationMode() = IsolationMode.InstancePerTest
|
||||
|
||||
init {
|
||||
val imageStorage = ImageStorage(
|
||||
maxSize = 5,
|
||||
cacheDirectory = tempdir().toPath(),
|
||||
database = Database.connect("jdbc:sqlite:${tempfile()}"),
|
||||
autoPrune = false,
|
||||
val imageStorage = autoClose(
|
||||
ImageStorage(
|
||||
maxSize = 5,
|
||||
cacheDirectory = tempdir().toPath(),
|
||||
database = Database.connect("jdbc:sqlite:${tempfile()}"),
|
||||
autoPrune = false,
|
||||
)
|
||||
)
|
||||
|
||||
val testMeta = ImageMetadata("a", "a", 123)
|
||||
@ -81,7 +83,8 @@ class ImageStorageTest : FreeSpec() {
|
||||
writer.stream.write(ByteArray(12))
|
||||
writer.abort()
|
||||
|
||||
"should not update size" {
|
||||
"should not update size even if calculated" {
|
||||
imageStorage.calculateSize()
|
||||
imageStorage.size.shouldBeZero()
|
||||
}
|
||||
}
|
||||
@ -157,14 +160,17 @@ class ImageStorageSlowTest : FreeSpec() {
|
||||
override fun isolationMode() = IsolationMode.InstancePerTest
|
||||
|
||||
init {
|
||||
val imageStorage = ImageStorage(
|
||||
maxSize = 4097,
|
||||
cacheDirectory = tempdir().toPath(),
|
||||
database = Database.connect("jdbc:sqlite:${tempfile()}"),
|
||||
val imageStorage = autoClose(
|
||||
ImageStorage(
|
||||
maxSize = 4097,
|
||||
cacheDirectory = tempdir().toPath(),
|
||||
database = Database.connect("jdbc:sqlite:${tempfile()}"),
|
||||
)
|
||||
)
|
||||
|
||||
"autoPrune" - {
|
||||
"should update size eventually" {
|
||||
println("1 - $imageStorage")
|
||||
val writer = imageStorage.storeImage("test", ImageMetadata("a", "a", 4096))
|
||||
writer.shouldNotBeNull()
|
||||
|
||||
@ -177,6 +183,7 @@ class ImageStorageSlowTest : FreeSpec() {
|
||||
}
|
||||
|
||||
"should prune if insufficient size eventually" {
|
||||
println("2 - $imageStorage")
|
||||
imageStorage.maxSize = 10000
|
||||
|
||||
val writer = imageStorage.storeImage("test", ImageMetadata("a", "a", 123))
|
||||
@ -185,6 +192,7 @@ class ImageStorageSlowTest : FreeSpec() {
|
||||
writer.stream.write(ByteArray(8192))
|
||||
writer.commit(8192).shouldBeTrue()
|
||||
|
||||
imageStorage.calculateSize()
|
||||
eventually(Duration.minutes(5)) {
|
||||
imageStorage.size.shouldBeZero()
|
||||
}
|
||||
|
||||
@ -116,11 +116,13 @@ class ImageServerTest : FreeSpec() {
|
||||
}
|
||||
|
||||
"with real cache" - {
|
||||
val storage = ImageStorage(
|
||||
maxSize = 100000,
|
||||
cacheDirectory = tempdir().toPath(),
|
||||
database = Database.connect("jdbc:sqlite:${tempfile()}"),
|
||||
autoPrune = false,
|
||||
val storage = autoClose(
|
||||
ImageStorage(
|
||||
maxSize = 100000,
|
||||
cacheDirectory = tempdir().toPath(),
|
||||
database = Database.connect("jdbc:sqlite:${tempfile()}"),
|
||||
autoPrune = false,
|
||||
)
|
||||
)
|
||||
|
||||
val server = ImageServer(
|
||||
@ -161,11 +163,13 @@ class ImageServerTest : FreeSpec() {
|
||||
"failed upstream responses" - {
|
||||
val client = mockk<HttpHandler>()
|
||||
|
||||
val storage = ImageStorage(
|
||||
maxSize = 100000,
|
||||
cacheDirectory = tempdir().toPath(),
|
||||
database = Database.connect("jdbc:sqlite:${tempfile()}"),
|
||||
autoPrune = false,
|
||||
val storage = autoClose(
|
||||
ImageStorage(
|
||||
maxSize = 100000,
|
||||
cacheDirectory = tempdir().toPath(),
|
||||
database = Database.connect("jdbc:sqlite:${tempfile()}"),
|
||||
autoPrune = false,
|
||||
)
|
||||
)
|
||||
|
||||
val server = ImageServer(
|
||||
|
||||
Loading…
Reference in New Issue
Block a user