From 5265b767cb50005fc6346d5cc93ea1d5f7f5523b Mon Sep 17 00:00:00 2001 From: TobiGr Date: Sat, 14 Oct 2023 18:31:33 +0200 Subject: [PATCH 1/2] Fix notifiying about old "new" streams Add tests for `FeedDAO.unlinkStreamsOlderThan(:offsetDateTime) ` Closes #10237 --- .../schabi/newpipe/database/FeedDAOTest.kt | 124 ++++++++++++++++++ .../newpipe/database/feed/dao/FeedDAO.kt | 30 +++-- 2 files changed, 144 insertions(+), 10 deletions(-) create mode 100644 app/src/androidTest/java/org/schabi/newpipe/database/FeedDAOTest.kt diff --git a/app/src/androidTest/java/org/schabi/newpipe/database/FeedDAOTest.kt b/app/src/androidTest/java/org/schabi/newpipe/database/FeedDAOTest.kt new file mode 100644 index 000000000..9fa87c31b --- /dev/null +++ b/app/src/androidTest/java/org/schabi/newpipe/database/FeedDAOTest.kt @@ -0,0 +1,124 @@ +package org.schabi.newpipe.database + +import android.content.Context +import androidx.room.Room +import androidx.test.core.app.ApplicationProvider +import io.reactivex.rxjava3.core.Single +import org.junit.After +import org.junit.Assert.assertEquals +import org.junit.Assert.assertNotNull +import org.junit.Before +import org.junit.Test +import org.schabi.newpipe.database.feed.dao.FeedDAO +import org.schabi.newpipe.database.feed.model.FeedEntity +import org.schabi.newpipe.database.feed.model.FeedGroupEntity +import org.schabi.newpipe.database.stream.StreamWithState +import org.schabi.newpipe.database.stream.dao.StreamDAO +import org.schabi.newpipe.database.stream.model.StreamEntity +import org.schabi.newpipe.database.subscription.SubscriptionDAO +import org.schabi.newpipe.database.subscription.SubscriptionEntity +import org.schabi.newpipe.extractor.ServiceList +import org.schabi.newpipe.extractor.channel.ChannelInfo +import org.schabi.newpipe.extractor.stream.StreamType +import java.io.IOException +import java.time.OffsetDateTime +import kotlin.streams.toList + +class FeedDAOTest { + private lateinit var db: AppDatabase + private lateinit var feedDAO: FeedDAO + private lateinit var streamDAO: StreamDAO + private lateinit var subscriptionDAO: SubscriptionDAO + + private val serviceId = ServiceList.YouTube.serviceId + + private val stream1 = StreamEntity(1, serviceId, "https://youtube.com/watch?v=1", "stream 1", StreamType.VIDEO_STREAM, 1000, "channel-1", "https://youtube.com/channel/1", "https://i.ytimg.com/vi/1/hqdefault.jpg", 100, "2023-01-01", OffsetDateTime.parse("2023-01-01T00:00:00Z")) + private val stream2 = StreamEntity(2, serviceId, "https://youtube.com/watch?v=2", "stream 2", StreamType.VIDEO_STREAM, 1000, "channel-1", "https://youtube.com/channel/1", "https://i.ytimg.com/vi/1/hqdefault.jpg", 100, "2023-01-02", OffsetDateTime.parse("2023-01-02T00:00:00Z")) + private val stream3 = StreamEntity(3, serviceId, "https://youtube.com/watch?v=3", "stream 3", StreamType.LIVE_STREAM, 1000, "channel-1", "https://youtube.com/channel/1", "https://i.ytimg.com/vi/1/hqdefault.jpg", 100, "2023-01-03", OffsetDateTime.parse("2023-01-03T00:00:00Z")) + private val stream4 = StreamEntity(4, serviceId, "https://youtube.com/watch?v=4", "stream 4", StreamType.VIDEO_STREAM, 1000, "channel-2", "https://youtube.com/channel/2", "https://i.ytimg.com/vi/1/hqdefault.jpg", 100, "2023-08-10", OffsetDateTime.parse("2023-08-10T00:00:00Z")) + private val stream5 = StreamEntity(5, serviceId, "https://youtube.com/watch?v=5", "stream 5", StreamType.VIDEO_STREAM, 1000, "channel-2", "https://youtube.com/channel/2", "https://i.ytimg.com/vi/1/hqdefault.jpg", 100, "2023-08-20", OffsetDateTime.parse("2023-08-20T00:00:00Z")) + private val stream6 = StreamEntity(6, serviceId, "https://youtube.com/watch?v=6", "stream 6", StreamType.VIDEO_STREAM, 1000, "channel-3", "https://youtube.com/channel/3", "https://i.ytimg.com/vi/1/hqdefault.jpg", 100, "2023-09-01", OffsetDateTime.parse("2023-09-01T00:00:00Z")) + private val stream7 = StreamEntity(7, serviceId, "https://youtube.com/watch?v=7", "stream 7", StreamType.VIDEO_STREAM, 1000, "channel-4", "https://youtube.com/channel/4", "https://i.ytimg.com/vi/1/hqdefault.jpg", 100, "2023-08-10", OffsetDateTime.parse("2023-08-10T00:00:00Z")) + + private val allStreams = listOf( + stream1, stream2, stream3, stream4, stream5, stream6, stream7 + ) + + @Before + fun createDb() { + val context = ApplicationProvider.getApplicationContext() + db = Room.inMemoryDatabaseBuilder( + context, AppDatabase::class.java + ).build() + feedDAO = db.feedDAO() + streamDAO = db.streamDAO() + subscriptionDAO = db.subscriptionDAO() + } + + @After + @Throws(IOException::class) + fun closeDb() { + db.close() + } + + @Test + fun testUnlinkStreamsOlderThan_KeepOne() { + setupUnlinkDelete("2023-08-15T00:00:00Z") + val streams = feedDAO.getStreams( + FeedGroupEntity.GROUP_ALL_ID, includePlayed = true, includePartiallyPlayed = true, null + ) + .blockingGet() + val allowedStreams = listOf(stream3, stream5, stream6, stream7) + assertEqual(streams, allowedStreams) + } + + @Test + fun testUnlinkStreamsOlderThan_KeepMultiple() { + setupUnlinkDelete("2023-08-01T00:00:00Z") + val streams = feedDAO.getStreams( + FeedGroupEntity.GROUP_ALL_ID, includePlayed = true, includePartiallyPlayed = true, null + ) + .blockingGet() + val allowedStreams = listOf(stream3, stream4, stream5, stream6, stream7) + assertEqual(streams, allowedStreams) + } + + private fun assertEqual(streams: List?, allowedStreams: List) { + assertNotNull(streams) + assertEquals(allowedStreams, streams!!.stream().map { it.stream }.toList().sortedBy { it.uid }) + } + + private fun setupUnlinkDelete(time: String) { + clearAndFillTables() + Single.fromCallable { + feedDAO.unlinkStreamsOlderThan(OffsetDateTime.parse(time)) + }.blockingSubscribe() + Single.fromCallable { + streamDAO.deleteOrphans() + }.blockingSubscribe() + } + + private fun clearAndFillTables() { + db.clearAllTables() + streamDAO.insertAll(allStreams) + subscriptionDAO.insertAll( + listOf( + SubscriptionEntity.from(ChannelInfo(serviceId, "1", "https://youtube.com/channel/1", "https://youtube.com/channel/1", "channel-1")), + SubscriptionEntity.from(ChannelInfo(serviceId, "2", "https://youtube.com/channel/2", "https://youtube.com/channel/2", "channel-2")), + SubscriptionEntity.from(ChannelInfo(serviceId, "3", "https://youtube.com/channel/3", "https://youtube.com/channel/3", "channel-3")), + SubscriptionEntity.from(ChannelInfo(serviceId, "4", "https://youtube.com/channel/4", "https://youtube.com/channel/4", "channel-4")), + ) + ) + feedDAO.insertAll( + listOf( + FeedEntity(1, 1), + FeedEntity(2, 1), + FeedEntity(3, 1), + FeedEntity(4, 2), + FeedEntity(5, 2), + FeedEntity(6, 3), + FeedEntity(7, 4), + ) + ) + } +} diff --git a/app/src/main/java/org/schabi/newpipe/database/feed/dao/FeedDAO.kt b/app/src/main/java/org/schabi/newpipe/database/feed/dao/FeedDAO.kt index 6947f66af..028ef04f7 100644 --- a/app/src/main/java/org/schabi/newpipe/database/feed/dao/FeedDAO.kt +++ b/app/src/main/java/org/schabi/newpipe/database/feed/dao/FeedDAO.kt @@ -93,18 +93,28 @@ abstract class FeedDAO { uploadDateBefore: OffsetDateTime? ): Maybe> + /** + * Remove links to streams that are older than the given date + * **but keep at least one stream per uploader**. + * + * One stream per uploader is kept because it is needed as reference + * when fetching new streams to check if they are new or not. + * @param offsetDateTime the newest date to keep, older streams are removed + */ @Query( """ - DELETE FROM feed WHERE - - feed.stream_id IN ( - SELECT s.uid FROM streams s - - INNER JOIN feed f - ON s.uid = f.stream_id - - WHERE s.upload_date < :offsetDateTime - ) + DELETE FROM feed + WHERE feed.stream_id IN (SELECT uid from ( + SELECT s.uid, + (SELECT MAX(upload_date) + FROM streams + WHERE uploader_url = s.uploader_url) max_upload_date + FROM streams s + INNER JOIN feed f + ON s.uid = f.stream_id + + WHERE s.upload_date < :offsetDateTime + AND s.upload_date <> max_upload_date)) """ ) abstract fun unlinkStreamsOlderThan(offsetDateTime: OffsetDateTime) From b4a0e08d9de66994d8b6dd0404f8ac5dbb870e5e Mon Sep 17 00:00:00 2001 From: Tobi Date: Thu, 23 Nov 2023 17:12:16 +0100 Subject: [PATCH 2/2] Update app/src/main/java/org/schabi/newpipe/database/feed/dao/FeedDAO.kt Co-authored-by: Stypox --- .../java/org/schabi/newpipe/database/feed/dao/FeedDAO.kt | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/app/src/main/java/org/schabi/newpipe/database/feed/dao/FeedDAO.kt b/app/src/main/java/org/schabi/newpipe/database/feed/dao/FeedDAO.kt index 028ef04f7..e7ed93497 100644 --- a/app/src/main/java/org/schabi/newpipe/database/feed/dao/FeedDAO.kt +++ b/app/src/main/java/org/schabi/newpipe/database/feed/dao/FeedDAO.kt @@ -107,8 +107,10 @@ abstract class FeedDAO { WHERE feed.stream_id IN (SELECT uid from ( SELECT s.uid, (SELECT MAX(upload_date) - FROM streams - WHERE uploader_url = s.uploader_url) max_upload_date + FROM streams s1 + INNER JOIN feed f1 + ON s1.uid = f1.stream_id + WHERE f1.subscription_id = f.subscription_id) max_upload_date FROM streams s INNER JOIN feed f ON s.uid = f.stream_id