Merge query for UpdateView & LibraryView (#741)

Remove the inline-code and move it to database

* merge queries for Updates view

* merge queries for Library view

* add comment

* fix spotless

* also migrate database
This commit is contained in:
Cuong-Tran 2025-03-03 12:21:43 +07:00
parent 778d901822
commit d4c4686ad1
No known key found for this signature in database
GPG key ID: 733AA7624B9315C2
10 changed files with 218 additions and 324 deletions

View file

@ -110,10 +110,4 @@ class AndroidDatabaseHandler(
val context = getCurrentDatabaseContext()
return withContext(context) { block(db) }
}
// SY -->
fun getLibraryQuery(condition: String = "M.favorite = 1") = LibraryQuery(driver, condition)
fun getUpdatesQuery(after: Long, limit: Long) = UpdatesQuery(driver, after, limit)
// SY <--
}

View file

@ -1,136 +0,0 @@
package tachiyomi.data
import app.cash.sqldelight.ExecutableQuery
import app.cash.sqldelight.db.QueryResult
import app.cash.sqldelight.db.SqlCursor
import app.cash.sqldelight.db.SqlDriver
import exh.source.MERGED_SOURCE_ID
import tachiyomi.view.LibraryView
private val mapper = { cursor: SqlCursor ->
LibraryView(
_id = cursor.getLong(0)!!,
source = cursor.getLong(1)!!,
url = cursor.getString(2)!!,
artist = cursor.getString(3),
author = cursor.getString(4),
description = cursor.getString(5),
genre = cursor.getString(6)?.let(StringListColumnAdapter::decode),
title = cursor.getString(7)!!,
status = cursor.getLong(8)!!,
thumbnail_url = cursor.getString(9),
favorite = cursor.getLong(10)!! == 1L,
last_update = cursor.getLong(11),
next_update = cursor.getLong(12),
initialized = cursor.getLong(13)!! == 1L,
viewer = cursor.getLong(14)!!,
chapter_flags = cursor.getLong(15)!!,
cover_last_modified = cursor.getLong(16)!!,
date_added = cursor.getLong(17)!!,
filtered_scanlators = null,
update_strategy = UpdateStrategyColumnAdapter.decode(cursor.getLong(19)!!),
calculate_interval = cursor.getLong(20)!!,
last_modified_at = cursor.getLong(21)!!,
favorite_modified_at = cursor.getLong(22),
version = cursor.getLong(23)!!,
is_syncing = cursor.getLong(24)!!,
totalCount = cursor.getLong(25)!!,
readCount = cursor.getDouble(26)!!,
latestUpload = cursor.getLong(27)!!,
chapterFetchedAt = cursor.getLong(28)!!,
lastRead = cursor.getLong(29)!!,
bookmarkCount = cursor.getDouble(30)!!,
category = cursor.getLong(31)!!,
)
}
class LibraryQuery(
val driver: SqlDriver,
val condition: String = "M.favorite = 1",
) : ExecutableQuery<LibraryView>(mapper) {
override fun <R> execute(mapper: (SqlCursor) -> QueryResult<R>): QueryResult<R> {
return driver.executeQuery(
null,
"""
SELECT
M.*,
coalesce(C.total, 0) AS totalCount,
coalesce(C.readCount, 0) AS readCount,
coalesce(C.latestUpload, 0) AS latestUpload,
coalesce(C.fetchedAt, 0) AS chapterFetchedAt,
coalesce(C.lastRead, 0) AS lastRead,
coalesce(C.bookmarkCount, 0) AS bookmarkCount,
coalesce(MC.category_id, 0) AS category
FROM mangas M
LEFT JOIN(
SELECT
chapters.manga_id,
count(*) AS total,
sum(read) AS readCount,
coalesce(max(chapters.date_upload), 0) AS latestUpload,
coalesce(max(history.last_read), 0) AS lastRead,
coalesce(max(chapters.date_fetch), 0) AS fetchedAt,
sum(chapters.bookmark) AS bookmarkCount
FROM chapters
LEFT JOIN excluded_scanlators
ON chapters.manga_id = excluded_scanlators.manga_id
AND chapters.scanlator = excluded_scanlators.scanlator
LEFT JOIN history
ON chapters._id = history.chapter_id
WHERE excluded_scanlators.scanlator IS NULL
GROUP BY chapters.manga_id
) AS C
ON M._id = C.manga_id
LEFT JOIN mangas_categories AS MC
ON MC.manga_id = M._id
WHERE $condition AND M.source <> $MERGED_SOURCE_ID
UNION
SELECT
M.*,
coalesce(C.total, 0) AS totalCount,
coalesce(C.readCount, 0) AS readCount,
coalesce(C.latestUpload, 0) AS latestUpload,
coalesce(C.fetchedAt, 0) AS chapterFetchedAt,
coalesce(C.lastRead, 0) AS lastRead,
coalesce(C.bookmarkCount, 0) AS bookmarkCount,
coalesce(MC.category_id, 0) AS category
FROM mangas M
LEFT JOIN (
SELECT merged.manga_id,merged.merge_id
FROM merged
GROUP BY merged.merge_id
) as ME
ON ME.merge_id = M._id
LEFT JOIN(
SELECT
ME.merge_id,
count(*) AS total,
sum(read) AS readCount,
coalesce(max(chapters.date_upload), 0) AS latestUpload,
coalesce(max(history.last_read), 0) AS lastRead,
coalesce(max(chapters.date_fetch), 0) AS fetchedAt,
sum(chapters.bookmark) AS bookmarkCount
FROM chapters
LEFT JOIN excluded_scanlators
ON chapters.manga_id = excluded_scanlators.manga_id
AND chapters.scanlator = excluded_scanlators.scanlator
LEFT JOIN history
ON chapters._id = history.chapter_id
LEFT JOIN merged as ME
ON ME.manga_id = chapters.manga_id
WHERE excluded_scanlators.scanlator IS NULL
GROUP BY ME.merge_id
) AS C
ON ME.merge_id = C.merge_id
LEFT JOIN mangas_categories AS MC
ON MC.manga_id = M._id
WHERE $condition AND M.source = $MERGED_SOURCE_ID;
""".trimIndent(),
mapper,
parameters = 0,
)
}
override fun toString(): String = "LibraryQuery.sq:get"
}

View file

@ -1,95 +0,0 @@
package tachiyomi.data
import app.cash.sqldelight.ExecutableQuery
import app.cash.sqldelight.db.QueryResult
import app.cash.sqldelight.db.SqlCursor
import app.cash.sqldelight.db.SqlDriver
import exh.source.MERGED_SOURCE_ID
import tachiyomi.view.UpdatesView
private val mapper = { cursor: SqlCursor ->
UpdatesView(
cursor.getLong(0)!!,
cursor.getString(1)!!,
cursor.getLong(2)!!,
cursor.getString(3)!!,
cursor.getString(4),
cursor.getLong(5)!! == 1L,
cursor.getLong(6)!! == 1L,
cursor.getLong(7)!!,
cursor.getLong(8)!!,
cursor.getLong(9)!! == 1L,
cursor.getString(10),
cursor.getLong(11)!!,
cursor.getLong(12)!!,
cursor.getLong(13)!!,
)
}
class UpdatesQuery(val driver: SqlDriver, val after: Long, val limit: Long) : ExecutableQuery<UpdatesView>(mapper) {
override fun <R> execute(mapper: (SqlCursor) -> QueryResult<R>): QueryResult<R> {
return driver.executeQuery(
null,
"""
SELECT
mangas._id AS mangaId,
mangas.title AS mangaTitle,
chapters._id AS chapterId,
chapters.name AS chapterName,
chapters.scanlator,
chapters.read,
chapters.bookmark,
chapters.last_page_read,
mangas.source,
mangas.favorite,
mangas.thumbnail_url AS thumbnailUrl,
mangas.cover_last_modified AS coverLastModified,
chapters.date_upload AS dateUpload,
chapters.date_fetch AS datefetch
FROM mangas JOIN chapters
ON mangas._id = chapters.manga_id
WHERE favorite = 1 AND source <> $MERGED_SOURCE_ID
AND date_fetch > date_added
AND dateUpload > :after
UNION
SELECT
mangas._id AS mangaId,
mangas.title AS mangaTitle,
chapters._id AS chapterId,
chapters.name AS chapterName,
chapters.scanlator,
chapters.read,
chapters.bookmark,
chapters.last_page_read,
mangas.source,
mangas.favorite,
mangas.thumbnail_url AS thumbnailUrl,
mangas.cover_last_modified AS coverLastModified,
chapters.date_upload AS dateUpload,
chapters.date_fetch AS datefetch
FROM mangas
LEFT JOIN (
SELECT merged.manga_id,merged.merge_id
FROM merged
GROUP BY merged.merge_id
) as ME
ON ME.merge_id = mangas._id
JOIN chapters
ON ME.manga_id = chapters.manga_id
WHERE favorite = 1 AND source = $MERGED_SOURCE_ID
AND date_fetch > date_added
AND dateUpload > :after
ORDER BY datefetch DESC
LIMIT :limit;
""".trimIndent(),
mapper,
2,
binders = {
bindLong(0, after)
bindLong(1, limit)
},
)
}
override fun toString(): String = "LibraryQuery.sq:get"
}

View file

@ -3,7 +3,6 @@ package tachiyomi.data.manga
import eu.kanade.tachiyomi.source.model.UpdateStrategy
import tachiyomi.domain.library.model.LibraryManga
import tachiyomi.domain.manga.model.Manga
import tachiyomi.view.LibraryView
object MangaMapper {
fun mapManga(
@ -138,41 +137,4 @@ object MangaMapper {
chapterFetchedAt = chapterFetchedAt,
lastRead = lastRead,
)
fun mapLibraryView(libraryView: LibraryView): LibraryManga {
return LibraryManga(
manga = Manga(
id = libraryView._id,
source = libraryView.source,
favorite = libraryView.favorite,
lastUpdate = libraryView.last_update ?: 0,
nextUpdate = libraryView.next_update ?: 0,
dateAdded = libraryView.date_added,
viewerFlags = libraryView.viewer,
chapterFlags = libraryView.chapter_flags,
coverLastModified = libraryView.cover_last_modified,
url = libraryView.url,
ogTitle = libraryView.title,
ogArtist = libraryView.artist,
ogAuthor = libraryView.author,
ogDescription = libraryView.description,
ogGenre = libraryView.genre,
ogStatus = libraryView.status,
ogThumbnailUrl = libraryView.thumbnail_url,
updateStrategy = libraryView.update_strategy,
initialized = libraryView.initialized,
fetchInterval = libraryView.calculate_interval.toInt(),
lastModifiedAt = libraryView.last_modified_at,
favoriteModifiedAt = libraryView.favorite_modified_at,
version = libraryView.version,
),
category = libraryView.category,
totalChapters = libraryView.totalCount,
readCount = libraryView.readCount.toLong(),
bookmarkCount = libraryView.bookmarkCount.toLong(),
latestUpload = libraryView.latestUpload,
chapterFetchedAt = libraryView.chapterFetchedAt,
lastRead = libraryView.lastRead,
)
}
}

View file

@ -1,10 +1,8 @@
package tachiyomi.data.manga
import kotlinx.coroutines.flow.Flow
import kotlinx.coroutines.flow.map
import logcat.LogPriority
import tachiyomi.core.common.util.system.logcat
import tachiyomi.data.AndroidDatabaseHandler
import tachiyomi.data.DatabaseHandler
import tachiyomi.data.StringListColumnAdapter
import tachiyomi.data.UpdateStrategyColumnAdapter
@ -56,17 +54,11 @@ class MangaRepositoryImpl(
}
override suspend fun getLibraryManga(): List<LibraryManga> {
return handler.awaitListExecutable {
(handler as AndroidDatabaseHandler).getLibraryQuery()
}.map(MangaMapper::mapLibraryView)
// return handler.awaitList { libraryViewQueries.library(MangaMapper::mapLibraryManga) }
return handler.awaitList { libraryViewQueries.library(MangaMapper::mapLibraryManga) }
}
override fun getLibraryMangaAsFlow(): Flow<List<LibraryManga>> {
return handler.subscribeToList { libraryViewQueries.library(MangaMapper::mapLibraryManga) }
// SY -->
.map { getLibraryManga() }
// SY <--
}
override fun getFavoritesBySourceId(sourceId: Long): Flow<List<Manga>> {
@ -203,9 +195,7 @@ class MangaRepositoryImpl(
}
override suspend fun getReadMangaNotInLibraryView(): List<LibraryManga> {
return handler.awaitListExecutable {
(handler as AndroidDatabaseHandler).getLibraryQuery("M.favorite = 0 AND C.readCount != 0")
}.map(MangaMapper::mapLibraryView)
return handler.awaitList { libraryViewQueries.readMangaNonLibrary(MangaMapper::mapLibraryManga) }
}
// SY <--
}

View file

@ -1,13 +1,10 @@
package tachiyomi.data.updates
import kotlinx.coroutines.flow.Flow
import kotlinx.coroutines.flow.map
import tachiyomi.data.AndroidDatabaseHandler
import tachiyomi.data.DatabaseHandler
import tachiyomi.domain.manga.model.MangaCover
import tachiyomi.domain.updates.model.UpdatesWithRelations
import tachiyomi.domain.updates.repository.UpdatesRepository
import tachiyomi.view.UpdatesView
class UpdatesRepositoryImpl(
private val databaseHandler: DatabaseHandler,
@ -31,11 +28,6 @@ class UpdatesRepositoryImpl(
override fun subscribeAll(after: Long, limit: Long): Flow<List<UpdatesWithRelations>> {
return databaseHandler.subscribeToList {
updatesViewQueries.getRecentUpdates(after, limit, ::mapUpdatesWithRelations)
}.map {
databaseHandler.awaitListExecutable {
(databaseHandler as AndroidDatabaseHandler).getUpdatesQuery(after, limit)
}
.map(::mapUpdatesView)
}
}
@ -67,7 +59,7 @@ class UpdatesRepositoryImpl(
favorite: Boolean,
thumbnailUrl: String?,
coverLastModified: Long,
dateUpload: Long,
@Suppress("UNUSED_PARAMETER") dateUpload: Long,
dateFetch: Long,
): UpdatesWithRelations = UpdatesWithRelations(
mangaId = mangaId,
@ -90,26 +82,4 @@ class UpdatesRepositoryImpl(
lastModified = coverLastModified,
),
)
fun mapUpdatesView(updatesView: UpdatesView): UpdatesWithRelations {
return UpdatesWithRelations(
mangaId = updatesView.mangaId,
ogMangaTitle = updatesView.mangaTitle,
chapterId = updatesView.chapterId,
chapterName = updatesView.chapterName,
scanlator = updatesView.scanlator,
read = updatesView.read,
bookmark = updatesView.bookmark,
lastPageRead = updatesView.last_page_read,
sourceId = updatesView.source,
dateFetch = updatesView.datefetch,
coverData = MangaCover(
mangaId = updatesView.mangaId,
sourceId = updatesView.source,
isMangaFavorite = updatesView.favorite,
ogUrl = updatesView.thumbnailUrl,
lastModified = updatesView.coverLastModified,
),
)
}
}

View file

@ -0,0 +1,129 @@
-- NOTE: must update MERGED_SOURCE_ID (6969) here
DROP VIEW IF EXISTS libraryView;
DROP VIEW IF EXISTS updatesView;
CREATE VIEW libraryView AS
SELECT
M.*,
coalesce(C.total, 0) AS totalCount,
coalesce(C.readCount, 0) AS readCount,
coalesce(C.latestUpload, 0) AS latestUpload,
coalesce(C.fetchedAt, 0) AS chapterFetchedAt,
coalesce(C.lastRead, 0) AS lastRead,
coalesce(C.bookmarkCount, 0) AS bookmarkCount,
coalesce(MC.category_id, 0) AS category
FROM mangas M
LEFT JOIN(
SELECT
chapters.manga_id,
count(*) AS total,
sum(read) AS readCount,
coalesce(max(chapters.date_upload), 0) AS latestUpload,
coalesce(max(history.last_read), 0) AS lastRead,
coalesce(max(chapters.date_fetch), 0) AS fetchedAt,
sum(chapters.bookmark) AS bookmarkCount,
excluded_scanlators.scanlator AS ex_scanlator
FROM chapters
LEFT JOIN excluded_scanlators
ON chapters.manga_id = excluded_scanlators.manga_id
AND chapters.scanlator = excluded_scanlators.scanlator
LEFT JOIN history
ON chapters._id = history.chapter_id
WHERE ex_scanlator IS NULL
GROUP BY chapters.manga_id
) AS C
ON M._id = C.manga_id
LEFT JOIN mangas_categories AS MC
ON MC.manga_id = M._id
WHERE M.source <> 6969
UNION
SELECT
M.*,
coalesce(C.total, 0) AS totalCount,
coalesce(C.readCount, 0) AS readCount,
coalesce(C.latestUpload, 0) AS latestUpload,
coalesce(C.fetchedAt, 0) AS chapterFetchedAt,
coalesce(C.lastRead, 0) AS lastRead,
coalesce(C.bookmarkCount, 0) AS bookmarkCount,
coalesce(MC.category_id, 0) AS category
FROM mangas M
LEFT JOIN (
SELECT merged.manga_id,merged.merge_id
FROM merged
GROUP BY merged.merge_id
) AS ME
ON ME.merge_id = M._id
LEFT JOIN(
SELECT
ME.merge_id,
count(*) AS total,
sum(read) AS readCount,
coalesce(max(chapters.date_upload), 0) AS latestUpload,
coalesce(max(history.last_read), 0) AS lastRead,
coalesce(max(chapters.date_fetch), 0) AS fetchedAt,
sum(chapters.bookmark) AS bookmarkCount,
excluded_scanlators.scanlator AS ex_scanlator
FROM chapters
LEFT JOIN excluded_scanlators
ON chapters.manga_id = excluded_scanlators.manga_id
AND chapters.scanlator = excluded_scanlators.scanlator
LEFT JOIN history
ON chapters._id = history.chapter_id
LEFT JOIN merged ME
ON ME.manga_id = chapters.manga_id
WHERE ex_scanlator IS NULL
GROUP BY ME.merge_id
) AS C
ON M._id = C.merge_id -- ON ME.merge_id = C.merge_id
LEFT JOIN mangas_categories AS MC
ON MC.manga_id = M._id
WHERE M.source = 6969;
CREATE VIEW updatesView AS
SELECT
mangas._id AS mangaId,
mangas.title AS mangaTitle,
chapters._id AS chapterId,
chapters.name AS chapterName,
chapters.scanlator,
chapters.read,
chapters.bookmark,
chapters.last_page_read,
mangas.source,
mangas.favorite,
mangas.thumbnail_url AS thumbnailUrl,
mangas.cover_last_modified AS coverLastModified,
chapters.date_upload AS dateUpload,
chapters.date_fetch AS datefetch
FROM mangas JOIN chapters
ON mangas._id = chapters.manga_id
WHERE favorite = 1 AND source <> 6969
AND date_fetch > date_added
UNION
SELECT
mangas._id AS mangaId,
mangas.title AS mangaTitle,
chapters._id AS chapterId,
chapters.name AS chapterName,
chapters.scanlator,
chapters.read,
chapters.bookmark,
chapters.last_page_read,
mangas.source,
mangas.favorite,
mangas.thumbnail_url AS thumbnailUrl,
mangas.cover_last_modified AS coverLastModified,
chapters.date_upload AS dateUpload,
chapters.date_fetch AS datefetch
FROM mangas
LEFT JOIN (
SELECT merged.manga_id,merged.merge_id
FROM merged
GROUP BY merged.merge_id
) AS ME
ON ME.merge_id = mangas._id
JOIN chapters
ON ME.manga_id = chapters.manga_id
WHERE favorite = 1 AND source = 6969
AND date_fetch > date_added
ORDER BY datefetch DESC;

View file

@ -1,3 +1,4 @@
-- NOTE: must update MERGED_SOURCE_ID (6969) here
CREATE VIEW libraryView AS
SELECT
M.*,
@ -17,21 +18,70 @@ LEFT JOIN(
coalesce(max(chapters.date_upload), 0) AS latestUpload,
coalesce(max(history.last_read), 0) AS lastRead,
coalesce(max(chapters.date_fetch), 0) AS fetchedAt,
sum(chapters.bookmark) AS bookmarkCount
sum(chapters.bookmark) AS bookmarkCount,
excluded_scanlators.scanlator AS ex_scanlator
FROM chapters
LEFT JOIN excluded_scanlators
ON chapters.manga_id = excluded_scanlators.manga_id
AND chapters.scanlator = excluded_scanlators.scanlator
LEFT JOIN history
ON chapters._id = history.chapter_id
WHERE excluded_scanlators.scanlator IS NULL
WHERE ex_scanlator IS NULL
GROUP BY chapters.manga_id
) AS C
ON M._id = C.manga_id
LEFT JOIN mangas_categories AS MC
ON MC.manga_id = M._id
WHERE M.favorite = 1;
WHERE M.source <> 6969
UNION
SELECT
M.*,
coalesce(C.total, 0) AS totalCount,
coalesce(C.readCount, 0) AS readCount,
coalesce(C.latestUpload, 0) AS latestUpload,
coalesce(C.fetchedAt, 0) AS chapterFetchedAt,
coalesce(C.lastRead, 0) AS lastRead,
coalesce(C.bookmarkCount, 0) AS bookmarkCount,
coalesce(MC.category_id, 0) AS category
FROM mangas M
LEFT JOIN (
SELECT merged.manga_id,merged.merge_id
FROM merged
GROUP BY merged.merge_id
) AS ME
ON ME.merge_id = M._id
LEFT JOIN(
SELECT
ME.merge_id,
count(*) AS total,
sum(read) AS readCount,
coalesce(max(chapters.date_upload), 0) AS latestUpload,
coalesce(max(history.last_read), 0) AS lastRead,
coalesce(max(chapters.date_fetch), 0) AS fetchedAt,
sum(chapters.bookmark) AS bookmarkCount,
excluded_scanlators.scanlator AS ex_scanlator
FROM chapters
LEFT JOIN excluded_scanlators
ON chapters.manga_id = excluded_scanlators.manga_id
AND chapters.scanlator = excluded_scanlators.scanlator
LEFT JOIN history
ON chapters._id = history.chapter_id
LEFT JOIN merged ME
ON ME.manga_id = chapters.manga_id
WHERE ex_scanlator IS NULL
GROUP BY ME.merge_id
) AS C
ON M._id = C.merge_id -- ON ME.merge_id = C.merge_id
LEFT JOIN mangas_categories AS MC
ON MC.manga_id = M._id
WHERE M.source = 6969;
library:
SELECT *
FROM libraryView;
FROM libraryView
WHERE libraryView.favorite = 1;
readMangaNonLibrary:
SELECT *
FROM libraryView
WHERE libraryView.favorite = 0 AND libraryView.readCount != 0;

View file

@ -1,3 +1,4 @@
-- NOTE: must update MERGED_SOURCE_ID (6969) here
CREATE VIEW updatesView AS
SELECT
mangas._id AS mangaId,
@ -16,9 +17,36 @@ SELECT
chapters.date_fetch AS datefetch
FROM mangas JOIN chapters
ON mangas._id = chapters.manga_id
WHERE favorite = 1
WHERE favorite = 1 AND source <> 6969
AND date_fetch > date_added
ORDER BY date_fetch DESC;
UNION
SELECT
mangas._id AS mangaId,
mangas.title AS mangaTitle,
chapters._id AS chapterId,
chapters.name AS chapterName,
chapters.scanlator,
chapters.read,
chapters.bookmark,
chapters.last_page_read,
mangas.source,
mangas.favorite,
mangas.thumbnail_url AS thumbnailUrl,
mangas.cover_last_modified AS coverLastModified,
chapters.date_upload AS dateUpload,
chapters.date_fetch AS datefetch
FROM mangas
LEFT JOIN (
SELECT merged.manga_id,merged.merge_id
FROM merged
GROUP BY merged.merge_id
) AS ME
ON ME.merge_id = mangas._id
JOIN chapters
ON ME.manga_id = chapters.manga_id
WHERE favorite = 1 AND source = 6969
AND date_fetch > date_added
ORDER BY datefetch DESC;
getRecentUpdates:
SELECT *

View file

@ -13,6 +13,8 @@ const val PURURIN_SOURCE_ID = 2221515250486218861L
const val TSUMINO_SOURCE_ID = 6707338697138388238L
const val EIGHTMUSES_SOURCE_ID = 1802675169972965535L
const val HBROWSE_SOURCE_ID = 1401584337232758222L
// If MERGED_SOURCE_ID is to update, remember to update updatesView.sq and libraryView.sq too
const val MERGED_SOURCE_ID = LEWD_SOURCE_SERIES + 69
// KMK -->