copilot/komikku
1
0
Fork 0
Code Issues Pull requests Projects Releases 5 Packages Wiki Activity Actions

chore: improve google drive sync. (jobobby04/TachiyomiSY#1200)

Browse source 
improve google drive sync, removes the lock, change to protobuf, and potentially fix deviceId not being unique, since it wasn't appState...

(cherry picked from commit c2eece0fff7c8b4395bf0aa02c7ef6012f9bbf8d)
This commit is contained in:
KaiserBh 2024-06-27 07:31:12 +10:00 committed by Cuong-Tran
parent e079b9bfd8
commit 94c0154ee3
No known key found for this signature in database
GPG key ID: 733AA7624B9315C2
2 changed files with 35 additions and 133 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
app/src/main/java/eu/kanade/domain/sync/SyncPreferences.kt
View file

@ -13,7 +13,7 @@ class SyncPreferences(
fun clientAPIKey() = preferenceStore.getString("sync_client_api_key", "")
fun lastSyncTimestamp() = preferenceStore.getLong(Preference.appStateKey("last_sync_timestamp"), 0L)
fun lastSyncEtag() = preferenceStore.getString("sync_etag", "")
fun lastSyncEtag() = preferenceStore.getString("sync_etag", "")
fun syncInterval() = preferenceStore.getInt("sync_interval", 0)
fun syncService() = preferenceStore.getInt("sync_service", 0)
@ -29,7 +29,7 @@ class SyncPreferences(
)
fun uniqueDeviceID(): String {
val uniqueIDPreference = preferenceStore.getString("unique_device_id", "")
val uniqueIDPreference = preferenceStore.getString(Preference.appStateKey("unique_device_id"), "")
// Retrieve the current value of the preference
var uniqueID = uniqueIDPreference.get()

164
app/src/main/java/eu/kanade/tachiyomi/data/sync/service/GoogleDriveSyncService.kt
View file

@ -10,7 +10,6 @@ import com.google.api.client.googleapis.auth.oauth2.GoogleAuthorizationCodeToken
import com.google.api.client.googleapis.auth.oauth2.GoogleClientSecrets
import com.google.api.client.googleapis.auth.oauth2.GoogleCredential
import com.google.api.client.googleapis.auth.oauth2.GoogleTokenResponse
import com.google.api.client.http.ByteArrayContent
import com.google.api.client.http.InputStreamContent
import com.google.api.client.http.javanet.NetHttpTransport
import com.google.api.client.json.JsonFactory
@ -20,11 +19,10 @@ import com.google.api.services.drive.DriveScopes
import com.google.api.services.drive.model.File
import eu.kanade.domain.sync.SyncPreferences
import eu.kanade.tachiyomi.data.backup.models.Backup
import kotlinx.coroutines.delay
import eu.kanade.tachiyomi.data.backup.models.BackupSerializer
import kotlinx.coroutines.launch
import kotlinx.serialization.json.Json
import kotlinx.serialization.json.decodeFromStream
import kotlinx.serialization.json.encodeToStream
import kotlinx.serialization.protobuf.ProtoBuf
import logcat.LogPriority
import logcat.logcat
import tachiyomi.core.common.i18n.stringResource
@ -37,7 +35,6 @@ import uy.kohesive.injekt.api.get
import java.io.IOException
import java.io.PipedInputStream
import java.io.PipedOutputStream
import java.time.Instant
import java.util.zip.GZIPInputStream
import java.util.zip.GZIPOutputStream
@ -64,19 +61,19 @@ class GoogleDriveSyncService(context: Context, json: Json, syncPreferences: Sync
private val appName = context.stringResource(MR.strings.app_name)
private val remoteFileName = "${appName}_sync_data.gz"
private val lockFileName = "${appName}_sync.lock"
private val remoteFileName = "${appName}_sync.proto.gz"
private val googleDriveService = GoogleDriveService(context)
private val protoBuf: ProtoBuf = Injekt.get()
override suspend fun doSync(syncData: SyncData): Backup? {
beforeSync()
try {
val remoteSData = pullSyncData()
if (remoteSData != null ){
if (remoteSData != null) {
// Get local unique device ID
val localDeviceId = syncPreferences.uniqueDeviceID()
val lastSyncDeviceId = remoteSData.deviceId
@ -90,7 +87,7 @@ class GoogleDriveSyncService(context: Context, json: Json, syncPreferences: Sync
return if (lastSyncDeviceId == localDeviceId) {
pushSyncData(syncData)
syncData.backup
}else{
} else {
// Merge the local and remote sync data
val mergedSyncData = mergeSyncData(syncData, remoteSData)
pushSyncData(mergedSyncData)
@ -107,64 +104,12 @@ class GoogleDriveSyncService(context: Context, json: Json, syncPreferences: Sync
}
private suspend fun beforeSync() {
try {
googleDriveService.refreshToken()
val drive = googleDriveService.driveService
?: throw Exception(context.stringResource(SYMR.strings.google_drive_not_signed_in))
var backoff = 1000L
var retries = 0 // Retry counter
val maxRetries = 10 // Maximum number of retries
while (retries < maxRetries) {
val lockFiles = findLockFile(drive)
logcat(LogPriority.DEBUG) { "Found ${lockFiles.size} lock file(s)" }
when {
lockFiles.isEmpty() -> {
logcat(LogPriority.DEBUG) { "No lock file found, creating a new one" }
createLockFile(drive)
break
}
lockFiles.size == 1 -> {
val lockFile = lockFiles.first()
val createdTime = Instant.parse(lockFile.createdTime.toString())
val ageMinutes = java.time.Duration.between(createdTime, Instant.now()).toMinutes()
logcat(LogPriority.DEBUG) { "Lock file age: $ageMinutes minutes" }
if (ageMinutes <= 3) {
logcat(LogPriority.DEBUG) { "Lock file is new, proceeding with sync" }
break
} else {
logcat(LogPriority.DEBUG) { "Lock file is old, deleting and creating a new one" }
deleteLockFile(drive)
createLockFile(drive)
break
}
}
else -> {
logcat(LogPriority.DEBUG) { "Multiple lock files found, applying backoff" }
delay(backoff) // Apply backoff strategy
backoff = (backoff * 2).coerceAtMost(16000L)
logcat(LogPriority.DEBUG) { "Backoff increased to $backoff milliseconds" }
}
}
retries++ // Increment retry counter
logcat(LogPriority.DEBUG) { "Loop iteration complete, retry count: $retries, backoff time: $backoff" }
}
if (retries >= maxRetries) {
logcat(LogPriority.ERROR) { "Max retries reached, exiting sync process" }
throw Exception("Max retries reached.")
}
} catch (e: Exception) {
logcat(LogPriority.ERROR, throwable = e) { "Error in GoogleDrive beforeSync" }
throw Exception(context.stringResource(SYMR.strings.error_before_sync_gdrive, e.message ?: "Exception"), e)
}
googleDriveService.refreshToken()
}
private fun pullSyncData(): SyncData? {
val drive = googleDriveService.driveService ?:
throw Exception(context.stringResource(SYMR.strings.google_drive_not_signed_in))
val drive = googleDriveService.driveService
?: throw Exception(context.stringResource(SYMR.strings.google_drive_not_signed_in))
val fileList = getAppDataFileList(drive)
if (fileList.isEmpty()) {
@ -178,7 +123,10 @@ class GoogleDriveSyncService(context: Context, json: Json, syncPreferences: Sync
try {
drive.files().get(gdriveFileId).executeMediaAsInputStream().use { inputStream ->
GZIPInputStream(inputStream).use { gzipInputStream ->
return Json.decodeFromStream(SyncData.serializer(), gzipInputStream)
val byteArray = gzipInputStream.readBytes()
val backup = protoBuf.decodeFromByteArray(BackupSerializer, byteArray)
val deviceId = fileList[0].appProperties["deviceId"] ?: ""
return SyncData(deviceId = deviceId, backup = backup)
}
}
} catch (e: Exception) {
@ -192,29 +140,40 @@ class GoogleDriveSyncService(context: Context, json: Json, syncPreferences: Sync
?: throw Exception(context.stringResource(SYMR.strings.google_drive_not_signed_in))
val fileList = getAppDataFileList(drive)
val backup = syncData.backup ?: return
val byteArray = protoBuf.encodeToByteArray(BackupSerializer, backup)
if (byteArray.isEmpty()) {
throw IllegalStateException(context.stringResource(MR.strings.empty_backup_error))
}
PipedOutputStream().use { pos ->
PipedInputStream(pos).use { pis ->
withIOContext {
// Start a coroutine or a background thread to write JSON to the PipedOutputStream
launch {
GZIPOutputStream(pos).use { gzipOutputStream ->
Json.encodeToStream(SyncData.serializer(), syncData, gzipOutputStream)
gzipOutputStream.write(byteArray)
}
}
val mediaContent = InputStreamContent("application/octet-stream", pis)
if (fileList.isNotEmpty()) {
val fileId = fileList[0].id
val mediaContent = InputStreamContent("application/gzip", pis)
drive.files().update(fileId, null, mediaContent).execute()
val fileMetadata = File().apply {
name = remoteFileName
mimeType = "application/octet-stream"
appProperties = mapOf("deviceId" to syncData.deviceId)
}
drive.files().update(fileId, fileMetadata, mediaContent).execute()
logcat(LogPriority.DEBUG) { "Updated existing sync data file in Google Drive with file ID: $fileId" }
} else {
val fileMetadata = File().apply {
name = remoteFileName
mimeType = "application/gzip"
mimeType = "application/octet-stream"
parents = listOf("appDataFolder")
appProperties = mapOf("deviceId" to syncData.deviceId)
}
val mediaContent = InputStreamContent("application/gzip", pis)
val uploadedFile = drive.files().create(fileMetadata, mediaContent)
.setFields("id")
.execute()
@ -228,12 +187,12 @@ class GoogleDriveSyncService(context: Context, json: Json, syncPreferences: Sync
private fun getAppDataFileList(drive: Drive): MutableList<File> {
try {
// Search for the existing file by name in the appData folder
val query = "mimeType='application/gzip' and name = '$remoteFileName'"
val query = "mimeType='application/x-gzip' and name = '$remoteFileName'"
val fileList = drive.files()
.list()
.setSpaces("appDataFolder")
.setQ(query)
.setFields("files(id, name, createdTime)")
.setFields("files(id, name, createdTime, appProperties)")
.execute()
.files
logcat { "AppData folder file list: $fileList" }
@ -245,63 +204,6 @@ class GoogleDriveSyncService(context: Context, json: Json, syncPreferences: Sync
}
}
private fun createLockFile(drive: Drive) {
try {
val fileMetadata = File().apply {
name = lockFileName
mimeType = "text/plain"
parents = listOf("appDataFolder")
}
// Create an empty content to upload as the lock file
val emptyContent = ByteArrayContent.fromString("text/plain", "")
val file = drive.files().create(fileMetadata, emptyContent)
.setFields("id, name, createdTime")
.execute()
logcat { "Created lock file with ID: ${file.id}" }
} catch (e: Exception) {
logcat(LogPriority.ERROR, throwable = e) { "Error creating lock file" }
throw Exception(e.message, e)
}
}
private fun findLockFile(drive: Drive): MutableList<File> {
return try {
val query = "mimeType='text/plain' and name = '$lockFileName'"
val fileList = drive.files()
.list()
.setSpaces("appDataFolder")
.setQ(query)
.setFields("files(id, name, createdTime)")
.execute().files
logcat { "Lock file search result: $fileList" }
fileList
} catch (e: Exception) {
logcat(LogPriority.ERROR, throwable = e) { "Error finding lock file" }
mutableListOf()
}
}
private fun deleteLockFile(drive: Drive) {
try {
val lockFiles = findLockFile(drive)
if (lockFiles.isNotEmpty()) {
for (file in lockFiles) {
drive.files().delete(file.id).execute()
logcat { "Deleted lock file with ID: ${file.id}" }
}
} else {
logcat { "No lock file found to delete." }
}
} catch (e: Exception) {
logcat(LogPriority.ERROR, throwable = e) { "Error deleting lock file" }
throw Exception(context.stringResource(SYMR.strings.error_deleting_google_drive_lock_file), e)
}
}
suspend fun deleteSyncDataFromGoogleDrive(): DeleteSyncDataStatus {
val drive = googleDriveService.driveService