Retry failed Matrix backfill messages and persist import progress
Build and Release APK / Build APK (push) Successful in 4m5s

This commit is contained in:
2026-10-18 00:44:23 +02:00
parent 77d3ff7e1e
commit 1a140650af
8 changed files with 268 additions and 29 deletions
@@ -20,6 +20,9 @@ import ch.skew.remotrix.data.RemotrixDB
import ch.skew.remotrix.data.RemotrixSettings
import ch.skew.remotrix.data.forwardRuleDB.ForwardRule
import ch.skew.remotrix.data.roomIdDB.RoomIdData
import ch.skew.remotrix.bridge.BackfillRunner
import ch.skew.remotrix.data.backfillDB.BackfillProgress
import kotlinx.coroutines.CancellationException
import ch.skew.remotrix.bridge.closeMatrixClient
import ch.skew.remotrix.bridge.resetBridge
import ch.skew.remotrix.bridge.RoomResolver
@@ -405,7 +408,7 @@ class CommandService: Service() {
return null
}
private suspend fun sendMsg(sender: String, payload: String, log: Boolean, outgoing: Boolean = false) {
private suspend fun sendMsg(sender: String, payload: String, log: Boolean, outgoing: Boolean = false): Result<Unit> {
if(currentStatus.get() != StatusType.Running) {
load()
}
@@ -421,7 +424,7 @@ class CommandService: Service() {
MsgStatus.MESSAGE_DROPPED,
null
)
return
return Result.success(Unit) // Intentionally excluded by forwarding settings.
}
// If match is not found, default account is chosen.
val sendAs = if (match !== null) match else defaultAccount
@@ -433,7 +436,7 @@ class CommandService: Service() {
null,
null
)
return
return Result.failure(IllegalStateException("No suitable Matrix forwarder"))
}
val client = pair.first
while (client.syncState.first() != net.folivo.trixnity.clientserverapi.client.SyncState.RUNNING) {
@@ -449,7 +452,7 @@ class CommandService: Service() {
null,
null
)
return
return Result.failure(it)
}
val roomId = RoomId(resolved.roomId)
if (resolved.created) {
@@ -469,11 +472,17 @@ class CommandService: Service() {
}
}
// Message is sent successfully, mark the pre-recorded message as successfully sent
if(currentLog != -1L) db.getLogDao().setSuccess(
currentLog,
MsgStatus.MESSAGE_SENT,
sendAs
)
// A logging failure must not cause an already queued SMS to be queued twice.
if (currentLog != -1L) {
try {
db.getLogDao().setSuccess(currentLog, MsgStatus.MESSAGE_SENT, sendAs)
} catch (e: CancellationException) {
throw e
} catch (e: Exception) {
Log.w("CommandService", "Could not record forwarding success", e)
}
}
return Result.success(Unit)
}
private fun stopAll() {
@@ -586,8 +595,9 @@ class CommandService: Service() {
// Backfill already routes each message to its selected account in sendMsg.
// Running it for every client duplicates the entire history.
if (clients.isNotEmpty() && !settings.getBackfillDone.first()) {
performBackfill(settings.getBackfillLimit.first())
settings.saveBackfillDone(true)
if (performBackfill(settings.getBackfillLimit.first())) {
settings.saveBackfillDone(true)
}
}
}
if(clients.isNotEmpty()){
@@ -697,19 +707,25 @@ class CommandService: Service() {
}
}
private suspend fun performBackfill(limit: Int) {
val bridge = ch.skew.remotrix.bridge.MatrixBridgeProvider.bridge ?: return
val messagesToSend = bridge.getBackfillMessages(limit)
Log.d("CommandService", "Starting backfill with ${messagesToSend.size} messages")
for ((address, body, isOutgoing) in messagesToSend) {
try {
sendMsg(address, body, false, isOutgoing)
} catch (e: Exception) {
Log.e("CommandService", "Error backfilling message for $address", e)
private suspend fun performBackfill(limit: Int): Boolean {
val bridge = ch.skew.remotrix.bridge.MatrixBridgeProvider.bridge ?: return false
val messages = bridge.getBackfillMessages(limit)
val progress = db.getBackfillProgressDao()
Log.i("CommandService", "Starting backfill with ${messages.size} messages")
BackfillRunner(
isHandled = { progress.isHandled(it) },
markHandled = { progress.insert(BackfillProgress(it)) },
send = { sendMsg(it.address, it.body, false, it.outgoing) },
onRetry = { index, error, delay ->
// Do not log SMS contents, addresses, or server error bodies.
Log.w("CommandService", "Backfill $index/${messages.size} failed (${error.javaClass.simpleName}); retry in ${delay}ms")
},
onProgress = { completed, total ->
Log.i("CommandService", "Backfill queued or previously handled: $completed/$total")
}
kotlinx.coroutines.delay(500) // Small delay to prevent rate limits
}
).run(messages)
Log.i("CommandService", "Backfill completed: ${messages.size} messages handled")
return true
}
private suspend fun sendSMS(sender: Int, roomId: RoomId, payload: String): Boolean {
@@ -0,0 +1,50 @@
package ch.skew.remotrix.bridge
import kotlinx.coroutines.CancellationException
import kotlinx.coroutines.NonCancellable
import kotlinx.coroutines.delay
import kotlinx.coroutines.withContext
data class BackfillMessage(
val id: String,
val address: String,
val body: String,
val outgoing: Boolean
)
/** Stops at a failed message: later messages must never hide an incomplete import. */
class BackfillRunner(
private val isHandled: suspend (String) -> Boolean,
private val markHandled: suspend (String) -> Unit,
private val send: suspend (BackfillMessage) -> Result<Unit>,
private val waitBeforeRetry: suspend (Long) -> Unit = { delay(it) },
private val onRetry: (Int, Throwable, Long) -> Unit = { _, _, _ -> },
private val onProgress: (Int, Int) -> Unit = { _, _ -> }
) {
suspend fun run(messages: List<BackfillMessage>) {
for ((index, message) in messages.withIndex()) {
if (!isHandled(message.id)) {
var retryDelay = 1_000L
while (true) {
val result = try {
send(message)
} catch (e: CancellationException) {
throw e
} catch (e: Exception) {
Result.failure(e)
}
val error = result.exceptionOrNull()
if (error == null) break
generateSequence(error) { it.cause }
.filterIsInstance<CancellationException>().firstOrNull()?.let { throw it }
onRetry(index + 1, error, retryDelay)
waitBeforeRetry(retryDelay)
retryDelay = (retryDelay * 2).coerceAtMost(60_000L)
}
// Once queued, preserve progress even if the service is being stopped.
withContext(NonCancellable) { markHandled(message.id) }
}
onProgress(index + 1, messages.size)
}
}
}
@@ -2,7 +2,7 @@ package ch.skew.remotrix.bridge
interface MatrixBridge {
suspend fun sendNewMessages(toAddress: String, body: String, fromMatrix: Boolean)
suspend fun getBackfillMessages(limit: Int): List<Triple<String, String, Boolean>> // sender, payload, isOutgoing
suspend fun getBackfillMessages(limit: Int): List<BackfillMessage>
}
object MatrixBridgeProvider {
@@ -1,5 +1,7 @@
package ch.skew.remotrix.data
import ch.skew.remotrix.data.backfillDB.BackfillProgress
import ch.skew.remotrix.data.backfillDB.BackfillProgressDao
import android.content.ContentValues
import android.content.Context
import android.database.sqlite.SQLiteDatabase
@@ -24,12 +26,14 @@ import ch.skew.remotrix.data.roomIdDB.RoomIdData
ForwardRule::class,
RoomIdData::class,
LogData::class,
DebugLogData::class
DebugLogData::class,
BackfillProgress::class
],
version = 3
version = 4
)
abstract class RemotrixDB: RoomDatabase(){
abstract fun getBackfillProgressDao(): BackfillProgressDao
abstract fun getAccountDao(): AccountDao
abstract fun getForwardRuleDao(): ForwardRuleDao
abstract fun getRoomIdDao(): RoomIdDao
@@ -51,8 +55,13 @@ abstract class RemotrixDB: RoomDatabase(){
.setJournalMode(androidx.room.RoomDatabase.JournalMode.WRITE_AHEAD_LOGGING)
.addMigrations(migration1To2)
.addMigrations(migration2To3)
.addMigrations(migration3To4)
.build()
private val migration3To4 = Migration(3, 4) {
it.execSQL("CREATE TABLE IF NOT EXISTS backfill_progress (messageId TEXT NOT NULL PRIMARY KEY)")
}
private val migration2To3 = Migration(2, 3) {
it.execSQL(
"CREATE TABLE IF NOT EXISTS debug_log (" +
@@ -0,0 +1,20 @@
package ch.skew.remotrix.data.backfillDB
import androidx.room.Dao
import androidx.room.Entity
import androidx.room.Insert
import androidx.room.OnConflictStrategy
import androidx.room.PrimaryKey
import androidx.room.Query
@Entity(tableName = "backfill_progress")
data class BackfillProgress(@PrimaryKey val messageId: String)
@Dao
interface BackfillProgressDao {
@Query("SELECT EXISTS(SELECT 1 FROM backfill_progress WHERE messageId = :id)")
suspend fun isHandled(id: String): Boolean
@Insert(onConflict = OnConflictStrategy.IGNORE)
suspend fun insert(progress: BackfillProgress)
}
@@ -0,0 +1,124 @@
package ch.skew.remotrix.bridge
import ch.skew.remotrix.classes.RoomCreationError
import ch.skew.remotrix.classes.MsgStatus
import kotlinx.coroutines.CancellationException
import kotlinx.coroutines.runBlocking
import org.junit.Assert.*
import org.junit.Test
class BackfillRunnerTest {
private fun message(id: String) = BackfillMessage(id, "123", "same body", false)
@Test
fun failedRoomCreationRetriesSameMessageBeforeAdvancing() = runBlocking {
val handled = mutableSetOf<String>()
val attempts = mutableListOf<String>()
val delays = mutableListOf<Long>()
var failures = 2
BackfillRunner(
isHandled = { it in handled },
markHandled = { handled.add(it); Unit },
send = {
attempts.add(it.id)
if (failures-- > 0) Result.failure(RoomCreationError(IllegalStateException("rate limited"), MsgStatus.CANNOT_CREATE_ROOM))
else Result.success(Unit)
},
waitBeforeRetry = { delays.add(it); Unit }
).run(listOf(message("sms:1"), message("sms:2")))
assertEquals(listOf("sms:1", "sms:1", "sms:1", "sms:2"), attempts)
assertEquals(listOf(1000L, 2000L), delays)
assertEquals(setOf("sms:1", "sms:2"), handled)
}
@Test
fun restartSkipsPersistedMessagesAndRetainsIdenticalTextsWithDifferentIds() = runBlocking {
val handled = mutableSetOf<String>()
val attempts = mutableListOf<String>()
val messages = listOf(message("sms:1"), message("sms:2"), message("mms:1"))
fun runner(stop: Boolean) = BackfillRunner(
isHandled = { it in handled },
markHandled = { handled.add(it); Unit },
send = {
if (stop && it.id == "sms:2") throw CancellationException()
attempts.add(it.id)
Result.success(Unit)
}
)
try {
runner(true).run(messages)
fail("Import should stop")
} catch (_: CancellationException) { }
assertEquals(setOf("sms:1"), handled)
runner(false).run(messages)
assertEquals(listOf("sms:1", "sms:2", "mms:1"), attempts)
}
@Test
fun cancellationReturnedAsFailureIsNotRetriedOrMarkedHandled() = runBlocking {
var marked = false
BackfillRunner(
isHandled = { false }, markHandled = { marked = true },
send = { Result.failure(CancellationException()) },
waitBeforeRetry = { fail("Cancellation must not be retried") }
).let {
try { it.run(listOf(message("sms:1"))); fail("Must propagate cancellation") }
catch (_: CancellationException) { }
}
assertFalse(marked)
}
@Test
fun cancellationDuringRetryLeavesImportIncomplete() = runBlocking {
var finished = false
var marked = false
try {
BackfillRunner(
isHandled = { false }, markHandled = { marked = true },
send = { Result.failure(IllegalStateException("room unavailable")) },
waitBeforeRetry = { throw CancellationException() }
).run(listOf(message("sms:1")))
finished = true
} catch (_: CancellationException) { }
assertFalse(finished)
assertFalse(marked)
}
@Test
fun thrownFailureIsRetriedWithCappedBackoffAndNoDelayAfterSuccess() = runBlocking {
var attempts = 0
val delays = mutableListOf<Long>()
BackfillRunner(
isHandled = { false }, markHandled = { },
send = { if (++attempts <= 9) throw IllegalStateException(); Result.success(Unit) },
waitBeforeRetry = { delays.add(it); Unit }
).run(listOf(message("sms:1")))
assertEquals(listOf(1000L, 2000L, 4000L, 8000L, 16000L, 32000L, 60000L, 60000L, 60000L), delays)
}
@Test
fun checkpointFailureStopsImportInsteadOfResendingQueuedMessage() = runBlocking {
var sends = 0
try {
BackfillRunner(
isHandled = { false }, markHandled = { throw IllegalStateException("disk full") },
send = { sends++; Result.success(Unit) },
waitBeforeRetry = { fail("Must not resend after a checkpoint failure") }
).run(listOf(message("sms:1"), message("sms:2")))
fail("Import must remain incomplete")
} catch (_: IllegalStateException) { }
assertEquals(1, sends)
}
@Test
fun cancellationWrappedByRoomCreationIsPropagated() = runBlocking {
try {
BackfillRunner(
isHandled = { false }, markHandled = { fail("Must not mark cancelled message") },
send = { Result.failure(RoomCreationError(CancellationException(), MsgStatus.CANNOT_CREATE_ROOM)) },
waitBeforeRetry = { fail("Must not retry cancellation") }
).run(listOf(message("sms:1")))
fail("Must propagate cancellation")
} catch (_: CancellationException) { }
}
}