Add a deduplicate message feature

- Adds a new class of features for message management.
- Adds a manual option to find and delete duplicate messages
- Adds a setting to automatically delete duplicates during sync
- Adds a sha256 hashing function

Co-authored-by: octoshrimpy <octoshrimpy@users.noreply.github.com>
This commit is contained in:
Inhishonor
2026-01-10 13:34:37 -08:00
parent b4bce2c1a9
commit c1d824abbc
24 changed files with 539 additions and 3 deletions
@@ -62,7 +62,11 @@ import dev.octoshrimpy.quik.receiver.SendDelayedMessageReceiver.Companion.MESSAG
import dev.octoshrimpy.quik.util.ImageUtils
import dev.octoshrimpy.quik.util.PhoneNumberUtils
import dev.octoshrimpy.quik.util.Preferences
import dev.octoshrimpy.quik.util.sha256
import dev.octoshrimpy.quik.util.tryOrNull
import io.reactivex.Flowable
import io.reactivex.subjects.BehaviorSubject
import io.reactivex.subjects.Subject
import io.realm.Case
import io.realm.Realm
import io.realm.RealmList
@@ -87,6 +91,9 @@ open class MessageRepositoryImpl @Inject constructor(
private val cursorToPart: CursorToPart,
) : MessageRepository {
override val deduplicationProgress: Subject<MessageRepository.DeduplicationProgress> =
BehaviorSubject.createDefault(MessageRepository.DeduplicationProgress.Idle)
companion object {
const val TELEPHONY_UPDATE_CHUNK_SIZE = 200
}
@@ -1025,5 +1032,61 @@ open class MessageRepositoryImpl @Inject constructor(
uri -> context.contentResolver.delete(uri, null, null)
}
}
}
override fun deduplicateMessages(): Flowable<MessageRepository.DeduplicationResult> =
Flowable.fromCallable {
val duplicateIds = findDuplicateMessages()
if (duplicateIds.isEmpty()) {
MessageRepository.DeduplicationResult.NoDuplicates
} else {
deduplicationProgress.onNext(MessageRepository.DeduplicationProgress.Running(0, 0, true))
deleteMessages(duplicateIds)
MessageRepository.DeduplicationResult.Success
}
}
.onErrorReturn { MessageRepository.DeduplicationResult.Failure(it) }
.doFinally {
deduplicationProgress.onNext(MessageRepository.DeduplicationProgress.Idle)
}
private fun findDuplicateMessages(): List<Long> {
val seenSignatures = HashSet<String>()
val duplicateIds = ArrayList<Long>()
Realm.getDefaultInstance().use { realm ->
val allMessages = realm.where(Message::class.java)
.sort("id", Sort.ASCENDING)
.findAll()
val max = allMessages.size
var progress = 0
allMessages.forEach { message ->
++progress
tryOrNull {
if (progress % 100 == 0 || progress == max) {
deduplicationProgress.onNext(
MessageRepository.DeduplicationProgress.Running(max, progress, false)
)
}
val signature = messageHash(message)
if (!seenSignatures.add(signature)) {
duplicateIds.add(message.id)
}
}
}
}
return duplicateIds
}
private fun messageHash(message: Message): String {
val signatureString= buildString {
append(message.address).append('|')
append(message.dateSent).append('|')
append(message.boxId).append('|')
append(message.body).append('|')
append(message.attachmentTypeString)
}
return sha256(signatureString)
}
}
@@ -45,8 +45,10 @@ import dev.octoshrimpy.quik.model.MmsPart
import dev.octoshrimpy.quik.model.PhoneNumber
import dev.octoshrimpy.quik.model.Recipient
import dev.octoshrimpy.quik.model.SyncLog
import dev.octoshrimpy.quik.interactor.DeduplicateMessages
import dev.octoshrimpy.quik.util.PhoneNumberUtils
import dev.octoshrimpy.quik.util.tryOrNull
import io.reactivex.android.schedulers.AndroidSchedulers
import io.reactivex.subjects.BehaviorSubject
import io.reactivex.subjects.Subject
import io.realm.Realm
@@ -54,6 +56,7 @@ import io.realm.RealmList
import io.realm.Sort
import timber.log.Timber
import javax.inject.Inject
import javax.inject.Provider
import javax.inject.Singleton
@Singleton
@@ -69,6 +72,7 @@ class SyncRepositoryImpl @Inject constructor(
private val cursorToContactGroupMember: CursorToContactGroupMember,
private val keys: KeyManager,
private val phoneNumberUtils: PhoneNumberUtils,
private val messageRepo: Provider<MessageRepository>,
private val rxPrefs: RxSharedPreferences,
private val reactions: EmojiReactionRepository,
) : SyncRepository {
@@ -234,6 +238,25 @@ class SyncRepositoryImpl @Inject constructor(
// Now that we have all the messages, we can scan for emoji reactions
reactions.deleteAndReparseAllEmojiReactions(realm)
if (rxPrefs.getBoolean("autoDeduplicateMessages").get()) {
DeduplicateMessages(messageRepo.get())
.buildObservable(Unit)
.observeOn(AndroidSchedulers.mainThread())
.subscribe { result ->
when (result) {
is MessageRepository.DeduplicationResult.NoDuplicates -> {
Timber.i("No duplicate messages found.")
}
is MessageRepository.DeduplicationResult.Success -> {
Timber.i("Deleted duplicate messages")
}
is MessageRepository.DeduplicationResult.Failure -> {
Timber.e(result.error, "Deduplication failed")
}
}
}
}
realm.insert(SyncLog())
}, {
handlerThread.quitSafely()