forked from upstream-mirrors/quik
Add a deduplicate message feature
- Adds a new class of features for message management. - Adds a manual option to find and delete duplicate messages - Adds a setting to automatically delete duplicates during sync - Adds a sha256 hashing function Co-authored-by: octoshrimpy <octoshrimpy@users.noreply.github.com>
This commit is contained in:
@@ -62,7 +62,11 @@ import dev.octoshrimpy.quik.receiver.SendDelayedMessageReceiver.Companion.MESSAG
|
||||
import dev.octoshrimpy.quik.util.ImageUtils
|
||||
import dev.octoshrimpy.quik.util.PhoneNumberUtils
|
||||
import dev.octoshrimpy.quik.util.Preferences
|
||||
import dev.octoshrimpy.quik.util.sha256
|
||||
import dev.octoshrimpy.quik.util.tryOrNull
|
||||
import io.reactivex.Flowable
|
||||
import io.reactivex.subjects.BehaviorSubject
|
||||
import io.reactivex.subjects.Subject
|
||||
import io.realm.Case
|
||||
import io.realm.Realm
|
||||
import io.realm.RealmList
|
||||
@@ -87,6 +91,9 @@ open class MessageRepositoryImpl @Inject constructor(
|
||||
private val cursorToPart: CursorToPart,
|
||||
) : MessageRepository {
|
||||
|
||||
override val deduplicationProgress: Subject<MessageRepository.DeduplicationProgress> =
|
||||
BehaviorSubject.createDefault(MessageRepository.DeduplicationProgress.Idle)
|
||||
|
||||
companion object {
|
||||
const val TELEPHONY_UPDATE_CHUNK_SIZE = 200
|
||||
}
|
||||
@@ -1025,5 +1032,61 @@ open class MessageRepositoryImpl @Inject constructor(
|
||||
uri -> context.contentResolver.delete(uri, null, null)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
override fun deduplicateMessages(): Flowable<MessageRepository.DeduplicationResult> =
|
||||
Flowable.fromCallable {
|
||||
val duplicateIds = findDuplicateMessages()
|
||||
if (duplicateIds.isEmpty()) {
|
||||
MessageRepository.DeduplicationResult.NoDuplicates
|
||||
} else {
|
||||
deduplicationProgress.onNext(MessageRepository.DeduplicationProgress.Running(0, 0, true))
|
||||
deleteMessages(duplicateIds)
|
||||
MessageRepository.DeduplicationResult.Success
|
||||
}
|
||||
}
|
||||
.onErrorReturn { MessageRepository.DeduplicationResult.Failure(it) }
|
||||
.doFinally {
|
||||
deduplicationProgress.onNext(MessageRepository.DeduplicationProgress.Idle)
|
||||
}
|
||||
|
||||
private fun findDuplicateMessages(): List<Long> {
|
||||
val seenSignatures = HashSet<String>()
|
||||
val duplicateIds = ArrayList<Long>()
|
||||
|
||||
Realm.getDefaultInstance().use { realm ->
|
||||
val allMessages = realm.where(Message::class.java)
|
||||
.sort("id", Sort.ASCENDING)
|
||||
.findAll()
|
||||
|
||||
val max = allMessages.size
|
||||
var progress = 0
|
||||
|
||||
allMessages.forEach { message ->
|
||||
++progress
|
||||
tryOrNull {
|
||||
if (progress % 100 == 0 || progress == max) {
|
||||
deduplicationProgress.onNext(
|
||||
MessageRepository.DeduplicationProgress.Running(max, progress, false)
|
||||
)
|
||||
}
|
||||
val signature = messageHash(message)
|
||||
if (!seenSignatures.add(signature)) {
|
||||
duplicateIds.add(message.id)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return duplicateIds
|
||||
}
|
||||
|
||||
private fun messageHash(message: Message): String {
|
||||
val signatureString= buildString {
|
||||
append(message.address).append('|')
|
||||
append(message.dateSent).append('|')
|
||||
append(message.boxId).append('|')
|
||||
append(message.body).append('|')
|
||||
append(message.attachmentTypeString)
|
||||
}
|
||||
return sha256(signatureString)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -45,8 +45,10 @@ import dev.octoshrimpy.quik.model.MmsPart
|
||||
import dev.octoshrimpy.quik.model.PhoneNumber
|
||||
import dev.octoshrimpy.quik.model.Recipient
|
||||
import dev.octoshrimpy.quik.model.SyncLog
|
||||
import dev.octoshrimpy.quik.interactor.DeduplicateMessages
|
||||
import dev.octoshrimpy.quik.util.PhoneNumberUtils
|
||||
import dev.octoshrimpy.quik.util.tryOrNull
|
||||
import io.reactivex.android.schedulers.AndroidSchedulers
|
||||
import io.reactivex.subjects.BehaviorSubject
|
||||
import io.reactivex.subjects.Subject
|
||||
import io.realm.Realm
|
||||
@@ -54,6 +56,7 @@ import io.realm.RealmList
|
||||
import io.realm.Sort
|
||||
import timber.log.Timber
|
||||
import javax.inject.Inject
|
||||
import javax.inject.Provider
|
||||
import javax.inject.Singleton
|
||||
|
||||
@Singleton
|
||||
@@ -69,6 +72,7 @@ class SyncRepositoryImpl @Inject constructor(
|
||||
private val cursorToContactGroupMember: CursorToContactGroupMember,
|
||||
private val keys: KeyManager,
|
||||
private val phoneNumberUtils: PhoneNumberUtils,
|
||||
private val messageRepo: Provider<MessageRepository>,
|
||||
private val rxPrefs: RxSharedPreferences,
|
||||
private val reactions: EmojiReactionRepository,
|
||||
) : SyncRepository {
|
||||
@@ -234,6 +238,25 @@ class SyncRepositoryImpl @Inject constructor(
|
||||
// Now that we have all the messages, we can scan for emoji reactions
|
||||
reactions.deleteAndReparseAllEmojiReactions(realm)
|
||||
|
||||
if (rxPrefs.getBoolean("autoDeduplicateMessages").get()) {
|
||||
DeduplicateMessages(messageRepo.get())
|
||||
.buildObservable(Unit)
|
||||
.observeOn(AndroidSchedulers.mainThread())
|
||||
.subscribe { result ->
|
||||
when (result) {
|
||||
is MessageRepository.DeduplicationResult.NoDuplicates -> {
|
||||
Timber.i("No duplicate messages found.")
|
||||
}
|
||||
is MessageRepository.DeduplicationResult.Success -> {
|
||||
Timber.i("Deleted duplicate messages")
|
||||
}
|
||||
is MessageRepository.DeduplicationResult.Failure -> {
|
||||
Timber.e(result.error, "Deduplication failed")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
realm.insert(SyncLog())
|
||||
}, {
|
||||
handlerThread.quitSafely()
|
||||
|
||||
Reference in New Issue
Block a user