Fix binary deduplication #715

CHANGELOG

@@ -1,5 +1,6 @@
 KeePassDX(2.9.14)
- * Dark Themes (WARNING: You must reselect your theme if upgrading from an old installation)
+ * Dark Themes (WARNING: You must reselect your theme if upgrading from an old installation) #532 #714
+ * Fix binary deduplication #715
 
 KeePassDX(2.9.13)
  * Binary image viewer #473 #749

app/src/androidTest/java/com/kunzisoft/keepass/tests/stream/BinaryAttachmentTest.kt

@@ -11,8 +11,6 @@ import org.junit.Test
 import java.io.DataInputStream
 import java.io.File
 import java.io.InputStream
-import java.lang.Exception
-import java.security.MessageDigest
 
 class BinaryAttachmentTest {
 
@@ -132,20 +130,6 @@ class BinaryAttachmentTest {
         }
     }
 
-    private fun BinaryAttachment.md5(): String {
-        val md = MessageDigest.getInstance("MD5")
-        return this.getInputDataStream(loadedKey).use { fis ->
-            val buffer = ByteArray(DEFAULT_BUFFER_SIZE)
-            generateSequence {
-                when (val bytesRead = fis.read(buffer)) {
-                    -1 -> null
-                    else -> bytesRead
-                }
-            }.forEach { bytesRead -> md.update(buffer, 0, bytesRead) }
-            md.digest().joinToString("") { "%02x".format(it) }
-        }
-    }
-
     companion object {
         private const val TEST_FILE_CACHE_A = "testA"
         private const val TEST_FILE_CACHE_B = "testB"

app/src/main/java/com/kunzisoft/keepass/database/element/database/BinaryAttachment.kt

@@ -28,6 +28,7 @@ import com.kunzisoft.keepass.database.element.Database
 import com.kunzisoft.keepass.stream.readAllBytes
 import org.apache.commons.io.output.CountingOutputStream
 import java.io.*
+import java.security.MessageDigest
 import java.util.zip.GZIPInputStream
 import java.util.zip.GZIPOutputStream
 import javax.crypto.Cipher
@@ -176,6 +177,25 @@ class BinaryAttachment : Parcelable {
             throw IOException("Unable to delete temp file " + dataFile!!.absolutePath)
     }
 
+    /**
+     * MD5 of the raw encrypted file in temp folder, only to compare binary data
+     */
+    fun md5(): String {
+        val md = MessageDigest.getInstance("MD5")
+        if (dataFile == null)
+            return ""
+        return FileInputStream(dataFile).use { fis ->
+            val buffer = ByteArray(DEFAULT_BUFFER_SIZE)
+            generateSequence {
+                when (val bytesRead = fis.read(buffer)) {
+                    -1 -> null
+                    else -> bytesRead
+                }
+            }.forEach { bytesRead -> md.update(buffer, 0, bytesRead) }
+            md.digest().joinToString("") { "%02x".format(it) }
+        }
+    }
+
     override fun equals(other: Any?): Boolean {
         if (this === other)
             return true

app/src/main/java/com/kunzisoft/keepass/database/element/database/BinaryPool.kt

@@ -99,7 +99,13 @@ class BinaryPool {
     private fun orderedBinaries(): List<KeyBinary> {
         val keyBinaryList = ArrayList<KeyBinary>()
         for ((key, binary) in pool) {
-            keyBinaryList.add(KeyBinary(key, binary))
+            // Don't deduplicate
+            val existentBinary = keyBinaryList.find { it.binary.md5() == binary.md5() }
+            if (existentBinary == null) {
+                keyBinaryList.add(KeyBinary(binary, key))
+            } else {
+                existentBinary.addKey(key)
+            }
         }
         return keyBinaryList
     }
@@ -108,7 +114,7 @@ class BinaryPool {
      * To register a binary with a ref corresponding to an ordered index
      */
     fun getBinaryIndexFromKey(key: Int): Int? {
-        val index = orderedBinaries().indexOfFirst { it.key == key }
+        val index = orderedBinaries().indexOfFirst { it.keys.contains(key) }
         return if (index < 0)
             null
         else
@@ -118,8 +124,10 @@ class BinaryPool {
     /**
      * Different from doForEach, provide an ordered index to each binary
      */
-    fun doForEachOrderedBinary(action: (index: Int, keyBinary: KeyBinary) -> Unit) {
+    fun doForEachOrderedBinary(action: (index: Int, binary: BinaryAttachment) -> Unit) {
-        orderedBinaries().forEachIndexed(action)
+        orderedBinaries().forEachIndexed { index, keyBinary ->
+            action.invoke(index, keyBinary.binary)
+        }
     }
 
     /**
@@ -149,7 +157,16 @@ class BinaryPool {
     }
 
     /**
-     * Utility data class to order binaries
+     * Utility class to order binaries
      */
-    data class KeyBinary(val key: Int, val binary: BinaryAttachment)
+    private class KeyBinary(val binary: BinaryAttachment, key: Int) {
+        val keys = HashSet<Int>()
+        init {
+            addKey(key)
+        }
+
+        fun addKey(key: Int) {
+            keys.add(key)
+        }
+    }
 }

app/src/main/java/com/kunzisoft/keepass/database/file/output/DatabaseOutputKDBX.kt

@@ -137,24 +137,23 @@ class DatabaseOutputKDBX(private val mDatabaseKDBX: DatabaseKDBX,
         dataOutputStream.writeInt(streamKeySize)
         dataOutputStream.write(header.innerRandomStreamKey)
 
-        database.binaryPool.doForEachOrderedBinary { _, keyBinary ->
+        val binaryCipherKey = database.loadedCipherKey
-            val protectedBinary = keyBinary.binary
+                ?: throw IOException("Unable to retrieve cipher key to write binaries")
-            val binaryCipherKey = database.loadedCipherKey
+        database.binaryPool.doForEachOrderedBinary { _, binary ->
-                    ?: throw IOException("Unable to retrieve cipher key to write binaries")
             // Force decompression to add binary in header
-            protectedBinary.decompress(binaryCipherKey)
+            binary.decompress(binaryCipherKey)
             // Write type binary
             dataOutputStream.writeByte(DatabaseHeaderKDBX.PwDbInnerHeaderV4Fields.Binary)
             // Write size
-            dataOutputStream.writeUInt(UnsignedInt.fromKotlinLong(protectedBinary.length + 1))
+            dataOutputStream.writeUInt(UnsignedInt.fromKotlinLong(binary.length + 1))
             // Write protected flag
             var flag = DatabaseHeaderKDBX.KdbxBinaryFlags.None
-            if (protectedBinary.isProtected) {
+            if (binary.isProtected) {
                 flag = flag or DatabaseHeaderKDBX.KdbxBinaryFlags.Protected
             }
             dataOutputStream.writeByte(flag)
 
-            protectedBinary.getInputDataStream(binaryCipherKey).use { inputStream ->
+            binary.getInputDataStream(binaryCipherKey).use { inputStream ->
                 inputStream.readAllBytes { buffer ->
                     dataOutputStream.write(buffer)
                 }
@@ -499,10 +498,9 @@ class DatabaseOutputKDBX(private val mDatabaseKDBX: DatabaseKDBX,
         xml.startTag(null, DatabaseKDBXXML.ElemBinaries)
 
         // Use indexes because necessarily (binary header ref is the order)
-        mDatabaseKDBX.binaryPool.doForEachOrderedBinary { index, keyBinary ->
+        mDatabaseKDBX.binaryPool.doForEachOrderedBinary { index, binary ->
             xml.startTag(null, DatabaseKDBXXML.ElemBinary)
             xml.attribute(null, DatabaseKDBXXML.AttrId, index.toString())
-            val binary = keyBinary.binary
             if (binary.length > 0) {
                 if (binary.isCompressed) {
                     xml.attribute(null, DatabaseKDBXXML.AttrCompressed, DatabaseKDBXXML.ValTrue)

fastlane/metadata/android/en-US/changelogs/58.txt

@@ -1 +1,2 @@
- * Dark Themes (WARNING: You must reselect your theme if upgrading from an old installation)
+ * Dark Themes (WARNING: You must reselect your theme if upgrading from an old installation) #532 #714
+ * Fix binary deduplication #715

fastlane/metadata/android/fr-FR/changelogs/58.txt

@@ -1 +1,2 @@
- * Thèmes sombres (ATTENTION: Vous devez reselectionner votre theme si mise à niveau d'une ancienne installation)
+ * Thèmes sombres (ATTENTION: Vous devez reselectionner votre theme si mise à niveau d'une ancienne installation) #532 #714
+ * Correction de la duplication des binaires #715