From dd8d114711210b74f11f909b9ab3b40533c67434 Mon Sep 17 00:00:00 2001 From: J-Jamet Date: Thu, 27 Aug 2020 13:49:48 +0200 Subject: [PATCH] Change save binaries compression for database 3.1 & 4 --- .../element/database/BinaryAttachment.kt | 10 ++++- .../database/element/database/BinaryPool.kt | 2 +- .../database/file/input/DatabaseInputKDBX.kt | 44 ++++++++++--------- .../output/DatabaseInnerHeaderOutputKDBX.kt | 9 ++-- .../file/output/DatabaseOutputKDBX.kt | 32 +++----------- .../AttachmentFileNotificationService.kt | 6 +-- 6 files changed, 46 insertions(+), 57 deletions(-) diff --git a/app/src/main/java/com/kunzisoft/keepass/database/element/database/BinaryAttachment.kt b/app/src/main/java/com/kunzisoft/keepass/database/element/database/BinaryAttachment.kt index 96ef57f91..55a41c0b0 100644 --- a/app/src/main/java/com/kunzisoft/keepass/database/element/database/BinaryAttachment.kt +++ b/app/src/main/java/com/kunzisoft/keepass/database/element/database/BinaryAttachment.kt @@ -72,6 +72,14 @@ class BinaryAttachment : Parcelable { } } + @Throws(IOException::class) + fun getUnGzipInputDataStream(): InputStream { + return if (isCompressed) + GZIPInputStream(getInputDataStream()) + else + getInputDataStream() + } + @Throws(IOException::class) fun getOutputDataStream(): OutputStream { return when { @@ -110,7 +118,7 @@ class BinaryAttachment : Parcelable { if (isCompressed) { val fileBinaryDecompress = File(concreteDataFile.parent, concreteDataFile.name + "_temp") FileOutputStream(fileBinaryDecompress).use { outputStream -> - GZIPInputStream(getInputDataStream()).use { inputStream -> + getUnGzipInputDataStream().use { inputStream -> inputStream.readBytes(bufferSize) { buffer -> outputStream.write(buffer) } diff --git a/app/src/main/java/com/kunzisoft/keepass/database/element/database/BinaryPool.kt b/app/src/main/java/com/kunzisoft/keepass/database/element/database/BinaryPool.kt index 5b186cbbd..f4868b20d 100644 --- a/app/src/main/java/com/kunzisoft/keepass/database/element/database/BinaryPool.kt +++ b/app/src/main/java/com/kunzisoft/keepass/database/element/database/BinaryPool.kt @@ -57,7 +57,7 @@ class BinaryPool { binaryAttachment.clear() } - fun findUnusedKey(): Int { + private fun findUnusedKey(): Int { var unusedKey = 0 while (pool[unusedKey] != null) unusedKey++ diff --git a/app/src/main/java/com/kunzisoft/keepass/database/file/input/DatabaseInputKDBX.kt b/app/src/main/java/com/kunzisoft/keepass/database/file/input/DatabaseInputKDBX.kt index 84fe13463..5cf767a3d 100644 --- a/app/src/main/java/com/kunzisoft/keepass/database/file/input/DatabaseInputKDBX.kt +++ b/app/src/main/java/com/kunzisoft/keepass/database/file/input/DatabaseInputKDBX.kt @@ -233,8 +233,10 @@ class DatabaseInputKDBX(cacheDirectory: File, var data = ByteArray(0) if (size > 0) { - if (fieldId != DatabaseHeaderKDBX.PwDbInnerHeaderV4Fields.Binary) + if (fieldId != DatabaseHeaderKDBX.PwDbInnerHeaderV4Fields.Binary) { + // TODO OOM here data = dataInputStream.readBytes(size) + } } var result = true @@ -249,12 +251,11 @@ class DatabaseInputKDBX(cacheDirectory: File, header.innerRandomStreamKey = data } DatabaseHeaderKDBX.PwDbInnerHeaderV4Fields.Binary -> { - val byteLength = size - 1 // Read in a file val protectedFlag = dataInputStream.readBytes(1)[0].toInt() != 0 - // Unknown compression at this level - val compression = mDatabase.compressionAlgorithm == CompressionAlgorithm.GZip - val protectedBinary = mDatabase.buildNewBinary(cacheDirectory, protectedFlag, compression) + val byteLength = size - 1 + // No compression at this level + val protectedBinary = mDatabase.buildNewBinary(cacheDirectory, protectedFlag, false) protectedBinary.getOutputDataStream().use { outputStream -> dataInputStream.readBytes(byteLength, DatabaseKDBX.BUFFER_SIZE_BYTES) { buffer -> outputStream.write(buffer) @@ -940,27 +941,28 @@ class DatabaseInputKDBX(cacheDirectory: File, // Reference Id to a binary already present in binary pool val ref = xpp.getAttributeValue(null, DatabaseKDBXXML.AttrRef) - if (ref != null) { - xpp.next() // Consume end tag - - val id = Integer.parseInt(ref) - return mDatabase.binaryPool[id] - } - + // New id to a binary val key = xpp.getAttributeValue(null, DatabaseKDBXXML.AttrId) - return if (key != null) { - createBinary(key.toIntOrNull(), xpp) - } - // New binary to retrieve - else { - createBinary(null, xpp) + return when { + ref != null -> { + xpp.next() // Consume end tag + val id = Integer.parseInt(ref) + mDatabase.binaryPool[id] + } + key != null -> { + createBinary(key.toIntOrNull(), xpp) + } + else -> { + // New binary to retrieve + createBinary(null, xpp) + } } } @Throws(IOException::class, XmlPullParserException::class) private fun createBinary(binaryId: Int?, xpp: XmlPullParser): BinaryAttachment? { - var compressed: Boolean = mDatabase.compressionAlgorithm == CompressionAlgorithm.GZip + var compressed = false var protected = false if (xpp.attributeCount > 0) { @@ -980,10 +982,10 @@ class DatabaseInputKDBX(cacheDirectory: File, return null val data = Base64.decode(base64, BASE_64_FLAG) - // Force compression in this specific case + // Build the new binary and compress val binaryAttachment = mDatabase.buildNewBinary(cacheDirectory, protected, compressed, binaryId) binaryAttachment.getOutputDataStream().use { outputStream -> - outputStream.write(data) + outputStream.write(data) } return binaryAttachment } diff --git a/app/src/main/java/com/kunzisoft/keepass/database/file/output/DatabaseInnerHeaderOutputKDBX.kt b/app/src/main/java/com/kunzisoft/keepass/database/file/output/DatabaseInnerHeaderOutputKDBX.kt index e14a1ff16..842f71e8e 100644 --- a/app/src/main/java/com/kunzisoft/keepass/database/file/output/DatabaseInnerHeaderOutputKDBX.kt +++ b/app/src/main/java/com/kunzisoft/keepass/database/file/output/DatabaseInnerHeaderOutputKDBX.kt @@ -54,11 +54,14 @@ class DatabaseInnerHeaderOutputKDBX(private val database: DatabaseKDBX, } dataOutputStream.write(DatabaseHeaderKDBX.PwDbInnerHeaderV4Fields.Binary.toInt()) - dataOutputStream.writeInt(protectedBinary.length().toInt() + 1) // TODO verify + dataOutputStream.writeInt(protectedBinary.length().toInt() + 1) dataOutputStream.write(flag.toInt()) - protectedBinary.getInputDataStream().readBytes(BUFFER_SIZE_BYTES) { buffer -> - dataOutputStream.write(buffer) + // if was compressed in cache, uncompress it + protectedBinary.getUnGzipInputDataStream().use { inputStream -> + inputStream.readBytes(BUFFER_SIZE_BYTES) { buffer -> + dataOutputStream.write(buffer) + } } } diff --git a/app/src/main/java/com/kunzisoft/keepass/database/file/output/DatabaseOutputKDBX.kt b/app/src/main/java/com/kunzisoft/keepass/database/file/output/DatabaseOutputKDBX.kt index 6ae0038f3..6a17df95f 100644 --- a/app/src/main/java/com/kunzisoft/keepass/database/file/output/DatabaseOutputKDBX.kt +++ b/app/src/main/java/com/kunzisoft/keepass/database/file/output/DatabaseOutputKDBX.kt @@ -55,7 +55,6 @@ import java.io.OutputStream import java.security.NoSuchAlgorithmException import java.security.SecureRandom import java.util.* -import java.util.zip.GZIPInputStream import java.util.zip.GZIPOutputStream import javax.crypto.Cipher import javax.crypto.CipherOutputStream @@ -422,7 +421,6 @@ class DatabaseOutputKDBX(private val mDatabaseKDBX: DatabaseKDBX, private fun writeBinary(binary : BinaryAttachment) { val binaryLength = binary.length() if (binaryLength > 0) { - if (binary.isProtected) { xml.attribute(null, DatabaseKDBXXML.AttrProtected, DatabaseKDBXXML.ValTrue) @@ -433,21 +431,11 @@ class DatabaseOutputKDBX(private val mDatabaseKDBX: DatabaseKDBX, xml.text(charArray, 0, charArray.size) } } else { - // Force binary compression from database (compression was harmonized during import) - if (mDatabaseKDBX.compressionAlgorithm === CompressionAlgorithm.GZip) { + if (binary.isCompressed) { xml.attribute(null, DatabaseKDBXXML.AttrCompressed, DatabaseKDBXXML.ValTrue) } - - // Force decompression in this specific case - val binaryInputStream = if (mDatabaseKDBX.compressionAlgorithm == CompressionAlgorithm.None - && binary.isCompressed) { - GZIPInputStream(binary.getInputDataStream()) - } else { - binary.getInputDataStream() - } - // Write the XML - binaryInputStream.readBytes(BUFFER_SIZE_BYTES) { buffer -> + binary.getInputDataStream().readBytes(BUFFER_SIZE_BYTES) { buffer -> val charArray = String(Base64.encode(buffer, BASE_64_FLAG)).toCharArray() xml.text(charArray, 0, charArray.size) } @@ -560,23 +548,15 @@ class DatabaseOutputKDBX(private val mDatabaseKDBX: DatabaseKDBX, @Throws(IllegalArgumentException::class, IllegalStateException::class, IOException::class) private fun writeEntryBinaries(binaries: LinkedHashMap) { - binaries.forEach { + for ((label, poolId) in binaries) { xml.startTag(null, DatabaseKDBXXML.ElemBinary) xml.startTag(null, DatabaseKDBXXML.ElemKey) - xml.text(safeXmlString(it.key)) + xml.text(safeXmlString(label)) xml.endTag(null, DatabaseKDBXXML.ElemKey) xml.startTag(null, DatabaseKDBXXML.ElemValue) - xml.attribute(null, DatabaseKDBXXML.AttrRef, it.value.toString()) - /* - // By default use only pool data in head to save binaries - val ref = mDatabaseKDBX.binaryPool.findKey(it.binaryAttachment) - if (ref != null) { - xml.attribute(null, DatabaseKDBXXML.AttrRef, ref.toString()) - } else { - writeBinary(it.binaryAttachment) - } - */ + // Use only pool data in Meta to save binaries + xml.attribute(null, DatabaseKDBXXML.AttrRef, poolId.toString()) xml.endTag(null, DatabaseKDBXXML.ElemValue) xml.endTag(null, DatabaseKDBXXML.ElemBinary) diff --git a/app/src/main/java/com/kunzisoft/keepass/notifications/AttachmentFileNotificationService.kt b/app/src/main/java/com/kunzisoft/keepass/notifications/AttachmentFileNotificationService.kt index 54ee785f8..b0a0b0630 100644 --- a/app/src/main/java/com/kunzisoft/keepass/notifications/AttachmentFileNotificationService.kt +++ b/app/src/main/java/com/kunzisoft/keepass/notifications/AttachmentFileNotificationService.kt @@ -362,11 +362,7 @@ class AttachmentFileNotificationService: LockNotificationService() { var dataDownloaded = 0L val fileSize = binaryAttachment.length() UriUtil.getUriOutputStream(contentResolver, attachmentToUploadUri)?.use { outputStream -> - if (binaryAttachment.isCompressed) { - GZIPInputStream(binaryAttachment.getInputDataStream()) - } else { - binaryAttachment.getInputDataStream() - }.use { inputStream -> + binaryAttachment.getUnGzipInputDataStream().use { inputStream -> inputStream.readBytes(bufferSize) { buffer -> outputStream.write(buffer) dataDownloaded += buffer.size