Passing encryption key via a callback

CHANGELOG.md

@@ -6,6 +6,7 @@
 ### Enhancements
 * [Sync] Added option to use managed WebSockets via OkHttp instead of Realm's built-in WebSocket client for Sync traffic (Only Android and JVM targets for now). Managed WebSockets offer improved support for proxies and firewalls that require authentication. This feature is currently opt-in and can be enabled by using `AppConfiguration.usePlatformNetworking()`. Managed WebSockets will become the default in a future version. (PR [#1528](https://github.com/realm/realm-kotlin/pull/1528)).
 * `AutoClientResetFailed` exception now reports as the throwable cause any user exceptions that might occur during a client reset. (Issue [#1580](https://github.com/realm/realm-kotlin/issues/1580))
+* Added an experimental configuration API which will allow to pass the encryption key using a callback https://github.com/realm/realm-kotlin/pull/1636.
 
 ### Fixed
 * Cache notification callback JNI references at startup to ensure that symbols can be resolved in core callbacks. (Issue [#1577](https://github.com/realm/realm-kotlin/issues/1577))

packages/cinterop/src/commonMain/kotlin/io/realm/kotlin/internal/interop/RealmInterop.kt

@@ -192,6 +192,7 @@ expect object RealmInterop {
     fun realm_config_set_schema(config: RealmConfigurationPointer, schema: RealmSchemaPointer)
     fun realm_config_set_max_number_of_active_versions(config: RealmConfigurationPointer, maxNumberOfVersions: Long)
     fun realm_config_set_encryption_key(config: RealmConfigurationPointer, encryptionKey: ByteArray)
+    fun realm_config_set_encryption_key_from_pointer(config: RealmConfigurationPointer, aesEncryptionKeyAddress: Long)
     fun realm_config_get_encryption_key(config: RealmConfigurationPointer): ByteArray?
     fun realm_config_set_should_compact_on_launch_function(config: RealmConfigurationPointer, callback: CompactOnLaunchCallback)
     fun realm_config_set_migration_function(config: RealmConfigurationPointer, callback: MigrationCallback)

packages/cinterop/src/jvm/kotlin/io/realm/kotlin/internal/interop/RealmInterop.kt

@@ -184,6 +184,10 @@ actual object RealmInterop {
         realmc.realm_config_set_encryption_key(config.cptr(), encryptionKey, encryptionKey.size.toLong())
     }
 
+    actual fun realm_config_set_encryption_key_from_pointer(config: RealmConfigurationPointer, aesEncryptionKeyAddress: Long) {
+        realmc.realm_config_set_encryption_key_from_pointer(config.cptr(), aesEncryptionKeyAddress)
+    }
+
     actual fun realm_config_get_encryption_key(config: RealmConfigurationPointer): ByteArray? {
         val key = ByteArray(ENCRYPTION_KEY_LENGTH)
         val keyLength: Long = realmc.realm_config_get_encryption_key(config.cptr(), key)

packages/cinterop/src/nativeDarwin/kotlin/io/realm/kotlin/internal/interop/RealmInterop.kt

@@ -54,6 +54,7 @@ import kotlinx.cinterop.CPointerVar
 import kotlinx.cinterop.CPointerVarOf
 import kotlinx.cinterop.CValue
 import kotlinx.cinterop.CVariable
+import kotlinx.cinterop.ExperimentalForeignApi
 import kotlinx.cinterop.LongVar
 import kotlinx.cinterop.MemScope
 import kotlinx.cinterop.StableRef
@@ -77,6 +78,7 @@ import kotlinx.cinterop.readValue
 import kotlinx.cinterop.refTo
 import kotlinx.cinterop.set
 import kotlinx.cinterop.staticCFunction
+import kotlinx.cinterop.toCPointer
 import kotlinx.cinterop.toCStringArray
 import kotlinx.cinterop.toCValues
 import kotlinx.cinterop.toKString
@@ -419,6 +421,16 @@ actual object RealmInterop {
         }
     }
 
+    @OptIn(ExperimentalForeignApi::class)
+    actual fun realm_config_set_encryption_key_from_pointer(config: RealmConfigurationPointer, aesEncryptionKeyAddress: Long) {
+        memScoped { // Ensure memory cleanup
+            val ptr = aesEncryptionKeyAddress.toCPointer<ByteVarOf<Byte>>()
+            val encryptionKey = ByteArray(64)
+            memcpy(encryptionKey.refTo(0), ptr, 64u)
+            realm_config_set_encryption_key(config, encryptionKey)
+        }
+    }
+
     actual fun realm_config_get_encryption_key(config: RealmConfigurationPointer): ByteArray? {
         memScoped {
             val encryptionKey = ByteArray(ENCRYPTION_KEY_LENGTH)

packages/jni-swig-stub/src/main/jni/realm_api_helpers.cpp

@@ -937,6 +937,12 @@ void realm_sync_websocket_closed(int64_t observer_ptr, bool was_clean, int error
     realm_sync_socket_websocket_closed(reinterpret_cast<realm_websocket_observer_t*>(observer_ptr), was_clean, static_cast<realm_web_socket_errno_e>(error_code), reason);
 }
 
+void realm_config_set_encryption_key_from_pointer(realm_config_t* config, int64_t aesKeyAddress) {
+    uint8_t key_array[64];
+    std::memcpy(key_array, reinterpret_cast<uint8_t*>(aesKeyAddress), 64);
+    realm_config_set_encryption_key(config, key_array, 64);
+}
+
 realm_sync_socket_t* realm_sync_websocket_new(int64_t sync_client_config_ptr, jobject websocket_transport) {
     auto jenv = get_env(false); // Always called from JVM
     realm_sync_socket_t* socket_provider = realm_sync_socket_new(jenv->NewGlobalRef(websocket_transport), /*userdata*/

packages/jni-swig-stub/src/main/jni/realm_api_helpers.h

@@ -161,4 +161,5 @@ bool realm_sync_websocket_message(int64_t observer_ptr, jbyteArray data, size_t
 
 void realm_sync_websocket_closed(int64_t observer_ptr, bool was_clean, int error_code, const char* reason);
 
+void realm_config_set_encryption_key_from_pointer(realm_config_t* config, int64_t aesKeyAddress);
 #endif //TEST_REALM_API_HELPERS_H

packages/library-base/src/commonMain/kotlin/io/realm/kotlin/Configuration.kt

@@ -17,6 +17,7 @@
 package io.realm.kotlin
 
 import io.realm.kotlin.Configuration.SharedBuilder
+import io.realm.kotlin.annotations.ExperimentalEncryptionCallbackApi
 import io.realm.kotlin.internal.MISSING_PLUGIN_MESSAGE
 import io.realm.kotlin.internal.REALM_FILE_EXTENSION
 import io.realm.kotlin.internal.platform.PATH_SEPARATOR
@@ -106,6 +107,24 @@ public data class InitialRealmFileConfiguration(
     val checksum: String?
 )
 
+@ExperimentalEncryptionCallbackApi
+public interface EncryptionKeyCallback {
+    /**
+     * Provides the native memory address of the 64 byte array containing the key used to encrypt and decrypt the Realm file.
+     * This can be called multiple times internally, so the key needs to be the same for between calls.
+     *
+     * Note: The Realm SDK is not responsible of checking that the pointer is a valid 64 byte array, providing an invalid address will probably
+     *       causes a segmentation fault and will crash the app.
+     */
+    public fun keyPointer(): Long
+
+    /**
+     * This callback will be invoked by Realm after it's open. This hint to the user that the key provided in [keyPointer] can now be released.
+     * This will be called once the Realm is open and it's safe to dispose of the encryption key.
+     */
+    public fun releaseKey()
+}
+
 /**
  * Base configuration options shared between all realm configuration types.
  */
@@ -153,6 +172,14 @@ public interface Configuration {
      */
     public val encryptionKey: ByteArray?
 
+    /**
+     * Native memory address of the 64 byte array containing the key used to encrypt and decrypt the Realm file.
+     *
+     * @return null on unencrypted Realms.
+     */
+    @OptIn(ExperimentalEncryptionCallbackApi::class)
+    public val encryptionKeyAsCallback: EncryptionKeyCallback?
+
     /**
      * Callback that determines if the realm file should be compacted as part of opening it.
      *
@@ -234,6 +261,8 @@ public interface Configuration {
         protected var writeDispatcher: CoroutineDispatcher? = null
         protected var schemaVersion: Long = 0
         protected var encryptionKey: ByteArray? = null
+        @OptIn(ExperimentalEncryptionCallbackApi::class)
+        protected var encryptionKeyAsCallback: EncryptionKeyCallback? = null
         protected var compactOnLaunchCallback: CompactOnLaunchCallback? = null
         protected var initialDataCallback: InitialDataCallback? = null
         protected var inMemory: Boolean = false
@@ -354,6 +383,51 @@ public interface Configuration {
         public fun encryptionKey(encryptionKey: ByteArray): S =
             apply { this.encryptionKey = validateEncryptionKey(encryptionKey) } as S
 
+        /**
+         * Similar to [encryptionKey] but instead this will read the encryption key from native memory.
+         * This can enhance the security of the app, since it reduces the window where the key is available in clear
+         * in memory (avoid memory dump attack). Once the Realm is open, one can zero-out the memory region holding the key
+         * as it will be already passed to the C++ storage engine.
+         *
+         * There's also extra protection for JVM Windows target, where the underlying storage engine uses the Windows Kernel
+         * to encrypt/decrypt the Realm's encryption key before each usage.
+         *
+         * Note: The RealmConfiguration doesn't take ownership of this native memory, the caller is responsible of disposing it
+         * appropriately after the Realm is open using the [EncryptionKeyCallback.releaseKey].
+         *
+         * @param encryptionKeyAsCallback Callback providing address/pointer to a 64-byte array containing the AES encryption key.
+         * This array should be in native memory to avoid copying the key into garbage collected heap memory (for JVM targets).
+         *
+         * One way to create such an array in JVM is to use JNI or use `sun.misc.Unsafe` as follow:
+         *
+         *```
+         * import sun.misc.Unsafe
+         *
+         * val field = Unsafe::class.java.getDeclaredField("theUnsafe")
+         * field.isAccessible = true
+         * val unsafe: Unsafe = field.get(null) as Unsafe
+         *
+         * val key = Random.nextBytes(64) // Replace with your actual AES key
+         * val keyPointer: Long = unsafe.allocateMemory(key.size.toLong())
+         *  for (i in key.indices) { // Write the key bytes to native memory
+         *      unsafe.putByte(keyPointer + i, key[i])
+         *  }
+         *
+         * val encryptedConf = RealmConfiguration
+         *      .Builder(schema = setOf(Sample::class))
+         *             .encryptionKey(object : EncryptionKeyCallback {
+         *                 override fun keyPointer() = keyPointer
+         *                 override fun releaseKey() = unsafe.freeMemory(keyPointer)
+         *             })
+         *             .build()
+         *
+         * val realm = Realm.open(encryptedConf)
+         *```
+         */
+        @OptIn(ExperimentalEncryptionCallbackApi::class)
+        public fun encryptionKey(encryptionKeyAsCallback: EncryptionKeyCallback): S =
+            apply { this.encryptionKeyAsCallback = encryptionKeyAsCallback } as S
+
         /**
          * Sets a callback for controlling whether the realm should be compacted when opened.
          *

packages/library-base/src/commonMain/kotlin/io/realm/kotlin/RealmConfiguration.kt

@@ -16,6 +16,7 @@
 
 package io.realm.kotlin
 
+import io.realm.kotlin.annotations.ExperimentalEncryptionCallbackApi
 import io.realm.kotlin.internal.ContextLogger
 import io.realm.kotlin.internal.RealmConfigurationImpl
 import io.realm.kotlin.internal.platform.appFilesDirectory
@@ -185,6 +186,8 @@ public interface RealmConfiguration : Configuration {
                 writerDispatcherFactory,
                 schemaVersion,
                 encryptionKey,
+                @OptIn(ExperimentalEncryptionCallbackApi::class)
+                encryptionKeyAsCallback,
                 deleteRealmIfMigrationNeeded,
                 compactOnLaunchCallback,
                 migration,

packages/library-base/src/commonMain/kotlin/io/realm/kotlin/annotations/ExperimentalEncryptionCallbackApi.kt

@@ -0,0 +1,35 @@
+/*
+ * Copyright 2023 Realm Inc.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package io.realm.kotlin.annotations
+
+/**
+ * This annotation mark Realm API for encryption callback **experimental**, i.e.
+ * there are no guarantees given that this API cannot change without warning between minor and
+ * major versions. They will not change between patch versions.
+ *
+ * For all other purposes these APIs are considered stable, i.e. they undergo the same testing
+ * as other parts of the API and should behave as documented with no bugs. It is primarily
+ * marked as experimental because we are unsure if this API provide value and solve the use
+ * cases that people have. If not, they will be changed or removed altogether.
+ */
+@MustBeDocumented
+@Target(
+    AnnotationTarget.CLASS,
+    AnnotationTarget.PROPERTY,
+    AnnotationTarget.FUNCTION,
+    AnnotationTarget.TYPEALIAS
+)
+@RequiresOptIn(level = RequiresOptIn.Level.ERROR)
+public annotation class ExperimentalEncryptionCallbackApi

packages/library-base/src/commonMain/kotlin/io/realm/kotlin/internal/ConfigurationImpl.kt

@@ -17,9 +17,11 @@
 package io.realm.kotlin.internal
 
 import io.realm.kotlin.CompactOnLaunchCallback
+import io.realm.kotlin.EncryptionKeyCallback
 import io.realm.kotlin.InitialDataCallback
 import io.realm.kotlin.InitialRealmFileConfiguration
 import io.realm.kotlin.LogConfiguration
+import io.realm.kotlin.annotations.ExperimentalEncryptionCallbackApi
 import io.realm.kotlin.dynamic.DynamicMutableRealm
 import io.realm.kotlin.dynamic.DynamicMutableRealmObject
 import io.realm.kotlin.dynamic.DynamicRealm
@@ -60,8 +62,10 @@ public open class ConfigurationImpl(
     schemaVersion: Long,
     schemaMode: SchemaMode,
     private val userEncryptionKey: ByteArray?,
+    @OptIn(ExperimentalEncryptionCallbackApi::class)
+    override val encryptionKeyAsCallback: EncryptionKeyCallback?,
     compactOnLaunchCallback: CompactOnLaunchCallback?,
-    private val userMigration: RealmMigration?,
+    userMigration: RealmMigration?,
     automaticBacklinkHandling: Boolean,
     initialDataCallback: InitialDataCallback?,
     override val isFlexibleSyncConfiguration: Boolean,
@@ -230,6 +234,11 @@ public open class ConfigurationImpl(
                 RealmInterop.realm_config_set_encryption_key(nativeConfig, key)
             }
 
+            @OptIn(ExperimentalEncryptionCallbackApi::class)
+            encryptionKeyAsCallback?.let {
+                RealmInterop.realm_config_set_encryption_key_from_pointer(nativeConfig, it.keyPointer())
+            }
+
             RealmInterop.realm_config_set_in_memory(nativeConfig, inMemory)
 
             nativeConfig

packages/library-base/src/commonMain/kotlin/io/realm/kotlin/internal/RealmConfigurationImpl.kt

@@ -17,10 +17,12 @@
 package io.realm.kotlin.internal
 
 import io.realm.kotlin.CompactOnLaunchCallback
+import io.realm.kotlin.EncryptionKeyCallback
 import io.realm.kotlin.InitialDataCallback
 import io.realm.kotlin.InitialRealmFileConfiguration
 import io.realm.kotlin.LogConfiguration
 import io.realm.kotlin.RealmConfiguration
+import io.realm.kotlin.annotations.ExperimentalEncryptionCallbackApi
 import io.realm.kotlin.internal.interop.SchemaMode
 import io.realm.kotlin.internal.util.CoroutineDispatcherFactory
 import io.realm.kotlin.migration.RealmMigration
@@ -40,6 +42,8 @@ internal class RealmConfigurationImpl(
     writeDispatcherFactory: CoroutineDispatcherFactory,
     schemaVersion: Long,
     encryptionKey: ByteArray?,
+    @OptIn(ExperimentalEncryptionCallbackApi::class)
+    encryptionKeyAsCallback: EncryptionKeyCallback?,
     override val deleteRealmIfMigrationNeeded: Boolean,
     compactOnLaunchCallback: CompactOnLaunchCallback?,
     migration: RealmMigration?,
@@ -62,6 +66,8 @@ internal class RealmConfigurationImpl(
         false -> SchemaMode.RLM_SCHEMA_MODE_AUTOMATIC
     },
     encryptionKey,
+    @OptIn(ExperimentalEncryptionCallbackApi::class)
+    encryptionKeyAsCallback,
     compactOnLaunchCallback,
     migration,
     automaticBacklinkHandling,

packages/library-base/src/commonMain/kotlin/io/realm/kotlin/internal/RealmImpl.kt

@@ -19,6 +19,7 @@ package io.realm.kotlin.internal
 import io.realm.kotlin.Configuration
 import io.realm.kotlin.MutableRealm
 import io.realm.kotlin.Realm
+import io.realm.kotlin.annotations.ExperimentalEncryptionCallbackApi
 import io.realm.kotlin.dynamic.DynamicRealm
 import io.realm.kotlin.internal.dynamic.DynamicRealmImpl
 import io.realm.kotlin.internal.interop.ClassKey
@@ -42,11 +43,12 @@ import kotlinx.atomicfu.AtomicRef
 import kotlinx.atomicfu.atomic
 import kotlinx.coroutines.CoroutineScope
 import kotlinx.coroutines.SupervisorJob
+import kotlinx.coroutines.async
+import kotlinx.coroutines.awaitAll
 import kotlinx.coroutines.cancel
 import kotlinx.coroutines.channels.BufferOverflow
 import kotlinx.coroutines.flow.Flow
 import kotlinx.coroutines.flow.MutableSharedFlow
-import kotlinx.coroutines.flow.collect
 import kotlinx.coroutines.flow.map
 import kotlinx.coroutines.flow.withIndex
 import kotlinx.coroutines.launch
@@ -138,6 +140,24 @@ public class RealmImpl private constructor(
             }
 
             realmScope.launch {
+                @OptIn(ExperimentalEncryptionCallbackApi::class)
+                configuration.encryptionKeyAsCallback?.let {
+                    // if we're using an encryption key as a callback, we preemptively open the notifier and writer Realm
+                    // with the given configuration because the key might be deleted from memory after the Realm is open.
+
+                    // These touches the notifier and writer lazy initialised Realms to open them with the provided configuration.
+                    awaitAll(
+                        async(notificationScheduler.dispatcher) {
+                            notifier.realm.version().version
+                        },
+                        async(writeScheduler.dispatcher) {
+                            writer.realm.version().version
+                        }
+                    )
+
+                    it.releaseKey()
+                }
+
                 notifier.realmChanged().collect {
                     removeInitialRealmReference()
                     // Closing this reference might be done by the GC:
@@ -270,7 +290,6 @@ public class RealmImpl private constructor(
             current = initialRealmReference.value?.uncheckedVersion(),
             active = versionTracker.versions()
         )
-
         return VersionInfo(
             main = mainVersions,
             notifier = notifier.versions(),

packages/library-sync/src/commonMain/kotlin/io/realm/kotlin/mongodb/sync/SyncConfiguration.kt

@@ -20,6 +20,7 @@ import io.realm.kotlin.LogConfiguration
 import io.realm.kotlin.MutableRealm
 import io.realm.kotlin.Realm
 import io.realm.kotlin.TypedRealm
+import io.realm.kotlin.annotations.ExperimentalEncryptionCallbackApi
 import io.realm.kotlin.internal.ConfigurationImpl
 import io.realm.kotlin.internal.ContextLogger
 import io.realm.kotlin.internal.ObjectIdImpl
@@ -565,6 +566,8 @@ public interface SyncConfiguration : Configuration {
                 schemaVersion,
                 SchemaMode.RLM_SCHEMA_MODE_ADDITIVE_DISCOVERED,
                 encryptionKey,
+                @OptIn(ExperimentalEncryptionCallbackApi::class)
+                encryptionKeyAsCallback,
                 compactOnLaunchCallback,
                 null, // migration is not relevant for sync,
                 false, // automatic backlink handling is not relevant for sync

packages/test-base/build.gradle.kts

@@ -126,6 +126,13 @@ android {
         }
     }
 
+    externalNativeBuild {
+        cmake {
+            version = Versions.cmake
+            path = project.file("src/androidMain/cpp/CMakeLists.txt")
+        }
+    }
+
     buildTypes {
         // LibraryBuildType is not minifiable, but the current dependency from test-sync doesn't
         // allow test-base to be configured as a library. To test test-base with minification

packages/test-base/src/androidMain/cpp/CMakeLists.txt

@@ -0,0 +1 @@
+add_library(android_jni_test_helper SHARED android_jni_helper.cpp)