Initial support for additional properties (#7519)

* Initial support for additional properties

This will allow some initial experiments in the SDKs. This is not a
fileformat breaking change as we just use already existing features.

---------

Co-authored-by: Claus Rørbech <claus.rorbech@gmail.com>
Co-authored-by: Kenneth Geisshirt <kenneth.geisshirt@mongodb.com>

bindgen/spec.yml

@@ -440,6 +440,9 @@ records:
       schema_mode:
         type: SchemaMode
         default: SchemaMode::Automatic
+      flexible_schema:
+        type: bool
+        default: false
       disable_format_upgrade:
         type: bool
         default: false

src/realm.h

@@ -925,6 +925,11 @@ RLM_API bool realm_config_get_cached(realm_config_t*) RLM_API_NOEXCEPT;
  */
 RLM_API void realm_config_set_automatic_backlink_handling(realm_config_t*, bool) RLM_API_NOEXCEPT;
 
+/**
+ * Allow realm objects in the realm to have additional properties that are not defined in the schema.
+ */
+RLM_API void realm_config_set_flexible_schema(realm_config_t*, bool) RLM_API_NOEXCEPT;
+
 /**
  * Create a custom scheduler object from callback functions.
  *
@@ -1640,6 +1645,13 @@ RLM_API realm_object_t* realm_object_from_thread_safe_reference(const realm_t*,
  */
 RLM_API bool realm_get_value(const realm_object_t*, realm_property_key_t, realm_value_t* out_value);
 
+/**
+ * Get the value for a property.
+ *
+ * @return True if no exception occurred.
+ */
+RLM_API bool realm_get_value_by_name(const realm_object_t*, const char* property_name, realm_value_t* out_value);
+
 /**
  * Get the values for several properties.
  *
@@ -1675,6 +1687,41 @@ RLM_API bool realm_get_values(const realm_object_t*, size_t num_values, const re
  */
 RLM_API bool realm_set_value(realm_object_t*, realm_property_key_t, realm_value_t new_value, bool is_default);
 
+/**
+ * Set the value for a property. Property need not be defined in schema if flexible
+ * schema is enabled in configuration
+ *
+ * @param property_name The name of the property.
+ * @param new_value The new value for the property.
+ * @return True if no exception occurred.
+ */
+RLM_API bool realm_set_value_by_name(realm_object_t*, const char* property_name, realm_value_t new_value);
+
+/**
+ * Examines if the object has a property with the given name.
+ * @param out_has_property will be true if the property exists.
+ * @return True if no exception occurred.
+ */
+RLM_API bool realm_has_property(realm_object_t*, const char* property_name, bool* out_has_property);
+
+/**
+ * Get a list of properties set on the object that are not defined in the schema.
+ *
+ * @param out_prop_names A pointer to an array of const char* of size @a max. If the pointer is NULL,
+ *                       no names will be copied, but @a out_n will be set to the required size.
+ * @param max size of @a out_prop_names
+ * @param out_n number of names actually returned.
+ */
+RLM_API void realm_get_additional_properties(realm_object_t*, const char** out_prop_names, size_t max, size_t* out_n);
+
+/**
+ * Erases a property from an object. You can't erase a property that is defined in the current schema.
+ *
+ * @param property_name The name of the property.
+ * @return True if the property was removed.
+ */
+RLM_API bool realm_erase_additional_property(realm_object_t*, const char* property_name);
+
 /**
  * Assign a JSON formatted string to a Mixed property. Underlying structures will be created as needed
  *
@@ -1696,6 +1743,8 @@ RLM_API realm_object_t* realm_set_embedded(realm_object_t*, realm_property_key_t
  */
 RLM_API realm_list_t* realm_set_list(realm_object_t*, realm_property_key_t);
 RLM_API realm_dictionary_t* realm_set_dictionary(realm_object_t*, realm_property_key_t);
+RLM_API realm_list_t* realm_set_list_by_name(realm_object_t*, const char* property_name);
+RLM_API realm_dictionary_t* realm_set_dictionary_by_name(realm_object_t*, const char* property_name);
 
 /** Return the object linked by the given property
  *
@@ -1748,6 +1797,15 @@ RLM_API bool realm_set_values(realm_object_t*, size_t num_values, const realm_pr
  */
 RLM_API realm_list_t* realm_get_list(realm_object_t*, realm_property_key_t);
 
+/**
+ * Get a list instance for the property of an object by name.
+ *
+ * Note: It is up to the caller to call `realm_release()` on the returned list.
+ *
+ * @return A non-null pointer if no exception occurred.
+ */
+RLM_API realm_list_t* realm_get_list_by_name(realm_object_t*, const char*);
+
 /**
  * Create a `realm_list_t` from a pointer to a `realm::List`, copy-constructing
  * the internal representation.
@@ -2253,6 +2311,15 @@ RLM_API realm_set_t* realm_set_from_thread_safe_reference(const realm_t*, realm_
  */
 RLM_API realm_dictionary_t* realm_get_dictionary(realm_object_t*, realm_property_key_t);
 
+/**
+ * Get a dictionary instance for the property of an object by name.
+ *
+ * Note: It is up to the caller to call `realm_release()` on the returned dictionary.
+ *
+ * @return A non-null pointer if no exception occurred.
+ */
+RLM_API realm_dictionary_t* realm_get_dictionary_by_name(realm_object_t*, const char*);
+
 /**
  * Create a `realm_dictionary_t` from a pointer to a `realm::object_store::Dictionary`,
  * copy-constructing the internal representation.

src/realm/db.cpp

@@ -2806,7 +2806,8 @@ void DB::async_request_write_mutex(TransactionRef& tr, util::UniqueFunction<void
 }
 
 inline DB::DB(Private, const DBOptions& options)
-    : m_upgrade_callback(std::move(options.upgrade_callback))
+    : m_allow_flexible_schema(options.allow_flexible_schema)
+    , m_upgrade_callback(std::move(options.upgrade_callback))
     , m_log_id(util::gen_log_id(this))
 {
     if (options.enable_async_writes) {

src/realm/db.hpp

@@ -498,6 +498,7 @@ class DB : public std::enable_shared_from_this<DB> {
     SharedInfo* m_info = nullptr;
     bool m_wait_for_change_enabled = true; // Initially wait_for_change is enabled
     bool m_write_transaction_open GUARDED_BY(m_mutex) = false;
+    bool m_allow_flexible_schema;
     std::string m_db_path;
     int m_file_format_version = 0;
     util::InterprocessMutex m_writemutex;

src/realm/db_options.hpp

@@ -106,6 +106,9 @@ struct DBOptions {
     /// will clear and reinitialize the file.
     bool clear_on_invalid_file = false;
 
+    /// Allow setting properties not supported by a specific column on an object
+    bool allow_flexible_schema = false;
+
     /// sys_tmp_dir will be used if the temp_dir is empty when creating DBOptions.
     /// It must be writable and allowed to create pipe/fifo file on it.
     /// set_sys_tmp_dir is not a thread-safe call and it is only supposed to be called once

src/realm/group.cpp

@@ -69,12 +69,13 @@ Group::Group()
 }
 
 
-Group::Group(const std::string& file_path, const char* encryption_key)
+Group::Group(const std::string& file_path, const char* encryption_key, bool allow_additional_properties)
     : m_local_alloc(new SlabAlloc) // Throws
     , m_alloc(*m_local_alloc)
     , m_top(m_alloc)
     , m_tables(m_alloc)
     , m_table_names(m_alloc)
+    , m_allow_additional_properties(allow_additional_properties)
 {
     init_array_parents();
 
@@ -760,6 +761,10 @@ Table* Group::do_add_table(StringData name, Table::Type table_type, bool do_repl
     Table* table = create_table_accessor(j);
     table->do_set_table_type(table_type);
 
+    if (m_allow_additional_properties && name.begins_with(g_class_name_prefix)) {
+        table->do_add_additional_prop_column();
+    }
+
     return table;
 }
 

src/realm/group.hpp

@@ -117,7 +117,8 @@ class Group : public ArrayParent {
     /// types that are derived from FileAccessError, the
     /// derived exception type is thrown. Note that InvalidDatabase is
     /// among these derived exception types.
-    explicit Group(const std::string& file, const char* encryption_key = nullptr);
+    explicit Group(const std::string& file, const char* encryption_key = nullptr,
+                   bool allow_additional_properties = false);
 
     /// Attach this Group instance to the specified memory buffer.
     ///
@@ -599,6 +600,7 @@ class Group : public ArrayParent {
     mutable int m_num_tables = 0;
     bool m_attached = false;
     bool m_is_writable = true;
+    bool m_allow_additional_properties = false;
     static std::optional<int> fake_target_file_format;
 
     util::UniqueFunction<void(const CascadeNotification&)> m_notify_handler;

src/realm/impl/array_writer.hpp

@@ -30,9 +30,7 @@ class ArrayWriterBase {
     bool only_modified = true;
     bool compress = true;
     const Table* table;
-    virtual ~ArrayWriterBase()
-    {
-    }
+    virtual ~ArrayWriterBase() {}
 
     /// Write the specified array data and its checksum into free
     /// space.

src/realm/obj.cpp

@@ -631,6 +631,30 @@ BinaryData Obj::_get<BinaryData>(ColKey::Idx col_ndx) const
     return ArrayBinary::get(alloc.translate(ref), m_row_ndx, alloc);
 }
 
+bool Obj::has_property(StringData prop_name) const
+{
+    if (m_table->get_column_key(prop_name))
+        return true;
+    if (auto ck = m_table->m_additional_prop_col) {
+        Dictionary dict(*this, ck);
+        return dict.contains(prop_name);
+    }
+    return false;
+}
+
+std::vector<StringData> Obj::get_additional_properties() const
+{
+    std::vector<StringData> ret;
+
+    if (auto ck = m_table->m_additional_prop_col) {
+        Dictionary dict(*this, ck);
+        dict.for_all_keys<StringData>([&ret](StringData key) {
+            ret.push_back(key);
+        });
+    }
+    return ret;
+}
+
 Mixed Obj::get_any(ColKey col_key) const
 {
     m_table->check_column(col_key);
@@ -676,6 +700,19 @@ Mixed Obj::get_any(ColKey col_key) const
     return {};
 }
 
+Mixed Obj::get_additional_prop(StringData prop_name) const
+{
+    if (auto ck = m_table->m_additional_prop_col) {
+        Dictionary dict(*this, ck);
+        if (auto val = dict.try_get(prop_name)) {
+            return *val;
+        }
+    }
+    throw InvalidArgument(ErrorCodes::InvalidProperty,
+                          util::format("Property '%1.%2' does not exist", m_table->get_class_name(), prop_name));
+    return {};
+}
+
 Mixed Obj::get_primary_key() const
 {
     auto col = m_table->get_primary_key_column();
@@ -1107,7 +1144,8 @@ StablePath Obj::get_stable_path() const noexcept
 void Obj::add_index(Path& path, const CollectionParent::Index& index) const
 {
     if (path.empty()) {
-        path.emplace_back(get_table()->get_column_key(index));
+        auto ck = m_table->get_column_key(index);
+        path.emplace_back(ck);
     }
     else {
         StringData col_name = get_table()->get_column_name(index);
@@ -1229,6 +1267,32 @@ Obj& Obj::set<Mixed>(ColKey col_key, Mixed value, bool is_default)
     return *this;
 }
 
+Obj& Obj::erase_additional_prop(StringData prop_name)
+{
+    bool erased = false;
+    if (auto ck = m_table->m_additional_prop_col) {
+        Dictionary dict(*this, ck);
+        erased = dict.try_erase(prop_name);
+    }
+    if (!erased) {
+        throw InvalidArgument(ErrorCodes::InvalidProperty, util::format("Could not erase property: %1", prop_name));
+    }
+    return *this;
+}
+
+Obj& Obj::set_additional_prop(StringData prop_name, const Mixed& value)
+{
+    if (auto ck = m_table->m_additional_prop_col) {
+        Dictionary dict(*this, ck);
+        dict.insert(prop_name, value);
+    }
+    else {
+        throw InvalidArgument(ErrorCodes::InvalidProperty,
+                              util::format("Property '%1.%2' does not exist", m_table->get_class_name(), prop_name));
+    }
+    return *this;
+}
+
 Obj& Obj::set_any(ColKey col_key, Mixed value, bool is_default)
 {
     if (value.is_null()) {
@@ -1983,19 +2047,41 @@ Dictionary Obj::get_dictionary(ColKey col_key) const
 
 Obj& Obj::set_collection(ColKey col_key, CollectionType type)
 {
-    REALM_ASSERT(col_key.get_type() == col_type_Mixed);
     if ((col_key.is_dictionary() && type == CollectionType::Dictionary) ||
         (col_key.is_list() && type == CollectionType::List)) {
         return *this;
     }
     if (type == CollectionType::Set) {
         throw IllegalOperation("Set nested in Mixed is not supported");
     }
+    if (col_key.get_type() != col_type_Mixed) {
+        throw IllegalOperation("Collection can only be nested in Mixed");
+    }
     set(col_key, Mixed(0, type));
 
     return *this;
 }
 
+Obj& Obj::set_collection(StringData prop_name, CollectionType type)
+{
+    if (auto ck = get_column_key(prop_name)) {
+        return set_collection(ck, type);
+    }
+    return set_additional_collection(prop_name, type);
+}
+
+Obj& Obj::set_additional_collection(StringData prop_name, CollectionType type)
+{
+    if (auto ck = m_table->m_additional_prop_col) {
+        Dictionary dict(*this, ck);
+        dict.insert_collection(prop_name, type);
+    }
+    else {
+        throw InvalidArgument(ErrorCodes::InvalidProperty, util::format("Property not found: %1", prop_name));
+    }
+    return *this;
+}
+
 DictionaryPtr Obj::get_dictionary_ptr(ColKey col_key) const
 {
     return std::make_shared<Dictionary>(get_dictionary(col_key));
@@ -2011,14 +2097,33 @@ Dictionary Obj::get_dictionary(StringData col_name) const
     return get_dictionary(get_column_key(col_name));
 }
 
-CollectionPtr Obj::get_collection_ptr(const Path& path) const
+CollectionBasePtr Obj::get_collection_ptr(const Path& path) const
 {
     REALM_ASSERT(path.size() > 0);
     // First element in path must be column name
     auto col_key = path[0].is_col_key() ? path[0].get_col_key() : m_table->get_column_key(path[0].get_key());
-    REALM_ASSERT(col_key);
+
+    CollectionBasePtr collection;
     size_t level = 1;
-    CollectionBasePtr collection = get_collection_ptr(col_key);
+    if (col_key) {
+        collection = get_collection_ptr(col_key);
+    }
+    else {
+        if (auto ck = m_table->m_additional_prop_col) {
+            auto prop_name = path[0].get_key();
+            Dictionary dict(*this, ck);
+            auto ref = dict.get(prop_name);
+            if (ref.is_type(type_List)) {
+                collection = dict.get_list(prop_name);
+            }
+            else if (ref.is_type(type_Dictionary)) {
+                collection = dict.get_dictionary(prop_name);
+            }
+            else {
+                throw InvalidArgument("Wrong path");
+            }
+        }
+    }
 
     while (level < path.size()) {
         auto& path_elem = path[level];
@@ -2044,7 +2149,7 @@ CollectionPtr Obj::get_collection_ptr(const Path& path) const
     return collection;
 }
 
-CollectionPtr Obj::get_collection_by_stable_path(const StablePath& path) const
+CollectionBasePtr Obj::get_collection_by_stable_path(const StablePath& path) const
 {
     // First element in path is phony column key
     ColKey col_key = m_table->get_column_key(path[0]);
@@ -2108,7 +2213,7 @@ CollectionBasePtr Obj::get_collection_ptr(ColKey col_key) const
 
 CollectionBasePtr Obj::get_collection_ptr(StringData col_name) const
 {
-    return get_collection_ptr(get_column_key(col_name));
+    return get_collection_ptr(Path{{col_name}});
 }
 
 LinkCollectionPtr Obj::get_linkcollection_ptr(ColKey col_key) const