From 4875fa2063bee7ba909ae99a408ba6976b6204f1 Mon Sep 17 00:00:00 2001 From: Bastien Gerard Date: Thu, 20 Aug 2026 16:55:21 +0200 Subject: [PATCH 1/8] Add test cases upfront to avoid behavior-change --- tests/fields/test_dict_field.py | 188 +++++++++++++++++++++++++++++++- 1 file changed, 187 insertions(+), 1 deletion(-) diff --git a/tests/fields/test_dict_field.py b/tests/fields/test_dict_field.py index c2c6ea1fd..1ff1bcfbf 100644 --- a/tests/fields/test_dict_field.py +++ b/tests/fields/test_dict_field.py @@ -1,5 +1,7 @@ +from enum import Enum + import pytest -from bson import InvalidDocument +from bson import DBRef, InvalidDocument, ObjectId from mongoengine import * from mongoengine.base import BaseDict @@ -10,6 +12,132 @@ from tests.utils import MongoDBTestCase, get_as_pymongo +class TestDictFieldToPython: + def test_to_python__untyped_dict_contains_document__converts_to_dbref(self): + class Referenced(Document): + pass + + referenced = Referenced(id=ObjectId()) + + converted = DictField().to_python( + { + "referenced": referenced, + "nested": [{"referenced": referenced}], + } + ) + + assert isinstance(converted["referenced"], DBRef) + assert converted["referenced"] == DBRef( + Referenced._get_collection_name(), referenced.id + ) + assert isinstance(converted["nested"][0]["referenced"], DBRef) + + def test_to_python__untyped_dict_contains_convertible_value__converts_value(self): + class Convertible: + def to_python(self): + return "converted" + + converted = DictField().to_python({"value": Convertible()}) + + assert converted == {"value": "converted"} + + def test_to_python__auto_dereferencing_disabled__propagates_to_nested_field(self): + class Referenced(Document): + pass + + class Embedded(EmbeddedDocument): + referenced = ReferenceField(Referenced) + + field = DictField(EmbeddedDocumentField(Embedded)) + field.set_auto_dereferencing(False) + + converted = field.to_python({"value": {"referenced": ObjectId()}}) + + assert converted["value"]._fields["referenced"]._auto_dereference is False + + def test_to_python__typed_dict_receives_truthy_non_dict__raises_validation_error( + self, + ): + class Model(Document): + values = DictField(IntField()) + + with pytest.raises( + ValidationError, match="Only dictionaries may be used in a DictField" + ): + Model(values=[1]).validate() + + @pytest.mark.parametrize( + "field,value", + [ + pytest.param(DictField(null=True), {}, id="top-level-nullable"), + pytest.param(ListField(DictField()), [{}], id="nested-dict-field"), + pytest.param(ListField(MapField(IntField())), [{}], id="nested-map-field"), + ], + ) + def test_to_python__value_contains_empty_dict__preserves_empty_dict( + self, field, value + ): + assert field.to_python(value) == value + + def test_to_python__typed_dict_of_primitives__preserves_shape(self): + """Large primitive dict should round-trip unchanged (perf fast path).""" + field = DictField(IntField()) + value = {f"k{i}": i for i in range(1000)} + + converted = field.to_python(value) + + assert converted == value + + def test_to_python__untyped_dict_contains_dbref__preserves_dbref(self): + oid = ObjectId() + dbref = DBRef("collection", oid) + + converted = DictField().to_python({"ref": dbref}) + + assert converted == {"ref": dbref} + assert isinstance(converted["ref"], DBRef) + + def test_to_python__dict_subclass_with_falsy_bool__preserves_entries(self): + """A dict subclass whose __bool__ is False must not be silently dropped.""" + + class FalsyDict(dict): + def __bool__(self): + return False + + value = FalsyDict({"a": 1, "b": 2}) + assert not value # sanity check + + converted = DictField(IntField()).to_python(value) + + assert dict(converted) == {"a": 1, "b": 2} + + def test_to_python__mapfield_typed_dict_of_primitives__preserves_shape(self): + """MapField inherits DictField.to_python; primitive dict must round-trip.""" + converted = MapField(IntField()).to_python({"a": 1, "b": 2}) + + assert converted == {"a": 1, "b": 2} + + def test_to_python__mapfield_delegates_to_nested_field(self): + class Doubling(IntField): + def to_python(self, value): + return value * 2 + + converted = MapField(Doubling()).to_python({"a": 1, "b": 2}) + + assert converted == {"a": 2, "b": 4} + + def test_to_python__mapfield_receives_truthy_non_dict__raises_validation_error( + self, + ): + class Model(Document): + values = MapField(IntField()) + + with pytest.raises( + ValidationError, match="Only dictionaries may be used in a DictField" + ): + Model(values=[1]).validate() + + class TestDictField(MongoDBTestCase): def test_storage(self): class BlogPost(Document): @@ -388,3 +516,61 @@ class Simple(Document): assert isinstance(s.mapping7["someint"][0]["d"], Doc) assert isinstance(s.mapping8["someint"][0]["d"][0], Doc) assert isinstance(s.mapping9["someint"][0]["d"][0], Doc) + + def test_dictfield_with_embeddeddocument_field_roundtrip(self): + """Ensure DictField(EmbeddedDocumentField) rebuilds the embedded instance.""" + + class Setting(EmbeddedDocument): + value = StringField() + + class Simple(Document): + mapping = DictField(EmbeddedDocumentField(Setting)) + + Simple.drop_collection() + + Simple(mapping={"a": Setting(value="foo"), "b": Setting(value="bar")}).save() + + reloaded = Simple.objects.first() + assert isinstance(reloaded.mapping["a"], Setting) + assert isinstance(reloaded.mapping["b"], Setting) + assert reloaded.mapping["a"].value == "foo" + assert reloaded.mapping["b"].value == "bar" + + def test_dictfield_reads_non_dict_stored_in_db_schema_drift(self): + """Reading a document whose DB value is not a dict must not blow up. + + Data written by a different tool or an older schema may end up with a + non-dict value on a DictField. Preserving the current tolerant behavior + avoids breaking existing systems when the field's ``to_python`` is + optimized. + """ + + class Model(Document): + m = DictField(field=IntField()) + + Model.drop_collection() + + Model._get_collection().insert_one({"_id": 1, "m": [{"a": 1}]}) + Model._get_collection().insert_one({"_id": 2, "m": "some-string"}) + + loaded = {doc.id: doc.m for doc in Model.objects.order_by("id")} + assert loaded == {1: [{"a": 1}], 2: "some-string"} + + def test_dictfield_with_enumfield_roundtrip(self): + """DictField(EnumField) must reconstruct enum members on read.""" + + class Status(Enum): + NEW = "new" + DONE = "done" + + class Model(Document): + mapping = DictField(EnumField(Status)) + + Model.drop_collection() + + Model(mapping={"a": Status.NEW, "b": Status.DONE}).save() + + reloaded = Model.objects.first() + assert reloaded.mapping == {"a": Status.NEW, "b": Status.DONE} + assert isinstance(reloaded.mapping["a"], Status) + assert isinstance(reloaded.mapping["b"], Status) From 27f39d64717e8e7a4c5c5039f47e9d79797f62ff Mon Sep 17 00:00:00 2001 From: Bastien Gerard Date: Thu, 20 Aug 2026 17:02:02 +0200 Subject: [PATCH 2/8] polish tests cases for dict perf improvements --- tests/fields/test_dict_field.py | 36 +++++++++------------------------ 1 file changed, 9 insertions(+), 27 deletions(-) diff --git a/tests/fields/test_dict_field.py b/tests/fields/test_dict_field.py index 1ff1bcfbf..ca5f98b3b 100644 --- a/tests/fields/test_dict_field.py +++ b/tests/fields/test_dict_field.py @@ -1,5 +1,3 @@ -from enum import Enum - import pytest from bson import DBRef, InvalidDocument, ObjectId @@ -79,10 +77,12 @@ def test_to_python__value_contains_empty_dict__preserves_empty_dict( ): assert field.to_python(value) == value - def test_to_python__typed_dict_of_primitives__preserves_shape(self): - """Large primitive dict should round-trip unchanged (perf fast path).""" - field = DictField(IntField()) - value = {f"k{i}": i for i in range(1000)} + def test_to_python__untyped_nested_primitives__preserves_shape(self): + field = DictField() + value = { + "numbers": [1, 2], + "nested": {"enabled": True, "name": "test"}, + } converted = field.to_python(value) @@ -104,12 +104,13 @@ class FalsyDict(dict): def __bool__(self): return False - value = FalsyDict({"a": 1, "b": 2}) + value = FalsyDict({"a": "1", "b": "2"}) assert not value # sanity check converted = DictField(IntField()).to_python(value) - assert dict(converted) == {"a": 1, "b": 2} + assert type(converted) is dict + assert converted == {"a": 1, "b": 2} def test_to_python__mapfield_typed_dict_of_primitives__preserves_shape(self): """MapField inherits DictField.to_python; primitive dict must round-trip.""" @@ -555,22 +556,3 @@ class Model(Document): loaded = {doc.id: doc.m for doc in Model.objects.order_by("id")} assert loaded == {1: [{"a": 1}], 2: "some-string"} - - def test_dictfield_with_enumfield_roundtrip(self): - """DictField(EnumField) must reconstruct enum members on read.""" - - class Status(Enum): - NEW = "new" - DONE = "done" - - class Model(Document): - mapping = DictField(EnumField(Status)) - - Model.drop_collection() - - Model(mapping={"a": Status.NEW, "b": Status.DONE}).save() - - reloaded = Model.objects.first() - assert reloaded.mapping == {"a": Status.NEW, "b": Status.DONE} - assert isinstance(reloaded.mapping["a"], Status) - assert isinstance(reloaded.mapping["b"], Status) From 46051a88866f585d6ee87369f6b564a6c7083cdc Mon Sep 17 00:00:00 2001 From: Noah Stapp Date: Mon, 23 Jun 2025 16:58:42 -0400 Subject: [PATCH 3/8] WIP --- mongoengine/base/datastructures.py | 57 ++++++++++++++++++++++++++ mongoengine/fields.py | 64 ++++++++++++++++++++++++++++++ 2 files changed, 121 insertions(+) diff --git a/mongoengine/base/datastructures.py b/mongoengine/base/datastructures.py index dcb8438c7..e6d9a42c8 100644 --- a/mongoengine/base/datastructures.py +++ b/mongoengine/base/datastructures.py @@ -472,3 +472,60 @@ def __getattr__(self, name): def __repr__(self): return f"" + + +class RawDict: + def __init__(self, data, deserialize_method): + self._data = data + self.deserialize_method = deserialize_method + + def deserialize(self): + return self.deserialize_method(self._data) + + def __setitem__(self, key, item): + self._data[key] = item + + def __getitem__(self, key): + return self._data[key] + + def __repr__(self): + return repr(self._data) + + def __len__(self): + return len(self._data) + + def __delitem__(self, key): + del self._data[key] + + def clear(self): + return self._data.clear() + + def copy(self): + return self._data.copy() + + def has_key(self, k): + return k in self._data + + def update(self, *args, **kwargs): + return self._data.update(*args, **kwargs) + + def keys(self): + return self._data.keys() + + def values(self): + return self._data.values() + + def items(self): + return self._data.items() + + def pop(self, *args): + return self._data.pop(*args) + + def __cmp__(self, dict_): + return self.__cmp__(self._data, dict_) + + def __contains__(self, item): + return item in self._data + + def __iter__(self): + return iter(self._data) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 0f5ee5402..5ece09fe0 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -16,6 +16,8 @@ from bson.decimal128 import Decimal128, create_decimal128_context from pymongo import ReturnDocument +from mongoengine.base.datastructures import RawDict + try: import dateutil except ImportError: @@ -81,6 +83,7 @@ "SortedListField", "EmbeddedDocumentListField", "DictField", + "LazyDictField", "MapField", "ReferenceField", "CachedReferenceField", @@ -1044,6 +1047,7 @@ def key_starts_with_dollar(d): return True +# TODO: make a LazyDictField that lazily deferences on access class DictField(ComplexBaseField): """A dictionary field that wraps a standard Python dictionary. This is similar to an embedded document, but the structure is not defined. @@ -1098,6 +1102,66 @@ def prepare_query_value(self, op, value): return super().prepare_query_value(op, value) +class LazyDictField(ComplexBaseField): + """A lazy dictionary field that wraps a standard Python dictionary. + Unlike the :class:`~mongoengine.fields.DictField`, it will + **not** be automatically deserialized. Manual deserialization must be triggered + using the ``deserialize()`` method. + + .. note:: + Required means it cannot be empty - as the default for DictFields is {} + """ + + def __init__(self, field=None, *args, **kwargs): + kwargs.setdefault("default", dict) + super().__init__(*args, field=field, **kwargs) + self.set_auto_dereferencing(False) + + def validate(self, value): + """Make sure that a list of valid fields is being used.""" + if not isinstance(value, dict): + self.error("Only dictionaries may be used in a DictField") + + if key_not_string(value): + msg = "Invalid dictionary key - documents must have only string keys" + self.error(msg) + + # Following condition applies to MongoDB >= 3.6 + # older Mongo has stricter constraints but + # it will be rejected upon insertion anyway + # Having a validation that depends on the MongoDB version + # is not straightforward as the field isn't aware of the connected Mongo + if key_starts_with_dollar(value): + self.error( + 'Invalid dictionary key name - keys may not startswith "$" characters' + ) + super().validate(value) + + def lookup_member(self, member_name): + return DictField(db_field=member_name) + + def prepare_query_value(self, op, value): + match_operators = [*STRING_OPERATORS] + + if op in match_operators and isinstance(value, str): + return StringField().prepare_query_value(op, value) + + if hasattr( + self.field, "field" + ): # Used for instance when using DictField(ListField(IntField())) + if op in ("set", "unset") and isinstance(value, dict): + return { + k: self.field.prepare_query_value(op, v) for k, v in value.items() + } + return self.field.prepare_query_value(op, value) + + return super().prepare_query_value(op, value) + + def to_python(self, value): + self._data = RawDict(value, super().to_python) + return self._data + + class MapField(DictField): """A field that maps a name to a specified field type. Similar to a DictField, except the 'value' of each item must match the specified From ad82cad92a5bfdfa2e411fcade87be7e5f60673b Mon Sep 17 00:00:00 2001 From: Noah Stapp Date: Tue, 24 Jun 2025 10:27:05 -0400 Subject: [PATCH 4/8] INTPYTHON-617 - Improve DictField to_python performance --- mongoengine/fields.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 5ece09fe0..eb3948634 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -1047,7 +1047,6 @@ def key_starts_with_dollar(d): return True -# TODO: make a LazyDictField that lazily deferences on access class DictField(ComplexBaseField): """A dictionary field that wraps a standard Python dictionary. This is similar to an embedded document, but the structure is not defined. @@ -1101,6 +1100,14 @@ def prepare_query_value(self, op, value): return super().prepare_query_value(op, value) + def to_python(self, value): + to_python = getattr(self.field, "to_python", None) + return ( + {k: to_python(v) for k, v in value.items()} + if to_python and value + else value or None + ) + class LazyDictField(ComplexBaseField): """A lazy dictionary field that wraps a standard Python dictionary. From 4c210f5d2abbf5454578df97a5fba9530d0e19b7 Mon Sep 17 00:00:00 2001 From: Noah Stapp Date: Tue, 24 Jun 2025 10:28:16 -0400 Subject: [PATCH 5/8] Cleanup --- mongoengine/base/datastructures.py | 57 --------------------------- mongoengine/fields.py | 63 ------------------------------ 2 files changed, 120 deletions(-) diff --git a/mongoengine/base/datastructures.py b/mongoengine/base/datastructures.py index e6d9a42c8..dcb8438c7 100644 --- a/mongoengine/base/datastructures.py +++ b/mongoengine/base/datastructures.py @@ -472,60 +472,3 @@ def __getattr__(self, name): def __repr__(self): return f"" - - -class RawDict: - def __init__(self, data, deserialize_method): - self._data = data - self.deserialize_method = deserialize_method - - def deserialize(self): - return self.deserialize_method(self._data) - - def __setitem__(self, key, item): - self._data[key] = item - - def __getitem__(self, key): - return self._data[key] - - def __repr__(self): - return repr(self._data) - - def __len__(self): - return len(self._data) - - def __delitem__(self, key): - del self._data[key] - - def clear(self): - return self._data.clear() - - def copy(self): - return self._data.copy() - - def has_key(self, k): - return k in self._data - - def update(self, *args, **kwargs): - return self._data.update(*args, **kwargs) - - def keys(self): - return self._data.keys() - - def values(self): - return self._data.values() - - def items(self): - return self._data.items() - - def pop(self, *args): - return self._data.pop(*args) - - def __cmp__(self, dict_): - return self.__cmp__(self._data, dict_) - - def __contains__(self, item): - return item in self._data - - def __iter__(self): - return iter(self._data) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index eb3948634..e44a64f4b 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -16,8 +16,6 @@ from bson.decimal128 import Decimal128, create_decimal128_context from pymongo import ReturnDocument -from mongoengine.base.datastructures import RawDict - try: import dateutil except ImportError: @@ -83,7 +81,6 @@ "SortedListField", "EmbeddedDocumentListField", "DictField", - "LazyDictField", "MapField", "ReferenceField", "CachedReferenceField", @@ -1109,66 +1106,6 @@ def to_python(self, value): ) -class LazyDictField(ComplexBaseField): - """A lazy dictionary field that wraps a standard Python dictionary. - Unlike the :class:`~mongoengine.fields.DictField`, it will - **not** be automatically deserialized. Manual deserialization must be triggered - using the ``deserialize()`` method. - - .. note:: - Required means it cannot be empty - as the default for DictFields is {} - """ - - def __init__(self, field=None, *args, **kwargs): - kwargs.setdefault("default", dict) - super().__init__(*args, field=field, **kwargs) - self.set_auto_dereferencing(False) - - def validate(self, value): - """Make sure that a list of valid fields is being used.""" - if not isinstance(value, dict): - self.error("Only dictionaries may be used in a DictField") - - if key_not_string(value): - msg = "Invalid dictionary key - documents must have only string keys" - self.error(msg) - - # Following condition applies to MongoDB >= 3.6 - # older Mongo has stricter constraints but - # it will be rejected upon insertion anyway - # Having a validation that depends on the MongoDB version - # is not straightforward as the field isn't aware of the connected Mongo - if key_starts_with_dollar(value): - self.error( - 'Invalid dictionary key name - keys may not startswith "$" characters' - ) - super().validate(value) - - def lookup_member(self, member_name): - return DictField(db_field=member_name) - - def prepare_query_value(self, op, value): - match_operators = [*STRING_OPERATORS] - - if op in match_operators and isinstance(value, str): - return StringField().prepare_query_value(op, value) - - if hasattr( - self.field, "field" - ): # Used for instance when using DictField(ListField(IntField())) - if op in ("set", "unset") and isinstance(value, dict): - return { - k: self.field.prepare_query_value(op, v) for k, v in value.items() - } - return self.field.prepare_query_value(op, value) - - return super().prepare_query_value(op, value) - - def to_python(self, value): - self._data = RawDict(value, super().to_python) - return self._data - - class MapField(DictField): """A field that maps a name to a specified field type. Similar to a DictField, except the 'value' of each item must match the specified From 12a045bfc0c3491cfecf3b36e1060122afc63b9b Mon Sep 17 00:00:00 2001 From: Noah Stapp Date: Tue, 19 Aug 2025 13:27:12 -0700 Subject: [PATCH 6/8] Refresh CI --- mongoengine/fields.py | 1 + 1 file changed, 1 insertion(+) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index e44a64f4b..18c4e3fc3 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -1110,6 +1110,7 @@ class MapField(DictField): """A field that maps a name to a specified field type. Similar to a DictField, except the 'value' of each item must match the specified field type. + """ def __init__(self, field=None, *args, **kwargs): From fe78355d59301e5cd3d4bbb7349827cee0c36a6c Mon Sep 17 00:00:00 2001 From: Noah Stapp Date: Tue, 19 Aug 2025 13:27:37 -0700 Subject: [PATCH 7/8] Refresh CI --- mongoengine/fields.py | 1 - 1 file changed, 1 deletion(-) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 18c4e3fc3..e44a64f4b 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -1110,7 +1110,6 @@ class MapField(DictField): """A field that maps a name to a specified field type. Similar to a DictField, except the 'value' of each item must match the specified field type. - """ def __init__(self, field=None, *args, **kwargs): From c9d84e7aaed06a4cdb58cad86a1450fc713a3a1e Mon Sep 17 00:00:00 2001 From: Noah Stapp Date: Wed, 24 Sep 2025 11:37:19 -0400 Subject: [PATCH 8/8] to_python should return falsy values --- mongoengine/fields.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index e44a64f4b..6abb9a7de 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -1098,12 +1098,12 @@ def prepare_query_value(self, op, value): return super().prepare_query_value(op, value) def to_python(self, value): + if value is None: + return None to_python = getattr(self.field, "to_python", None) - return ( - {k: to_python(v) for k, v in value.items()} - if to_python and value - else value or None - ) + if not to_python or not value: + return value + return {k: to_python(v) for k, v in value.items()} class MapField(DictField):