Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
23 commits
Select commit Hold shift + click to select a range
1da1bc1
INTPYTHON-736 Convert simple $expr queries to $match queries
Jibola Aug 21, 2025
0c27833
try without deepcopy
timgraham Sep 7, 2025
22a43d4
getField support overlay
Jibola Sep 8, 2025
edc0746
added getfield tests to expression conversion
Jibola Sep 8, 2025
088ffb2
Merge branch 'main' into expr-support-getfield
Jibola Sep 8, 2025
cfd13b6
fixed lint issues and lingering class usage
Jibola Sep 8, 2025
7ab6b37
include tests for getfield on all basic ops
Jibola Sep 8, 2025
07e6ffb
fixed match and op conversion tests
Jibola Sep 8, 2025
bdc6ae0
removed mutation of getfield even when values aren't converted
Jibola Sep 10, 2025
4257582
fix assert not opmtmizable test within test_op_expressions
Jibola Sep 10, 2025
a88fbe7
removed the getfield mutation on in test
Jibola Sep 10, 2025
e62e299
fixed case where already nested fields (without getField) would not g…
Jibola Sep 10, 2025
00426a7
Merge branch 'main' into expr-support-getfield
Jibola Sep 10, 2025
8331701
fixed path name reconcilitation issue on in JSONField tests
Jibola Sep 10, 2025
77f9d34
fix InExpression convert_path_name
Jibola Sep 10, 2025
f783c8e
handle the case of None being passed
Jibola Sep 10, 2025
797da9e
make converesion explicitly change for null equality
Jibola Sep 10, 2025
b8c99ca
add null existence checks for expr conversions
Jibola Sep 10, 2025
db17c7a
add explicit : true check
Jibola Sep 10, 2025
aacdd4d
provide path in the null wrapping
Jibola Sep 10, 2025
49c34d6
add age path to InExpression
Jibola Sep 10, 2025
6a14b1d
fix lint error
Jibola Sep 10, 2025
377f9ea
Merge branch 'main' into expr-support-getfield
Jibola Sep 11, 2025
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
76 changes: 59 additions & 17 deletions django_mongodb_backend/query_conversion/expression_converters.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,45 @@ class BaseConverter:
def convert(cls, expr):
raise NotImplementedError("Subclasses must implement this method.")

@classmethod
def is_simple_path_name(cls, field_name):
return (
isinstance(field_name, str)
and field_name != ""
and field_name.startswith("$")
# Case for catching variables
and not field_name.startswith("$$")
)

@classmethod
def is_simple_get_field(cls, get_field_object):
if not isinstance(get_field_object, dict):
return False

get_field_expr = get_field_object.get("$getField")

if (
isinstance(get_field_expr, dict)
and "input" in get_field_expr
and "field" in get_field_expr
):
input_expr = get_field_expr["input"]
field_name = get_field_expr["field"]
return cls.convert_path_name(input_expr) and (
isinstance(field_name, str) and "$" not in field_name and "." not in field_name
)
return False

@classmethod
def convert_path_name(cls, field_name):
if cls.is_simple_path_name(field_name):
return field_name[1:]
if cls.is_simple_get_field(field_name):
get_field_input = field_name["$getField"]["input"]
get_field_field_name = field_name["$getField"]["field"]
return f"{cls.convert_path_name(get_field_input)}.{get_field_field_name}"
return None

@classmethod
def is_simple_value(cls, value):
"""Is the value is a simple type (not a dict)?"""
Expand All @@ -14,7 +53,6 @@ def is_simple_value(cls, value):
return False
if isinstance(value, (list, tuple, set)):
return all(cls.is_simple_value(v) for v in value)
# TODO: Support `$getField` conversion.
return not isinstance(value, dict)


Expand All @@ -27,7 +65,7 @@ class BinaryConverter(BaseConverter):
{"$gt": ["$price", 100]}
}
is converted to:
{"$gt": ["price", 100]}
{"price": {"$gt": 100}}
"""

operator: str
Expand All @@ -37,15 +75,14 @@ def convert(cls, args):
if isinstance(args, list) and len(args) == 2:
field_expr, value = args
# Check if first argument is a simple field reference.
if (
isinstance(field_expr, str)
and field_expr.startswith("$")
and cls.is_simple_value(value)
):
field_name = field_expr[1:] # Remove the $ prefix.
if (field_name := cls.convert_path_name(field_expr)) and cls.is_simple_value(value):
if cls.operator == "$eq":
return {field_name: value}
return {field_name: {cls.operator: value}}
query = {field_name: value}
else:
query = {field_name: {cls.operator: value}}
if value is None:
query = {"$and": [{field_name: {"$exists": True}}, query]}
return query
return None


Expand Down Expand Up @@ -96,13 +133,18 @@ class InConverter(BaseConverter):
def convert(cls, in_args):
if isinstance(in_args, list) and len(in_args) == 2:
field_expr, values = in_args
# Check if first argument is a simple field reference.
if isinstance(field_expr, str) and field_expr.startswith("$"):
field_name = field_expr[1:] # Remove the $ prefix.
if isinstance(values, (list, tuple, set)) and all(
cls.is_simple_value(v) for v in values
):
return {field_name: {"$in": values}}
# Check if first argument is a simple field reference
# Check if second argument is a list of simple values
if (field_name := cls.convert_path_name(field_expr)) and (
isinstance(values, list | tuple | set)
Copy link
Preview

Copilot AI Sep 10, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Use explicit tuple syntax (list, tuple, set) instead of union operator | for better compatibility with older Python versions.

Suggested change
isinstance(values, list | tuple | set)
isinstance(values, (list, tuple, set))

Copilot uses AI. Check for mistakes.

and all(cls.is_simple_value(v) for v in values)
):
core_check = {field_name: {"$in": values}}
return (
{"$and": [{field_name: {"$exists": True}}, core_check]}
if None in values
else core_check
)
return None


Expand Down
46 changes: 46 additions & 0 deletions tests/expression_converter_/test_match_conversion.py
Original file line number Diff line number Diff line change
Expand Up @@ -213,3 +213,49 @@ def test_deeply_nested_logical_operator_with_variable(self):
}
]
self.assertOptimizerEqual(expr, expected)

def test_getfield_usage_on_dual_binary_operator(self):
expr = {
"$expr": {
"$gt": [
{"$getField": {"input": "$price", "field": "value"}},
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can we optimize this down to remove the $getField when the field names don't contain $ or .? Something like:

{  
  "$match": {  
    "$expr": {  
      "$gt": ["$price.value", "$discounted_price.value"]  
    }  
  }  
}  

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We could. I'm trying to think if there's any case where this would be an issue if it mistakenly resolved it.

Copy link
Contributor Author

@Jibola Jibola Sep 8, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This would have to be a two-phase conversion. I think it may even be best to still change it in our actual query code.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm assuming here that this change ends up being needed. Holding off until we either get customer complaints or the refactor can't solve this makes more sense to me for now.

Copy link
Contributor Author

@Jibola Jibola Sep 8, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yeah, I tried to make it work (and I think I managed to), but the silent mutation of $getField to $field is worrying to me. Even if I manage to make things pass tests, it just feels out of scope of the optimizers goal, which is to explicitly change things from $expr to $match.

{"$getField": {"input": "$discounted_price", "field": "value"}},
]
}
}
expected = [
{
"$match": {
"$expr": {
"$gt": [
{"$getField": {"input": "$price", "field": "value"}},
{"$getField": {"input": "$discounted_price", "field": "value"}},
]
}
}
}
]
self.assertOptimizerEqual(expr, expected)

def test_getfield_usage_on_onesided_binary_operator(self):
expr = {"$expr": {"$gt": [{"$getField": {"input": "$price", "field": "value"}}, 100]}}
# This should create a proper match condition with no $expr
expected = [{"$match": {"price.value": {"$gt": 100}}}]
self.assertOptimizerEqual(expr, expected)

def test_nested_getfield_usage_on_onesided_binary(self):
expr = {
"$expr": {
"$gt": [
{
"$getField": {
"input": {"$getField": {"input": "$item", "field": "price"}},
"field": "value",
}
},
100,
]
}
}
expected = [{"$match": {"item.price.value": {"$gt": 100}}}]
self.assertOptimizerEqual(expr, expected)
Loading