diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/VariantSubPathPruning.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/VariantSubPathPruning.java index 5b081ce2c2477c..82f71b9db29bc3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/VariantSubPathPruning.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/VariantSubPathPruning.java @@ -32,7 +32,6 @@ import org.apache.doris.nereids.trees.expressions.functions.ExpressionTrait; import org.apache.doris.nereids.trees.expressions.functions.Function; import org.apache.doris.nereids.trees.expressions.functions.scalar.ElementAt; -import org.apache.doris.nereids.trees.expressions.literal.Literal; import org.apache.doris.nereids.trees.expressions.literal.StringLikeLiteral; import org.apache.doris.nereids.trees.expressions.literal.VarcharLiteral; import org.apache.doris.nereids.trees.expressions.visitor.DefaultExpressionVisitor; @@ -790,14 +789,13 @@ protected static Pair> extractSlotToSubPathPair(Elem if (!(elementAt.left() instanceof ElementAt || elementAt.left() instanceof SlotReference)) { return null; } + // Storage sub-paths address object keys only. An integer index selects an array element of the + // VARIANT value, so the sub-path stops before it and the index is applied to the extracted value. Expression key = elementAt.right(); - if (key instanceof StringLikeLiteral) { - subPath.add(((StringLikeLiteral) key).getStringValue()); - } else if (key instanceof Literal && key.getDataType().isIntegerLikeType()) { - subPath.add(((Literal) key).getStringValue()); - } else { + if (!(key instanceof StringLikeLiteral)) { return null; } + subPath.add(((StringLikeLiteral) key).getStringValue()); if (elementAt.left() instanceof SlotReference) { // ElementAt's left child is SlotReference // reverse subPath because we put them by reverse order diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PruneNestedColumnTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PruneNestedColumnTest.java index 4d7a8a4488e3b7..b94b21de7a0ee0 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PruneNestedColumnTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PruneNestedColumnTest.java @@ -186,6 +186,23 @@ public void testVariantMultiProjectionAccessPaths() throws Exception { )); } + @Test + public void testVariantIntegerIndexStopsSubColumnPath() throws Exception { + // DORIS-28435: an integer index selects an array element, so only the object-key prefix becomes a storage + // sub-column; a sub-column items.1 would look up a missing object key and return NULL. + assertVariantSubColumnSlots("select element_at(element_at(v, 'items'), 1), v['items'][-1] from variant_tbl", + ImmutableList.of(ImmutableList.of("items"))); + } + + @Test + public void testVariantIntegerIndexPredicateStopsSubColumnPath() throws Exception { + // Filters stop at the integer index too, also when a string key follows it; the object-key sub-column + // items.1, spelled like the index, stays a separate slot. + assertVariantSubColumnSlots("select id from variant_tbl" + + " where v['items'][1] = 2 and v['items'][-1]['k'] = 1 and v['items']['1'] = 'x'", + ImmutableList.of(ImmutableList.of("items"), ImmutableList.of("items", "1"))); + } + @Test public void testVariantPredicateAccessPath() throws Exception { assertColumn("select 1 from variant_tbl where v['k'] is not null", diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/VariantPruningLogicTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/VariantPruningLogicTest.java index 65a61f8a0ff37d..2b17d366bbda96 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/VariantPruningLogicTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/VariantPruningLogicTest.java @@ -61,10 +61,12 @@ protected void runBeforeAll() throws Exception { @Test public void testVariantNumericIndexSubPath() throws Exception { + // A numeric index selects an array element, not an object key, so the storage sub-column stops at arr and + // [0]['x'] is applied to it at runtime; the access path still names the whole expression. assertVariantSubColumnSlots( "select v['arr'][0]['x'] from variant_tbl", ImmutableList.of( - ImmutableList.of("arr", "0", "x") + ImmutableList.of("arr") ) ); assertAllAccessPathsContain( diff --git a/regression-test/data/variant_p0/element_function.out b/regression-test/data/variant_p0/element_function.out index 095c7b2035676a..007761a663b063 100644 --- a/regression-test/data/variant_p0/element_function.out +++ b/regression-test/data/variant_p0/element_function.out @@ -2,3 +2,37 @@ -- !sql -- 1 +-- !nested_integer_index -- +1 2 4 \N \N 3 \N +2 [5,6] 7 \N \N s \N +3 \N \N \N \N \N key one +4 \N \N \N \N \N \N +5 {"k":1} {"k":2} \N \N {"k":2} \N +6 2 4 \N \N null \N + +-- !nested_integer_index_const -- +2 4 + +-- !nested_integer_index_filter -- +1 +6 + +-- !nested_integer_index_filter_negative -- +1 3 +6 null + +-- !nested_integer_index_filter_null -- +3 +4 + +-- !nested_integer_index_filter_object -- +5 + +-- !nested_integer_index_filter_with_key -- +1 +3 +6 + +-- !nested_integer_index_filter_topn -- +1 [2,3,4] + diff --git a/regression-test/suites/variant_p0/element_function.groovy b/regression-test/suites/variant_p0/element_function.groovy index ed4cd1503ea6aa..e0c65ec6a4817c 100644 --- a/regression-test/suites/variant_p0/element_function.groovy +++ b/regression-test/suites/variant_p0/element_function.groovy @@ -59,4 +59,77 @@ suite("regression_test_variant_element_at", "p0") { def obj = sql """select sort_json_object_keys(json_extract( cast(${variantV2Function}('{"o":{"name":"john"}}') as json), '\$.o'))""" assertEquals('{"name":"john"}', obj[0][0]) + + // The cases below need Variant V2. On the legacy V1 path element_at resolves an integer index as an + // object key, so indexing an array stored in a VARIANT column returns NULL there. + if (!getFeConfig("enable_variant_v2").toBoolean()) { + return + } + + // DORIS-28435: an integer index on a VARIANT array that was itself extracted with element_at is 1-based like + // ARRAY element_at and counts from the end when negative; 0, out-of-range indexes, non-array values and + // missing paths give NULL, while a string index still reads object keys. On a stored column the planner + // must not turn the integer index into the storage sub-path items.1. + sql "DROP TABLE IF EXISTS element_at_nested_index_test" + sql """ + CREATE TABLE element_at_nested_index_test ( + id INT, + json_variant VARIANT + ) + DUPLICATE KEY(id) + DISTRIBUTED BY HASH(id) BUCKETS 1 + PROPERTIES ("replication_num" = "1") + """ + sql """INSERT INTO element_at_nested_index_test VALUES + (1, ${variantV2Function}('{"items": [2, 3, 4]}')), + (2, ${variantV2Function}('{"items": [[5, 6], "s", 7]}')), + (3, ${variantV2Function}('{"items": {"1": "key one"}}')), + (4, ${variantV2Function}('{"other": 1}')), + (5, ${variantV2Function}('{"items": [{"k": 1}, {"k": 2}]}')), + (6, ${variantV2Function}('{"items": [2, null, 4]}'))""" + order_qt_nested_integer_index """ + SELECT id, + element_at(element_at(json_variant, 'items'), 1), + element_at(element_at(json_variant, 'items'), -1), + element_at(element_at(json_variant, 'items'), 0), + element_at(element_at(json_variant, 'items'), 4), + json_variant['items'][2], + element_at(element_at(json_variant, 'items'), '1') + FROM element_at_nested_index_test + """ + qt_nested_integer_index_const """ + SELECT element_at(element_at(${variantV2Function}('{"items": [2, 3, 4]}'), 'items'), 1), + element_at(element_at(${variantV2Function}('{"items": [2, 3, 4]}'), 'items'), -1) + """ + + // Filters read items and apply the index the same way: as a pushed-down predicate, after a string key that + // follows the index, next to the object-key sub-column items.1 in one filter, and under a TopN. + order_qt_nested_integer_index_filter """ + SELECT id FROM element_at_nested_index_test + WHERE CAST(element_at(element_at(json_variant, 'items'), 1) AS INT) = 2 + """ + order_qt_nested_integer_index_filter_negative """ + SELECT id, json_variant['items'][2] FROM element_at_nested_index_test + WHERE CAST(json_variant['items'][-1] AS INT) = 4 + """ + order_qt_nested_integer_index_filter_null """ + SELECT id FROM element_at_nested_index_test + WHERE element_at(element_at(json_variant, 'items'), 1) IS NULL + """ + order_qt_nested_integer_index_filter_object """ + SELECT id FROM element_at_nested_index_test WHERE CAST(json_variant['items'][1]['k'] AS INT) = 1 + """ + order_qt_nested_integer_index_filter_with_key """ + SELECT id FROM element_at_nested_index_test + WHERE CAST(json_variant['items']['1'] AS STRING) = 'key one' OR CAST(json_variant['items'][1] AS INT) = 2 + """ + qt_nested_integer_index_filter_topn """ + SELECT id, CAST(json_variant['items'] AS STRING) FROM element_at_nested_index_test + WHERE CAST(json_variant['items'][1] AS INT) = 2 ORDER BY id LIMIT 1 + """ + // MATCH needs a storage column, and a path with an integer index is not one. + test { + sql "SELECT id FROM element_at_nested_index_test WHERE json_variant['items'][2] MATCH_ANY 's'" + exception "Only support match left operand is SlotRef" + } }