Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,6 @@
import org.apache.doris.nereids.trees.expressions.functions.ExpressionTrait;
import org.apache.doris.nereids.trees.expressions.functions.Function;
import org.apache.doris.nereids.trees.expressions.functions.scalar.ElementAt;
import org.apache.doris.nereids.trees.expressions.literal.Literal;
import org.apache.doris.nereids.trees.expressions.literal.StringLikeLiteral;
import org.apache.doris.nereids.trees.expressions.literal.VarcharLiteral;
import org.apache.doris.nereids.trees.expressions.visitor.DefaultExpressionVisitor;
Expand Down Expand Up @@ -790,14 +789,13 @@ protected static Pair<SlotReference, List<String>> extractSlotToSubPathPair(Elem
if (!(elementAt.left() instanceof ElementAt || elementAt.left() instanceof SlotReference)) {
return null;
}
// Storage sub-paths address object keys only. An integer index selects an array element of the
// VARIANT value, so the sub-path stops before it and the index is applied to the extracted value.
Expression key = elementAt.right();
if (key instanceof StringLikeLiteral) {
subPath.add(((StringLikeLiteral) key).getStringValue());
} else if (key instanceof Literal && key.getDataType().isIntegerLikeType()) {
subPath.add(((Literal) key).getStringValue());
} else {
if (!(key instanceof StringLikeLiteral)) {
return null;
}
subPath.add(((StringLikeLiteral) key).getStringValue());
if (elementAt.left() instanceof SlotReference) {
// ElementAt's left child is SlotReference
// reverse subPath because we put them by reverse order
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -186,6 +186,23 @@ public void testVariantMultiProjectionAccessPaths() throws Exception {
));
}

@Test
public void testVariantIntegerIndexStopsSubColumnPath() throws Exception {
// DORIS-28435: an integer index selects an array element, so only the object-key prefix becomes a storage
// sub-column; a sub-column items.1 would look up a missing object key and return NULL.
assertVariantSubColumnSlots("select element_at(element_at(v, 'items'), 1), v['items'][-1] from variant_tbl",
ImmutableList.of(ImmutableList.of("items")));
}

@Test
public void testVariantIntegerIndexPredicateStopsSubColumnPath() throws Exception {
// Filters stop at the integer index too, also when a string key follows it; the object-key sub-column
// items.1, spelled like the index, stays a separate slot.
assertVariantSubColumnSlots("select id from variant_tbl"
+ " where v['items'][1] = 2 and v['items'][-1]['k'] = 1 and v['items']['1'] = 'x'",
ImmutableList.of(ImmutableList.of("items"), ImmutableList.of("items", "1")));
}

@Test
public void testVariantPredicateAccessPath() throws Exception {
assertColumn("select 1 from variant_tbl where v['k'] is not null",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -61,10 +61,12 @@ protected void runBeforeAll() throws Exception {

@Test
public void testVariantNumericIndexSubPath() throws Exception {
// A numeric index selects an array element, not an object key, so the storage sub-column stops at arr and
// [0]['x'] is applied to it at runtime; the access path still names the whole expression.
assertVariantSubColumnSlots(
"select v['arr'][0]['x'] from variant_tbl",
ImmutableList.of(
ImmutableList.of("arr", "0", "x")
ImmutableList.of("arr")
)
);
assertAllAccessPathsContain(
Expand Down
34 changes: 34 additions & 0 deletions regression-test/data/variant_p0/element_function.out
Original file line number Diff line number Diff line change
Expand Up @@ -2,3 +2,37 @@
-- !sql --
1

-- !nested_integer_index --
1 2 4 \N \N 3 \N
2 [5,6] 7 \N \N s \N
3 \N \N \N \N \N key one
4 \N \N \N \N \N \N
5 {"k":1} {"k":2} \N \N {"k":2} \N
6 2 4 \N \N null \N

-- !nested_integer_index_const --
2 4

-- !nested_integer_index_filter --
1
6

-- !nested_integer_index_filter_negative --
1 3
6 null

-- !nested_integer_index_filter_null --
3
4

-- !nested_integer_index_filter_object --
5

-- !nested_integer_index_filter_with_key --
1
3
6

-- !nested_integer_index_filter_topn --
1 [2,3,4]

73 changes: 73 additions & 0 deletions regression-test/suites/variant_p0/element_function.groovy
Original file line number Diff line number Diff line change
Expand Up @@ -59,4 +59,77 @@ suite("regression_test_variant_element_at", "p0") {
def obj = sql """select sort_json_object_keys(json_extract(
cast(${variantV2Function}('{"o":{"name":"john"}}') as json), '\$.o'))"""
assertEquals('{"name":"john"}', obj[0][0])

// The cases below need Variant V2. On the legacy V1 path element_at resolves an integer index as an
// object key, so indexing an array stored in a VARIANT column returns NULL there.
if (!getFeConfig("enable_variant_v2").toBoolean()) {
return
}

// DORIS-28435: an integer index on a VARIANT array that was itself extracted with element_at is 1-based like
// ARRAY element_at and counts from the end when negative; 0, out-of-range indexes, non-array values and
// missing paths give NULL, while a string index still reads object keys. On a stored column the planner
// must not turn the integer index into the storage sub-path items.1.
sql "DROP TABLE IF EXISTS element_at_nested_index_test"
sql """
CREATE TABLE element_at_nested_index_test (
id INT,
json_variant VARIANT
)
DUPLICATE KEY(id)
DISTRIBUTED BY HASH(id) BUCKETS 1
PROPERTIES ("replication_num" = "1")
"""
sql """INSERT INTO element_at_nested_index_test VALUES
(1, ${variantV2Function}('{"items": [2, 3, 4]}')),
(2, ${variantV2Function}('{"items": [[5, 6], "s", 7]}')),
(3, ${variantV2Function}('{"items": {"1": "key one"}}')),
(4, ${variantV2Function}('{"other": 1}')),
(5, ${variantV2Function}('{"items": [{"k": 1}, {"k": 2}]}')),
(6, ${variantV2Function}('{"items": [2, null, 4]}'))"""
order_qt_nested_integer_index """
SELECT id,
element_at(element_at(json_variant, 'items'), 1),
element_at(element_at(json_variant, 'items'), -1),
element_at(element_at(json_variant, 'items'), 0),
element_at(element_at(json_variant, 'items'), 4),
json_variant['items'][2],
element_at(element_at(json_variant, 'items'), '1')
FROM element_at_nested_index_test
"""
qt_nested_integer_index_const """
SELECT element_at(element_at(${variantV2Function}('{"items": [2, 3, 4]}'), 'items'), 1),
element_at(element_at(${variantV2Function}('{"items": [2, 3, 4]}'), 'items'), -1)
"""

// Filters read items and apply the index the same way: as a pushed-down predicate, after a string key that
// follows the index, next to the object-key sub-column items.1 in one filter, and under a TopN.
order_qt_nested_integer_index_filter """
SELECT id FROM element_at_nested_index_test
WHERE CAST(element_at(element_at(json_variant, 'items'), 1) AS INT) = 2
"""
order_qt_nested_integer_index_filter_negative """
SELECT id, json_variant['items'][2] FROM element_at_nested_index_test
WHERE CAST(json_variant['items'][-1] AS INT) = 4
"""
order_qt_nested_integer_index_filter_null """
SELECT id FROM element_at_nested_index_test
WHERE element_at(element_at(json_variant, 'items'), 1) IS NULL
"""
order_qt_nested_integer_index_filter_object """
SELECT id FROM element_at_nested_index_test WHERE CAST(json_variant['items'][1]['k'] AS INT) = 1
"""
order_qt_nested_integer_index_filter_with_key """
SELECT id FROM element_at_nested_index_test
WHERE CAST(json_variant['items']['1'] AS STRING) = 'key one' OR CAST(json_variant['items'][1] AS INT) = 2
"""
qt_nested_integer_index_filter_topn """
SELECT id, CAST(json_variant['items'] AS STRING) FROM element_at_nested_index_test
WHERE CAST(json_variant['items'][1] AS INT) = 2 ORDER BY id LIMIT 1
"""
// MATCH needs a storage column, and a path with an integer index is not one.
test {
sql "SELECT id FROM element_at_nested_index_test WHERE json_variant['items'][2] MATCH_ANY 's'"
exception "Only support match left operand is SlotRef"
}
}
Loading