Skip to content

Commit 38b2368

Browse files
github-actions[bot]Pxl
github-actions[bot]
and
Pxl
authored
branch-3.1: [Feature](function) support function array_flatten #47404 (#59319)
Cherry-picked from #47404 --------- Co-authored-by: Pxl <xl@selectdb.com>
1 parent d5f02e7 commit 38b2368

9 files changed

Lines changed: 261 additions & 1 deletion

File tree

‎be/src/vec/functions/array/function_array_flatten.cpp‎

Lines changed: 92 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,92 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
#include "common/status.h"
19+
#include "vec/aggregate_functions/aggregate_function.h"
20+
#include "vec/columns/column.h"
21+
#include "vec/columns/column_array.h"
22+
#include "vec/columns/column_nullable.h"
23+
#include "vec/common/assert_cast.h"
24+
#include "vec/core/block.h"
25+
#include "vec/core/column_numbers.h"
26+
#include "vec/core/column_with_type_and_name.h"
27+
#include "vec/core/types.h"
28+
#include "vec/data_types/data_type.h"
29+
#include "vec/data_types/data_type_array.h"
30+
#include "vec/functions/function.h"
31+
#include "vec/functions/simple_function_factory.h"
32+
33+
namespace doris::vectorized {
34+
#include "common/compile_check_begin.h"
35+
36+
class FunctionArrayFlatten : public IFunction {
37+
public:
38+
static constexpr auto name = "array_flatten";
39+
static FunctionPtr create() { return std::make_shared<FunctionArrayFlatten>(); }
40+
41+
/// Get function name.
42+
String get_name() const override { return name; }
43+
44+
size_t get_number_of_arguments() const override { return 1; }
45+
46+
DataTypePtr get_return_type_impl(const DataTypes& arguments) const override {
47+
DataTypePtr arg = arguments[0];
48+
while (is_array(arg)) {
49+
arg = remove_nullable(assert_cast<const DataTypeArray*>(arg.get())->get_nested_type());
50+
}
51+
return std::make_shared<DataTypeArray>(make_nullable(arg));
52+
}
53+
54+
Status execute_impl(FunctionContext* context, Block& block, const ColumnNumbers& arguments,
55+
size_t result, size_t input_rows_count) const override {
56+
auto src_column =
57+
block.get_by_position(arguments[0]).column->convert_to_full_column_if_const();
58+
auto* src_column_array_ptr =
59+
assert_cast<ColumnArray*>(remove_nullable(src_column)->assume_mutable().get());
60+
ColumnArray* nested_src_column_array_ptr = src_column_array_ptr;
61+
62+
auto result_column_offsets =
63+
assert_cast<ColumnArray::ColumnOffsets&>(src_column_array_ptr->get_offsets_column())
64+
.clone();
65+
auto* offsets = assert_cast<ColumnArray::ColumnOffsets*>(result_column_offsets.get())
66+
->get_data()
67+
.data();
68+
69+
while (src_column_array_ptr->get_data_ptr()->is_column_array()) {
70+
nested_src_column_array_ptr = assert_cast<ColumnArray*>(
71+
remove_nullable(src_column_array_ptr->get_data_ptr())->assume_mutable().get());
72+
73+
for (size_t i = 0; i < input_rows_count; ++i) {
74+
offsets[i] = nested_src_column_array_ptr->get_offsets()[offsets[i] - 1];
75+
}
76+
src_column_array_ptr = nested_src_column_array_ptr;
77+
}
78+
79+
block.replace_by_position(
80+
result, ColumnArray::create(assert_cast<const ColumnNullable&>(
81+
nested_src_column_array_ptr->get_data())
82+
.clone(),
83+
std::move(result_column_offsets)));
84+
return Status::OK();
85+
}
86+
};
87+
88+
void register_function_array_flatten(SimpleFunctionFactory& factory) {
89+
factory.register_function<FunctionArrayFlatten>();
90+
}
91+
92+
} // namespace doris::vectorized

‎be/src/vec/functions/array/function_array_register.cpp‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@
2121
#include "vec/functions/simple_function_factory.h"
2222

2323
namespace doris::vectorized {
24-
24+
void register_function_array_flatten(SimpleFunctionFactory&);
2525
void register_function_array_shuffle(SimpleFunctionFactory&);
2626
void register_function_array_exists(SimpleFunctionFactory&);
2727
void register_function_array_element(SimpleFunctionFactory&);
@@ -59,6 +59,7 @@ void register_function_array_splits(SimpleFunctionFactory&);
5959
void register_function_array_match(SimpleFunctionFactory&);
6060

6161
void register_function_array(SimpleFunctionFactory& factory) {
62+
register_function_array_flatten(factory);
6263
register_function_array_shuffle(factory);
6364
register_function_array_exists(factory);
6465
register_function_array_element(factory);

‎fe/fe-core/src/main/java/org/apache/doris/catalog/BuiltinScalarFunctions.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,7 @@
4141
import org.apache.doris.nereids.trees.expressions.functions.scalar.ArrayFilter;
4242
import org.apache.doris.nereids.trees.expressions.functions.scalar.ArrayFirst;
4343
import org.apache.doris.nereids.trees.expressions.functions.scalar.ArrayFirstIndex;
44+
import org.apache.doris.nereids.trees.expressions.functions.scalar.ArrayFlatten;
4445
import org.apache.doris.nereids.trees.expressions.functions.scalar.ArrayIntersect;
4546
import org.apache.doris.nereids.trees.expressions.functions.scalar.ArrayJoin;
4647
import org.apache.doris.nereids.trees.expressions.functions.scalar.ArrayLast;
@@ -523,6 +524,7 @@ public class BuiltinScalarFunctions implements FunctionHelper {
523524
scalar(ArrayFilter.class, "array_filter"),
524525
scalar(ArrayFirst.class, "array_first"),
525526
scalar(ArrayFirstIndex.class, "array_first_index"),
527+
scalar(ArrayFlatten.class, "array_flatten"),
526528
scalar(ArrayIntersect.class, "array_intersect"),
527529
scalar(ArrayJoin.class, "array_join"),
528530
scalar(ArrayLast.class, "array_last"),

‎fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ArrayFlatten.java‎

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,67 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
package org.apache.doris.nereids.trees.expressions.functions.scalar;
19+
20+
import org.apache.doris.catalog.FunctionSignature;
21+
import org.apache.doris.nereids.trees.expressions.Expression;
22+
import org.apache.doris.nereids.trees.expressions.functions.CustomSignature;
23+
import org.apache.doris.nereids.trees.expressions.functions.PropagateNullable;
24+
import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor;
25+
import org.apache.doris.nereids.types.ArrayType;
26+
import org.apache.doris.nereids.types.DataType;
27+
28+
import com.google.common.base.Preconditions;
29+
30+
import java.util.List;
31+
32+
/**
33+
* ScalarFunction 'array_flatten'
34+
*/
35+
public class ArrayFlatten extends ScalarFunction
36+
implements CustomSignature, PropagateNullable {
37+
38+
/**
39+
* constructor with 1 arguments.
40+
*/
41+
public ArrayFlatten(Expression arg) {
42+
super("array_flatten", arg);
43+
}
44+
45+
@Override
46+
public FunctionSignature customSignature() {
47+
DataType dataType = getArgument(0).getDataType();
48+
while (dataType instanceof ArrayType) {
49+
dataType = ((ArrayType) dataType).getItemType();
50+
}
51+
return FunctionSignature.ret(ArrayType.of(dataType)).args(getArgument(0).getDataType());
52+
}
53+
54+
/**
55+
* withChildren.
56+
*/
57+
@Override
58+
public ArrayFlatten withChildren(List<Expression> children) {
59+
Preconditions.checkArgument(children.size() == 1);
60+
return new ArrayFlatten(children.get(0));
61+
}
62+
63+
@Override
64+
public <R, C> R accept(ExpressionVisitor<R, C> visitor, C context) {
65+
return visitor.visitArrayFlatten(this, context);
66+
}
67+
}

‎fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ScalarFunctionVisitor.java‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,7 @@
4242
import org.apache.doris.nereids.trees.expressions.functions.scalar.ArrayExists;
4343
import org.apache.doris.nereids.trees.expressions.functions.scalar.ArrayFilter;
4444
import org.apache.doris.nereids.trees.expressions.functions.scalar.ArrayFirstIndex;
45+
import org.apache.doris.nereids.trees.expressions.functions.scalar.ArrayFlatten;
4546
import org.apache.doris.nereids.trees.expressions.functions.scalar.ArrayIntersect;
4647
import org.apache.doris.nereids.trees.expressions.functions.scalar.ArrayJoin;
4748
import org.apache.doris.nereids.trees.expressions.functions.scalar.ArrayLastIndex;
@@ -683,6 +684,10 @@ default R visitArrayShuffle(ArrayShuffle arrayShuffle, C context) {
683684
return visitScalarFunction(arrayShuffle, context);
684685
}
685686

687+
default R visitArrayFlatten(ArrayFlatten arrayFlatten, C context) {
688+
return visitScalarFunction(arrayFlatten, context);
689+
}
690+
686691
default R visitArrayMap(ArrayMap arraySort, C context) {
687692
return visitScalarFunction(arraySort, context);
688693
}

‎regression-test/data/nereids_function_p0/scalar_function/Array.out‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16924,3 +16924,33 @@ false false
1692416924
-- !sql --
1692516925
false false
1692616926

16927+
-- !sql --
16928+
[1, 2, 3, 4, 5]
16929+
16930+
-- !sql --
16931+
[]
16932+
16933+
-- !sql --
16934+
[1]
16935+
16936+
-- !sql --
16937+
[1, 2, 3]
16938+
16939+
-- !sql --
16940+
[1, 2, 3, 4, 5]
16941+
16942+
-- !sql --
16943+
[null, null]
16944+
16945+
-- !sql --
16946+
[1, 2, 3, 4, 5]
16947+
16948+
-- !sql --
16949+
[1, 2, 3, 4, 5]
16950+
16951+
-- !sql --
16952+
[1, 2, 3, 4, 5, 6, 7, 8, 9]
16953+
16954+
-- !sql --
16955+
[1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12]
16956+

‎regression-test/data/query_p0/sql_functions/array_functions/array_flatten.out‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
-- This file is automatically generated. You should know what you did if you want to edit this
2+
-- !test --
3+
1 [1, 2, 3] [1, 2, 3] [1, 2, 3] [1, 2, 3] ["a", "b", "c"]
4+
2 \N \N [] \N ["b", null]
5+
3 [1, 2, null] [null] [null, 2] [null, null, 3] [null, "aaaab", "ccc"]
6+

‎regression-test/suites/nereids_function_p0/scalar_function/Array.groovy‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1416,4 +1416,15 @@ suite("nereids_scalar_fn_Array") {
14161416
// map_contains_value
14171417
qt_sql """ select map_contains_value(map(1,1), 257), map_contains_value(map(1,2), 258);"""
14181418

1419+
qt_sql """select array_flatten([[1,2,3],[4,5]]);"""
1420+
qt_sql """select array_flatten([[],[]]);"""
1421+
qt_sql """select array_flatten([[1],[]]);"""
1422+
qt_sql """select array_flatten([[1,2,3],null]);"""
1423+
qt_sql """select array_flatten([[1,2,3],null,[4,5]]);"""
1424+
qt_sql """select array_flatten([null,null]);"""
1425+
qt_sql """select array_flatten([[1,2,3,4,5]]);"""
1426+
qt_sql """select array_flatten([[[1,2,3,4,5]]]);;"""
1427+
qt_sql """select array_flatten([ [[1,2,3,4,5]],[[6,7],[8,9]] ]);"""
1428+
qt_sql """select array_flatten([[[[[[1,2,3,4,5],[6,7],[8,9],[10,11],[12]]]]]]);"""
1429+
14191430
}

‎regression-test/suites/query_p0/sql_functions/array_functions/array_flatten.groovy‎

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
suite("array_flatten") {
19+
sql """DROP TABLE IF EXISTS t_array_flatten"""
20+
sql """
21+
CREATE TABLE IF NOT EXISTS t_array_flatten (
22+
`k1` int(11) NULL COMMENT "",
23+
`a1` array<tinyint(4)> NULL COMMENT "",
24+
`aaa1` array<array<array<tinyint(4)>>> NULL COMMENT "",
25+
`aa3` array<array<int(11)>> NOT NULL COMMENT "",
26+
`aa5` array<array<largeint(40)>> NULL COMMENT "",
27+
`aa14` array<array<string>> NULL COMMENT ""
28+
29+
) ENGINE=OLAP
30+
DUPLICATE KEY(`k1`)
31+
DISTRIBUTED BY HASH(`k1`) BUCKETS 1
32+
PROPERTIES (
33+
"replication_allocation" = "tag.location.default: 1",
34+
"storage_format" = "V2"
35+
)
36+
"""
37+
sql """ INSERT INTO t_array_flatten VALUES(1, [1, 2, 3],[[[1]],[[2],[3]]],[[1,2],[3]],[[1,2],[3]],[['a'],['b','c']]) """
38+
sql """ INSERT INTO t_array_flatten VALUES(2, null,null,[],null,[null,['b',null]]) """
39+
sql """ INSERT INTO t_array_flatten VALUES(3, [1, 2, null],[[[]],[[null],[]]],[[null,2],[]],[[null,null],[3]],[[null],['aaaab','ccc']]) """
40+
41+
42+
43+
qt_test """
44+
select k1, array_flatten(a1), array_flatten(aaa1), array_flatten(aa3), array_flatten(aa5), array_flatten(aa14) from t_array_flatten order by k1;
45+
"""
46+
}

0 commit comments

Comments
 (0)