Skip to content

[fix](json-functions)fix json-replace/insert/set/array behavior with complex type #50308

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Apr 24, 2025
+529 −55
Merged

[fix](json-functions)fix json-replace/insert/set/array behavior with complex type #50308

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions be/src/vec/functions/function_json.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -645,7 +645,7 @@ struct FunctionJsonArrayImpl {
rapidjson::Document::AllocatorType& allocator,
const std::vector<const ColumnUInt8*>& nullmaps) {
for (int i = 0; i < data_columns.size() - 1; i++) {
constexpr_int_match<'0', '6', Reducer>::run(type_flags[i], objects, allocator,
constexpr_int_match<'0', '7', Reducer>::run(type_flags[i], objects, allocator,
data_columns[i], nullmaps[i]);
}
}
Expand Down Expand Up @@ -1523,7 +1523,7 @@ class FunctionJsonModifyImpl : public IFunction {
const std::vector<const ColumnUInt8*>& nullmaps,
std::vector<bool>& column_is_consts) {
for (auto col = 1; col + 1 < data_columns.size() - 1; col += 2) {
constexpr_int_match<'0', '6', Reducer>::run(
constexpr_int_match<'0', '7', Reducer>::run(
type_flags[col + 1], objects, json_paths[col / 2], data_columns[col + 1],
nullmaps[col + 1], column_is_consts[col + 1]);
}
Expand Down
33 changes: 19 additions & 14 deletions .../src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/JsonArray.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,24 +20,20 @@
import org.apache.doris.catalog.FunctionSignature;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.functions.AlwaysNotNullable;
import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature;
import org.apache.doris.nereids.trees.expressions.functions.CustomSignature;
import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor;
import org.apache.doris.nereids.types.DataType;
import org.apache.doris.nereids.types.JsonType;
import org.apache.doris.nereids.types.VarcharType;
import org.apache.doris.nereids.util.ExpressionUtils;

import com.google.common.collect.ImmutableList;

import java.util.ArrayList;
import java.util.List;

/**
* ScalarFunction 'json_array'. This class is generated by GenerateFunction.
*/
public class JsonArray extends ScalarFunction
implements ExplicitlyCastableSignature, AlwaysNotNullable {

public static final List<FunctionSignature> SIGNATURES = ImmutableList.of(
FunctionSignature.ret(VarcharType.SYSTEM_DEFAULT).varArgs(VarcharType.SYSTEM_DEFAULT)
);
public class JsonArray extends ScalarFunction implements CustomSignature, AlwaysNotNullable {

/**
* constructor with 0 or more arguments.
Expand All @@ -46,6 +42,20 @@ public JsonArray(Expression... varArgs) {
super("json_array", ExpressionUtils.mergeArguments(varArgs));
}

@Override
public FunctionSignature customSignature() {
List<DataType> arguments = new ArrayList<>();
for (int i = 0; i < arity(); i++) {
if (getArgumentType(i).isComplexType() || getArgumentType(i).isJsonType()) {
// keep origin type for BE Serialization
arguments.add(JsonType.INSTANCE);
} else {
arguments.add(VarcharType.SYSTEM_DEFAULT);
}
}
return FunctionSignature.of(VarcharType.SYSTEM_DEFAULT, arguments);
}

/**
* withChildren.
*/
Expand All @@ -54,11 +64,6 @@ public JsonArray withChildren(List<Expression> children) {
return new JsonArray(children.toArray(new Expression[0]));
}

@Override
public List<FunctionSignature> getSignatures() {
return SIGNATURES;
}

@Override
public <R, C> R accept(ExpressionVisitor<R, C> visitor, C context) {
return visitor.visitJsonArray(this, context);
Expand Down
33 changes: 20 additions & 13 deletions ...src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/JsonInsert.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,26 +19,23 @@

import org.apache.doris.catalog.FunctionSignature;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature;
import org.apache.doris.nereids.trees.expressions.functions.CustomSignature;
import org.apache.doris.nereids.trees.expressions.functions.PropagateNullable;
import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor;
import org.apache.doris.nereids.types.DataType;
import org.apache.doris.nereids.types.JsonType;
import org.apache.doris.nereids.types.VarcharType;
import org.apache.doris.nereids.util.ExpressionUtils;

import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;

import java.util.ArrayList;
import java.util.List;

/**
* ScalarFunction 'json_insert'. This class is generated by GenerateFunction.
*/
public class JsonInsert extends ScalarFunction
implements ExplicitlyCastableSignature, PropagateNullable {

public static final List<FunctionSignature> SIGNATURES =
ImmutableList.of(FunctionSignature.ret(VarcharType.SYSTEM_DEFAULT)
.varArgs(VarcharType.SYSTEM_DEFAULT, VarcharType.SYSTEM_DEFAULT, VarcharType.SYSTEM_DEFAULT));
public class JsonInsert extends ScalarFunction implements CustomSignature, PropagateNullable {

/**
* constructor with 3 or more arguments.
Expand All @@ -47,6 +44,21 @@ public JsonInsert(Expression arg0, Expression arg1, Expression arg2, Expression.
super("json_insert", ExpressionUtils.mergeArguments(arg0, arg1, arg2, varArgs));
}

@Override
public FunctionSignature customSignature() {
List<DataType> arguments = new ArrayList<>();
arguments.add(VarcharType.SYSTEM_DEFAULT); // json_str
for (int i = 1; i < arity(); i++) {
if ((i & 1) == 0 && (getArgumentType(i).isComplexType() || getArgumentType(i).isJsonType())) {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

what does (i & 1) == 0 mean?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

(i & 1) == 0 determines the even position, which exactly corresponds to the position of the value parameter. pass the origin value type to BE and should not cast to varchar

// keep origin type for BE Serialization
arguments.add(JsonType.INSTANCE);
} else {
arguments.add(VarcharType.SYSTEM_DEFAULT);
}
}
return FunctionSignature.of(VarcharType.SYSTEM_DEFAULT, arguments);
}

/**
* withChildren.
*/
Expand All @@ -57,11 +69,6 @@ public JsonInsert withChildren(List<Expression> children) {
children.subList(3, children.size()).toArray(new Expression[0]));
}

@Override
public List<FunctionSignature> getSignatures() {
return SIGNATURES;
}

@Override
public <R, C> R accept(ExpressionVisitor<R, C> visitor, C context) {
return visitor.visitJsonInsert(this, context);
Expand Down
33 changes: 20 additions & 13 deletions ...rc/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/JsonReplace.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,26 +19,23 @@

import org.apache.doris.catalog.FunctionSignature;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature;
import org.apache.doris.nereids.trees.expressions.functions.CustomSignature;
import org.apache.doris.nereids.trees.expressions.functions.PropagateNullable;
import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor;
import org.apache.doris.nereids.types.DataType;
import org.apache.doris.nereids.types.JsonType;
import org.apache.doris.nereids.types.VarcharType;
import org.apache.doris.nereids.util.ExpressionUtils;

import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;

import java.util.ArrayList;
import java.util.List;

/**
* ScalarFunction 'json_replace'. This class is generated by GenerateFunction.
*/
public class JsonReplace extends ScalarFunction
implements ExplicitlyCastableSignature, PropagateNullable {

public static final List<FunctionSignature> SIGNATURES =
ImmutableList.of(FunctionSignature.ret(VarcharType.SYSTEM_DEFAULT)
.varArgs(VarcharType.SYSTEM_DEFAULT, VarcharType.SYSTEM_DEFAULT, VarcharType.SYSTEM_DEFAULT));
public class JsonReplace extends ScalarFunction implements CustomSignature, PropagateNullable {

/**
* constructor with 3 or more arguments.
Expand All @@ -47,6 +44,21 @@ public JsonReplace(Expression arg0, Expression arg1, Expression arg2, Expression
super("json_replace", ExpressionUtils.mergeArguments(arg0, arg1, arg2, varArgs));
}

@Override
public FunctionSignature customSignature() {
List<DataType> arguments = new ArrayList<>();
arguments.add(VarcharType.SYSTEM_DEFAULT); // json_str
for (int i = 1; i < arity(); i++) {
if ((i & 1) == 0 && (getArgumentType(i).isComplexType() || getArgumentType(i).isJsonType())) {
// keep origin type for BE Serialization
arguments.add(JsonType.INSTANCE);
} else {
arguments.add(VarcharType.SYSTEM_DEFAULT);
}
}
return FunctionSignature.of(VarcharType.SYSTEM_DEFAULT, arguments);
}

/**
* withChildren.
*/
Expand All @@ -57,11 +69,6 @@ public JsonReplace withChildren(List<Expression> children) {
children.subList(3, children.size()).toArray(new Expression[0]));
}

@Override
public List<FunctionSignature> getSignatures() {
return SIGNATURES;
}

@Override
public <R, C> R accept(ExpressionVisitor<R, C> visitor, C context) {
return visitor.visitJsonReplace(this, context);
Expand Down
33 changes: 20 additions & 13 deletions ...re/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/JsonSet.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,26 +19,23 @@

import org.apache.doris.catalog.FunctionSignature;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature;
import org.apache.doris.nereids.trees.expressions.functions.CustomSignature;
import org.apache.doris.nereids.trees.expressions.functions.PropagateNullable;
import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor;
import org.apache.doris.nereids.types.DataType;
import org.apache.doris.nereids.types.JsonType;
import org.apache.doris.nereids.types.VarcharType;
import org.apache.doris.nereids.util.ExpressionUtils;

import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;

import java.util.ArrayList;
import java.util.List;

/**
* ScalarFunction 'json_set'. This class is generated by GenerateFunction.
*/
public class JsonSet extends ScalarFunction
implements ExplicitlyCastableSignature, PropagateNullable {

public static final List<FunctionSignature> SIGNATURES =
ImmutableList.of(FunctionSignature.ret(VarcharType.SYSTEM_DEFAULT)
.varArgs(VarcharType.SYSTEM_DEFAULT, VarcharType.SYSTEM_DEFAULT, VarcharType.SYSTEM_DEFAULT));
public class JsonSet extends ScalarFunction implements CustomSignature, PropagateNullable {

/**
* constructor with 3 or more arguments.
Expand All @@ -47,6 +44,21 @@ public JsonSet(Expression arg0, Expression arg1, Expression arg2, Expression...
super("json_set", ExpressionUtils.mergeArguments(arg0, arg1, arg2, varArgs));
}

@Override
public FunctionSignature customSignature() {
List<DataType> arguments = new ArrayList<>();
arguments.add(VarcharType.SYSTEM_DEFAULT); // json_str
for (int i = 1; i < arity(); i++) {
if ((i & 1) == 0 && (getArgumentType(i).isComplexType() || getArgumentType(i).isJsonType())) {
// keep origin type for BE Serialization
arguments.add(JsonType.INSTANCE);
} else {
arguments.add(VarcharType.SYSTEM_DEFAULT);
}
}
return FunctionSignature.of(VarcharType.SYSTEM_DEFAULT, arguments);
}

/**
* withChildren.
*/
Expand All @@ -57,11 +69,6 @@ public JsonSet withChildren(List<Expression> children) {
children.subList(3, children.size()).toArray(new Expression[0]));
}

@Override
public List<FunctionSignature> getSignatures() {
return SIGNATURES;
}

@Override
public <R, C> R accept(ExpressionVisitor<R, C> visitor, C context) {
return visitor.visitJsonSet(this, context);
Expand Down
58 changes: 58 additions & 0 deletions regression-test/data/query_p0/sql_functions/json_function/test_query_json_array.out
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,3 +6,61 @@
["k0",4,"k1",null,"k2",null,"k3","test","k4","2022-01-01 11:11:11","k5",null,"k6","k6"]
["k0",5,"k1",1,"k2",true,"k3","test","k4","2022-01-01 11:11:11","k5",null,"k6","k6"]

-- !sql_array --
[["\\"aaa\\"","\\"bbb\\""]]

-- !sql_array --
[["aaa","bbb"]]

-- !sql_array --
[[1,2]]

-- !sql_array --
[[1.1,2.2]]

-- !sql_array --
[[1.1,2.0]]

-- !sql_array --
[[1.0,1.2]]

-- !sql_map --
[{"a":"b","c":"d"}]

-- !sql_map --
[{"a":1,"c":2}]

-- !sql_map --
[{"a":1.1,"c":2.2}]

-- !sql_map --
[{"a":1.1,"c":2.0}]

-- !sql_map --
[{"a":1.0,"c":1.2}]

-- !sql_struct --
[{"name":"a","age":1}]

-- !sql_struct --
[{"name":"a","age":1.1}]

-- !sql_struct --
[{"name":"a","age":1.0}]

-- !sql_json --
[{"a":"b"}]

-- !sql_json --
[{"a":1}]

-- !sql_json --
[{"a":1.1}]

-- !sql3 --
[1,null,null,null,null]
[2,["a","b"],{"a":"b"},{"name":"a","age":1},{"a":"b"}]
[3,["\\"a\\"","\\"b\\""],{"\\"a\\"":"\\"b\\"","\\"c\\"":"\\"d\\""},{"name":"\\"a\\"","age":1},{"c":"d"}]
[4,["1","2"],{"1":"2"},{"name":"2","age":1},{"a":"b"}]
[5,["1","2","3","3"],{"1":"2","3":"4"},{"name":"a","age":1},{"a":"b"}]

Loading
Loading