Skip to content

Commit

Permalink
Add semi and cross semi join kind (#8263)
Browse files Browse the repository at this point in the history 
ref #8262
  • Loading branch information
@gengliqi
gengliqi committed Nov 3, 2023
1 parent 8685769 commit f4b5154
Show file tree
Hide file tree
Showing 14 changed files with 185 additions and 106 deletions.
2 changes: 2 additions & 0 deletions dbms/src/DataStreams/HashJoinBuildBlockInputStream.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,11 +54,13 @@ void HashJoinBuildBlockInputStream::appendInfo(FmtBuffer & buffer) const
{ASTTableJoin::Kind::Full, "Full"},
{ASTTableJoin::Kind::Cross, "Cross"},
{ASTTableJoin::Kind::Comma, "Comma"},
{ASTTableJoin::Kind::Semi, "Semi"},
{ASTTableJoin::Kind::Anti, "Anti"},
{ASTTableJoin::Kind::LeftOuterSemi, "Left_Semi"},
{ASTTableJoin::Kind::LeftOuterAnti, "Left_Anti"},
{ASTTableJoin::Kind::Cross_LeftOuter, "Cross_Left"},
{ASTTableJoin::Kind::Cross_RightOuter, "Cross_Right"},
{ASTTableJoin::Kind::Cross_Semi, "Cross_Semi"},
{ASTTableJoin::Kind::Cross_Anti, "Cross_Anti"},
{ASTTableJoin::Kind::Cross_LeftOuterSemi, "Cross_LeftSemi"},
{ASTTableJoin::Kind::Cross_LeftOuterAnti, "Cross_LeftAnti"},
Expand Down
3 changes: 1 addition & 2 deletions dbms/src/Flash/Coprocessor/DAGQueryBlockInterpreter.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -260,7 +260,7 @@ void DAGQueryBlockInterpreter::handleJoin(
= tiflash_join.genJoinOutputColumns(left_input_header, right_input_header, match_helper_name);
/// add necessary transformation if the join key is an expression

bool is_tiflash_right_join = tiflash_join.isTiFlashRightOuterJoin();
bool is_tiflash_right_join = isRightOuterJoin(tiflash_join.kind);

JoinNonEqualConditions join_non_equal_conditions;
// prepare probe side
Expand Down Expand Up @@ -331,7 +331,6 @@ void DAGQueryBlockInterpreter::handleJoin(
probe_key_names,
build_key_names,
tiflash_join.kind,
tiflash_join.strictness,
log->identifier(),
enableFineGrainedShuffle(fine_grained_shuffle_count),
fine_grained_shuffle_count,
Expand Down
10 changes: 4 additions & 6 deletions dbms/src/Flash/Coprocessor/JoinInterpreterHelper.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -73,7 +73,7 @@ std::pair<ASTTableJoin::Kind, size_t> getJoinKindAndBuildSideIndex(
{{tipb::JoinType::TypeRightOuterJoin, 0}, {ASTTableJoin::Kind::LeftOuter, 0}},
{{tipb::JoinType::TypeRightOuterJoin, 1}, {ASTTableJoin::Kind::RightOuter, 1}},
{{tipb::JoinType::TypeSemiJoin, 0}, {ASTTableJoin::Kind::RightSemi, 0}},
{{tipb::JoinType::TypeSemiJoin, 1}, {ASTTableJoin::Kind::Inner, 1}},
{{tipb::JoinType::TypeSemiJoin, 1}, {ASTTableJoin::Kind::Semi, 1}},
{{tipb::JoinType::TypeAntiSemiJoin, 0}, {ASTTableJoin::Kind::RightAnti, 0}},
{{tipb::JoinType::TypeAntiSemiJoin, 1}, {ASTTableJoin::Kind::Anti, 1}},
{{tipb::JoinType::TypeLeftOuterSemiJoin, 1}, {ASTTableJoin::Kind::LeftOuterSemi, 1}},
Expand All @@ -89,7 +89,7 @@ std::pair<ASTTableJoin::Kind, size_t> getJoinKindAndBuildSideIndex(
{{tipb::JoinType::TypeLeftOuterJoin, 1}, {ASTTableJoin::Kind::Cross_LeftOuter, 1}},
{{tipb::JoinType::TypeRightOuterJoin, 0}, {ASTTableJoin::Kind::Cross_LeftOuter, 0}},
{{tipb::JoinType::TypeRightOuterJoin, 1}, {ASTTableJoin::Kind::Cross_LeftOuter, 0}},
{{tipb::JoinType::TypeSemiJoin, 1}, {ASTTableJoin::Kind::Cross, 1}},
{{tipb::JoinType::TypeSemiJoin, 1}, {ASTTableJoin::Kind::Cross_Semi, 1}},
{{tipb::JoinType::TypeAntiSemiJoin, 1}, {ASTTableJoin::Kind::Cross_Anti, 1}},
{{tipb::JoinType::TypeLeftOuterSemiJoin, 1}, {ASTTableJoin::Kind::Cross_LeftOuterSemi, 1}},
{{tipb::JoinType::TypeAntiLeftOuterSemiJoin, 1}, {ASTTableJoin::Kind::Cross_LeftOuterAnti, 1}}};
Expand Down Expand Up @@ -212,8 +212,6 @@ TiFlashJoin::TiFlashJoin(const tipb::Join & join_, bool is_test) // NOLINT(cppco
, join_key_collators(getJoinKeyCollators(join_, join_key_types, is_test))
{
std::tie(kind, build_side_index) = getJoinKindAndBuildSideIndex(join);
strictness
= (isSemiJoin() && !isRightSemiFamily(kind)) ? ASTTableJoin::Strictness::Any : ASTTableJoin::Strictness::All;
}

String TiFlashJoin::genMatchHelperName(const Block & header1, const Block & header2) const
Expand Down Expand Up @@ -333,9 +331,9 @@ NamesAndTypes TiFlashJoin::genJoinOutputColumns(
};

append_output_columns(left_input_header, join.join_type() == tipb::JoinType::TypeRightOuterJoin);
if (!isSemiJoin())
if (!isSemiFamily() && !isLeftOuterSemiFamily())
{
/// for semi join, the columns from right table will be ignored
/// for (left outer) semi join, the columns from right table will be ignored
append_output_columns(right_input_header, join.join_type() == tipb::JoinType::TypeLeftOuterJoin);
}

Expand Down
24 changes: 7 additions & 17 deletions dbms/src/Flash/Coprocessor/JoinInterpreterHelper.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -92,11 +92,11 @@ struct JoinNonEqualConditions
ExpressionActionsPtr null_aware_eq_cond_expr;

/// Validate this JoinNonEqualConditions and return error message if any.
String validate(ASTTableJoin::Kind kind) const
const char * validate(ASTTableJoin::Kind kind) const
{
if (unlikely(!left_filter_column.empty() && !isLeftOuterJoin(kind)))
if unlikely (!left_filter_column.empty() && !isLeftOuterJoin(kind))
return "non left join with left conditions";
if (unlikely(!right_filter_column.empty() && !isRightOuterJoin(kind)))
if unlikely (!right_filter_column.empty() && !isRightOuterJoin(kind))
return "non right join with right conditions";

if unlikely ((!other_cond_name.empty() || !other_eq_cond_from_in_name.empty()) && other_cond_expr == nullptr)
Expand All @@ -113,7 +113,7 @@ struct JoinNonEqualConditions
return "null-aware semi join should not have other_eq_cond_from_in_name";
}

return "";
return nullptr;
}
};

Expand All @@ -131,19 +131,17 @@ struct TiFlashJoin
JoinKeyTypes join_key_types;
TiDB::TiDBCollators join_key_collators;

ASTTableJoin::Strictness strictness;

/// (cartesian) (anti) left outer semi join.
bool isLeftOuterSemiFamily() const
{
return join.join_type() == tipb::JoinType::TypeLeftOuterSemiJoin
|| join.join_type() == tipb::JoinType::TypeAntiLeftOuterSemiJoin;
}

bool isSemiJoin() const
/// (anti) semi join.
bool isSemiFamily() const
{
return join.join_type() == tipb::JoinType::TypeSemiJoin || join.join_type() == tipb::JoinType::TypeAntiSemiJoin
|| isLeftOuterSemiFamily();
return join.join_type() == tipb::JoinType::TypeSemiJoin || join.join_type() == tipb::JoinType::TypeAntiSemiJoin;
}

const google::protobuf::RepeatedPtrField<tipb::Expr> & getBuildJoinKeys() const
Expand All @@ -166,14 +164,6 @@ struct TiFlashJoin
return build_side_index == 0 ? join.right_conditions() : join.left_conditions();
}

bool isTiFlashLeftOuterJoin() const
{
return kind == ASTTableJoin::Kind::LeftOuter || kind == ASTTableJoin::Kind::Cross_LeftOuter;
}

/// Cross_RightOuter join will be converted to Cross_LeftOuter join, so no need to check Cross_RightOuter
bool isTiFlashRightOuterJoin() const { return kind == ASTTableJoin::Kind::RightOuter; }

/// return a name that is unique in header1 and header2 for left outer semi family join,
/// return "" for everything else.
String genMatchHelperName(const Block & header1, const Block & header2) const;
Expand Down
4 changes: 2 additions & 2 deletions dbms/src/Flash/Coprocessor/tests/gtest_join_get_kind_and_build_index.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -81,7 +81,7 @@ TEST(JoinKindAndBuildIndexTestRunner, TestCrossJoins)

/// Cross Semi/Anti, expects right table as build side only, otherwise throws exceptions
result = JoinInterpreterHelper::getJoinKindAndBuildSideIndex(tipb::JoinType::TypeSemiJoin, 1, false, 0);
ASSERT_TRUE(result.first == ASTTableJoin::Kind::Cross && result.second == 1);
ASSERT_TRUE(result.first == ASTTableJoin::Kind::Cross_Semi && result.second == 1);
result = JoinInterpreterHelper::getJoinKindAndBuildSideIndex(tipb::JoinType::TypeAntiSemiJoin, 1, false, 0);
ASSERT_TRUE(result.first == ASTTableJoin::Kind::Cross_Anti && result.second == 1);

Expand Down Expand Up @@ -121,7 +121,7 @@ TEST(JoinKindAndBuildIndexTestRunner, TestEqualJoins)

/// Semi/Anti
result = JoinInterpreterHelper::getJoinKindAndBuildSideIndex(tipb::JoinType::TypeSemiJoin, 1, false, 1);
ASSERT_TRUE(result.first == ASTTableJoin::Kind::Inner && result.second == 1);
ASSERT_TRUE(result.first == ASTTableJoin::Kind::Semi && result.second == 1);
result = JoinInterpreterHelper::getJoinKindAndBuildSideIndex(tipb::JoinType::TypeSemiJoin, 0, false, 1);
ASSERT_TRUE(result.first == ASTTableJoin::Kind::RightSemi && result.second == 0);
result = JoinInterpreterHelper::getJoinKindAndBuildSideIndex(tipb::JoinType::TypeAntiSemiJoin, 1, false, 1);
Expand Down
3 changes: 1 addition & 2 deletions dbms/src/Flash/Planner/Plans/PhysicalJoin.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -89,7 +89,7 @@ PhysicalPlanNodePtr PhysicalJoin::build(

/// add necessary transformation if the join key is an expression

bool is_tiflash_right_join = tiflash_join.isTiFlashRightOuterJoin();
bool is_tiflash_right_join = isRightOuterJoin(tiflash_join.kind);

JoinNonEqualConditions join_non_equal_conditions;
// prepare probe side
Expand Down Expand Up @@ -169,7 +169,6 @@ PhysicalPlanNodePtr PhysicalJoin::build(
probe_key_names,
build_key_names,
tiflash_join.kind,
tiflash_join.strictness,
join_req_id,
fine_grained_shuffle.enable(),
fine_grained_shuffle.stream_count,
Expand Down
112 changes: 108 additions & 4 deletions dbms/src/Interpreters/CrossJoinProbeHelper.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -242,6 +242,102 @@ struct CrossJoinAdder<ASTTableJoin::Kind::Cross_Anti, ASTTableJoin::Strictness::
return ret;
}
};
template <>
struct CrossJoinAdder<ASTTableJoin::Kind::Cross_Semi, ASTTableJoin::Strictness::Any>
{
static bool addFound(
MutableColumns & dst_columns,
size_t num_existing_columns,
ColumnRawPtrs & src_left_columns,
size_t num_columns_to_add,
size_t i,
const Blocks & blocks,
IColumn::Filter * is_row_matched,
IColumn::Offset & current_offset,
IColumn::Offsets * expanded_row_size_after_join,
size_t right_rows_to_be_added,
size_t max_block_size)
{
return CrossJoinAdder<ASTTableJoin::Kind::Cross, ASTTableJoin::Strictness::Any>::addFound(
dst_columns,
num_existing_columns,
src_left_columns,
num_columns_to_add,
i,
blocks,
is_row_matched,
current_offset,
expanded_row_size_after_join,
right_rows_to_be_added,
max_block_size);
;
}
static bool addNotFound(
MutableColumns & /* dst_columns */,
size_t /* num_existing_columns */,
ColumnRawPtrs & /* src_left_columns */,
size_t /* num_columns_to_add */,
size_t /* i */,
IColumn::Filter * /* is_row_matched */,
IColumn::Offset & /* current_offset */,
IColumn::Offsets * /* expanded_row_size_after_join */,
size_t /*max_block_size*/)
{
return false;
}
};
template <>
struct CrossJoinAdder<ASTTableJoin::Kind::Cross_Semi, ASTTableJoin::Strictness::All>
{
static bool addFound(
MutableColumns & dst_columns,
size_t num_existing_columns,
ColumnRawPtrs & src_left_columns,
size_t num_columns_to_add,
size_t i,
const Blocks & blocks,
IColumn::Filter * is_row_matched,
IColumn::Offset & current_offset,
IColumn::Offsets * expanded_row_size_after_join,
size_t right_rows_to_be_added,
size_t max_block_size)
{
return CrossJoinAdder<ASTTableJoin::Kind::Cross, ASTTableJoin::Strictness::All>::addFound(
dst_columns,
num_existing_columns,
src_left_columns,
num_columns_to_add,
i,
blocks,
is_row_matched,
current_offset,
expanded_row_size_after_join,
right_rows_to_be_added,
max_block_size);
}
static bool addNotFound(
MutableColumns & dst_columns,
size_t num_existing_columns,
ColumnRawPtrs & src_left_columns,
size_t num_columns_to_add,
size_t i,
IColumn::Filter * is_row_matched,
IColumn::Offset & current_offset,
IColumn::Offsets * expanded_row_size_after_join,
size_t max_block_size)
{
return CrossJoinAdder<ASTTableJoin::Kind::Cross, ASTTableJoin::Strictness::All>::addNotFound(
dst_columns,
num_existing_columns,
src_left_columns,
num_columns_to_add,
i,
is_row_matched,
current_offset,
expanded_row_size_after_join,
max_block_size);
}
};
template <ASTTableJoin::Strictness STRICTNESS>
struct CrossJoinAdder<ASTTableJoin::Kind::Cross_LeftOuterSemi, STRICTNESS>
{
Expand Down Expand Up @@ -530,8 +626,6 @@ Block crossProbeBlockDeepCopyRightBlock(
#define DISPATCH(HAS_NULL_MAP) \
if (kind == Cross && strictness == All) \
return crossProbeBlockDeepCopyRightBlockImpl<Cross, All, HAS_NULL_MAP>(probe_process_info, right_blocks); \
else if (kind == Cross && strictness == Any) \
return crossProbeBlockDeepCopyRightBlockImpl<Cross, Any, HAS_NULL_MAP>(probe_process_info, right_blocks); \
else if (kind == Cross_LeftOuter && strictness == All) \
return crossProbeBlockDeepCopyRightBlockImpl<Cross_LeftOuter, All, HAS_NULL_MAP>( \
probe_process_info, \
Expand All @@ -540,6 +634,10 @@ Block crossProbeBlockDeepCopyRightBlock(
return crossProbeBlockDeepCopyRightBlockImpl<Cross_LeftOuter, Any, HAS_NULL_MAP>( \
probe_process_info, \
right_blocks); \
else if (kind == Cross_Semi && strictness == All) \
return crossProbeBlockDeepCopyRightBlockImpl<Cross_Semi, All, HAS_NULL_MAP>(probe_process_info, right_blocks); \
else if (kind == Cross_Semi && strictness == Any) \
return crossProbeBlockDeepCopyRightBlockImpl<Cross_Semi, Any, HAS_NULL_MAP>(probe_process_info, right_blocks); \
else if (kind == Cross_Anti && strictness == All) \
return crossProbeBlockDeepCopyRightBlockImpl<Cross_Anti, All, HAS_NULL_MAP>(probe_process_info, right_blocks); \
else if (kind == Cross_Anti && strictness == Any) \
Expand Down Expand Up @@ -585,8 +683,6 @@ std::pair<Block, bool> crossProbeBlockShallowCopyRightBlock(
#define DISPATCH(HAS_NULL_MAP) \
if (kind == Cross && strictness == All) \
return crossProbeBlockShallowCopyRightBlockImpl<Cross, All, HAS_NULL_MAP>(probe_process_info, right_blocks); \
else if (kind == Cross && strictness == Any) \
return crossProbeBlockShallowCopyRightBlockImpl<Cross, Any, HAS_NULL_MAP>(probe_process_info, right_blocks); \
else if (kind == Cross_LeftOuter && strictness == All) \
return crossProbeBlockShallowCopyRightBlockImpl<Cross_LeftOuter, All, HAS_NULL_MAP>( \
probe_process_info, \
Expand All @@ -595,6 +691,14 @@ std::pair<Block, bool> crossProbeBlockShallowCopyRightBlock(
return crossProbeBlockShallowCopyRightBlockImpl<Cross_LeftOuter, Any, HAS_NULL_MAP>( \
probe_process_info, \
right_blocks); \
else if (kind == Cross_Semi && strictness == All) \
return crossProbeBlockShallowCopyRightBlockImpl<Cross_Semi, All, HAS_NULL_MAP>( \
probe_process_info, \
right_blocks); \
else if (kind == Cross_Semi && strictness == Any) \
return crossProbeBlockShallowCopyRightBlockImpl<Cross_Semi, Any, HAS_NULL_MAP>( \
probe_process_info, \
right_blocks); \
else if (kind == Cross_Anti && strictness == All) \
return crossProbeBlockShallowCopyRightBlockImpl<Cross_Anti, All, HAS_NULL_MAP>( \
probe_process_info, \
Expand Down

0 comments on commit f4b5154

Please sign in to comment.