Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Clear extra input (Bias, ResidualData) in OpMaker of conv2d #47579

Merged
merged 18 commits into from
Dec 6, 2022
Merged

Clear extra input (Bias, ResidualData) in OpMaker of conv2d #47579

Show file tree
Hide file tree
Changes from 17 commits
Commits
Show all changes
18 commits
Select commit Hold shift + click to select a range
533e273
delete Bias and ResidualData in OpMaker of conv2d
zyfncg Nov 2, 2022
a2b55fe
delete extra input of conv3d
zyfncg Nov 2, 2022
39bd7ce
refactor pass of conv_bias_fusion
zyfncg Nov 17, 2022
ca679de
Merge branch 'develop' of https://github.com/PaddlePaddle/Paddle into…
zyfncg Nov 17, 2022
6c8ca04
fix mkldnn dependency
zyfncg Nov 17, 2022
6344a4a
fix mkldnn compile
zyfncg Nov 18, 2022
dffcf8f
fix test_conv_bias_mkldnn_fuse_pass
zyfncg Nov 18, 2022
9634422
Merge branch 'develop' of https://github.com/PaddlePaddle/Paddle into…
zyfncg Nov 20, 2022
61ff0ce
police some code
zyfncg Nov 21, 2022
e6da463
remove useless log
zyfncg Nov 21, 2022
0c4fec1
Merge branch 'develop' of https://github.com/PaddlePaddle/Paddle into…
zyfncg Nov 22, 2022
bc4fcad
fix analyzer_vit_ocr_tester
zyfncg Nov 22, 2022
f5e1cc5
fix conv_activation_mkldnn_fuse_pass
zyfncg Nov 23, 2022
cbcfb3c
fix test_analyzer_ocr
zyfncg Nov 24, 2022
68dba4b
add fused_conv_sig
zyfncg Nov 28, 2022
3c0d756
fix performence regression
zyfncg Nov 29, 2022
03c86be
Merge branch 'develop' of https://github.com/PaddlePaddle/Paddle into…
zyfncg Nov 29, 2022
5d82a37
fix performance regression
zyfncg Dec 2, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion paddle/fluid/framework/ir/graph_helper.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -713,7 +713,7 @@ static void GetGraphOpDesc(const std::vector<Node *> &nodes,
UpdateControlOpSkipEagerDeletionVars(*n, graph, graph_idx, n->Name());
}
ops->emplace_back(*n->Op());
VLOG(4) << n->ToString();
VLOG(5) << n->ToString();
}
// delete no OpDesc op
}
Expand Down
13 changes: 7 additions & 6 deletions paddle/fluid/framework/ir/graph_pattern_detector.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2043,8 +2043,9 @@ PDNode *patterns::Flatten2Matmul::operator()() {
return matmul_out;
}

PDNode *patterns::ConvResidual::operator()(bool with_residual_data) {
auto conv_op = pattern->NewNode(conv_op_repr())->assert_is_op("conv2d");
PDNode *patterns::ConvResidual::operator()(const std::string &conv_type,
bool with_residual_data) {
auto conv_op = pattern->NewNode(conv_op_repr())->assert_is_op(conv_type);

if (!with_residual_data) {
conv_op->assert_more([&](Node *x) {
Expand All @@ -2057,22 +2058,22 @@ PDNode *patterns::ConvResidual::operator()(bool with_residual_data) {

auto input_var = pattern->NewNode(conv_input_repr())
->AsInput()
->assert_is_op_input("conv2d", "Input");
->assert_is_op_input(conv_type, "Input");

auto filter_var = pattern->NewNode(conv_filter_repr())
->AsInput()
->assert_is_op_input("conv2d", "Filter");
->assert_is_op_input(conv_type, "Filter");

auto output_var = pattern->NewNode(conv_output_repr())
->AsOutput()
->assert_is_op_output("conv2d", "Output");
->assert_is_op_output(conv_type, "Output");

std::vector<PDNode *> links_from{input_var, filter_var};

if (with_residual_data) {
auto res_conn_var = pattern->NewNode(conv_residual_data_repr())
->AsInput()
->assert_is_op_input("conv2d", "ResidualData");
->assert_is_op_input(conv_type, "ResidualData");
links_from.push_back(res_conn_var);
}

Expand Down
2 changes: 1 addition & 1 deletion paddle/fluid/framework/ir/graph_pattern_detector.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1036,7 +1036,7 @@ struct ConvResidual : public PatternBase {
ConvResidual(PDPattern* pattern, const std::string& name_scope)
: PatternBase(pattern, name_scope, "conv_residual") {}

PDNode* operator()(bool with_residual_data);
PDNode* operator()(const std::string& conv_type, bool with_residual_data);

PATTERN_DECL_NODE(conv_op);
PATTERN_DECL_NODE(conv_input);
Expand Down
41 changes: 40 additions & 1 deletion paddle/fluid/framework/ir/mkldnn/conv_activation_mkldnn_fuse_pass.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ using string::PrettyLogDetail;

void ConvActivationMkldnnFusePass::ApplyImpl(Graph* graph) const {
auto act_types = phi::funcs::GetSupportedActivations();
std::vector<std::string> conv_types = {"conv2d"};
std::vector<std::string> conv_types = {"conv2d", "fused_conv2d"};

for (auto& act_type : act_types) {
FuseConvConcatAct(graph, act_type);
Expand Down Expand Up @@ -218,6 +218,45 @@ ConvActivationMkldnnFusePass::ConvActivationMkldnnFusePass() {
.IsStringIn({"NCHW", "NHWC", "AnyLayout"})
.End();

AddOpCompat(OpCompat("fused_conv2d"))
.AddInput("Input")
.IsTensor()
.End()
.AddInput("Filter")
.IsTensor()
.End()
.AddInput("Bias")
.IsOptional()
.IsTensor()
.End()
.AddInput("ResidualData")
.IsTensor()
.IsOptional()
.End()
.AddOutput("Output")
.IsTensor()
.End()
.AddAttr("strides")
.IsType<std::vector<int>>()
.End()
.AddAttr("paddings")
.IsType<std::vector<int>>()
.End()
.AddAttr("padding_algorithm")
.IsOptional()
.IsStringIn({"EXPLICIT", "SAME", "VALID"})
.End()
.AddAttr("groups")
.IsNumGE(1)
.End()
.AddAttr("dilations")
.IsType<std::vector<int>>()
.End()
.AddAttr("data_format")
.IsOptional()
.IsStringIn({"NCHW", "NHWC", "AnyLayout"})
.End();

AddOpCompat(OpCompat("concat"))
.AddInput("X")
.End()
Expand Down
87 changes: 83 additions & 4 deletions paddle/fluid/framework/ir/mkldnn/conv_bias_mkldnn_fuse_pass.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,40 @@ ConvBiasFusePass::ConvBiasFusePass() {
.IsStringIn({"NCHW", "NHWC", "AnyLayout"})
.End();

AddOpCompat(OpCompat("fused_conv2d"))
.AddInput("Input")
.IsTensor()
.End()
.AddInput("Filter")
.IsTensor()
.End()
.AddInput("Bias")
.IsTensor()
.IsOptional()
.End()
.AddOutput("Output")
.IsTensor()
.End()
.AddAttr("strides")
.IsType<std::vector<int>>()
.End()
.AddAttr("paddings")
.IsType<std::vector<int>>()
.End()
.AddAttr("padding_algorithm")
.IsOptional()
.IsStringIn({"EXPLICIT", "SAME", "VALID"})
.End()
.AddAttr("groups")
.IsNumGE(1)
.End()
.AddAttr("dilations")
.IsType<std::vector<int>>()
.End()
.AddAttr("data_format")
.IsStringIn({"NCHW", "NHWC", "AnyLayout"})
.End();

AddOpCompat(OpCompat("elementwise_add"))
.AddInput("X")
.IsTensor()
Expand Down Expand Up @@ -165,6 +199,40 @@ Conv3DBiasFusePass::Conv3DBiasFusePass() {
.IsStringIn({"NDHWC", "NCDHW"})
.End();

AddOpCompat(OpCompat("fused_conv3d"))
.AddInput("Input")
.IsTensor()
.End()
.AddInput("Filter")
.IsTensor()
.End()
.AddInput("Bias")
.IsTensor()
.IsOptional()
.End()
.AddOutput("Output")
.IsTensor()
.End()
.AddAttr("strides")
.IsType<std::vector<int>>()
.End()
.AddAttr("paddings")
.IsType<std::vector<int>>()
.End()
.AddAttr("padding_algorithm")
.IsOptional()
.IsStringIn({"EXPLICIT", "SAME", "VALID"})
.End()
.AddAttr("groups")
.IsNumGE(1)
.End()
.AddAttr("dilations")
.IsType<std::vector<int>>()
.End()
.AddAttr("data_format")
.IsStringIn({"NCHW", "NHWC", "AnyLayout"})
.End();

AddOpCompat(OpCompat("elementwise_add"))
.AddInput("X")
.IsTensor()
Expand Down Expand Up @@ -203,6 +271,16 @@ phi::DenseTensor tensor_apply_eltwise(const phi::DenseTensor& vec_a,
}

void ConvBiasFusePass::ApplyImpl(ir::Graph* graph) const {
FuseConvBias(graph, type(), fused_type());
if (type() != fused_type()) {
// Is the second pass useful?
FuseConvBias(graph, fused_type(), fused_type());
}
Comment on lines +276 to +278
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

What's the purpose of executing this fuse pass twice?

Copy link
Contributor Author

@zyfncg zyfncg Nov 28, 2022 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

After conv2d is split, we need to modify the related passes. The following case may be exist:

// Bebore this PR:
conv2d -> op1 -> elementwise_add   --- pass1--->  conv2d -> elementwise_add  ---conv_bias_mkldnn_fuse_pass--->  conv2d

// After conv2d is split: 
conv2d -> op1 -> elementwise_add   --- pass1--->  conv2d -> elementwise_add  ---conv_bias_mkldnn_fuse_pass---> fused_conv2d

conv2d -> op2 -> elementwise_add   --- pass2--->  fused_conv2d -> elementwise_add  ---conv_bias_mkldnn_fuse_pass---> fused_conv2d

Currently, the conv2d can be used recursively in the fuse pass. I'm not sure whether it's a design to optimize performance, but in order to keep performance, conv_bias_mkldnn_fuse_pass need to support conv2d and fused_conv2d as input operator. So the first time of fuse pass is processing conv2d and the second time of fuse pass is processing fused_conv2d.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ok, that makes sense 👍 Thank you

}

void ConvBiasFusePass::FuseConvBias(ir::Graph* graph,
const std::string& conv_type,
const std::string& fused_conv) const {
PADDLE_ENFORCE_NOT_NULL(
graph, platform::errors::InvalidArgument("Graph cannot be nullptr."));
FusePassBase::Init(name_scope_, graph);
Expand All @@ -216,9 +294,9 @@ void ConvBiasFusePass::ApplyImpl(ir::Graph* graph) const {
gpd.mutable_pattern()
->NewNode(patterns::PDNodeName(name_scope_, "conv_input"))
->AsInput()
->assert_is_op_input(type(), "Input");
->assert_is_op_input(conv_type, "Input");
patterns::ConvBias conv_bias_pattern(gpd.mutable_pattern(), name_scope_);
conv_bias_pattern(conv_input, type());
conv_bias_pattern(conv_input, conv_type);
int found_conv_bias_count = 0;
auto handler = [&](const GraphPatternDetector::subgraph_t& subgraph,
Graph* g) {
Expand Down Expand Up @@ -249,7 +327,7 @@ void ConvBiasFusePass::ApplyImpl(ir::Graph* graph) const {
// check if fuse can be done and if MKL-DNN should be used
FuseOptions fuse_option = FindFuseOption(*conv, *eltwise);
if (fuse_option == DO_NOT_FUSE || fuse_option == FUSE_NATIVE) {
VLOG(3) << "do not perform " + type() + "+bias fuse";
VLOG(3) << "do not perform " + conv_type + "+bias fuse";
return;
}

Expand Down Expand Up @@ -294,7 +372,7 @@ void ConvBiasFusePass::ApplyImpl(ir::Graph* graph) const {
desc.SetInput("Filter", std::vector<std::string>({conv_weight->Name()}));
desc.SetInput("Bias", std::vector<std::string>({eltwise_bias->Name()}));
desc.SetOutput("Output", std::vector<std::string>({eltwise_out->Name()}));
desc.SetType(type());
desc.SetType(fused_conv);

for (auto& attr : conv->Op()->GetAttrMap()) {
desc.SetAttr(attr.first, attr.second);
Expand Down Expand Up @@ -323,6 +401,7 @@ void ConvBiasFusePass::ApplyImpl(ir::Graph* graph) const {
type());
}
}

} // namespace ir
} // namespace framework
} // namespace paddle
Expand Down
8 changes: 8 additions & 0 deletions paddle/fluid/framework/ir/mkldnn/conv_bias_mkldnn_fuse_pass.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,24 +32,32 @@ class ConvBiasFusePass : public FusePassBase {
ConvBiasFusePass();
virtual ~ConvBiasFusePass() {}
virtual std::string type() const { return "conv2d"; }
virtual std::string fused_type() const { return "fused_conv2d"; }

protected:
void ApplyImpl(ir::Graph* graph) const override;
void FuseConvBias(ir::Graph* graph,
const std::string& conv_type,
const std::string& fused_conv) const;

const std::string name_scope_{"conv_bias_mkldnn_fuse"};
};

/*
* Fuse the Conv3D and Elementwise_add to a Conv3DBiasOp.
*/
class Conv2DTransposeBiasFusePass : public ConvBiasFusePass {
public:
Conv2DTransposeBiasFusePass();
std::string type() const override { return "conv2d_transpose"; }
std::string fused_type() const override { return "conv2d_transpose"; }
};

class Conv3DBiasFusePass : public ConvBiasFusePass {
public:
Conv3DBiasFusePass();
std::string type() const override { return "conv3d"; }
std::string fused_type() const override { return "fused_conv3d"; }
};
} // namespace ir
} // namespace framework
Expand Down
3 changes: 2 additions & 1 deletion paddle/fluid/framework/ir/mkldnn/conv_bias_mkldnn_fuse_pass_tester.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -139,7 +139,8 @@ void MainTest(bool convWithExistingBias) {
int conv_bias_count = 0;

for (auto* node : graph->Nodes()) {
if (node->IsOp() && node->Op()->Type() == "conv2d") {
if (node->IsOp() && (node->Op()->Type() == "conv2d" ||
node->Op()->Type() == "fused_conv2d")) {
auto* op = node->Op();
ASSERT_TRUE(op->HasAttr("use_mkldnn"));
EXPECT_TRUE(PADDLE_GET_CONST(bool, op->GetAttr("use_mkldnn")));
Expand Down
11 changes: 7 additions & 4 deletions paddle/fluid/framework/ir/mkldnn/cpu_quantize_pass.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -388,11 +388,12 @@ void CPUQuantizePass::GetQuantInfo(Graph* graph) const {
}

void CPUQuantizePass::QuantizeConv(Graph* graph,
const std::string& conv_type,
bool with_residual_data) const {
GraphPatternDetector gpd;
auto pattern = gpd.mutable_pattern();
patterns::ConvResidual conv_pattern{pattern, name_scope_};
conv_pattern(with_residual_data);
conv_pattern(conv_type, with_residual_data);

int quantize_conv_count = 0;
auto handler = [&](const GraphPatternDetector::subgraph_t& subgraph,
Expand Down Expand Up @@ -510,7 +511,7 @@ void CPUQuantizePass::QuantizeConv(Graph* graph,
AddStatis(quantize_conv_count);

LogQuantizedOpsCounter(
"conv2d",
conv_type,
quantize_conv_count,
((with_residual_data) ? "with residual connection" : ""));
}
Expand Down Expand Up @@ -1247,8 +1248,10 @@ void CPUQuantizePass::ApplyImpl(ir::Graph* graph) const {
platform::errors::InvalidArgument("Scope cannot be nullptr."));

GetQuantInfo(graph);
QuantizeConv(graph, false /* with_residual_data */);
QuantizeConv(graph, true /* with_residual_data */);
QuantizeConv(graph, "conv2d", false /* with_residual_data */);
QuantizeConv(graph, "conv2d", true /* with_residual_data */);
QuantizeConv(graph, "fused_conv2d", false /* with_residual_data */);
QuantizeConv(graph, "fused_conv2d", true /* with_residual_data */);
QuantizePool(graph);
QuantizeConcat(graph);
QuantizePriorBox(graph);
Expand Down
4 changes: 3 additions & 1 deletion paddle/fluid/framework/ir/mkldnn/cpu_quantize_pass.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,9 @@ class CPUQuantizePass : public FusePassBase {
protected:
void ApplyImpl(ir::Graph* graph) const override;

void QuantizeConv(Graph* graph, bool with_residual_data) const;
void QuantizeConv(Graph* graph,
const std::string& conv_type,
bool with_residual_data) const;
void QuantizeFc(Graph* graph, bool with_residual_data) const;
void QuantizePool(Graph* graph) const;
void QuantizeConcat(Graph* graph) const;
Expand Down
27 changes: 8 additions & 19 deletions paddle/fluid/framework/ir/mkldnn/cpu_quantize_placement_pass.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,25 +25,14 @@ class Graph;
void CPUQuantizePlacementPass::ApplyImpl(ir::Graph* graph) const {
VLOG(3) << "Marks operators which are to be quantized.";
std::unordered_set<std::string> supported_op_types =
std::unordered_set<std::string>({"concat",
"conv2d",
"depthwise_conv2d",
"elementwise_add",
"elementwise_mul",
"elementwise_sub",
"fc",
"matmul",
"nearest_interp",
"nearest_interp_v2",
"pool2d",
"prior_box",
"reshape2",
"transpose2",
"fusion_gru",
"fusion_lstm",
"multi_gru",
"slice",
"split"});
std::unordered_set<std::string>(
{"concat", "conv2d", "depthwise_conv2d",
"fused_conv2d", "fused_conv3d", "elementwise_add",
"elementwise_mul", "elementwise_sub", "fc",
"matmul", "nearest_interp", "nearest_interp_v2",
"pool2d", "prior_box", "reshape2",
"transpose2", "fusion_gru", "fusion_lstm",
"multi_gru", "slice", "split"});
const auto& excluded_ids_list =
Get<std::unordered_set<int>>("quantize_excluded_op_ids");
const auto& op_types_list =
Expand Down
2 changes: 1 addition & 1 deletion paddle/fluid/framework/ir/mkldnn/params_quantization_mkldnn_pass.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -120,7 +120,7 @@ void ParamsQuantizationMkldnnPass::QuantizeConv(ir::Graph* graph,
bool with_residual_data) const {
GraphPatternDetector gpd;
patterns::ConvResidual conv_pattern(gpd.mutable_pattern(), name_scope_);
conv_pattern(with_residual_data);
conv_pattern("conv2d", with_residual_data);

int params_to_int8_conv_found = 0;

Expand Down
2 changes: 1 addition & 1 deletion paddle/fluid/framework/ir/pass.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -146,7 +146,7 @@ class Pass {
}
attrs_[attr_name] = attr;
attr_dels_[attr_name] = [attr, attr_name]() {
VLOG(3) << "deleting " << attr_name;
VLOG(8) << "deleting " << attr_name;
delete attr;
};
}
Expand Down
2 changes: 1 addition & 1 deletion paddle/fluid/framework/op_desc.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -979,7 +979,7 @@ struct SetAttrDescVisitor {
};

void OpDesc::Flush() {
VLOG(4) << "Flush "
VLOG(8) << "Flush "
<< " " << Type() << " " << need_update_;
if (need_update_) {
this->desc_.mutable_inputs()->Clear();
Expand Down
Loading