Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix operator type record in profiler [cherry-pick PR44582] #44654

Merged
merged 2 commits into from
Aug 2, 2022
Merged

Fix operator type record in profiler [cherry-pick PR44582] #44654

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
91b762f
fix record event for operator type in new dygraph (#44582)
rainyfly Jul 26, 2022
87277af
fix file mode
rainyfly Jul 27, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
348 changes: 228 additions & 120 deletions paddle/fluid/eager/auto_code_generator/eager_generator.cc
View file
Open in desktop

Large diffs are not rendered by default.

24 changes: 13 additions & 11 deletions paddle/fluid/eager/auto_code_generator/final_state_generator/python_c_gen.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,11 +1,11 @@
# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
#
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#
# http://www.apache.org/licenses/LICENSE-2.0
#
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
Expand Down Expand Up @@ -71,7 +71,7 @@ def FindParsingFunctionFromAttributeType(atype):


RECORD_EVENT_TEMPLATE = \
" paddle::platform::RecordEvent {}(\"{} {}\", paddle::platform::TracerEventType::Operator, 1);"
"paddle::platform::RecordEvent {}(\"{} {}\", paddle::platform::TracerEventType::UserDefined, 1);"


RETURN_INPLACE_PYOBJECT_TEMPLATE = \
Expand Down Expand Up @@ -253,6 +253,7 @@ def FindParsingFunctionFromAttributeType(atype):
## Generator Classes ##
#######################
class PythonCSingleFunctionGenerator(FunctionGeneratorBase):

def __init__(self, forward_api_contents, namespace):
# Members from Parent:
#self.namespace
Expand All @@ -265,7 +266,7 @@ def __init__(self, forward_api_contents, namespace):
#self.forward_outputs_position_map
#self.optional_inputs
#self.no_need_buffers
#self.intermediate_outputs
#self.intermediate_outputs
#self.inplace_map
FunctionGeneratorBase.__init__(self, forward_api_contents, namespace)

Expand Down Expand Up @@ -327,16 +328,16 @@ def GeneratePythonCFunction(self):
set_device_str = FUNCTION_SET_DEVICE_TEMPLATE.format(expected_place_str)

# Generate Dygraph Function Call Logic
num_args = len(forward_inputs_position_map.keys()) + len(
orig_forward_attrs_list)
num_args = len(
forward_inputs_position_map.keys()) + len(orig_forward_attrs_list)
dygraph_function_call_list = ["" for i in range(num_args)]
for name, (_, pos) in forward_inputs_position_map.items():
dygraph_function_call_list[pos] = f"{name}"
for name, _, _, pos in orig_forward_attrs_list:
dygraph_function_call_list[pos] = f"{name}"
dygraph_function_call_str = ",".join(dygraph_function_call_list)

# Generate Python-C Function Definitions
# Generate Python-C Function Definitions
if is_forward_only:
fwd_function_name = FUNCTION_NAME_TEMPLATE.format(
"paddle::experimental::", namespace, forward_api_name)
Expand Down Expand Up @@ -441,8 +442,9 @@ def run(self):


class PythonCYamlGenerator(YamlGeneratorBase):

def __init__(self, path):
# Parent members:
# Parent members:
# self.namespace
# self.api_yaml_path
# self.forward_api_list
Expand All @@ -457,8 +459,8 @@ def GeneratePythonCFunctions(self):
forward_api_list = self.forward_api_list

for forward_api_content in forward_api_list:
f_generator = PythonCSingleFunctionGenerator(forward_api_content,
namespace)
f_generator = PythonCSingleFunctionGenerator(
forward_api_content, namespace)
status = f_generator.run()

if status == True:
Expand Down
68 changes: 44 additions & 24 deletions paddle/fluid/eager/backward.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,10 +30,10 @@
namespace egr {

/*
* GeneralGrad is Helpper class to implement custom grad operation between
* outputs and inputs.
*
* **/
* GeneralGrad is Helpper class to implement custom grad operation between
* outputs and inputs.
*
* **/
class GeneralGrad {
public:
static GeneralGrad& Instance() { return *general_grad_; }
Expand Down Expand Up @@ -64,7 +64,8 @@ class GeneralGrad {
paddle::platform::errors::Fatal(
"There is no grad op for %s:[%d] or it's"
"stop_gradient=True.",
msg, i));
msg,
i));
if (is_no_grad_vars) {
(no_grad_var_nodes_inputmeta_map)[target_node] = auto_grad_meta;
} else { // normal input
Expand Down Expand Up @@ -248,7 +249,8 @@ class GeneralGrad {

std::vector<paddle::experimental::Tensor> GetResults(
const std::vector<paddle::experimental::Tensor>& inputs,
bool allow_unused, bool create_graph) {
bool allow_unused,
bool create_graph) {
VLOG(6) << "Running in GetResults";
if (inputs.empty()) return {};

Expand Down Expand Up @@ -276,7 +278,8 @@ class GeneralGrad {
tensor_auto_grad_meta->SetStopGradient(!create_graph);
results.emplace_back(iter->second);
} else {
PADDLE_ENFORCE_EQ(allow_unused, true,
PADDLE_ENFORCE_EQ(allow_unused,
true,
paddle::platform::errors::InvalidArgument(
"The %d-th input does not appear in the backward "
"graph. Please check the input tensor or set "
Expand Down Expand Up @@ -493,7 +496,8 @@ std::unordered_map<GradNodeBase*, int> getInDegreeMap(
void EnforceGradNodeHasInput(GradNodeBase* node) {
VLOG(6) << "Running in EnforceGradNodeHasInput";
PADDLE_ENFORCE_NE(
node->IsTensorWrappersCleared(), true,
node->IsTensorWrappersCleared(),
true,
paddle::platform::errors::Fatal(
"The TensorWrappers of %s do not exist. This may be because:\n"
"You calculate backward twice for the same subgraph without "
Expand All @@ -509,10 +513,13 @@ void DuplicateCheck(const std::vector<paddle::experimental::Tensor>& inputs,
for (auto in : inputs) {
AutogradMeta* auto_grad_meta = EagerUtils::unsafe_autograd_meta(in);
PADDLE_ENFORCE_EQ(
visisted_ins.count(auto_grad_meta), 0,
visisted_ins.count(auto_grad_meta),
0,
paddle::platform::errors::AlreadyExists(
"%s contain duplicate tensor %s, please check %s carefully.", msg,
in.name(), msg));
"%s contain duplicate tensor %s, please check %s carefully.",
msg,
in.name(),
msg));
visisted_ins.insert(auto_grad_meta);
}
}
Expand All @@ -522,7 +529,8 @@ GeneralGrad* GeneralGrad::general_grad_ = new GeneralGrad();
std::vector<paddle::experimental::Tensor> RunBackward(
const std::vector<paddle::experimental::Tensor>& tensors, // output
const std::vector<paddle::experimental::Tensor>& grad_tensors,
bool retain_graph, bool create_graph = false,
bool retain_graph,
bool create_graph = false,
const std::vector<paddle::experimental::Tensor>& inputs = {},
bool allow_unused = false,
const std::vector<paddle::experimental::Tensor>& no_grad_vars = {}) {
Expand Down Expand Up @@ -631,8 +639,8 @@ std::vector<paddle::experimental::Tensor> RunBackward(

if (is_general_grad) {
// Prepare several vital preprocess for GeneralGrad
GeneralGrad::Instance().PreparedForGeneralGrad(inputs, no_grad_vars, &queue,
node_input_buffers_dict);
GeneralGrad::Instance().PreparedForGeneralGrad(
inputs, no_grad_vars, &queue, node_input_buffers_dict);
}

VLOG(6) << " startup_ops' size is :" << queue.size();
Expand All @@ -651,7 +659,8 @@ std::vector<paddle::experimental::Tensor> RunBackward(

paddle::platform::RecordEvent node_record_event(
std::string((*node).name()) + " grad_node",
paddle::platform::TracerEventType::Operator, 1);
paddle::platform::TracerEventType::Operator,
1);

if (queue.size() > 1 && node_in_degree_map[node] != 0) {
queue.pop();
Expand Down Expand Up @@ -716,7 +725,8 @@ std::vector<paddle::experimental::Tensor> RunBackward(
"Number of edges should be either empty ( for leaf node "
") or the same as number of output grad tensors, but we "
"got edges size is: %d, grad_output size is: %d",
edges.size(), grad_output_tensors.size()));
edges.size(),
grad_output_tensors.size()));

for (size_t i = 0; i < edges.size(); i++) {
for (size_t j = 0; j < edges[i].size(); j++) {
Expand All @@ -739,7 +749,8 @@ std::vector<paddle::experimental::Tensor> RunBackward(
}

PADDLE_ENFORCE_LT(
j, grad_output_tensors[i].size(),
j,
grad_output_tensors[i].size(),
paddle::platform::errors::Fatal(
"Rank of grad_output_tensors should be less than "
"grad_output_tensors[i].size(), which is: %d. This error may "
Expand Down Expand Up @@ -771,9 +782,10 @@ std::vector<paddle::experimental::Tensor> RunBackward(
VLOG(6) << "Sum grad inputs for edge slot: " << edge_rank.first
<< ", rank: " << edge_rank.second;

node_input_buffers_dict[next_node]->add(
edge_rank.first, edge_rank.second, grad_output_tensor,
create_graph);
node_input_buffers_dict[next_node]->add(edge_rank.first,
edge_rank.second,
grad_output_tensor,
create_graph);

// Update queue
node_in_degree_map[next_node]--;
Expand Down Expand Up @@ -810,7 +822,7 @@ void Backward(
bool retain_graph) {
VLOG(6) << "Run in Backward";
paddle::platform::RecordEvent backward_record_event(
"backward", paddle::platform::TracerEventType::Operator, 1);
"backward", paddle::platform::TracerEventType::UserDefined, 1);
RunBackward(tensors, grad_tensors, retain_graph);
phi::autotune::AutoTuneStatus::Instance().Update();
}
Expand All @@ -819,14 +831,22 @@ std::vector<paddle::experimental::Tensor> Grad(
const std::vector<paddle::experimental::Tensor>& tensors, // outputs
const std::vector<paddle::experimental::Tensor>& inputs,
const std::vector<paddle::experimental::Tensor>& grad_tensors,
bool retain_graph, bool create_graph, bool only_inputs, bool allow_unused,
bool retain_graph,
bool create_graph,
bool only_inputs,
bool allow_unused,
const std::vector<paddle::experimental::Tensor>& no_grad_vars) {
VLOG(6) << "Run in Grad";

DuplicateCheck(inputs, true /* is_input */);
DuplicateCheck(tensors, false /* is_input */);

return RunBackward(tensors, grad_tensors, retain_graph, create_graph, inputs,
allow_unused, no_grad_vars);
return RunBackward(tensors,
grad_tensors,
retain_graph,
create_graph,
inputs,
allow_unused,
no_grad_vars);
}
} // namespace egr
4 changes: 2 additions & 2 deletions paddle/fluid/platform/profiler/chrometracing_logger.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -588,7 +588,7 @@ void ChromeTracingLogger::StartLog() {
std::string(
R"JSON(
{
"id": %d, "name": "%s", "totalGlobalMem": %u,
"id": %d, "name": "%s", "totalGlobalMem": %llu,
"computeMajor": %d, "computeMinor": %d,
"maxThreadsPerBlock": %d, "maxThreadsPerMultiprocessor": %d,
"regsPerBlock": %d, "regsPerMultiprocessor": %d, "warpSize": %d,
Expand Down Expand Up @@ -618,7 +618,7 @@ void ChromeTracingLogger::StartLog() {
std::string(
R"JSON(
{
"id": %d, "name": "%s", "totalGlobalMem": %u,
"id": %d, "name": "%s", "totalGlobalMem": %llu,
"computeMajor": %d, "computeMinor": %d,
"maxThreadsPerBlock": %d, "maxThreadsPerMultiprocessor": %d,
"regsPerBlock": %d, "regsPerMultiprocessor": %d, "warpSize": %d,
Expand Down
Loading