From b1011b68e4b6ae0fa940a5e5863377a1b4714c65 Mon Sep 17 00:00:00 2001 From: Kexin Zhao Date: Wed, 27 Jun 2018 19:13:58 -0700 Subject: [PATCH 1/2] add accuracy func and test mnist accuracy --- src/example/mnist/create_mnist_recordio.py | 18 +++- src/example/mnist/test_mnist.cc | 96 +++++++++++++++++----- src/function.h | 38 ++++++++- src/variable.h | 1 + 4 files changed, 127 insertions(+), 26 deletions(-) diff --git a/src/example/mnist/create_mnist_recordio.py b/src/example/mnist/create_mnist_recordio.py index 18e73de..41b9398 100644 --- a/src/example/mnist/create_mnist_recordio.py +++ b/src/example/mnist/create_mnist_recordio.py @@ -18,7 +18,7 @@ def create_mnist_recordio_files(): - # Convert mnist to recordio file + # Convert mnist training set to recordio files with fluid.program_guard(fluid.Program(), fluid.Program()): reader = paddle.batch(mnist.train(), batch_size=32) feeder = fluid.DataFeeder( @@ -30,7 +30,21 @@ def create_mnist_recordio_files(): ], place=fluid.CPUPlace()) fluid.recordio_writer.convert_reader_to_recordio_file( - '/tmp/mnist.recordio', reader, feeder) + '/tmp/mnist_train.recordio', reader, feeder) + + # Convert mnist testing set to recordio files + with fluid.program_guard(fluid.Program(), fluid.Program()): + reader = paddle.batch(mnist.test(), batch_size=32) + feeder = fluid.DataFeeder( + feed_list=[ # order is image and label + fluid.layers.data( + name='image', shape=[1, 28, 28], dtype='float32'), + fluid.layers.data( + name='label', shape=[1], dtype='int64'), + ], + place=fluid.CPUPlace()) + fluid.recordio_writer.convert_reader_to_recordio_file( + '/tmp/mnist_test.recordio', reader, feeder) if __name__ == "__main__": diff --git a/src/example/mnist/test_mnist.cc b/src/example/mnist/test_mnist.cc index a2b2fc2..c38a1b4 100644 --- a/src/example/mnist/test_mnist.cc +++ b/src/example/mnist/test_mnist.cc @@ -13,13 +13,17 @@ // limitations under the License. #include +#include +#include #include "gtest/gtest.h" #include "src/function.h" +using paddle::tape::VariableHandle; using paddle::tape::Linear; using paddle::tape::SGD; using paddle::tape::Adam; +using paddle::tape::accuracy; using paddle::tape::mean; using paddle::tape::softmax; using paddle::tape::cross_entropy; @@ -35,35 +39,41 @@ bool is_file_exist(const std::string& fileName) { } TEST(Mnist, TestCPU) { - std::string filename = "/tmp/mnist.recordio"; - PADDLE_ENFORCE(is_file_exist(filename), - "file doesn't exist; have you run create_mnist_recordio.py"); - auto reader = CreateRecordioFileReader( - filename, {32, 1, 28, 28, 32, 1}, {4, 2}, {0, 0}); - - Linear linear1(784, 200, "relu"); - Linear linear2(200, 200, "relu"); - Linear linear3(200, 10, "relu"); + std::string filename1 = "/tmp/mnist_train.recordio"; + std::string filename2 = "/tmp/mnist_test.recordio"; + PADDLE_ENFORCE(is_file_exist(filename1), + "%s doesn't exist; have you run create_mnist_recordio.py", + filename1); + PADDLE_ENFORCE(is_file_exist(filename2), + "%s doesn't exist; have you run create_mnist_recordio.py", + filename2); + auto train_reader = CreateRecordioFileReader( + filename1, {32, 1, 28, 28, 32, 1}, {4, 2}, {0, 0}); + auto test_reader = CreateRecordioFileReader( + filename2, {32, 1, 28, 28, 32, 1}, {4, 2}, {0, 0}); + + Linear linear1(784, 200, "tanh"); + Linear linear2(200, 200, "tanh"); + Linear linear3(200, 10, "softmax"); Adam adam(0.001); + auto forward = [&](VariableHandle input) -> VariableHandle { + return linear3(linear2(linear1(input))); + }; + + int total_steps = 10000; int print_step = 100; - float avg_loss = 0.0; + float threshold = 0.90f; - for (int i = 0; i < 1000; ++i) { + for (int i = 0; i < total_steps; ++i) { reset_global_tape(); - auto data_label = ReadNext(reader); + auto data_label = ReadNext(train_reader, true); auto data = data_label[0]; auto label = data_label[1]; - auto predict = softmax(linear3(linear2(linear1(data)))); + auto predict = forward(data); auto loss = mean(cross_entropy(predict, label)); - - avg_loss += - loss->Value().Get().data()[0]; - if ((i + 1) % print_step == 0) { - LOG(INFO) << avg_loss / print_step; - avg_loss = 0; - } + auto precision = accuracy(predict, label); get_global_tape().Backward(loss); @@ -76,6 +86,52 @@ TEST(Mnist, TestCPU) { for (auto w : linear3.Params()) { adam.Update(w); } + + // Every time certain amount of batches have been processed, + // we test the average loss and accuracy on the test data set, + // we stop training when the accuracy hit some threshold + if ((i + 1) % print_step == 0) { + std::vector losses; + std::vector accuracies; + + while (true) { + reset_global_tape(); + + auto data_label = ReadNext(test_reader, false); + if (data_label.empty()) { + break; + } + + auto data = data_label[0]; + auto label = data_label[1]; + + auto predict = forward(data); + auto loss = mean(cross_entropy(predict, label)); + auto precision = accuracy(predict, label); + + get_global_tape().Forward(); + + losses.push_back( + loss->Get().data()[0]); + accuracies.push_back( + precision->Get().data()[0]); + } + + float avg_loss = + std::accumulate(losses.begin(), losses.end(), 0.0f) / losses.size(); + float avg_accu = + std::accumulate(accuracies.begin(), accuracies.end(), 0.0f) / + accuracies.size(); + + LOG(INFO) << "Pass #" << (i + 1) / print_step + << ", test set evaluation result: Avg loss is " << avg_loss + << ", Avg accuracy is " << avg_accu; + + if (avg_accu >= threshold) { + LOG(INFO) << "Meets target accuracy, stop training"; + break; + } + } } } diff --git a/src/function.h b/src/function.h index 65c75a7..c264cac 100644 --- a/src/function.h +++ b/src/function.h @@ -76,7 +76,7 @@ void init_params(VariableHandle v, class Linear { public: - Linear(int in_dim, int out_dim, const std::string &act) + Linear(int in_dim, int out_dim, const std::string &act = "") : w_(new Variable("LinearWeight")), b_(new Variable("LinearBias")), act_(act) { @@ -110,6 +110,9 @@ class Linear { {{"X", {pre_bias}}, {"Y", {b_}}}, {{"Out", {pre_act}}}, add_op_attrs); + if (act_.empty()) { + return pre_act; + } VariableHandle post_act(new Variable("linear")); get_global_tape().AddOp( act_, {{"X", {pre_act}}}, {{"Out", {post_act}}}, {}); @@ -337,6 +340,29 @@ class BatchNorm { std::string act_; }; +// Calculate the top k accuracy of the prediction against the label +VariableHandle accuracy(VariableHandle prediction, + VariableHandle label, + int k = 1) { + // Use top_k op to get top k prediction class labels + VariableHandle topk_values(new Variable("accuracy")); + VariableHandle topk_indices(new Variable("accuracy")); + get_global_tape().AddOp("top_k", + {{"X", {prediction}}}, + {{"Out", {topk_values}}, {"Indices", {topk_indices}}}, + {{"k", k}}); + + VariableHandle acc_out(new Variable("accuracy")); + VariableHandle correct(new Variable("accuracy")); + VariableHandle total(new Variable("accuracy")); + get_global_tape().AddOp( + "accuracy", + {{"Out", {topk_values}}, {"Indices", {topk_indices}}, {"Label", {label}}}, + {{"Accuracy", {acc_out}}, {"Correct", {correct}}, {"Total", {total}}}, + {}); + return acc_out; +} + VariableHandle pool2d(VariableHandle x, const framework::AttributeMap &attrs = {}) { VariableHandle out(new Variable("pool2d")); @@ -397,15 +423,19 @@ VariableHandle CreateRecordioFileReader(std::string filename, return reader; } -std::vector ReadNext(VariableHandle reader) { +std::vector ReadNext(VariableHandle reader, bool repeat) { PADDLE_ENFORCE(reader->Var().IsType()); paddle::framework::LoDTensorArray data_holder; reader->GetMutable()->ReadNext(&data_holder); if (data_holder.empty()) { reader->GetMutable()->ReInit(); - reader->GetMutable()->ReadNext( - &data_holder); + if (repeat) { + reader->GetMutable()->ReadNext( + &data_holder); + } else { + return {}; + } } PADDLE_ENFORCE(!data_holder.empty(), "Error reading file."); diff --git a/src/variable.h b/src/variable.h index 00a2bc2..e5d1968 100644 --- a/src/variable.h +++ b/src/variable.h @@ -15,6 +15,7 @@ #include #include +#include #include #include "paddle/fluid/framework/operator.h" // framework::kGradVarSuffix From 4c729c98f13f02b5d6e75e3644e79384567f8793 Mon Sep 17 00:00:00 2001 From: Kexin Zhao Date: Thu, 28 Jun 2018 10:37:30 -0700 Subject: [PATCH 2/2] address comments --- src/example/mnist/test_mnist.cc | 12 ------------ src/function.h | 27 +++++++++++++++++++-------- 2 files changed, 19 insertions(+), 20 deletions(-) diff --git a/src/example/mnist/test_mnist.cc b/src/example/mnist/test_mnist.cc index c38a1b4..80b3d9d 100644 --- a/src/example/mnist/test_mnist.cc +++ b/src/example/mnist/test_mnist.cc @@ -12,7 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include #include #include @@ -33,20 +32,9 @@ using paddle::tape::get_global_tape; using paddle::tape::CreateRecordioFileReader; using paddle::tape::ReadNext; -bool is_file_exist(const std::string& fileName) { - std::ifstream infile(fileName); - return infile.good(); -} - TEST(Mnist, TestCPU) { std::string filename1 = "/tmp/mnist_train.recordio"; std::string filename2 = "/tmp/mnist_test.recordio"; - PADDLE_ENFORCE(is_file_exist(filename1), - "%s doesn't exist; have you run create_mnist_recordio.py", - filename1); - PADDLE_ENFORCE(is_file_exist(filename2), - "%s doesn't exist; have you run create_mnist_recordio.py", - filename2); auto train_reader = CreateRecordioFileReader( filename1, {32, 1, 28, 28, 32, 1}, {4, 2}, {0, 0}); auto test_reader = CreateRecordioFileReader( diff --git a/src/function.h b/src/function.h index c264cac..f12341f 100644 --- a/src/function.h +++ b/src/function.h @@ -15,6 +15,7 @@ #pragma once #include +#include #include #include #include @@ -129,7 +130,7 @@ class Linear { class Convolution2D { public: - Convolution2D(int c_in, int c_out, int f, const std::string &act) + Convolution2D(int c_in, int c_out, int f, const std::string &act = "") : w_(new Variable("ConvolutionWeight")), b_(new Variable("ConvolutionBias")), act_(act) { @@ -165,6 +166,9 @@ class Convolution2D { {{"X", {pre_bias}}, {"Y", {b_}}}, {{"Out", {pre_act}}}, add_op_attrs); + if (act_.empty()) { + return pre_act; + } VariableHandle post_act(new Variable("conv")); get_global_tape().AddOp( act_, {{"X", {pre_act}}}, {{"Out", {post_act}}}, {}); @@ -285,7 +289,7 @@ class Adam { class BatchNorm { public: - BatchNorm(int channel_in, const std::string &act) + explicit BatchNorm(int channel_in, const std::string &act = "") : scale_(new Variable("BatchNormScale")), bias_(new Variable("BatchNormBias")), mean_(new Variable("BatchNormMean")), @@ -322,7 +326,9 @@ class BatchNorm { {"SavedMean", {tmp_mean}}, {"SavedVariance", {tmp_var}}}, attrs); - + if (act_.empty()) { + return pre_act; + } VariableHandle post_act(new Variable("batch_norm")); get_global_tape().AddOp( act_, {{"X", {pre_act}}}, {{"Out", {post_act}}}, {}); @@ -408,6 +414,11 @@ VariableHandle CreateRecordioFileReader(std::string filename, std::vector shape_concat, std::vector ranks, std::vector lod_levels) { + std::ifstream infile(filename); + PADDLE_ENFORCE(infile.good(), + "%s doesn't exist; have you run create_mnist_recordio.py?", + filename); + VariableHandle reader(new paddle::tape::Variable("reader")); framework::OpDesc op_desc = CreateOpDesc("create_recordio_file_reader", @@ -430,14 +441,14 @@ std::vector ReadNext(VariableHandle reader, bool repeat) { reader->GetMutable()->ReadNext(&data_holder); if (data_holder.empty()) { reader->GetMutable()->ReInit(); - if (repeat) { - reader->GetMutable()->ReadNext( - &data_holder); - } else { + reader->GetMutable()->ReadNext( + &data_holder); + PADDLE_ENFORCE(!data_holder.empty(), "Error reading file."); + if (!repeat) { + reader->GetMutable()->ReInit(); return {}; } } - PADDLE_ENFORCE(!data_holder.empty(), "Error reading file."); std::vector rval; for (size_t i = 0; i < data_holder.size(); ++i) {