From 61ce72ecd46fac745a87c0f291f7e1f2e2a352f2 Mon Sep 17 00:00:00 2001 From: Fabian Kaczmarczyck Date: Mon, 12 Oct 2020 13:34:26 +0200 Subject: [PATCH 1/3] adds run functionality, including time + commit for results --- .bazelrc | 2 +- BUILD | 8 ++++++++ README.md | 22 +++++++++++++++------- docs/results.md | 12 ++++++++++++ run.sh | 20 ++++++++++++++++++++ src/device_tracker.cc | 32 ++++++++++++++++++++++---------- src/device_tracker.h | 5 +++-- src/device_tracker_test.cc | 17 ++++++++++------- src/fido2_conformance_main.cc | 7 +++++++ src/hid/hid_device.cc | 13 +++++++++++++ src/hid/hid_device.h | 3 +++ src/stamp.cc | 17 +++++++++++++++++ tools/workspace_status.sh | 20 ++++++++++++++++++++ 13 files changed, 151 insertions(+), 27 deletions(-) create mode 100644 docs/results.md create mode 100755 run.sh create mode 100644 src/stamp.cc create mode 100755 tools/workspace_status.sh diff --git a/.bazelrc b/.bazelrc index 037e0e1..c120aa8 100644 --- a/.bazelrc +++ b/.bazelrc @@ -1 +1 @@ -build --cxxopt='-std=c++17' +build --cxxopt='-std=c++17' --stamp --workspace_status_command=tools/workspace_status.sh diff --git a/BUILD b/BUILD index 10062be..76f493f 100644 --- a/BUILD +++ b/BUILD @@ -90,8 +90,10 @@ cc_library( deps = [ ":constants", ":parameter_check", + ":stamp", "//third_party/chromium_components_cbor:cbor", "@com_github_nlohmann_json//:json", + "@com_google_absl//absl/time", ], ) @@ -134,6 +136,11 @@ cc_library( ], ) +cc_library( + name = "stamp", + linkstamp = "src/stamp.cc" +) + cc_binary( name = "fido2_conformance", srcs = ["src/fido2_conformance_main.cc"], @@ -146,3 +153,4 @@ cc_binary( "@com_google_glog//:glog", ], ) + diff --git a/README.md b/README.md index f35fd7e..5dffcc2 100644 --- a/README.md +++ b/README.md @@ -23,16 +23,26 @@ using git. ### How to run Running the tool without comments lists all avaiable devices. Select the device -you want to test by passing `--token_path`. +you want to test by passing `--token_path`. If only one CTAP2 compatible device +is plugged in, you can simply run: + +```shell +./run.sh +``` + +For more control, try i.e.: ```shell bazel run //:fido2_conformance -bazel run //:fido2_conformance -- --token_path=/dev/hidraw5 +bazel run //:fido2_conformance -- --token_path=/dev/hidraw0 ``` :warning: Please do not plug in other security keys with the same product ID, or the tool might contact the wrong device during testing. +While running the test tool, you will be prompted to touch or replug your +security key multiple times, to test various features. + ### Supported features At the moment, we only support USB HID as a transport. We test the commands from @@ -42,10 +52,8 @@ but plan to add tests for supported extensions and ### Results -While running the test tool, you will be prompted to touch or replug your -security key multiple times, to test various features. After finishing all -tests, you see a printed summary of your results in your terminal, and a report -file is created in the `results` directory. +For more information on checking or contributing test results, please check +[results.md](docs/results.md). ### Contributing @@ -53,5 +61,5 @@ If we didn't already test your security key or you have an updated version, please create a pull request with your result file! If you want to contribute code, please check -[Contributing.md](docs/contributing.md). +[contributing.md](docs/contributing.md). diff --git a/docs/results.md b/docs/results.md new file mode 100644 index 0000000..3af5a54 --- /dev/null +++ b/docs/results.md @@ -0,0 +1,12 @@ +## Results + +Results are stored in JSON files and generated after successful test tool runs. +It contains information about tests and other findings. Check the `results` +directory and look for your product ID. + +### Contributing your own results + +After finishing all tests, you see a printed summary of your results in your +terminal, and a report file is created in the `results` directory. You can +contribute this file to our collection of results with a pull request! + diff --git a/run.sh b/run.sh new file mode 100755 index 0000000..72b4d8c --- /dev/null +++ b/run.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# The underscore is the magic path that uses the first device found. You can +# also pass the desired path as a command line argument. +path=${1:-_} +bazel run //:fido2_conformance -- --token_path="$path" + diff --git a/src/device_tracker.cc b/src/device_tracker.cc index bb797eb..bdeb768 100644 --- a/src/device_tracker.cc +++ b/src/device_tracker.cc @@ -18,9 +18,12 @@ #include #include +#include "absl/time/clock.h" #include "src/parameter_check.h" #include "third_party/chromium_components_cbor/values.h" +extern const char build_scm_revision[]; + namespace fido2_tests { namespace { constexpr std::string_view kRelativeDir = "results/"; @@ -188,30 +191,39 @@ void DeviceTracker::ReportFindings() const { << " tests." << std::endl; } -nlohmann::json DeviceTracker::GenerateResultsJson() { +nlohmann::json DeviceTracker::GenerateResultsJson( + std::string_view commit_hash, std::string_view time_string) { int successful_test_count = successful_tests_.size(); int failed_test_count = failed_tests_.size(); int test_count = successful_test_count + failed_test_count; nlohmann::json results = { - {"Passed tests", successful_test_count}, - {"Total tests", test_count}, - {"Failed tests", failed_tests_}, - {"Reported problems", problems_}, - {"Reported observations", observations_}, - {"Counter", counter_checker_.ReportFindings()}, + {"passed_test_count", successful_test_count}, + {"total_test_count", test_count}, + {"failed_tests", failed_tests_}, + {"problems", problems_}, + {"observations", observations_}, + {"counter", counter_checker_.ReportFindings()}, + {"date", time_string}, + {"commit", commit_hash}, }; return results; } void DeviceTracker::SaveResultsToFile() { - std::filesystem::path results_path = - absl::StrCat(CreateSaveFileDirectory(), product_name_, kFileType); + absl::Time now = absl::Now(); + absl::TimeZone local = absl::LocalTimeZone(); + std::string time_string = absl::FormatTime("%Y-%m-%d", now, local); + + std::filesystem::path results_path = absl::StrCat( + CreateSaveFileDirectory(), product_name_, "_", time_string, kFileType); std::ofstream results_file; results_file.open(results_path); CHECK(results_file.is_open()) << "Unable to open file: " << results_path; - results_file << std::setw(2) << GenerateResultsJson() << std::endl; + results_file << std::setw(2) + << GenerateResultsJson(build_scm_revision, time_string) + << std::endl; } } // namespace fido2_tests diff --git a/src/device_tracker.h b/src/device_tracker.h index 093f51d..991ca75 100644 --- a/src/device_tracker.h +++ b/src/device_tracker.h @@ -78,11 +78,12 @@ class DeviceTracker { // observations, problems and tests. void ReportFindings() const; // Generates a JSON object with test results. - nlohmann::json GenerateResultsJson(); + nlohmann::json GenerateResultsJson(std::string_view commit_hash, + std::string_view time_string); // Saves the results to a JSON file. Creates a "results" directory, if // necessary. The file name will be derived from the product name as listed // through HID, or a default if none is found. Overwrites existing files of - // the same name. + // the same name. The commit is stamped into the binary and read here. void SaveResultsToFile(); private: diff --git a/src/device_tracker_test.cc b/src/device_tracker_test.cc index bc12f2b..829d634 100644 --- a/src/device_tracker_test.cc +++ b/src/device_tracker_test.cc @@ -113,14 +113,17 @@ TEST(DeviceTracker, TestGenerateResultsJson) { device_tracker.CheckAndReport(false, "FALSE_TEST"); device_tracker.CheckAndReport(true, "TRUE_TEST"); - nlohmann::json output = device_tracker.GenerateResultsJson(); + nlohmann::json output = + device_tracker.GenerateResultsJson("c0", "2020-01-01"); nlohmann::json expected_output = { - {"Passed tests", 1}, - {"Total tests", 2}, - {"Failed tests", {"FALSE_TEST"}}, - {"Reported problems", {"PROBLEM"}}, - {"Reported observations", {"OBSERVATION"}}, - {"Counter", "All counters were constant zero."}, + {"passed_test_count", 1}, + {"total_test_count", 2}, + {"failed_tests", {"FALSE_TEST"}}, + {"problems", {"PROBLEM"}}, + {"observations", {"OBSERVATION"}}, + {"counter", "All counters were constant zero."}, + {"date", "2020-01-01"}, + {"commit", "c0"}, }; EXPECT_EQ(output, expected_output); } diff --git a/src/fido2_conformance_main.cc b/src/fido2_conformance_main.cc index a371ad9..1c668b3 100644 --- a/src/fido2_conformance_main.cc +++ b/src/fido2_conformance_main.cc @@ -39,6 +39,7 @@ DEFINE_int32(num_credentials, 50, int main(int argc, char** argv) { google::InitGoogleLogging(argv[0]); gflags::ParseCommandLineFlags(&argc, &argv, true); + if (FLAGS_token_path.empty()) { std::cout << "Please add the --token_path flag for one of these devices:" << std::endl; @@ -46,6 +47,12 @@ int main(int argc, char** argv) { exit(0); } + if (FLAGS_token_path == "_") { + // This magic value is used by the run script for comfort. + FLAGS_token_path = fido2_tests::hid::FindFidoDevicePath(); + std::cout << "Testing device at path: " << FLAGS_token_path << std::endl; + } + fido2_tests::DeviceTracker tracker; std::unique_ptr device = absl::make_unique(&tracker, FLAGS_token_path, diff --git a/src/hid/hid_device.cc b/src/hid/hid_device.cc index cc338b7..d165e7f 100644 --- a/src/hid/hid_device.cc +++ b/src/hid/hid_device.cc @@ -525,5 +525,18 @@ void PrintFidoDevices() { hid_free_enumeration(devs); } +std::string FindFidoDevicePath() { + hid_device_info* devs = hid_enumerate(0, 0); // 0 means all devices. + std::string device_path; + for (hid_device_info* cur_dev = devs; cur_dev; cur_dev = cur_dev->next) { + if (cur_dev->usage_page == 0xf1d0 /* FIDO specific usage page*/) { + device_path = cur_dev->path; + break; + } + } + hid_free_enumeration(devs); + return device_path; +} + } // namespace hid } // namespace fido2_tests diff --git a/src/hid/hid_device.h b/src/hid/hid_device.h index 7a006e9..ec88506 100644 --- a/src/hid/hid_device.h +++ b/src/hid/hid_device.h @@ -61,6 +61,9 @@ struct __attribute__((__packed__)) Frame { // FIDO HID usage page (i.e. 0xf1d0) and prints their details on stdout. void PrintFidoDevices(); +// Utility function that returns the first suitable device path found. +std::string FindFidoDevicePath(); + class HidDevice : public DeviceInterface { public: // The constructor without the third parameter implicitly assumes false. diff --git a/src/stamp.cc b/src/stamp.cc new file mode 100644 index 0000000..51ca8de --- /dev/null +++ b/src/stamp.cc @@ -0,0 +1,17 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +extern const char build_scm_revision[]; + +const char build_scm_revision[] = BUILD_SCM_REVISION; diff --git a/tools/workspace_status.sh b/tools/workspace_status.sh new file mode 100755 index 0000000..12a379b --- /dev/null +++ b/tools/workspace_status.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# The underscore is the magic path that uses the first device found. You can +# also pass the desired path as a command line argument. + +echo BUILD_SCM_REVISION $(git rev-parse HEAD) + From 78682bdddcf0952cba25ca1ef5a01bd5e42600d6 Mon Sep 17 00:00:00 2001 From: Fabian Kaczmarczyck Date: Mon, 19 Oct 2020 15:29:35 +0200 Subject: [PATCH 2/3] tools for Windows --- .bazelrc | 2 +- README.md | 4 ++-- src/fido2_conformance_main.cc | 2 +- src/hid/hid_device.cc | 2 +- src/hid/hid_device.h | 2 +- tools/workspace_status.bat | 6 ++++++ tools/workspace_status.cmd | 16 ++++++++++++++++ tools/workspace_status.sh | 3 --- 8 files changed, 28 insertions(+), 9 deletions(-) create mode 100755 tools/workspace_status.bat create mode 100755 tools/workspace_status.cmd diff --git a/.bazelrc b/.bazelrc index c120aa8..ef88d3d 100644 --- a/.bazelrc +++ b/.bazelrc @@ -1 +1 @@ -build --cxxopt='-std=c++17' --stamp --workspace_status_command=tools/workspace_status.sh +build --cxxopt='-std=c++17' --stamp --workspace_status_command=tools/workspace_status.bat diff --git a/README.md b/README.md index 5dffcc2..c542f27 100644 --- a/README.md +++ b/README.md @@ -23,8 +23,8 @@ using git. ### How to run Running the tool without comments lists all avaiable devices. Select the device -you want to test by passing `--token_path`. If only one CTAP2 compatible device -is plugged in, you can simply run: +you want to test by passing `--token_path`. For Unix, ff only one CTAP2 +compatible device is plugged in, you can simply run: ```shell ./run.sh diff --git a/src/fido2_conformance_main.cc b/src/fido2_conformance_main.cc index 1c668b3..0b092ea 100644 --- a/src/fido2_conformance_main.cc +++ b/src/fido2_conformance_main.cc @@ -49,7 +49,7 @@ int main(int argc, char** argv) { if (FLAGS_token_path == "_") { // This magic value is used by the run script for comfort. - FLAGS_token_path = fido2_tests::hid::FindFidoDevicePath(); + FLAGS_token_path = fido2_tests::hid::FindFirstFidoDevicePath(); std::cout << "Testing device at path: " << FLAGS_token_path << std::endl; } diff --git a/src/hid/hid_device.cc b/src/hid/hid_device.cc index d165e7f..bb41994 100644 --- a/src/hid/hid_device.cc +++ b/src/hid/hid_device.cc @@ -525,7 +525,7 @@ void PrintFidoDevices() { hid_free_enumeration(devs); } -std::string FindFidoDevicePath() { +std::string FindFirstFidoDevicePath() { hid_device_info* devs = hid_enumerate(0, 0); // 0 means all devices. std::string device_path; for (hid_device_info* cur_dev = devs; cur_dev; cur_dev = cur_dev->next) { diff --git a/src/hid/hid_device.h b/src/hid/hid_device.h index ec88506..1bc1f53 100644 --- a/src/hid/hid_device.h +++ b/src/hid/hid_device.h @@ -62,7 +62,7 @@ struct __attribute__((__packed__)) Frame { void PrintFidoDevices(); // Utility function that returns the first suitable device path found. -std::string FindFidoDevicePath(); +std::string FindFirstFidoDevicePath(); class HidDevice : public DeviceInterface { public: diff --git a/tools/workspace_status.bat b/tools/workspace_status.bat new file mode 100755 index 0000000..23599ea --- /dev/null +++ b/tools/workspace_status.bat @@ -0,0 +1,6 @@ +:; ./tools/workspace_status.sh +:; exit +@ECHO OFF +cd tools +workspace_status.cmd + diff --git a/tools/workspace_status.cmd b/tools/workspace_status.cmd new file mode 100755 index 0000000..5086000 --- /dev/null +++ b/tools/workspace_status.cmd @@ -0,0 +1,16 @@ +:; Copyright 2020 Google LLC +:; +:; Licensed under the Apache License, Version 2.0 (the "License"); +:; you may not use this file except in compliance with the License. +:; You may obtain a copy of the License at +:; +:; http://www.apache.org/licenses/LICENSE-2.0 +:; +:; Unless required by applicable law or agreed to in writing, software +:; distributed under the License is distributed on an "AS IS" BASIS, +:; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +:; See the License for the specific language governing permissions and +:; limitations under the License. + +for /f "delims=" %%i in ('git rev-parse HEAD') do set BUILD_SCM_REVISION==%%i + diff --git a/tools/workspace_status.sh b/tools/workspace_status.sh index 12a379b..4d8ef8c 100755 --- a/tools/workspace_status.sh +++ b/tools/workspace_status.sh @@ -13,8 +13,5 @@ # See the License for the specific language governing permissions and # limitations under the License. -# The underscore is the magic path that uses the first device found. You can -# also pass the desired path as a command line argument. - echo BUILD_SCM_REVISION $(git rev-parse HEAD) From c7a43a4de7886077390e6268ded1d3bb14a3d539 Mon Sep 17 00:00:00 2001 From: Fabian Kaczmarczyck Date: Mon, 19 Oct 2020 17:34:48 +0200 Subject: [PATCH 3/3] fixes README typo --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index c542f27..3625c20 100644 --- a/README.md +++ b/README.md @@ -23,7 +23,7 @@ using git. ### How to run Running the tool without comments lists all avaiable devices. Select the device -you want to test by passing `--token_path`. For Unix, ff only one CTAP2 +you want to test by passing `--token_path`. For Unix, if only one CTAP2 compatible device is plugged in, you can simply run: ```shell