Easy run script and stamped binaries

.bazelrc

@@ -1 +1 @@
-build --cxxopt='-std=c++17'
+build --cxxopt='-std=c++17' --stamp --workspace_status_command=tools/workspace_status.bat

BUILD

@@ -90,8 +90,10 @@ cc_library(
     deps = [
         ":constants",
         ":parameter_check",
+        ":stamp",
         "//third_party/chromium_components_cbor:cbor",
         "@com_github_nlohmann_json//:json",
+        "@com_google_absl//absl/time",
     ],
 )
 
@@ -151,6 +153,11 @@ cc_library(
     ],
 )
 
+cc_library(
+    name = "stamp",
+    linkstamp = "src/stamp.cc"
+)
+
 cc_binary(
     name = "fido2_conformance",
     srcs = ["src/fido2_conformance_main.cc"],
@@ -164,3 +171,4 @@ cc_binary(
         "@com_google_glog//:glog",
     ],
 )
+

README.md

@@ -23,16 +23,26 @@ using git.
 ### How to run
 
 Running the tool without comments lists all avaiable devices. Select the device
-you want to test by passing `--token_path`.
+you want to test by passing `--token_path`. For Unix, if only one CTAP2
+compatible device is plugged in, you can simply run:
+
+```shell
+./run.sh
+```
+
+For more control, try i.e.:
 
 ```shell
 bazel run //:fido2_conformance
-bazel run //:fido2_conformance -- --token_path=/dev/hidraw5
+bazel run //:fido2_conformance -- --token_path=/dev/hidraw0
 ```
 
 :warning: Please do not plug in other security keys with the same product ID, or
 the tool might contact the wrong device during testing.
 
+While running the test tool, you will be prompted to touch or replug your
+security key multiple times, to test various features.
+
 ### Supported features
 
 At the moment, we only support USB HID as a transport. We test the commands from
@@ -42,16 +52,14 @@ but plan to add tests for supported extensions and
 
 ### Results
 
-While running the test tool, you will be prompted to touch or replug your
-security key multiple times, to test various features. After finishing all
-tests, you see a printed summary of your results in your terminal, and a report
-file is created in the `results` directory.
+For more information on checking or contributing test results, please check
+[results.md](docs/results.md).
 
 ### Contributing
 
 If we didn't already test your security key or you have an updated version,
 please create a pull request with your result file!
 
 If you want to contribute code, please check
-[Contributing.md](docs/contributing.md).
+[contributing.md](docs/contributing.md).
 

docs/results.md

@@ -0,0 +1,12 @@
+## Results
+
+Results are stored in JSON files and generated after successful test tool runs.
+It contains information about tests and other findings. Check the `results`
+directory and look for your product ID.
+
+### Contributing your own results
+
+After finishing all tests, you see a printed summary of your results in your
+terminal, and a report file is created in the `results` directory. You can
+contribute this file to our collection of results with a pull request!
+

run.sh

@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+# Copyright 2020 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# The underscore is the magic path that uses the first device found. You can
+# also pass the desired path as a command line argument.
+path=${1:-_}
+bazel run //:fido2_conformance -- --token_path="$path"
+

src/device_tracker.cc

@@ -18,9 +18,12 @@
 #include <fstream>
 #include <iostream>
 
+#include "absl/time/clock.h"
 #include "src/parameter_check.h"
 #include "third_party/chromium_components_cbor/values.h"
 
+extern const char build_scm_revision[];
+
 namespace fido2_tests {
 namespace {
 constexpr std::string_view kRelativeDir = "results/";
@@ -212,30 +215,39 @@ void DeviceTracker::ReportFindings() const {
             << " tests." << std::endl;
 }
 
-nlohmann::json DeviceTracker::GenerateResultsJson() {
+nlohmann::json DeviceTracker::GenerateResultsJson(
+    std::string_view commit_hash, std::string_view time_string) {
   int successful_test_count = successful_tests_.size();
   int failed_test_count = failed_tests_.size();
   int test_count = successful_test_count + failed_test_count;
 
   nlohmann::json results = {
-      {"Passed tests", successful_test_count},
-      {"Total tests", test_count},
-      {"Failed tests", failed_tests_},
-      {"Reported problems", problems_},
-      {"Reported observations", observations_},
-      {"Counter", counter_checker_.ReportFindings()},
+      {"passed_test_count", successful_test_count},
+      {"total_test_count", test_count},
+      {"failed_tests", failed_tests_},
+      {"problems", problems_},
+      {"observations", observations_},
+      {"counter", counter_checker_.ReportFindings()},
+      {"date", time_string},
+      {"commit", commit_hash},
   };
   return results;
 }
 
 void DeviceTracker::SaveResultsToFile() {
-  std::filesystem::path results_path =
-      absl::StrCat(CreateSaveFileDirectory(), product_name_, kFileType);
+  absl::Time now = absl::Now();
+  absl::TimeZone local = absl::LocalTimeZone();
+  std::string time_string = absl::FormatTime("%Y-%m-%d", now, local);
+
+  std::filesystem::path results_path = absl::StrCat(
+      CreateSaveFileDirectory(), product_name_, "_", time_string, kFileType);
   std::ofstream results_file;
   results_file.open(results_path);
   CHECK(results_file.is_open()) << "Unable to open file: " << results_path;
 
-  results_file << std::setw(2) << GenerateResultsJson() << std::endl;
+  results_file << std::setw(2)
+               << GenerateResultsJson(build_scm_revision, time_string)
+               << std::endl;
 }
 
 }  // namespace fido2_tests

src/device_tracker.h

@@ -87,11 +87,12 @@ class DeviceTracker {
   // observations, problems and tests.
   void ReportFindings() const;
   // Generates a JSON object with test results.
-  nlohmann::json GenerateResultsJson();
+  nlohmann::json GenerateResultsJson(std::string_view commit_hash,
+                                     std::string_view time_string);
   // Saves the results to a JSON file. Creates a "results" directory, if
   // necessary. The file name will be derived from the product name as listed
   // through HID, or a default if none is found. Overwrites existing files of
-  // the same name.
+  // the same name. The commit is stamped into the binary and read here.
   void SaveResultsToFile();
 
  private:

src/device_tracker_test.cc

@@ -113,14 +113,17 @@ TEST(DeviceTracker, TestGenerateResultsJson) {
   device_tracker.CheckAndReport(false, "FALSE_TEST");
   device_tracker.CheckAndReport(true, "TRUE_TEST");
 
-  nlohmann::json output = device_tracker.GenerateResultsJson();
+  nlohmann::json output =
+      device_tracker.GenerateResultsJson("c0", "2020-01-01");
   nlohmann::json expected_output = {
-      {"Passed tests", 1},
-      {"Total tests", 2},
-      {"Failed tests", {"FALSE_TEST"}},
-      {"Reported problems", {"PROBLEM"}},
-      {"Reported observations", {"OBSERVATION"}},
-      {"Counter", "All counters were constant zero."},
+      {"passed_test_count", 1},
+      {"total_test_count", 2},
+      {"failed_tests", {"FALSE_TEST"}},
+      {"problems", {"PROBLEM"}},
+      {"observations", {"OBSERVATION"}},
+      {"counter", "All counters were constant zero."},
+      {"date", "2020-01-01"},
+      {"commit", "c0"},
   };
   EXPECT_EQ(output, expected_output);
 }

src/fido2_conformance_main.cc

@@ -40,13 +40,20 @@ DEFINE_int32(num_credentials, 50,
 int main(int argc, char** argv) {
   google::InitGoogleLogging(argv[0]);
   gflags::ParseCommandLineFlags(&argc, &argv, true);
+
   if (FLAGS_token_path.empty()) {
     std::cout << "Please add the --token_path flag for one of these devices:"
               << std::endl;
     fido2_tests::hid::PrintFidoDevices();
     exit(0);
   }
 
+  if (FLAGS_token_path == "_") {
+    // This magic value is used by the run script for comfort.
+    FLAGS_token_path = fido2_tests::hid::FindFirstFidoDevicePath();
+    std::cout << "Testing device at path: " << FLAGS_token_path << std::endl;
+  }
+
   fido2_tests::DeviceTracker tracker;
   std::unique_ptr<fido2_tests::DeviceInterface> device =
       absl::make_unique<fido2_tests::hid::HidDevice>(&tracker, FLAGS_token_path,

src/hid/hid_device.cc

@@ -525,5 +525,18 @@ void PrintFidoDevices() {
   hid_free_enumeration(devs);
 }
 
+std::string FindFirstFidoDevicePath() {
+  hid_device_info* devs = hid_enumerate(0, 0);  // 0 means all devices.
+  std::string device_path;
+  for (hid_device_info* cur_dev = devs; cur_dev; cur_dev = cur_dev->next) {
+    if (cur_dev->usage_page == 0xf1d0 /* FIDO specific usage page*/) {
+      device_path = cur_dev->path;
+      break;
+    }
+  }
+  hid_free_enumeration(devs);
+  return device_path;
+}
+
 }  // namespace hid
 }  // namespace fido2_tests

src/hid/hid_device.h

@@ -61,6 +61,9 @@ struct __attribute__((__packed__)) Frame {
 // FIDO HID usage page (i.e. 0xf1d0) and prints their details on stdout.
 void PrintFidoDevices();
 
+// Utility function that returns the first suitable device path found.
+std::string FindFirstFidoDevicePath();
+
 class HidDevice : public DeviceInterface {
  public:
   // The constructor without the third parameter implicitly assumes false.

src/stamp.cc

@@ -0,0 +1,17 @@
+// Copyright 2019 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+extern const char build_scm_revision[];
+
+const char build_scm_revision[] = BUILD_SCM_REVISION;

tools/workspace_status.bat

@@ -0,0 +1,6 @@
+:; ./tools/workspace_status.sh
+:; exit
+@ECHO OFF
+cd tools
+workspace_status.cmd
+

tools/workspace_status.cmd

@@ -0,0 +1,16 @@
+:; Copyright 2020 Google LLC
+:;
+:; Licensed under the Apache License, Version 2.0 (the "License");
+:; you may not use this file except in compliance with the License.
+:; You may obtain a copy of the License at
+:;
+:;      http://www.apache.org/licenses/LICENSE-2.0
+:;
+:; Unless required by applicable law or agreed to in writing, software
+:; distributed under the License is distributed on an "AS IS" BASIS,
+:; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+:; See the License for the specific language governing permissions and
+:; limitations under the License.
+
+for /f "delims=" %%i in ('git rev-parse HEAD') do set BUILD_SCM_REVISION==%%i
+

tools/workspace_status.sh

@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+# Copyright 2020 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+echo BUILD_SCM_REVISION  $(git rev-parse HEAD)
+