Add --expected-failures option

Also add corresponding input to the action

Signed-off-by: Jussi Kukkonen <[email protected]>

action.yml

@@ -3,8 +3,12 @@ description: "TUF client conformance test suite"
 
 inputs:
   entrypoint:
-    description: "client wrapper to invoke"
+    description: "client-under-test CLI to invoke"
     required: true
+  expected-failures:
+    description: "Optional list test names expected to fail"
+    default: ""
+    required: false
 
 runs:
   using: "composite"
@@ -26,7 +30,10 @@ runs:
       id: sigstore-conformance
       env:
         ENTRYPOINT: ${{ inputs.entrypoint }}
+        EXPECTED_FAILURES: ${{ inputs.expected-failures }}
         TEST_LOCATION: ${{ github.action_path }}/tuf_conformance
       run: |
-        pytest -v "$TEST_LOCATION" --entrypoint "$ENTRYPOINT"
+        pytest -v "$TEST_LOCATION" \
+          --entrypoint "$ENTRYPOINT" \
+          --expected-failures "$EXPECTED_FAILURES"
       shell: bash

tuf_conformance/conftest.py

@@ -13,6 +13,14 @@ def pytest_addoption(parser) -> None:
         required=True,
         type=str,
     )
+    parser.addoption(
+        "--expected-failures",
+        action="store",
+        help="Optional space delimited list of test names expected to fail",
+        required=False,
+        type=str,
+    )
+
 
 @pytest.fixture
 def server():
@@ -34,4 +42,12 @@ def client(pytestconfig, server):
 
     return ClientRunner(entrypoint, server)
 
-
+
+@pytest.fixture(autouse=True)
+def conformance_xfail(pytestconfig, request):
+    xfail_option = pytestconfig.getoption("--expected-failures")
+    if xfail_option is None:
+        return
+
+    if request.node.originalname in xfail_option.split(" "):
+        request.node.add_marker(pytest.mark.xfail(strict=True))