From 8d81328768ce597ed837c1aeab0b3fd1f068af76 Mon Sep 17 00:00:00 2001 From: Frames White Date: Fri, 17 May 2024 23:47:18 +0800 Subject: [PATCH 1/4] Do not automatically create if missing --- README.md | 15 ++++------ src/test_reference.jl | 67 ++++++++++++++++++++++++------------------- test/runtests.jl | 12 +++++--- 3 files changed, 52 insertions(+), 42 deletions(-) diff --git a/README.md b/README.md index 30d79cc..00586f9 100644 --- a/README.md +++ b/README.md @@ -37,10 +37,9 @@ using ReferenceTests If you put the above code into your `test/runtests.jl` and execute the file in an interactive julia session (i.e. with -`include`), then it will trigger an interactive dialog if the -results don't match. This dialog allows the user to update the -reference files. If you do not want to be prompted, just -delete the reference data before running the tests. +`include`), then it will trigger an interactive dialog if the refrence is missing or the results don't match. +This dialog allows the user to create/update the reference files. +If you do not want to be prompted, set `ENV["JULIA_REFERENCETESTS_UPDATE"]=true` before running. ![readme1](https://user-images.githubusercontent.com/10854026/30002940-3ba480b0-90b6-11e7-93f6-148ac38bd695.png) @@ -91,13 +90,11 @@ test suite. These tests are easy to run via `pkg> test` but the child process used within `pkg> test` is non-interactive, so the update prompt will not show if there are mismatches. -To update references within a package test suite, there are three options: +To update/create references within a package test suite, there are two options: - Set the environment variable `JULIA_REFERENCETESTS_UPDATE` to `"true"` - and run `pkg> test`, which will force update any non-matches. You can then - check changes to any git-tracked reference images before commit. -- Delete any reference images you wish to update and run `pkg> test`, given - that missing references are created automatically. + and run `pkg> test`, which will force update any non-matches and create any missing files. + You can then check changes to any git-tracked reference images before commit. - Run the `test/runtests.jl` interactively. This may be easier using the [`TestEnv.jl`](https://github.com/JuliaTesting/TestEnv.jl) package, given that the test environment used by `pkg> test` is a merge of the diff --git a/src/test_reference.jl b/src/test_reference.jl index 98af86f..9e340a2 100644 --- a/src/test_reference.jl +++ b/src/test_reference.jl @@ -117,9 +117,7 @@ function test_reference( rendermode=nothing; kw...) where {F <: DataFormat, T} - reference_path = reference_file.filename - reference_dir, reference_filename = splitdir(reference_path) - + reference_filename = basename(reference_file.filename) actual = _convert(F, raw_actual; kw...) # infer the default rendermode here @@ -129,14 +127,14 @@ function test_reference( rendermode = default_rendermode(F, actual) end - if !isfile(reference_path) # when reference file doesn't exists - mkpath(reference_dir) - savefile(reference_file, actual) - @info """Reference file for \"$reference_filename\" did not exist. It has been created: + if !isfile(reference_file.filename) # when reference file doesn't exists + actual_path = save_mismatch_file(reference_file, actual) + + @info """Reference file for \"$reference_filename\" did not exist: $(render(rendermode, actual)) - """ new_reference = reference_path + """ actual=actual_path - @info "Please run the tests again for any changes to take effect" + handle_mismatch(reference_file, actual_path) return nothing # skip current test case end @@ -152,33 +150,44 @@ function test_reference( @test true # to increase test counter if reached else # When test fails # Saving actual file so user can look at it - actual_path = joinpath(mismatch_staging_dir(), reference_filename) - actual_file = typeof(reference_file)(actual_path) - savefile(actual_file, actual) + actual_path = save_mismatch_file(reference_file, actual) # Report to user. @info """Reference Test for \"$reference_filename\" failed: $(render(rendermode, reference, actual)) - """ reference = reference_path actual = actual_path - - if !isinteractive() && !force_update() - error(""" - To update the reference images either run the tests interactively with 'include(\"test/runtests.jl\")', - or to force-update all failing reference images set the environment variable `JULIA_REFERENCETESTS_UPDATE` - to "true" and re-run the tests via Pkg. - """) - end - - if force_update() || input_bool("Replace reference with actual result?") - mv(actual_path, reference_path; force=true) # overwrite old file it - @info "Please run the tests again for any changes to take effect" - else - @test false - end + """ reference=reference_file.filename actual=actual_path + handle_mismatch(reference_file, actual_path) + end +end + +function save_mismatch_file(reference_file::File{F}, actual) where F + # Saving actual file so user can look at it + actual_path = joinpath(mismatch_staging_dir(), basename(reference_file.filename)) + actual_file = File{F}(actual_path) + + savefile(actual_file, actual) + return actual_path +end + +function handle_mismatch(reference_file, actual_path) + if !isinteractive() && !force_update() + error(""" + To update the reference images either run the tests interactively with 'include(\"test/runtests.jl\")', + or to force-update/create all failing reference images set the environment variable `JULIA_REFERENCETESTS_UPDATE` + to "true" and re-run all tests. + """) + end + + if force_update() || input_bool("Replace reference with actual result?") + mkpath(dirname(reference_file.filename)) + mv(actual_path, reference_file.filename; force=true) # overwrite old file + @info "Please run the tests again for any changes to take effect" + else + @test false end end -force_update() = tryparse(Bool, get(ENV, "JULIA_REFERENCETESTS_UPDATE", "false")) === true +force_update() = tryparse(Bool, get(ENV, "JULIA_REFERENCETESTS_UPDATE", "false")) """ mismatch_staging_dir() diff --git a/test/runtests.jl b/test/runtests.jl index e30914a..8691307 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -162,13 +162,15 @@ end mktempdir() do path newfilename = joinpath(path, "newfilename.$ext") @assert !isfile(newfilename) - with_logger(test_logger) do - @test_reference newfilename val # this should create it + withenv("JULIA_REFERENCETESTS_UPDATE"=>true) do + with_logger(test_logger) do + @test_reference newfilename val # this should create it + end end @test isfile(newfilename) # Was created @test_reference newfilename val # Matches expected content end -end + end @testset "Create new image as txt" begin # This is a separate testset as need to use the `size` argument to ``@test_reference` @@ -176,7 +178,9 @@ end newfilename = joinpath(path, "new_camera.txt") @assert !isfile(newfilename) with_logger(test_logger) do - @test_reference newfilename camera size=(5,10) # this should create it + withenv("JULIA_REFERENCETESTS_UPDATE"=>true) do + @test_reference newfilename camera size=(5,10) # this should create it + end end @test isfile(newfilename) # Was created @test_reference newfilename camera size=(5,10) # Matches expected content From a660216b0ef49f807f23ea76f298ce19af1645d1 Mon Sep 17 00:00:00 2001 From: Frames White Date: Fri, 17 May 2024 23:47:59 +0800 Subject: [PATCH 2/4] Bump to release as 1.0 --- Project.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Project.toml b/Project.toml index 7884ebd..520431c 100644 --- a/Project.toml +++ b/Project.toml @@ -1,7 +1,7 @@ name = "ReferenceTests" uuid = "324d217c-45ce-50fc-942e-d289b448e8cf" authors = ["Christof Stocker ", "Frames White ", "Johnny Chen "] -version = "0.10.4" +version = "1.0.0" [deps] Colors = "5ae59095-9a9b-59fe-a467-6f913c188581" From 2c296d545d682aeae05726139667e1d00aa43949 Mon Sep 17 00:00:00 2001 From: Frames White Date: Sat, 18 May 2024 00:14:18 +0800 Subject: [PATCH 3/4] Update CI script to create missing files by setting env-var --- .github/workflows/UnitTest.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/UnitTest.yml b/.github/workflows/UnitTest.yml index 456d3d1..5ed9a9a 100644 --- a/.github/workflows/UnitTest.yml +++ b/.github/workflows/UnitTest.yml @@ -42,7 +42,9 @@ jobs: run: | julia --color=yes --check-bounds=yes --project -e "using Pkg; Pkg.test(coverage=true)" rm -rf test/references + export JULIA_REFERENCETESTS_UPDATE=true julia --color=yes --check-bounds=yes --project -e "using Pkg; Pkg.test(coverage=true)" + export JULIA_REFERENCETESTS_UPDATE=false julia --color=yes --check-bounds=yes --project -e "using Pkg; Pkg.test(coverage=true)" - name: Upload Mismatched Files as a Build Artifact uses: actions/upload-artifact@v3 From 2a0a4be41622fc27df5cb74b7583068909a56514 Mon Sep 17 00:00:00 2001 From: Frames White Date: Sat, 18 May 2024 00:25:35 +0800 Subject: [PATCH 4/4] Do not delete and regenerate during CI --- .github/workflows/UnitTest.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.github/workflows/UnitTest.yml b/.github/workflows/UnitTest.yml index 5ed9a9a..efb640c 100644 --- a/.github/workflows/UnitTest.yml +++ b/.github/workflows/UnitTest.yml @@ -41,11 +41,6 @@ jobs: shell: bash run: | julia --color=yes --check-bounds=yes --project -e "using Pkg; Pkg.test(coverage=true)" - rm -rf test/references - export JULIA_REFERENCETESTS_UPDATE=true - julia --color=yes --check-bounds=yes --project -e "using Pkg; Pkg.test(coverage=true)" - export JULIA_REFERENCETESTS_UPDATE=false - julia --color=yes --check-bounds=yes --project -e "using Pkg; Pkg.test(coverage=true)" - name: Upload Mismatched Files as a Build Artifact uses: actions/upload-artifact@v3 with: