From d7557d2fc292b8ed4a37e3e7d3adaefa96058c2c Mon Sep 17 00:00:00 2001
From: Penelope Yong <penelopeysm@gmail.com>
Date: Mon, 9 Sep 2024 15:27:20 +0100
Subject: [PATCH] Reorganise failing tests

 - Disabled reference tests 25 and 30, which require StatsPlots support,
   for all backends (in PlotsBase/src/examples.jl)
 - Separate tests that are skipped and tests that are known to be broken
   due to upstream issues
 - Re-enable reference test 41 as upstream issue has been fixed
   (https://github.com/JuliaLang/julia/issues/47261)
 - Disable reference test 50 because of upstream issue
   (https://github.com/jheinen/GR.jl/issues/550)
---
 PlotsBase/src/examples.jl        |  8 +++++---
 PlotsBase/test/runtests.jl       | 10 +++++-----
 PlotsBase/test/test_backends.jl  |  2 +-
 PlotsBase/test/test_reference.jl | 10 ++++++++--
 4 files changed, 19 insertions(+), 11 deletions(-)

diff --git a/PlotsBase/src/examples.jl b/PlotsBase/src/examples.jl
index 86c02710e..d11b4a342 100644
--- a/PlotsBase/src/examples.jl
+++ b/PlotsBase/src/examples.jl
@@ -1258,12 +1258,10 @@ _animation_examples = [02, 31]
 _backend_skips = Dict(
     :none => Int[],
     :pythonplot => Int[],
-    :gr => [25, 30],  # TODO: add back when StatsPlots is available
+    :gr => Int[],
     :plotlyjs => [
         21,
         24,
-        25,
-        30,
         49,
         50,
         51,
@@ -1314,6 +1312,10 @@ _backend_skips = Dict(
     ],
 )
 _backend_skips[:plotly] = _backend_skips[:plotlyjs]
+# 25 and 30 require StatsPlots, which doesn't support Plots v2 yet
+for backend in keys(_backend_skips)
+    append!(_backend_skips[backend], [25, 30])
+end
 
 # ---------------------------------------------------------------------------------
 # replace `f(args...)` with `f(rng, args...)` for `f ∈ (rand, randn)`
diff --git a/PlotsBase/test/runtests.jl b/PlotsBase/test/runtests.jl
index 6b5fa1b78..9552a16f7 100644
--- a/PlotsBase/test/runtests.jl
+++ b/PlotsBase/test/runtests.jl
@@ -54,17 +54,17 @@ is_ci() || @eval using Gtk  # see JuliaPlots/VisualRegressionTests.jl/issues/30
 
 ref_name(i) = "ref" * lpad(i, 3, '0')
 
-const blacklist = if VERSION.major == 1 && VERSION.minor ≥ 9
+const broken_examples = if VERSION.major == 1 && VERSION.minor ≥ 9
     [
-        25,
+        25, # FIXME: remove, when StatsPlots supports Plots v2
         30, # FIXME: remove, when StatsPlots supports Plots v2
-        41,
-    ]  # FIXME: github.com/JuliaLang/julia/issues/47261
+        41, # FIXME: https://github.com/JuliaLang/julia/issues/47261
+    ]
 else
     []
 end
 if Sys.isapple()
-    push!(blacklist, 50)  # FIXME: https://github.com/jheinen/GR.jl/issues/550
+    push!(broken_examples, 50)  # FIXME: https://github.com/jheinen/GR.jl/issues/550
 end
 
 for name ∈ (
diff --git a/PlotsBase/test/test_backends.jl b/PlotsBase/test/test_backends.jl
index 6c83ac87b..b30fc6b99 100644
--- a/PlotsBase/test/test_backends.jl
+++ b/PlotsBase/test/test_backends.jl
@@ -66,7 +66,7 @@ is_pkgeval() || @testset "Backends" begin
         @test filesize(fn) > 1_000
     end
     Sys.islinux() && for be ∈ TEST_BACKENDS
-        skip = vcat(PlotsBase._backend_skips[be], blacklist)
+        skip = vcat(PlotsBase._backend_skips[be], broken_examples)
         PlotsBase.test_examples(be; skip, callback, disp = is_ci(), strict = true)  # `ci` display for coverage
         closeall()
     end
diff --git a/PlotsBase/test/test_reference.jl b/PlotsBase/test/test_reference.jl
index a03389b30..4e3133af0 100644
--- a/PlotsBase/test/test_reference.jl
+++ b/PlotsBase/test/test_reference.jl
@@ -105,6 +105,7 @@ end
 function image_comparison_facts(
     pkg::Symbol;
     skip = [],          # skip these examples (int index)
+    broken = [],        # known broken examples (int index)
     only = nothing,     # limit to these examples (int index)
     debug = false,      # print debug information ?
     sigma = [1, 1],     # number of pixels to "blur"
@@ -112,7 +113,11 @@ function image_comparison_facts(
 )
     for i ∈ setdiff(1:length(PlotsBase._examples), skip)
         if only ≡ nothing || i in only
-            @test success(image_comparison_tests(pkg, i; debug, sigma, tol))
+            if i ∈ broken
+                @test_broken success(image_comparison_tests(pkg, i; debug, sigma, tol))
+            else
+                @test success(image_comparison_tests(pkg, i; debug, sigma, tol))
+            end
         end
     end
 end
@@ -141,7 +146,8 @@ end
         image_comparison_facts(
             :gr,
             tol = PLOTS_IMG_TOL,
-            skip = vcat(PlotsBase._backend_skips[:gr], blacklist),
+            skip = vcat(PlotsBase._backend_skips[:gr]),
+            broken = broken_examples,
         )
     end
 end