reftests: Allow minor differences to be tolerated

Based on an earlier patch by Michael Biebl, as used in Debian's GTK 3
packaging, with additional inspiration from librsvg's reftests.

Each .ui or .node reftest can have an accompanying .keyfile file
like this:

    [reftest]
    accepted-diff-level=2
    accepted-diff-pixels=100
    tolerated-diff-level=20
    tolerated-diff-pixels=1000

If the number of pixels that differ from the reference is no more than
accepted-diff-pixels, and each channel in each of those pixels differs
from the reference by no more than accepted-diff-level, then we consider
that to be a full success, and don't even save the .diff.png for
analysis.

If that check fails, but the number of pixels that differ is no more
than tolerated-diff-pixels and the differences are no more than
tolerated-diff-level, then we treat it as a success with warnings, save
the .diff.png for analysis, and use g_test_incomplete() to record the
test-case as "TODO".

Signed-off-by: Simon McVittie <smcv@debian.org>

testsuite/gsk/compare-render.c

@@ -2,7 +2,7 @@
 #include <glib/gstdio.h>
 #include <gtk/gtk.h>
 #include <stdlib.h>
-#include "reftest-compare.h"
+#include "../reftests/reftest-compare.h"
 
 static char *arg_output_dir = NULL;
 
@@ -98,6 +98,12 @@ get_output_file (const char *file,
   return result;
 }
 
+static char *
+get_test_keyfile (const char *node_file)
+{
+  return file_replace_extension (node_file, ".node", ".keyfile");
+}
+
 static void
 save_image (cairo_surface_t *surface,
             const char      *test_name,
@@ -232,14 +238,55 @@ main (int argc, char **argv)
     }
   else
     {
+      guint max_diff = 0;
+      guint pixels_changed = 0;
+      guint pixels = 0;
+
       /* Now compare the two */
-      diff_surface = reftest_compare_surfaces (rendered_surface, reference_surface);
+      diff_surface = reftest_compare_surfaces (rendered_surface, reference_surface,
+                                               &max_diff, &pixels_changed, &pixels);
 
       if (diff_surface)
         {
-          save_image (diff_surface, node_file, ".diff.png");
+          char *keyfile_path = get_test_keyfile (node_file);
+          GKeyFile *keyfile = g_key_file_new ();
+          guint64 tolerated_diff = 0;
+          guint64 tolerated_pixels = 0;
+          guint64 accepted_diff = 0;
+          guint64 accepted_pixels = 0;
+
+          if (keyfile_path != NULL && g_file_test (keyfile_path, G_FILE_TEST_EXISTS))
+            {
+              GError *error = NULL;
+              g_key_file_load_from_file (keyfile, keyfile_path, G_KEY_FILE_NONE, &error);
+              g_assert_no_error (error);
+              accepted_diff = g_key_file_get_uint64 (keyfile, "reftest", "accepted-diff-level", NULL);
+              g_print ("Maximum difference accepted: %" G_GUINT64_FORMAT " levels\n", accepted_diff);
+              accepted_pixels = g_key_file_get_uint64 (keyfile, "reftest", "accepted-diff-pixels", NULL);
+              g_print ("Different pixels accepted: %" G_GUINT64_FORMAT "\n", accepted_pixels);
+              tolerated_diff = g_key_file_get_uint64 (keyfile, "reftest", "tolerated-diff-level", NULL);
+              g_print ("Maximum difference tolerated: %" G_GUINT64_FORMAT " levels\n", tolerated_diff);
+              tolerated_pixels = g_key_file_get_uint64 (keyfile, "reftest", "tolerated-diff-pixels", NULL);
+              g_print ("Different pixels tolerated: %" G_GUINT64_FORMAT "\n", tolerated_pixels);
+            }
+
+          g_print ("%u (out of %u) pixels differ from reference by up to %u levels\n",
+                   pixels_changed, pixels, max_diff);
+
+          if (max_diff > accepted_diff || pixels_changed > accepted_pixels)
+            save_image (diff_surface, node_file, ".diff.png");
+
           cairo_surface_destroy (diff_surface);
-          success = FALSE;
+
+          if (max_diff <= accepted_diff && pixels_changed <= accepted_pixels)
+            g_print ("differences are within acceptable range\n");
+          else if (max_diff <= tolerated_diff && pixels_changed <= tolerated_pixels)
+            g_print ("not right, but close enough\n");
+          else
+            g_test_fail ();
+
+          g_key_file_unref (keyfile);
+          g_free (keyfile_path);
         }
     }
 

testsuite/gsk/meson.build

@@ -3,7 +3,7 @@ testdatadir = join_paths(installed_test_datadir, 'gsk')
 
 compare_render = executable(
   'compare-render',
-  ['compare-render.c', 'reftest-compare.c'],
+  ['compare-render.c', '../reftests/reftest-compare.c'],
   dependencies: libgtk_dep,
   c_args: common_cflags,
   install: get_option('install-tests'),

testsuite/gsk/reftest-compare.c

@@ -1,173 +0,0 @@
-/*
- * Copyright (C) 2011 Red Hat Inc.
- *
- * Author:
- *      Benjamin Otte <otte@gnome.org>
- *
- * This library is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Library General Public
- * License as published by the Free Software Foundation; either
- * version 2 of the License, or (at your option) any later version.
- *
- * This library is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * Library General Public License for more details.
- *
- * You should have received a copy of the GNU Library General Public
- * License along with this library. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#include "config.h"
-
-#include "reftest-compare.h"
-
-static void
-get_surface_size (cairo_surface_t *surface,
-                  int             *width,
-                  int             *height)
-{
-  cairo_t *cr;
-  double x1, x2, y1, y2;
-
-  cr = cairo_create (surface);
-  cairo_clip_extents (cr, &x1, &y1, &x2, &y2);
-  cairo_destroy (cr);
-
-  g_assert (x1 == 0 && y1 == 0);
-  g_assert (x2 > 0 && y2 > 0);
-  g_assert ((int) x2 == x2 && (int) y2 == y2);
-
-  *width = x2;
-  *height = y2;
-}
-
-static cairo_surface_t *
-coerce_surface_for_comparison (cairo_surface_t *surface,
-                               int              width,
-                               int              height)
-{
-  cairo_surface_t *coerced;
-  cairo_t *cr;
-
-  coerced = cairo_image_surface_create (CAIRO_FORMAT_ARGB32,
-                                        width,
-                                        height);
-  cr = cairo_create (coerced);
-  
-  cairo_set_source_surface (cr, surface, 0, 0);
-  cairo_set_operator (cr, CAIRO_OPERATOR_SOURCE);
-  cairo_paint (cr);
-
-  cairo_destroy (cr);
-
-  g_assert (cairo_surface_status (coerced) == CAIRO_STATUS_SUCCESS);
-
-  return coerced;
-}
-
-/* Compares two CAIRO_FORMAT_ARGB32 buffers, returning NULL if the
- * buffers are equal or a surface containing a diff between the two
- * surfaces.
- *
- * This function should be rewritten to compare all formats supported by
- * cairo_format_t instead of taking a mask as a parameter.
- *
- * This function is originally from cairo:test/buffer-diff.c.
- * Copyright © 2004 Richard D. Worth
- */
-static cairo_surface_t *
-buffer_diff_core (const guchar *buf_a,
-                  int           stride_a,
-        	  const guchar *buf_b,
-                  int           stride_b,
-        	  int		width,
-        	  int		height)
-{
-  int x, y;
-  guchar *buf_diff = NULL;
-  int stride_diff = 0;
-  cairo_surface_t *diff = NULL;
-
-  for (y = 0; y < height; y++)
-    {
-      const guint32 *row_a = (const guint32 *) (buf_a + y * stride_a);
-      const guint32 *row_b = (const guint32 *) (buf_b + y * stride_b);
-      guint32 *row = (guint32 *) (buf_diff + y * stride_diff);
-
-      for (x = 0; x < width; x++)
-        {
-          int channel;
-          guint32 diff_pixel = 0;
-
-          /* check if the pixels are the same */
-          if (row_a[x] == row_b[x])
-            continue;
-        
-          if (diff == NULL)
-            {
-              diff = cairo_image_surface_create (CAIRO_FORMAT_RGB24,
-                                                 width,
-                                                 height);
-              g_assert (cairo_surface_status (diff) == CAIRO_STATUS_SUCCESS);
-              buf_diff = cairo_image_surface_get_data (diff);
-              stride_diff = cairo_image_surface_get_stride (diff);
-              row = (guint32 *) (buf_diff + y * stride_diff);
-            }
-
-          /* calculate a difference value for all 4 channels */
-          for (channel = 0; channel < 4; channel++)
-            {
-              int value_a = (row_a[x] >> (channel*8)) & 0xff;
-              int value_b = (row_b[x] >> (channel*8)) & 0xff;
-              guint channel_diff;
-
-              channel_diff = ABS (value_a - value_b);
-              channel_diff *= 4;  /* emphasize */
-              if (channel_diff)
-                channel_diff += 128; /* make sure it's visible */
-              if (channel_diff > 255)
-                channel_diff = 255;
-              diff_pixel |= channel_diff << (channel*8);
-            }
-
-          if ((diff_pixel & 0x00ffffff) == 0)
-            {
-              /* alpha only difference, convert to luminance */
-              guint8 alpha = diff_pixel >> 24;
-              diff_pixel = alpha * 0x010101;
-            }
-          
-          row[x] = diff_pixel;
-      }
-  }
-
-  return diff;
-}
-
-cairo_surface_t *
-reftest_compare_surfaces (cairo_surface_t *surface1,
-                          cairo_surface_t *surface2)
-{
-  int w1, h1, w2, h2, w, h;
-  cairo_surface_t *coerced1, *coerced2, *diff;
-  
-  get_surface_size (surface1, &w1, &h1);
-  get_surface_size (surface2, &w2, &h2);
-  w = MAX (w1, w2);
-  h = MAX (h1, h2);
-  coerced1 = coerce_surface_for_comparison (surface1, w, h);
-  coerced2 = coerce_surface_for_comparison (surface2, w, h);
-
-  diff = buffer_diff_core (cairo_image_surface_get_data (coerced1),
-                           cairo_image_surface_get_stride (coerced1),
-                           cairo_image_surface_get_data (coerced2),
-                           cairo_image_surface_get_stride (coerced2),
-                           w, h);
-
-  cairo_surface_destroy (coerced1);
-  cairo_surface_destroy (coerced2);
-
-  return diff;
-}
-

testsuite/gsk/reftest-compare.h

@@ -1,30 +0,0 @@
-/* GTK - The GIMP Toolkit
- * Copyright (C) 2014 Red Hat, Inc.
- *
- * This library is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 2 of the License, or (at your option) any later version.
- *
- * This library is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public
- * License along with this library. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#ifndef __REFTEST_COMPARE_H__
-#define __REFTEST_COMPARE_H__
-
-#include <gtk/gtk.h>
-
-G_BEGIN_DECLS
-
-cairo_surface_t *       reftest_compare_surfaces        (cairo_surface_t        *surface1,
-                                                         cairo_surface_t        *surface2);
-
-G_END_DECLS
-
-#endif /* __REFTEST_COMPARE_H__ */

testsuite/reftests/gtk-reftest.c

@@ -290,12 +290,21 @@ save_image (cairo_surface_t *surface,
   g_free (filename);
 }
 
+static char *
+get_test_keyfile (const char *ui_file)
+{
+  return get_test_file (ui_file, ".keyfile", TRUE);
+}
+
 static void
 test_ui_file (GFile *file)
 {
   char *ui_file, *reference_file;
   cairo_surface_t *ui_image, *reference_image, *diff_image;
   GtkStyleProvider *provider;
+  guint max_diff = 0;
+  guint pixels_changed = 0;
+  guint pixels = 0;
 
   ui_file = g_file_get_path (file);
 
@@ -315,14 +324,51 @@ test_ui_file (GFile *file)
     }
   g_free (reference_file);
 
-  diff_image = reftest_compare_surfaces (ui_image, reference_image);
+  diff_image = reftest_compare_surfaces (ui_image, reference_image,
+                                         &max_diff, &pixels_changed, &pixels);
 
   save_image (ui_image, ui_file, ".out.png");
   save_image (reference_image, ui_file, ".ref.png");
+
   if (diff_image)
     {
-      save_image (diff_image, ui_file, ".diff.png");
-      g_test_fail ();
+      char *keyfile_path = get_test_keyfile (ui_file);
+      GKeyFile *keyfile = g_key_file_new ();
+      guint64 tolerated_diff = 0;
+      guint64 tolerated_pixels = 0;
+      guint64 accepted_diff = 0;
+      guint64 accepted_pixels = 0;
+
+      if (keyfile_path != NULL)
+        {
+          GError *error = NULL;
+          g_key_file_load_from_file (keyfile, keyfile_path, G_KEY_FILE_NONE, &error);
+          g_assert_no_error (error);
+          accepted_diff = g_key_file_get_uint64 (keyfile, "reftest", "accepted-diff-level", NULL);
+          g_test_message ("Maximum difference accepted: %" G_GUINT64_FORMAT " levels", accepted_diff);
+          accepted_pixels = g_key_file_get_uint64 (keyfile, "reftest", "accepted-diff-pixels", NULL);
+          g_test_message ("Different pixels accepted: %" G_GUINT64_FORMAT, accepted_pixels);
+          tolerated_diff = g_key_file_get_uint64 (keyfile, "reftest", "tolerated-diff-level", NULL);
+          g_test_message ("Maximum difference tolerated: %" G_GUINT64_FORMAT " levels", tolerated_diff);
+          tolerated_pixels = g_key_file_get_uint64 (keyfile, "reftest", "tolerated-diff-pixels", NULL);
+          g_test_message ("Different pixels tolerated: %" G_GUINT64_FORMAT, tolerated_pixels);
+        }
+
+      g_test_message ("%u (out of %u) pixels differ from reference by up to %u levels",
+                      pixels_changed, pixels, max_diff);
+
+      if (max_diff > accepted_diff || pixels_changed > accepted_pixels)
+        save_image (diff_image, ui_file, ".diff.png");
+
+      if (max_diff <= accepted_diff && pixels_changed <= accepted_pixels)
+        g_test_message ("differences are within acceptable range");
+      else if (max_diff <= tolerated_diff && pixels_changed <= tolerated_pixels)
+        g_test_incomplete ("not right, but close enough");
+      else
+        g_test_fail ();
+
+      g_key_file_unref (keyfile);
+      g_free (keyfile_path);
     }
 
   remove_extra_css (provider);

testsuite/reftests/image-compare.c

@@ -41,7 +41,7 @@ main (int argc, char **argv)
   image1 = cairo_image_surface_create_from_png (argv[1]);
   image2 = cairo_image_surface_create_from_png (argv[2]);
 
-  diff = reftest_compare_surfaces (image1, image2);
+  diff = reftest_compare_surfaces (image1, image2, NULL, NULL, NULL);
 
   if (opt_filename && diff)
     cairo_surface_write_to_png (diff, opt_filename);

testsuite/reftests/reftest-compare.c

@@ -83,12 +83,16 @@ buffer_diff_core (const guchar *buf_a,
         	  const guchar *buf_b,
                   int           stride_b,
         	  int		width,
-        	  int		height)
+        	  int		height,
+                  guint        *max_diff_out,
+                  guint        *pixels_changed_out)
 {
   int x, y;
   guchar *buf_diff = NULL;
   int stride_diff = 0;
   cairo_surface_t *diff = NULL;
+  guint max_diff = 0;
+  guint pixels_changed = 0;
 
   for (y = 0; y < height; y++)
     {
@@ -121,17 +125,23 @@ buffer_diff_core (const guchar *buf_a,
             {
               int value_a = (row_a[x] >> (channel*8)) & 0xff;
               int value_b = (row_b[x] >> (channel*8)) & 0xff;
-              guint channel_difff;
+              guint channel_diff;
 
-              channel_difff = ABS (value_a - value_b);
-              channel_difff *= 4;  /* emphasize */
-              if (channel_difff)
-                channel_difff += 128; /* make sure it's visible */
-              if (channel_difff > 255)
-                channel_difff = 255;
-              diff_pixel |= channel_difff << (channel * 8);
+              channel_diff = ABS (value_a - value_b);
+
+              if (channel_diff > max_diff)
+                max_diff = channel_diff;
+
+              channel_diff *= 4;  /* emphasize */
+              if (channel_diff)
+                channel_diff += 128; /* make sure it's visible */
+              if (channel_diff > 255)
+                channel_diff = 255;
+              diff_pixel |= channel_diff << (channel * 8);
             }
 
+          pixels_changed++;
+
           if ((diff_pixel & 0x00ffffff) == 0)
             {
               /* alpha only difference, convert to luminance */
@@ -143,12 +153,21 @@ buffer_diff_core (const guchar *buf_a,
       }
   }
 
+  if (max_diff_out != NULL)
+    *max_diff_out = max_diff;
+
+  if (pixels_changed_out != NULL)
+    *pixels_changed_out = pixels_changed;
+
   return diff;
 }
 
 cairo_surface_t *
 reftest_compare_surfaces (cairo_surface_t *surface1,
-                          cairo_surface_t *surface2)
+                          cairo_surface_t *surface2,
+                          guint           *max_diff_out,
+                          guint           *pixels_changed_out,
+                          guint           *pixels_out)
 {
   int w1, h1, w2, h2, w, h;
   cairo_surface_t *coerced1, *coerced2, *diff;
@@ -164,11 +183,14 @@ reftest_compare_surfaces (cairo_surface_t *surface1,
                            cairo_image_surface_get_stride (coerced1),
                            cairo_image_surface_get_data (coerced2),
                            cairo_image_surface_get_stride (coerced2),
-                           w, h);
+                           w, h, max_diff_out, pixels_changed_out);
 
   cairo_surface_destroy (coerced1);
   cairo_surface_destroy (coerced2);
 
+  if (pixels_out != NULL)
+    *pixels_out = w * h;
+
   return diff;
 }
 

testsuite/reftests/reftest-compare.h

@@ -24,7 +24,10 @@ G_BEGIN_DECLS
 
 G_MODULE_EXPORT
 cairo_surface_t *       reftest_compare_surfaces        (cairo_surface_t        *surface1,
-                                                         cairo_surface_t        *surface2);
+                                                         cairo_surface_t        *surface2,
+                                                         guint                  *max_diff_out,
+                                                         guint                  *pixels_changed_out,
+                                                         guint                  *pixels_out);
 
 G_END_DECLS