Tests: speed up render tests by running multiple in the same process

Blender startup time and shader compilation is a big factor when running
hundreds of tests, so now all renders in the same ctest run in the same
process. If a test crashes, the remaining tests in the same category will
be marked as skipped.

Benchmarked on a quad core with ctest -j8.

cycles: 118.1s -> 94.3s
eevee: 66.2s -> 29.2s
workbench: 31.7s -> 8.6s

tests/python/eevee_render_tests.py

@@ -49,57 +49,61 @@ if inside_blender:
         sys.exit(1)
 
 
-def render_file(filepath, output_filepath):
-    dirname = os.path.dirname(filepath)
-    basedir = os.path.dirname(dirname)
-    subject = os.path.basename(dirname)
-
-    frame_filepath = output_filepath + '0001.png'
-
+def render_files(filepaths, output_filepaths):
     command = [
         BLENDER,
         "--background",
         "-noaudio",
         "--factory-startup",
-        "--enable-autoexec",
-        filepath,
-        "-E", "BLENDER_EEVEE",
-        "-P",
-        os.path.realpath(__file__),
-        "-o", output_filepath,
-        "-F", "PNG",
-        "-f", "1"]
+        "--enable-autoexec"]
 
+    for filepath, output_filepath in zip(filepaths, output_filepaths):
+        frame_filepath = output_filepath + '0001.png'
+        if os.path.exists(frame_filepath):
+            os.remove(frame_filepath)
+
+        command.extend([
+            filepath,
+            "-E", "BLENDER_EEVEE",
+            "-P",
+            os.path.realpath(__file__),
+            "-o", output_filepath,
+            "-F", "PNG",
+            "-f", "1"])
+
+    error = None
     try:
         # Success
         output = subprocess.check_output(command)
-        if os.path.exists(frame_filepath):
-            shutil.copy(frame_filepath, output_filepath)
-            os.remove(frame_filepath)
         if VERBOSE:
             print(" ".join(command))
             print(output.decode("utf-8"))
-        return None
     except subprocess.CalledProcessError as e:
         # Error
-        if os.path.exists(frame_filepath):
-            os.remove(frame_filepath)
         if VERBOSE:
             print(" ".join(command))
             print(e.output.decode("utf-8"))
-        if b"Error: engine not found" in e.output:
-            return "NO_ENGINE"
-        elif b"blender probably wont start" in e.output:
-            return "NO_START"
-        return "CRASH"
+        error = "CRASH"
     except BaseException as e:
         # Crash
-        if os.path.exists(frame_filepath):
-            os.remove(frame_filepath)
         if VERBOSE:
             print(" ".join(command))
-            print(e)
-        return "CRASH"
+            print(e.decode("utf-8"))
+        error = "CRASH"
+
+    # Detect missing filepaths and consider those errors
+    errors = []
+    for output_filepath in output_filepaths:
+        frame_filepath = output_filepath + '0001.png'
+        if os.path.exists(frame_filepath):
+            shutil.copy(frame_filepath, output_filepath)
+            os.remove(frame_filepath)
+            errors.append(None)
+        else:
+            errors.append(error)
+            error = 'SKIPPED'
+
+    return errors
 
 
 def create_argparse():
@@ -129,7 +133,7 @@ def main():
     report.set_pixelated(True)
     report.set_reference_dir("eevee_renders")
     report.set_compare_engines('eevee', 'cycles')
-    ok = report.run(test_dir, render_file)
+    ok = report.run(test_dir, render_files)
 
     sys.exit(not ok)