123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128 |
- import argparse, json, os, os.path, sys
- def splitall(path):
- allparts = []
- while 1:
- parts = os.path.split(path)
- if parts[0] == path: # sentinel for absolute paths
- allparts.insert(0, parts[0])
- break
- elif parts[1] == path: # sentinel for relative paths
- allparts.insert(0, parts[1])
- break
- else:
- path = parts[0]
- allparts.insert(0, parts[1])
- return allparts
- def find_subdirectory(subdir, node):
- contents = node["contents"]
- for item in contents:
- if item["type"] == "directory" and item["name"] == subdir:
- return item
- raise RuntimeError("Missing subdir '%s'" % subdir)
- def find_directory(subdir, node):
- if subdir == '.' or subdir == None:
- return node
- for part in splitall(subdir):
- node = find_subdirectory(part, node)
- return node
- def get_baseline(tname, path, image, root):
- import re
- filename = os.path.join(path, image)
- relpath = os.path.relpath(filename, root)
- return tname + "_" + re.sub('[^0-9a-zA-Z.]+', '_', relpath)
- parser = argparse.ArgumentParser(\
- description="Validate Extrats generated in tests")
- parser.add_argument("--name", type=str, required=True,
- help="name of the test being validated")
- parser.add_argument("--subdir", type=str, default=None,
- help="sub directory to validate")
- parser.add_argument("--root", type=str, required=True,
- help ="root directory with results")
- parser.add_argument("--json", type=str, required=True,
- help="validation json file")
- parser.add_argument("--baseline-dir", type=str, default=None,
- help="directory for baselines for regression tests, if any")
- parser.add_argument("--temp-dir", type=str, default="/tmp",
- help="directory for results for regression test failures, if any")
- args = parser.parse_args()
- with open(args.json, 'r') as f:
- data = json.load(f)[0]
- node = data
- test_dir = args.root
- if args.subdir is not None:
- node = find_directory(args.subdir, node)
- test_dir = os.path.join(args.root, args.subdir)
- # Check that directory exists since `os.walk()` does not fail
- # for non extant directories.
- if not os.path.isdir(test_dir):
- raise RuntimeError("Missing directory: '%s'" % test_dir)
- # Validate directory structure.
- regressions_failure_count = 0
- for dirpath, dirs, files in os.walk(test_dir):
- # skip hidden files and directories
- # ref: https://stackoverflow.com/questions/13454164/os-walk-without-hidden-folders
- files = [f for f in files if not f[0] == '.']
- dirs[:] = [d for d in dirs if not d[0] == '.']
- relpath = os.path.relpath(dirpath, test_dir)
- currentnode = find_directory(relpath, node)
- if not currentnode:
- raise RuntimeError("Unexpected directory was found: '%s'" % dirpath)
- dirs = set(dirs)
- files = set(files)
- expected_dirs = set()
- expected_files = set()
- regression_test = []
- for item in currentnode["contents"]:
- if item["type"] == "directory":
- expected_dirs.add(item["name"])
- elif item["type"] == "file":
- expected_files.add(item["name"])
- if item.get("compare", False):
- regression_test.append(item["name"])
- if expected_dirs != dirs:
- missing = expected_dirs - dirs
- unexpected = dirs - expected_dirs
- raise RuntimeError(\
- "Mismatched directories under '%s' found.\n"\
- "Missing : %s\n" \
- "Unexpected: %s" % (dirpath, missing, unexpected))
- if expected_files != files:
- missing = expected_files - files
- unexpected = files - expected_files
- raise RuntimeError(\
- "Mismatched files under '%s' found.\n"\
- "Missing : %s\n" \
- "Unexpected: %s" % (dirpath, missing, unexpected))
- if regression_test:
- from vtkmodules.vtkTestingRendering import vtkTesting
- t = vtkTesting()
- for image in regression_test:
- t.CleanArguments()
- t.AddArgument("-V")
- baseline_name = get_baseline(args.name, dirpath, image, args.root)
- t.AddArgument(os.path.join(args.baseline_dir, baseline_name))
- t.AddArgument("-T")
- t.AddArgument(args.temp_dir)
- if t.RegressionTest(os.path.join(dirpath, image), 15) != t.PASSED:
- regressions_failure_count += 1
- print("\n")
- if regressions_failure_count:
- raise RuntimeError("ERROR: %d regression tests failed!" % regressions_failure_count)
|