validate_extracts.py 4.4 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128
  1. import argparse, json, os, os.path, sys
  2. def splitall(path):
  3. allparts = []
  4. while 1:
  5. parts = os.path.split(path)
  6. if parts[0] == path: # sentinel for absolute paths
  7. allparts.insert(0, parts[0])
  8. break
  9. elif parts[1] == path: # sentinel for relative paths
  10. allparts.insert(0, parts[1])
  11. break
  12. else:
  13. path = parts[0]
  14. allparts.insert(0, parts[1])
  15. return allparts
  16. def find_subdirectory(subdir, node):
  17. contents = node["contents"]
  18. for item in contents:
  19. if item["type"] == "directory" and item["name"] == subdir:
  20. return item
  21. raise RuntimeError("Missing subdir '%s'" % subdir)
  22. def find_directory(subdir, node):
  23. if subdir == '.' or subdir == None:
  24. return node
  25. for part in splitall(subdir):
  26. node = find_subdirectory(part, node)
  27. return node
  28. def get_baseline(tname, path, image, root):
  29. import re
  30. filename = os.path.join(path, image)
  31. relpath = os.path.relpath(filename, root)
  32. return tname + "_" + re.sub('[^0-9a-zA-Z.]+', '_', relpath)
  33. parser = argparse.ArgumentParser(\
  34. description="Validate Extrats generated in tests")
  35. parser.add_argument("--name", type=str, required=True,
  36. help="name of the test being validated")
  37. parser.add_argument("--subdir", type=str, default=None,
  38. help="sub directory to validate")
  39. parser.add_argument("--root", type=str, required=True,
  40. help ="root directory with results")
  41. parser.add_argument("--json", type=str, required=True,
  42. help="validation json file")
  43. parser.add_argument("--baseline-dir", type=str, default=None,
  44. help="directory for baselines for regression tests, if any")
  45. parser.add_argument("--temp-dir", type=str, default="/tmp",
  46. help="directory for results for regression test failures, if any")
  47. args = parser.parse_args()
  48. with open(args.json, 'r') as f:
  49. data = json.load(f)[0]
  50. node = data
  51. test_dir = args.root
  52. if args.subdir is not None:
  53. node = find_directory(args.subdir, node)
  54. test_dir = os.path.join(args.root, args.subdir)
  55. # Check that directory exists since `os.walk()` does not fail
  56. # for non extant directories.
  57. if not os.path.isdir(test_dir):
  58. raise RuntimeError("Missing directory: '%s'" % test_dir)
  59. # Validate directory structure.
  60. regressions_failure_count = 0
  61. for dirpath, dirs, files in os.walk(test_dir):
  62. # skip hidden files and directories
  63. # ref: https://stackoverflow.com/questions/13454164/os-walk-without-hidden-folders
  64. files = [f for f in files if not f[0] == '.']
  65. dirs[:] = [d for d in dirs if not d[0] == '.']
  66. relpath = os.path.relpath(dirpath, test_dir)
  67. currentnode = find_directory(relpath, node)
  68. if not currentnode:
  69. raise RuntimeError("Unexpected directory was found: '%s'" % dirpath)
  70. dirs = set(dirs)
  71. files = set(files)
  72. expected_dirs = set()
  73. expected_files = set()
  74. regression_test = []
  75. for item in currentnode["contents"]:
  76. if item["type"] == "directory":
  77. expected_dirs.add(item["name"])
  78. elif item["type"] == "file":
  79. expected_files.add(item["name"])
  80. if item.get("compare", False):
  81. regression_test.append(item["name"])
  82. if expected_dirs != dirs:
  83. missing = expected_dirs - dirs
  84. unexpected = dirs - expected_dirs
  85. raise RuntimeError(\
  86. "Mismatched directories under '%s' found.\n"\
  87. "Missing : %s\n" \
  88. "Unexpected: %s" % (dirpath, missing, unexpected))
  89. if expected_files != files:
  90. missing = expected_files - files
  91. unexpected = files - expected_files
  92. raise RuntimeError(\
  93. "Mismatched files under '%s' found.\n"\
  94. "Missing : %s\n" \
  95. "Unexpected: %s" % (dirpath, missing, unexpected))
  96. if regression_test:
  97. from vtkmodules.vtkTestingRendering import vtkTesting
  98. t = vtkTesting()
  99. for image in regression_test:
  100. t.CleanArguments()
  101. t.AddArgument("-V")
  102. baseline_name = get_baseline(args.name, dirpath, image, args.root)
  103. t.AddArgument(os.path.join(args.baseline_dir, baseline_name))
  104. t.AddArgument("-T")
  105. t.AddArgument(args.temp_dir)
  106. if t.RegressionTest(os.path.join(dirpath, image), 15) != t.PASSED:
  107. regressions_failure_count += 1
  108. print("\n")
  109. if regressions_failure_count:
  110. raise RuntimeError("ERROR: %d regression tests failed!" % regressions_failure_count)