#!/usr/bin/env python3 import subprocess import json import os import argparse import sys def load_test_case(filename: str) -> dict: file = open(filename, 'r') test_case = json.load(file) file.close() return test_case def test_test_case(test_case: dict) -> bool: filename = test_case["file"] filepath = os.path.join("tests", filename) print(f"-> Testing {filename}") compiler = test_case["compile"] extra_flags = compiler["flags"] returncode = compiler["exit-code"] compiler_stdout = compiler["stdout"] compiler_stderr = compiler["stderr"] command = " ".join(["build/sapphirec", filepath, "-o", ".tests-bin/exe"] + extra_flags) print(f"-> Running command: {command}") compile_task = subprocess.Popen(["build/sapphirec", filepath, "-o", ".tests-bin/exe"] + extra_flags, stdout=subprocess.PIPE, stderr=subprocess.PIPE) retcode = compile_task.wait() retstdout = compile_task.stdout.read().decode('utf-8') retstderr = compile_task.stderr.read().decode('utf-8') if retcode != returncode: print( f"-> Test failed: compiler exited with code {str(retcode)} (expected {str(returncode)})") return False if compiler_stdout != retstdout: print("-> Test failed: compiler stdout does not match test case") print(retstdout) return False if compiler_stderr != retstderr: print("-> Test failed: compiler stderr does not match test case") print(retstderr) return False print("-> Test succeeded") return True test_cases = [] def run_tests(): os.mkdir(".tests-bin") for filename in os.listdir("tests"): if filename.endswith(".json"): test_cases.append(load_test_case(os.path.join("tests", filename))) continue else: continue for test_case in test_cases: if not test_test_case(test_case): print("-> Test suite failed") os.rmdir(".tests-bin") sys.exit(1) else: print("-> Test suite succeeded") os.rmdir(".tests-bin") def create_test(filename: str, extra_flags: list): os.mkdir(".tests-bin") test_case = {} test_case["file"] = filename filepath = os.path.join("tests", filename) compiler = {} compiler["flags"] = extra_flags compile_task = subprocess.Popen(["build/sapphirec", filepath, "-o", ".tests-bin/exe"] + extra_flags, stdout=subprocess.PIPE, stderr=subprocess.PIPE) retcode = compile_task.wait() retstdout = compile_task.stdout.read().decode('utf-8') retstderr = compile_task.stderr.read().decode('utf-8') compiler["exit-code"] = retcode compiler["stdout"] = retstdout compiler["stderr"] = retstderr test_case["compile"] = compiler ofilepath = ".".join(filepath.split(".")[:-1]) + ".json" ofile = open(ofilepath, "w+") json.dump(test_case, ofile) ofile.close() os.rmdir(".tests-bin") if len(sys.argv) < 2: print("Please provide at least 1 argument (run-tests, create)") else: action = sys.argv[1] if action not in ["run-tests", "create"]: print("Action must be run-tests or create.") else: if action == "run-tests": run_tests() else: if len(sys.argv) == 2: print("You must provide a filename to create a test for.") else: extra_flags = [] if(len(sys.argv) > 3): extra_flags = sys.argv[3:] create_test(sys.argv[2], extra_flags)