#!/usr/bin/env python3 import subprocess import json import os import sys import shutil def load_test_case(filename: str) -> dict: file = open(filename, 'r') test_case = json.load(file) file.close() return test_case def test_test_case(test_case: dict) -> bool: filename = test_case["file"] filepath = os.path.join("tests", filename) print(f"-> Testing {filename}") compiler = test_case["compile"] extra_flags = compiler["flags"] returncode = compiler["exit-code"] compiler_stdout = compiler["stdout"] compiler_stderr = compiler["stderr"] command = " ".join(["build/sapphirec", filepath, "-o", ".tests-bin/output.o"] + extra_flags) print(f"-> Running command: {command}") compile_task = subprocess.Popen(["build/sapphirec", filepath, "-o", ".tests-bin/output.o"] + extra_flags, stdout=subprocess.PIPE, stderr=subprocess.PIPE) retcode = compile_task.wait() retstdout = compile_task.stdout.read().decode('utf-8') retstderr = compile_task.stderr.read().decode('utf-8') if retcode != returncode: print( f"-> Test failed: compiler exited with code {str(retcode)} (expected {str(returncode)})") return False if compiler_stdout != retstdout: print("-> Test failed: compiler stdout does not match test case") print(retstdout) return False if compiler_stderr != retstderr: print("-> Test failed: compiler stderr does not match test case") print(retstderr) return False runtime = test_case.get("run", False) if runtime is not False: print("-> Running command: gcc .tests-bin/output.o -o .tests-bin/output") link_task = subprocess.Popen(["gcc", ".tests-bin/output.o", "-o", ".tests-bin/output"] + extra_flags, stdout=subprocess.PIPE, stderr=subprocess.PIPE) if link_task.wait() != 0: print(f"-> Failed to link program") return False print(f"-> Running command: .tests-bin/output") runtime_task = subprocess.Popen([".tests-bin/output"], stdout=subprocess.PIPE, stderr=subprocess.PIPE) runtime_code = runtime_task.wait() runtime_stdout = runtime_task.stdout.read().decode('utf-8') runtime_stderr = runtime_task.stderr.read().decode('utf-8') test_code = runtime["exit-code"] if runtime_code != runtime["exit-code"]: print( f"-> Test failed: program exited with code {str(runtime_code)} (expected {str(test_code)})") return False if runtime_stdout != runtime["stdout"]: print("-> Test failed: program stdout does not match test case") print(retstdout) return False if runtime_stderr != runtime["stderr"]: print("-> Test failed: program stderr does not match test case") print(retstderr) return False print("-> Test succeeded") return True test_cases = [] def run_tests(): os.makedirs(".tests-bin", exist_ok=True) for filename in os.listdir("tests"): if filename.endswith(".json"): test_cases.append(load_test_case(os.path.join("tests", filename))) continue else: continue for test_case in test_cases: if not test_test_case(test_case): print("-> Test suite failed") shutil.rmtree(".tests-bin") sys.exit(1) else: print("-> Test suite succeeded") shutil.rmtree(".tests-bin") def create_test(filename: str, extra_flags: list): os.makedirs(".tests-bin", exist_ok=True) test_case = {} test_case["file"] = filename filepath = os.path.join("tests", filename) compiler = {} compiler["flags"] = extra_flags compile_task = subprocess.Popen(["build/sapphirec", filepath, "-o", ".tests-bin/output.o"] + extra_flags, stdout=subprocess.PIPE, stderr=subprocess.PIPE) retcode = compile_task.wait() retstdout = compile_task.stdout.read().decode('utf-8') retstderr = compile_task.stderr.read().decode('utf-8') compiler["exit-code"] = retcode compiler["stdout"] = retstdout compiler["stderr"] = retstderr test_case["compile"] = compiler if retcode == 0: link_task = subprocess.Popen(["gcc", ".tests-bin/output.o", "-o", ".tests-bin/output"] + extra_flags, stdout=subprocess.PIPE, stderr=subprocess.PIPE) link_status = link_task.wait() if link_status == 0: program = {} runtime_task = subprocess.Popen([".tests-bin/output"], stdout=subprocess.PIPE, stderr=subprocess.PIPE) runtime_code = runtime_task.wait() runtime_stdout = runtime_task.stdout.read().decode('utf-8') runtime_stderr = runtime_task.stderr.read().decode('utf-8') program["exit-code"] = runtime_code program["stdout"] = runtime_stdout program["stderr"] = runtime_stderr test_case["run"] = program ofilepath = ".".join(filepath.split(".")[:-1]) + ".json" ofile = open(ofilepath, "w+") json.dump(test_case, ofile) ofile.close() shutil.rmtree(".tests-bin") if len(sys.argv) < 2: print("Please provide at least 1 argument (run-tests, create)") else: action = sys.argv[1] if action not in ["run-tests", "create"]: print("Action must be run-tests or create.") else: if action == "run-tests": run_tests() else: if len(sys.argv) == 2: print("You must provide a filename to create a test for.") else: extra_flags = [] if(len(sys.argv) > 3): extra_flags = sys.argv[3:] create_test(sys.argv[2], extra_flags)