sapphire/test.py

#!/usr/bin/env python3
import subprocess
import json
import os
import sys
import shutil


def load_test_case(filename: str) -> dict:
    file = open(filename, 'r')
    test_case = json.load(file)
    file.close()
    return test_case


def test_test_case(test_case: dict) -> bool:
    filename = test_case["file"]
    filepath = os.path.join("tests", filename)
    print(f"-> Testing {filename}")
    compiler = test_case["compile"]
    extra_flags = compiler["flags"]
    returncode = compiler["exit-code"]
    compiler_stdout = compiler["stdout"]
    compiler_stderr = compiler["stderr"]
    command = " ".join(["build/sapphirec", filepath, "-o",
                       ".tests-bin/output.o"] + extra_flags)
    print(f"-> Running command: {command}")
    compile_task = subprocess.Popen(["build/sapphirec", filepath, "-o", ".tests-bin/output.o"] + extra_flags,
                                    stdout=subprocess.PIPE, stderr=subprocess.PIPE)
    retcode = compile_task.wait()
    retstdout = compile_task.stdout.read().decode('utf-8')
    retstderr = compile_task.stderr.read().decode('utf-8')
    if retcode != returncode:
        print(
            f"-> Test failed: compiler exited with code {str(retcode)} (expected {str(returncode)})")
        return False
    if compiler_stdout != retstdout:
        print("-> Test failed: compiler stdout does not match test case")
        print(retstdout)
        return False
    if compiler_stderr != retstderr:
        print("-> Test failed: compiler stderr does not match test case")
        print(retstderr)
        return False
    runtime = test_case.get("run", False)
    if runtime is not False:
        print("-> Running command: gcc .tests-bin/output.o -o .tests-bin/output")
        link_task = subprocess.Popen(["gcc", ".tests-bin/output.o", "-o", ".tests-bin/output"] + extra_flags,
                                     stdout=subprocess.PIPE, stderr=subprocess.PIPE)
        if link_task.wait() != 0:
            print(f"-> Failed to link program")
            return False
        print(f"-> Running command: .tests-bin/output")
        runtime_task = subprocess.Popen([".tests-bin/output"],
                                        stdout=subprocess.PIPE, stderr=subprocess.PIPE)
        runtime_code = runtime_task.wait()
        runtime_stdout = runtime_task.stdout.read().decode('utf-8')
        runtime_stderr = runtime_task.stderr.read().decode('utf-8')
        test_code = runtime["exit-code"]
        if runtime_code != runtime["exit-code"]:
            print(
                f"-> Test failed: program exited with code {str(runtime_code)} (expected {str(test_code)})")
            return False
        if runtime_stdout != runtime["stdout"]:
            print("-> Test failed: program stdout does not match test case")
            print(retstdout)
            return False
        if runtime_stderr != runtime["stderr"]:
            print("-> Test failed: program stderr does not match test case")
            print(retstderr)
            return False
    print("-> Test succeeded")
    return True


test_cases = []


def run_tests():
    os.makedirs(".tests-bin", exist_ok=True)
    for filename in os.listdir("tests"):
        if filename.endswith(".json"):
            test_cases.append(load_test_case(os.path.join("tests", filename)))
            continue
        else:
            continue
    for test_case in test_cases:
        if not test_test_case(test_case):
            print("-> Test suite failed")
            shutil.rmtree(".tests-bin")
            sys.exit(1)
    else:
        print("-> Test suite succeeded")
    shutil.rmtree(".tests-bin")


def create_test(filename: str, extra_flags: list):
    os.makedirs(".tests-bin", exist_ok=True)
    test_case = {}
    test_case["file"] = filename
    filepath = os.path.join("tests", filename)
    compiler = {}
    compiler["flags"] = extra_flags
    compile_task = subprocess.Popen(["build/sapphirec", filepath, "-o", ".tests-bin/output.o"] + extra_flags,
                                    stdout=subprocess.PIPE, stderr=subprocess.PIPE)
    retcode = compile_task.wait()
    retstdout = compile_task.stdout.read().decode('utf-8')
    retstderr = compile_task.stderr.read().decode('utf-8')
    compiler["exit-code"] = retcode
    compiler["stdout"] = retstdout
    compiler["stderr"] = retstderr
    test_case["compile"] = compiler
    if retcode == 0:
        link_task = subprocess.Popen(["gcc", ".tests-bin/output.o", "-o", ".tests-bin/output"] + extra_flags,
                                     stdout=subprocess.PIPE, stderr=subprocess.PIPE)
        link_status = link_task.wait()
        if link_status == 0:
            program = {}
            runtime_task = subprocess.Popen([".tests-bin/output"],
                                            stdout=subprocess.PIPE, stderr=subprocess.PIPE)
            runtime_code = runtime_task.wait()
            runtime_stdout = runtime_task.stdout.read().decode('utf-8')
            runtime_stderr = runtime_task.stderr.read().decode('utf-8')
            program["exit-code"] = runtime_code
            program["stdout"] = runtime_stdout
            program["stderr"] = runtime_stderr
            test_case["run"] = program
    ofilepath = ".".join(filepath.split(".")[:-1]) + ".json"
    ofile = open(ofilepath, "w+")
    json.dump(test_case, ofile)
    ofile.close()
    shutil.rmtree(".tests-bin")


if len(sys.argv) < 2:
    print("Please provide at least 1 argument (run-tests, create)")
else:
    action = sys.argv[1]
    if action not in ["run-tests", "create"]:
        print("Action must be run-tests or create.")
    else:
        if action == "run-tests":
            run_tests()
        else:
            if len(sys.argv) == 2:
                print("You must provide a filename to create a test for.")
            else:
                extra_flags = []
                if(len(sys.argv) > 3):
                    extra_flags = sys.argv[3:]
                create_test(sys.argv[2], extra_flags)