diff --git a/test.py b/test.py
new file mode 100755
index 0000000..57b523a
--- /dev/null
+++ b/test.py
@@ -0,0 +1,109 @@
+#!/usr/bin/env python3
+import subprocess
+import json
+import os
+import argparse
+import sys
+
+
+def load_test_case(filename: str) -> dict:
+    file = open(filename, 'r')
+    test_case = json.load(file)
+    file.close()
+    return test_case
+
+
+def test_test_case(test_case: dict) -> bool:
+    filename = test_case["file"]
+    filepath = os.path.join("tests", filename)
+    print(f"-> Testing {filename}")
+    compiler = test_case["compile"]
+    extra_flags = compiler["flags"]
+    returncode = compiler["exit-code"]
+    compiler_stdout = compiler["stdout"]
+    compiler_stderr = compiler["stderr"]
+    command = " ".join(["build/sapphirec", filepath, "-o",
+                       ".tests-bin/exe"] + extra_flags)
+    print(f"-> Running command: {command}")
+    compile_task = subprocess.Popen(["build/sapphirec", filepath, "-o", ".tests-bin/exe"] + extra_flags,
+                                    stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+    retcode = compile_task.wait()
+    retstdout = compile_task.stdout.read().decode('utf-8')
+    retstderr = compile_task.stderr.read().decode('utf-8')
+    if retcode != returncode:
+        print(
+            f"-> Test failed: compiler exited with code {str(retcode)} (expected {str(returncode)})")
+        return False
+    if compiler_stdout != retstdout:
+        print("-> Test failed: compiler stdout does not match test case")
+        print(retstdout)
+        return False
+    if compiler_stderr != retstderr:
+        print("-> Test failed: compiler stderr does not match test case")
+        print(retstderr)
+        return False
+    print("-> Test succeeded")
+    return True
+
+
+test_cases = []
+
+
+def run_tests():
+    os.mkdir(".tests-bin")
+    for filename in os.listdir("tests"):
+        if filename.endswith(".json"):
+            test_cases.append(load_test_case(os.path.join("tests", filename)))
+            continue
+        else:
+            continue
+    for test_case in test_cases:
+        if not test_test_case(test_case):
+            print("-> Test suite failed")
+            os.rmdir(".tests-bin")
+            sys.exit(1)
+    else:
+        print("-> Test suite succeeded")
+    os.rmdir(".tests-bin")
+
+
+def create_test(filename: str, extra_flags: list):
+    os.mkdir(".tests-bin")
+    test_case = {}
+    test_case["file"] = filename
+    filepath = os.path.join("tests", filename)
+    compiler = {}
+    compiler["flags"] = extra_flags
+    compile_task = subprocess.Popen(["build/sapphirec", filepath, "-o", ".tests-bin/exe"] + extra_flags,
+                                    stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+    retcode = compile_task.wait()
+    retstdout = compile_task.stdout.read().decode('utf-8')
+    retstderr = compile_task.stderr.read().decode('utf-8')
+    compiler["exit-code"] = retcode
+    compiler["stdout"] = retstdout
+    compiler["stderr"] = retstderr
+    test_case["compile"] = compiler
+    ofilepath = ".".join(filepath.split(".")[:-1]) + ".json"
+    ofile = open(ofilepath, "w+")
+    json.dump(test_case, ofile)
+    ofile.close()
+    os.rmdir(".tests-bin")
+
+
+if len(sys.argv) < 2:
+    print("Please provide at least 1 argument (run-tests, create)")
+else:
+    action = sys.argv[1]
+    if action not in ["run-tests", "create"]:
+        print("Action must be run-tests or create.")
+    else:
+        if action == "run-tests":
+            run_tests()
+        else:
+            if len(sys.argv) == 2:
+                print("You must provide a filename to create a test for.")
+            else:
+                extra_flags = []
+                if(len(sys.argv) > 3):
+                    extra_flags = sys.argv[3:]
+                create_test(sys.argv[2], extra_flags)
diff --git a/tests/import-inexistent.json b/tests/import-inexistent.json
new file mode 100644
index 0000000..1a946af
--- /dev/null
+++ b/tests/import-inexistent.json
@@ -0,0 +1 @@
+{"file": "import-inexistent.sp", "compile": {"flags": [], "exit-code": 1, "stdout": "", "stderr": "\u001b[1;1mtests/import-inexistent.sp:1:8: \u001b[31;49merror: \u001b[0;0mfile not found\n1    import penguin_boi;\n            \u001b[31;49m^\u001b[0;0m\n"}}
\ No newline at end of file
diff --git a/tests/import-inexistent.sp b/tests/import-inexistent.sp
new file mode 100644
index 0000000..f248604
--- /dev/null
+++ b/tests/import-inexistent.sp
@@ -0,0 +1 @@
+import penguin_boi;
\ No newline at end of file
diff --git a/tests/simple.json b/tests/simple.json
new file mode 100644
index 0000000..5ae853e
--- /dev/null
+++ b/tests/simple.json
@@ -0,0 +1 @@
+{"file": "simple.sp", "compile": {"flags": [], "exit-code": 0, "stdout": "ID:const (tests/simple.sp:1:1)\nID:outln (tests/simple.sp:1:7)\nID:from (tests/simple.sp:1:13)\nAT (tests/simple.sp:1:18)\nSTRING:'core/io' (tests/simple.sp:1:19)\nSEMICOLON (tests/simple.sp:1:28)\nID:let (tests/simple.sp:3:1)\nAT (tests/simple.sp:3:5)\nID:main (tests/simple.sp:3:6)\nID:in (tests/simple.sp:3:11)\nRBRACKET (tests/simple.sp:3:14)\nID:outln (tests/simple.sp:4:5)\nLPAREN (tests/simple.sp:4:10)\nSTRING:'Hello, world!' (tests/simple.sp:4:11)\nRPAREN (tests/simple.sp:4:26)\nSEMICOLON (tests/simple.sp:4:27)\nLBRACKET (tests/simple.sp:5:1)\nEOF (tests/simple.sp:5:2)\nOutput filename: .tests-bin/exe\nOutput target triple: x86_64-unknown-linux-gnu\n", "stderr": ""}}
\ No newline at end of file
diff --git a/tests/simple.sp b/tests/simple.sp
new file mode 100644
index 0000000..aae958a
--- /dev/null
+++ b/tests/simple.sp
@@ -0,0 +1,5 @@
+const outln from @'core/io';
+
+let @main in {
+    outln('Hello, world!');
+}
\ No newline at end of file
diff --git a/tests/wimport.json b/tests/wimport.json
new file mode 100644
index 0000000..a3b25ba
--- /dev/null
+++ b/tests/wimport.json
@@ -0,0 +1 @@
+{"file": "wimport.sp", "compile": {"flags": ["--wimport"], "exit-code": 0, "stdout": "\u001b[1;1mtests/wimport.sp:1:8: \u001b[33;49mwarning: \u001b[0;0mfile already imported, skipping\n1    import tests/wimport;\n            \u001b[33;49m^\u001b[0;0m\nEOF (tests/wimport.sp:1:22)\nOutput filename: .tests-bin/exe\nOutput target triple: x86_64-unknown-linux-gnu\n", "stderr": ""}}
\ No newline at end of file
diff --git a/tests/wimport.sp b/tests/wimport.sp
new file mode 100644
index 0000000..62b74a3
--- /dev/null
+++ b/tests/wimport.sp
@@ -0,0 +1 @@
+import tests/wimport;
\ No newline at end of file