test(wiptest): add a way to test wiptest (#699)

This adds a really basic test runner that will execute test files using
`bun wiptest` and compare the output to make sure it's correct. It could
definitely be improved, especially in the speed department, but it's at
least functional now, which is better than we had before!

1 files changed, 300 insertions, 0 deletions

diff --git a/test/wiptest/run.cpp b/test/wiptest/run.cpp
new file mode 100644
index 000000000..97c7c9cd9
--- /dev/null
+++ b/test/wiptest/run.cpp
@@ -0,0 +1,300 @@
+#include <dirent.h>
+#include <filesystem>
+#include <iostream>
+#include <fstream>
+#include <unistd.h>
+#include <vector>
+
+int passed = 0;
+int failed = 0;
+
+/**
+ * @brief Run an executable and capture the output.
+ *
+ * @param path The path to the executable.
+ * @param argv The null-terminated arguments to pass to the executable.
+ * @param output [out] The string into which to place the output. This string will NOT be cleared first.
+ * @param exitCode [out] The return code of the executable.
+ * @return -1 if something went wrong, otherwise 0.
+ */
+int exec(char const* const path, char const* const argv[], std::string* const output, int* const exitCode) {
+  int stdpipe[2];
+  if (pipe(stdpipe) == -1) {
+    std::cerr << "ERROR: Unable to capture pipe" << std::endl;
+    return -1;
+  }
+
+  pid_t pid = fork();
+  if (pid == -1) {
+    std::cerr << "ERROR: Unable to capture pipe for stderr"  << std::endl;
+    return -1;
+  } else if (pid == 0) {
+    close(stdpipe[0]); // Child doesn't need to read the pipe.
+    // Capture stdout and stderr
+    dup2(stdpipe[1], 1);
+    dup2(stdpipe[1], 2);
+    execv(const_cast<char const*>(path), const_cast<char* const*>(argv));
+    return -1; // This is just to make the compiler happy.
+  } else {
+    close(stdpipe[1]); // Parent doesn't need to write.
+
+    if (output != nullptr) {
+      char buf[8];
+      int count;
+      while((count = read(stdpipe[0], buf, 8)) > 0) {
+        output->append(buf, count);
+      }
+    }
+    close(stdpipe[0]); // Done
+
+    // parent
+    int status;
+    if (waitpid(pid, &status, 0) == -1) {
+      std::cerr << "ERROR: waitpid failed somehow" << std::endl;
+      return -1;
+    }
+
+    if (exitCode != nullptr)
+      *exitCode = WEXITSTATUS(status);
+
+    return 0;
+  }
+}
+
+int execTest(char const* const bunBin, char const* const testMatch, std::string* const output, int* const exitCode) {
+  char const* args[] = {bunBin, "wiptest", testMatch, NULL};
+  return exec(bunBin, args, output, exitCode);
+}
+
+void parseMacros(std::string const filePath, int* const expectPass, std::vector<std::string const>& expects, std::vector<std::string const>& expectNots, std::string& testPattern, std::vector<std::string const>& errors) {
+  std::ifstream fstream(filePath);
+
+  if (!fstream.is_open()) {
+    errors.push_back("Unable to open file");
+    return;
+  }
+
+  std::string line;
+  size_t idx;
+  char const* rest;
+  while (fstream) {
+    std::getline(fstream, line);
+    idx = line.find("// ");
+    if (idx == std::string::npos)
+      continue;
+
+    idx += 3;
+
+    if (idx >= line.length())
+      continue;
+
+    switch (line.at(idx)) {
+    // STATUS
+    case 'S':
+      rest = line.c_str() + (idx);
+      if (strncmp(rest, "STATUS: ", 8) != 0)
+        continue;
+      if (idx + 8 >= line.length())
+        continue;
+      rest += 8;
+      if (strcmp(rest, "PASS") == 0)
+        *expectPass = 1;
+      else if (strcmp(rest, "FAIL") == 0)
+        *expectPass = 0;
+      else {
+        std::string err = "Invalid STATUS: '";
+        err += rest;
+        err += "', must be PASS or FAIL";
+        errors.push_back(err);
+      }
+      break;
+
+    // EXPECT
+    // EXPECTNOT
+    case 'E': {
+      rest = line.c_str() + (idx);
+      if (strncmp(rest, "EXPECT", 6) != 0)
+        continue;
+      idx += 6;
+      if (idx >= line.length())
+        continue;
+      rest += 6;
+      bool isNot = strncmp(rest, "NOT", 3) == 0;
+      if (isNot) {
+        idx += 3;
+        if (idx >= line.length())
+          continue;
+        rest += 3;
+      }
+
+      if (strncmp(rest, ": ", 2) != 0)
+        continue;
+
+      idx += 2;
+      if (idx >= line.length()) {
+        std::string err = isNot ? "EXPECTNOT" : "EXPECT";
+        err += " must not be empty";
+        errors.push_back(err);
+        continue;
+      }
+      rest += 2;
+
+      if (isNot)
+        expectNots.push_back(rest);
+      else
+        expects.push_back(rest);
+      break;
+    }
+
+    // TESTPATTERN
+    case 'T':
+      rest = line.c_str() + (idx);
+      if (strncmp(rest, "TESTPATTERN: ", 13) != 0)
+        continue;
+      if (idx + 13 >= line.length()) {
+        errors.push_back("TESTPATTERN must not be empty");
+        continue;
+      }
+      rest += 13;
+      testPattern.assign(rest);
+      break;
+
+    default:
+      break;
+    }
+  }
+}
+
+/**
+ * @brief Execute a single test.
+ *
+ * @param bunBin The path to the Bun binary to execute.
+ * @param baseDir The directory in which the test file is stored.
+ * @param testFile The name of the test file.
+ * @return The output of the test command (stdout + stderr).
+ */
+void runTest(char const* const bunBin, char const* const baseDir, char const* const testFile) {
+  printf("Running test '%s'...", testFile);
+  std::string filePath;
+  filePath += baseDir;
+  filePath += "/";
+  filePath += testFile;
+
+  std::vector<std::string const> errors;
+
+  int expectPass = 2;
+  std::vector<std::string const> expects;
+  std::vector<std::string const> expectNots;
+  parseMacros(filePath, &expectPass, expects, expectNots, filePath, errors);
+
+  if (expectPass == 2)
+    errors.push_back("Missing STATUS macro");
+
+  if (expects.size() == 0 && expectNots.size() == 0)
+    errors.push_back("File must contain at least one EXPECT or EXPECTNOT macro");
+
+  // Only run the test suite if we haven't yet failed.
+  if (errors.size() == 0) {
+    int exitCode;
+    std::string output;
+    if (execTest(bunBin, filePath.c_str(), &output, &exitCode) < 0) {
+      errors.push_back("Unable to parse test file");
+    }
+
+    auto didPass = exitCode == 0;
+    if (expectPass != didPass) {
+      if (expectPass) {
+        std::string err = "Expected exit code to be 0, got ";
+        err += exitCode;
+        errors.push_back(err);
+      } else
+        errors.push_back("Expected non-zero exit code");
+    }
+
+
+    for (auto ex : expects) {
+      if (output.find(ex) == std::string::npos) {
+        std::string msg = "Output does not contain '";
+        msg += ex;
+        msg += "'";
+        errors.push_back(msg);
+      }
+    }
+    for (auto ex : expectNots){
+      if (output.find(ex) != std::string::npos) {
+        std::string msg = "Output contains '";
+        msg += ex;
+        msg += "'";
+        errors.push_back(msg);
+      }
+    }
+  }
+
+  if (errors.size() > 0) {
+    ++failed;
+    printf(" Fail\n");
+    for (auto err : errors)
+      printf("  ERROR: %s\n", err.c_str());
+    printf("\n");
+  } else {
+    ++passed;
+    printf(" Pass\n");
+  }
+}
+
+/**
+ * @brief Run all tests in a directory.
+ *
+ * @param dir The directory to run tests in.
+ * @param bunBin The path to the Bun binary to execute.
+ */
+int runAll(char const* const dir, char const* const bunBin) {
+  struct dirent *entry = nullptr;
+
+  auto* dp = opendir(dir);
+  if (dp == nullptr) {
+    std::cerr << "ERROR: Unable to open directory '" << dir << "'" << std::endl;
+    return -1;
+  }
+
+  while ((entry = readdir(dp))) {
+    if (!strcmp(entry->d_name, ".") || !strcmp(entry->d_name, ".."))
+      continue;
+    runTest(bunBin, dir, entry->d_name);
+  }
+
+  if (closedir(dp) < 0) {
+    std::cerr << "ERROR: Unable to close directory '" << dir << "'" << std::endl;
+    return -1;
+  }
+
+  return 0;
+}
+
+int main(int argc, char const* argv[]) {
+  if (argc != 2) {
+    std::cerr << "Must provide path to test files" << std::endl;
+    return 1;
+  }
+
+  auto bunBin = std::getenv("BUN_BIN");
+  if (bunBin == nullptr) {
+    std::cerr << "ERROR: `$BUN_BIN` is not defined. Either set it manually or run this file via `make`'" << std::endl;
+    return 1;
+  }
+
+  if (!std::filesystem::exists(bunBin)) {
+    std::cerr << "ERROR: " << bunBin << " does not exist. Did you forget to run `make dev`?" << std::endl;
+    return 1;
+  }
+
+  char testDir[PATH_MAX];
+  realpath(argv[1], testDir);
+
+  if (runAll(testDir, bunBin) < 0)
+    return -1;
+
+  printf("\n\n\nFinished running tests.\nTotal: %d\nPassed: %d\nFailed: %d\n", passed + failed, passed, failed);
+
+  return failed;
+}