Initial commit

tools/decompctx.py

@@ -0,0 +1,87 @@
+#!/usr/bin/env python3
+
+###
+# Generates a ctx.c file, usable for "Context" on https://decomp.me.
+#
+# Usage:
+#   python3 tools/decompctx.py src/file.cpp
+#
+# If changes are made, please submit a PR to
+# https://github.com/encounter/dtk-template
+###
+
+import argparse
+import os
+import re
+
+script_dir = os.path.dirname(os.path.realpath(__file__))
+root_dir = os.path.abspath(os.path.join(script_dir, ".."))
+src_dir = os.path.join(root_dir, "src")
+include_dir = os.path.join(root_dir, "include")
+
+include_pattern = re.compile(r'^#include\s*[<"](.+?)[>"]$')
+guard_pattern = re.compile(r'^#ifndef\s+(.*)$')
+
+defines = set()
+
+def import_h_file(in_file: str, r_path: str) -> str:
+    rel_path = os.path.join(root_dir, r_path, in_file)
+    inc_path = os.path.join(include_dir, in_file)
+    if os.path.exists(rel_path):
+      return import_c_file(rel_path)
+    elif os.path.exists(inc_path):
+      return import_c_file(inc_path)
+    else:
+      print("Failed to locate", in_file)
+      exit(1)
+
+def import_c_file(in_file) -> str:
+    in_file = os.path.relpath(in_file, root_dir)
+    out_text = ''
+
+    try:
+      with open(in_file, encoding="utf-8") as file:
+        out_text += process_file(in_file, list(file))
+    except Exception:
+      with open(in_file) as file:
+        out_text += process_file(in_file, list(file))
+    return out_text
+
+def process_file(in_file: str, lines) -> str:
+    out_text = ''
+    for idx, line in enumerate(lines):
+      guard_match = guard_pattern.match(line.strip())
+      if idx == 0:
+        if guard_match:
+          if guard_match[1] in defines:
+            break
+          defines.add(guard_match[1])
+        print("Processing file", in_file)
+      include_match = include_pattern.match(line.strip())
+      if include_match and not include_match[1].endswith(".s"):
+        out_text += f"/* \"{in_file}\" line {idx} \"{include_match[1]}\" */\n"
+        out_text += import_h_file(include_match[1], os.path.dirname(in_file))
+        out_text += f"/* end \"{include_match[1]}\" */\n"
+      else:
+        out_text += line
+
+    return out_text
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="""Create a context file which can be used for decomp.me"""
+    )
+    parser.add_argument(
+        "c_file",
+        help="""File from which to create context""",
+    )
+    args = parser.parse_args()
+
+    output = import_c_file(args.c_file)
+
+    with open(os.path.join(root_dir, "ctx.c"), "w", encoding="utf-8") as f:
+        f.write(output)
+
+
+if __name__ == "__main__":
+    main()

tools/download_tool.py

@@ -0,0 +1,89 @@
+#!/usr/bin/env python3
+
+###
+# Downloads various tools from GitHub releases.
+#
+# Usage:
+#   python3 tools/download_tool.py wibo build/tools/wibo --tag 1.0.0
+#
+# If changes are made, please submit a PR to
+# https://github.com/encounter/dtk-template
+###
+
+import argparse
+import io
+import os
+import platform
+import shutil
+import stat
+import sys
+import urllib.request
+import zipfile
+
+from pathlib import Path
+
+
+def dtk_url(tag):
+    uname = platform.uname()
+    suffix = ""
+    system = uname.system.lower()
+    if system == "darwin":
+        system = "macos"
+    elif system == "windows":
+        suffix = ".exe"
+    arch = uname.machine.lower()
+    if arch == "amd64":
+        arch = "x86_64"
+
+    repo = "https://github.com/encounter/decomp-toolkit"
+    return f"{repo}/releases/download/{tag}/dtk-{system}-{arch}{suffix}"
+
+
+def sjiswrap_url(tag):
+    repo = "https://github.com/encounter/sjiswrap"
+    return f"{repo}/releases/download/{tag}/sjiswrap-windows-x86.exe"
+
+
+def wibo_url(tag):
+    repo = "https://github.com/decompals/wibo"
+    return f"{repo}/releases/download/{tag}/wibo"
+
+
+def compilers_url(tag):
+    return f"https://files.decomp.dev/compilers_{tag}.zip"
+
+TOOLS = {
+    "dtk": dtk_url,
+    "sjiswrap": sjiswrap_url,
+    "wibo": wibo_url,
+    "compilers": compilers_url,
+}
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("tool", help="Tool name")
+    parser.add_argument("output", type=Path, help="output file path")
+    parser.add_argument("--tag", help="GitHub tag", required=True)
+    args = parser.parse_args()
+
+    url = TOOLS[args.tool](args.tag)
+    output = Path(args.output)
+
+    print(f"Downloading {url} to {output}")
+    req = urllib.request.Request(url, headers={"User-Agent": "Mozilla/5.0"})
+    with urllib.request.urlopen(req) as response:
+        if url.endswith(".zip"):
+            data = io.BytesIO(response.read())
+            with zipfile.ZipFile(data) as f:
+                f.extractall(output)
+            output.touch(mode=0o755)
+        else:
+            with open(output, "wb") as f:
+                shutil.copyfileobj(response, f)
+            st = os.stat(output)
+            os.chmod(output, st.st_mode | stat.S_IEXEC)
+
+
+if __name__ == "__main__":
+    main()

tools/ninja_syntax.py

@@ -0,0 +1,223 @@
+# Copyright 2011 Google Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Python module for generating .ninja files.
+
+Note that this is emphatically not a required piece of Ninja; it's
+just a helpful utility for build-file-generation systems that already
+use Python.
+"""
+
+import re
+import textwrap
+
+
+def escape_path(word):
+    return word.replace("$ ", "$$ ").replace(" ", "$ ").replace(":", "$:")
+
+
+class Writer(object):
+    def __init__(self, output, width=78):
+        self.output = output
+        self.width = width
+
+    def newline(self):
+        self.output.write("\n")
+
+    def comment(self, text):
+        for line in textwrap.wrap(
+            text, self.width - 2, break_long_words=False, break_on_hyphens=False
+        ):
+            self.output.write("# " + line + "\n")
+
+    def variable(self, key, value, indent=0):
+        if value is None:
+            return
+        if isinstance(value, list):
+            value = " ".join(filter(None, value))  # Filter out empty strings.
+        self._line("%s = %s" % (key, value), indent)
+
+    def pool(self, name, depth):
+        self._line("pool %s" % name)
+        self.variable("depth", depth, indent=1)
+
+    def rule(
+        self,
+        name,
+        command,
+        description=None,
+        depfile=None,
+        generator=False,
+        pool=None,
+        restat=False,
+        rspfile=None,
+        rspfile_content=None,
+        deps=None,
+    ):
+        self._line("rule %s" % name)
+        self.variable("command", command, indent=1)
+        if description:
+            self.variable("description", description, indent=1)
+        if depfile:
+            self.variable("depfile", depfile, indent=1)
+        if generator:
+            self.variable("generator", "1", indent=1)
+        if pool:
+            self.variable("pool", pool, indent=1)
+        if restat:
+            self.variable("restat", "1", indent=1)
+        if rspfile:
+            self.variable("rspfile", rspfile, indent=1)
+        if rspfile_content:
+            self.variable("rspfile_content", rspfile_content, indent=1)
+        if deps:
+            self.variable("deps", deps, indent=1)
+
+    def build(
+        self,
+        outputs,
+        rule,
+        inputs=None,
+        implicit=None,
+        order_only=None,
+        variables=None,
+        implicit_outputs=None,
+        pool=None,
+        dyndep=None,
+    ):
+        outputs = as_list(outputs)
+        out_outputs = [escape_path(x) for x in outputs]
+        all_inputs = [escape_path(x) for x in as_list(inputs)]
+
+        if implicit:
+            implicit = [escape_path(x) for x in as_list(implicit)]
+            all_inputs.append("|")
+            all_inputs.extend(implicit)
+        if order_only:
+            order_only = [escape_path(x) for x in as_list(order_only)]
+            all_inputs.append("||")
+            all_inputs.extend(order_only)
+        if implicit_outputs:
+            implicit_outputs = [escape_path(x) for x in as_list(implicit_outputs)]
+            out_outputs.append("|")
+            out_outputs.extend(implicit_outputs)
+
+        self._line(
+            "build %s: %s" % (" ".join(out_outputs), " ".join([rule] + all_inputs))
+        )
+        if pool is not None:
+            self._line("  pool = %s" % pool)
+        if dyndep is not None:
+            self._line("  dyndep = %s" % dyndep)
+
+        if variables:
+            if isinstance(variables, dict):
+                iterator = iter(variables.items())
+            else:
+                iterator = iter(variables)
+
+            for key, val in iterator:
+                self.variable(key, val, indent=1)
+
+        return outputs
+
+    def include(self, path):
+        self._line("include %s" % path)
+
+    def subninja(self, path):
+        self._line("subninja %s" % path)
+
+    def default(self, paths):
+        self._line("default %s" % " ".join(as_list(paths)))
+
+    def _count_dollars_before_index(self, s, i):
+        """Returns the number of '$' characters right in front of s[i]."""
+        dollar_count = 0
+        dollar_index = i - 1
+        while dollar_index > 0 and s[dollar_index] == "$":
+            dollar_count += 1
+            dollar_index -= 1
+        return dollar_count
+
+    def _line(self, text, indent=0):
+        """Write 'text' word-wrapped at self.width characters."""
+        leading_space = "  " * indent
+        while len(leading_space) + len(text) > self.width:
+            # The text is too wide; wrap if possible.
+
+            # Find the rightmost space that would obey our width constraint and
+            # that's not an escaped space.
+            available_space = self.width - len(leading_space) - len(" $")
+            space = available_space
+            while True:
+                space = text.rfind(" ", 0, space)
+                if space < 0 or self._count_dollars_before_index(text, space) % 2 == 0:
+                    break
+
+            if space < 0:
+                # No such space; just use the first unescaped space we can find.
+                space = available_space - 1
+                while True:
+                    space = text.find(" ", space + 1)
+                    if (
+                        space < 0
+                        or self._count_dollars_before_index(text, space) % 2 == 0
+                    ):
+                        break
+            if space < 0:
+                # Give up on breaking.
+                break
+
+            self.output.write(leading_space + text[0:space] + " $\n")
+            text = text[space + 1 :]
+
+            # Subsequent lines are continuations, so indent them.
+            leading_space = "  " * (indent + 2)
+
+        self.output.write(leading_space + text + "\n")
+
+    def close(self):
+        self.output.close()
+
+
+def as_list(input):
+    if input is None:
+        return []
+    if isinstance(input, list):
+        return input
+    return [input]
+
+
+def escape(string):
+    """Escape a string such that it can be embedded into a Ninja file without
+    further interpretation."""
+    assert "\n" not in string, "Ninja syntax does not allow newlines"
+    # We only have one special metacharacter: '$'.
+    return string.replace("$", "$$")
+
+
+def expand(string, vars, local_vars={}):
+    """Expand a string containing $vars as Ninja would.
+
+    Note: doesn't handle the full Ninja variable syntax, but it's enough
+    to make configure.py's use of it work.
+    """
+
+    def exp(m):
+        var = m.group(1)
+        if var == "$":
+            return "$"
+        return local_vars.get(var, vars.get(var, ""))
+
+    return re.sub(r"\$(\$|\w*)", exp, string)

tools/transform_dep.py

@@ -0,0 +1,84 @@
+#!/usr/bin/env python3
+
+###
+# Transforms .d files, converting Windows paths to Unix paths.
+# Allows usage of the mwcc -MMD flag on platforms other than Windows.
+#
+# Usage:
+#   python3 tools/transform_dep.py build/src/file.d build/src/file.d
+#
+# If changes are made, please submit a PR to
+# https://github.com/encounter/dtk-template
+###
+
+import argparse
+import os
+from platform import uname
+
+wineprefix = os.path.join(os.environ["HOME"], ".wine")
+if "WINEPREFIX" in os.environ:
+    wineprefix = os.environ["WINEPREFIX"]
+winedevices = os.path.join(wineprefix, "dosdevices")
+
+
+def in_wsl() -> bool:
+    return "microsoft-standard" in uname().release
+
+
+def import_d_file(in_file) -> str:
+    out_text = ""
+
+    with open(in_file) as file:
+        for idx, line in enumerate(file):
+            if idx == 0:
+                if line.endswith(" \\\n"):
+                    out_text += line[:-3].replace("\\", "/") + " \\\n"
+                else:
+                    out_text += line.replace("\\", "/")
+            else:
+                suffix = ""
+                if line.endswith(" \\\n"):
+                    suffix = " \\"
+                    path = line.lstrip()[:-3]
+                else:
+                    path = line.strip()
+                # lowercase drive letter
+                path = path[0].lower() + path[1:]
+                if path[0] == "z":
+                    # shortcut for z:
+                    path = path[2:].replace("\\", "/")
+                elif in_wsl():
+                    path = path[0:1] + path[2:]
+                    path = os.path.join("/mnt", path.replace("\\", "/"))
+                else:
+                    # use $WINEPREFIX/dosdevices to resolve path
+                    path = os.path.realpath(
+                        os.path.join(winedevices, path.replace("\\", "/"))
+                    )
+                out_text += "\t" + path + suffix + "\n"
+
+    return out_text
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="""Transform a .d file from Wine paths to normal paths"""
+    )
+    parser.add_argument(
+        "d_file",
+        help="""Dependency file in""",
+    )
+    parser.add_argument(
+        "d_file_out",
+        help="""Dependency file out""",
+    )
+    args = parser.parse_args()
+
+    output = import_d_file(args.d_file)
+
+    with open(args.d_file_out, "w", encoding="UTF-8") as f:
+        f.write(output)
+
+
+if __name__ == "__main__":
+    main()

tools/upload_progress.py

@@ -0,0 +1,76 @@
+#!/usr/bin/env python3
+
+###
+# Uploads progress information to https://github.com/decompals/frogress.
+#
+# Usage:
+#   python3 tools/upload_progress.py -b https://progress.decomp.club/ -p [project] -v [version] build/[version]/progress.json
+#
+# If changes are made, please submit a PR to
+# https://github.com/encounter/dtk-template
+###
+
+import argparse
+import json
+import os
+import requests
+import subprocess
+import sys
+
+
+def get_git_commit_timestamp() -> int:
+    return int(
+        subprocess.check_output(["git", "show", "-s", "--format=%ct"])
+        .decode("ascii")
+        .rstrip()
+    )
+
+
+def get_git_commit_sha() -> str:
+    return subprocess.check_output(["git", "rev-parse", "HEAD"]).decode("ascii").strip()
+
+
+def generate_url(args: argparse.Namespace) -> str:
+    url_components = [args.base_url.rstrip("/"), "data"]
+
+    for arg in [args.project, args.version]:
+        if arg != "":
+            url_components.append(arg)
+
+    return str.join("/", url_components) + "/"
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="Upload progress information.")
+    parser.add_argument("-b", "--base_url", help="API base URL", required=True)
+    parser.add_argument("-a", "--api_key", help="API key (env var PROGRESS_API_KEY)")
+    parser.add_argument("-p", "--project", help="Project slug", required=True)
+    parser.add_argument("-v", "--version", help="Version slug", required=True)
+    parser.add_argument("input", help="Progress JSON input")
+
+    args = parser.parse_args()
+    api_key = args.api_key or os.environ.get("PROGRESS_API_KEY")
+    if not api_key:
+        raise "API key required"
+    url = generate_url(args)
+
+    entries = []
+    with open(args.input, "r") as f:
+        data = json.load(f)
+        entries.append(
+            {
+                "timestamp": get_git_commit_timestamp(),
+                "git_hash": get_git_commit_sha(),
+                "categories": data,
+            }
+        )
+
+    print("Publishing entry to", url)
+    json.dump(entries[0], sys.stdout, indent=4)
+    print()
+    r = requests.post(url, json={
+        "api_key": api_key,
+        "entries": entries,
+    })
+    r.raise_for_status()
+    print("Done!")