diff --git a/.flake8 b/.flake8 new file mode 100644 index 00000000..3dd31e6e --- /dev/null +++ b/.flake8 @@ -0,0 +1,4 @@ +[flake8] +# E203: whitespace before ':' +# E501: line too long +extend-ignore = E203,E501 diff --git a/configure.py b/configure.py index 7fd2f1ea..11791df0 100644 --- a/configure.py +++ b/configure.py @@ -117,10 +117,11 @@ if not is_windows(): config.wrapper = args.wrapper # Tool versions +config.binutils_tag = "2.42-1" config.compilers_tag = "20231018" -config.dtk_tag = "v0.6.3" +config.dtk_tag = "v0.7.4" config.sjiswrap_tag = "v1.1.1" -config.wibo_tag = "0.6.3" +config.wibo_tag = "0.6.11" # Project config.config_path = Path("config") / config.version / "config.yml" diff --git a/tools/decompctx.py b/tools/decompctx.py index c42b1828..e86d5ef3 100644 --- a/tools/decompctx.py +++ b/tools/decompctx.py @@ -13,60 +13,74 @@ import argparse import os import re +from typing import List script_dir = os.path.dirname(os.path.realpath(__file__)) root_dir = os.path.abspath(os.path.join(script_dir, "..")) src_dir = os.path.join(root_dir, "src") -include_dir = os.path.join(root_dir, "include") +include_dirs = [ + os.path.join(root_dir, "include"), + # Add additional include directories here +] include_pattern = re.compile(r'^#include\s*[<"](.+?)[>"]$') -guard_pattern = re.compile(r'^#ifndef\s+(.*)$') +guard_pattern = re.compile(r"^#ifndef\s+(.*)$") defines = set() -def import_h_file(in_file: str, r_path: str) -> str: - rel_path = os.path.join(root_dir, r_path, in_file) - inc_path = os.path.join(include_dir, in_file) - if os.path.exists(rel_path): - return import_c_file(rel_path) - elif os.path.exists(inc_path): - return import_c_file(inc_path) - else: - print("Failed to locate", in_file) - exit(1) -def import_c_file(in_file) -> str: +def import_h_file(in_file: str, r_path: str, deps: List[str]) -> str: + rel_path = os.path.join(root_dir, r_path, in_file) + if os.path.exists(rel_path): + return import_c_file(rel_path, deps) + for include_dir in include_dirs: + inc_path = os.path.join(include_dir, in_file) + if os.path.exists(inc_path): + return import_c_file(inc_path, deps) + else: + print("Failed to locate", in_file) + return "" + + +def import_c_file(in_file: str, deps: List[str]) -> str: in_file = os.path.relpath(in_file, root_dir) - out_text = '' + deps.append(in_file) + out_text = "" try: - with open(in_file, encoding="utf-8") as file: - out_text += process_file(in_file, list(file)) + with open(in_file, encoding="utf-8") as file: + out_text += process_file(in_file, list(file), deps) except Exception: - with open(in_file) as file: - out_text += process_file(in_file, list(file)) + with open(in_file) as file: + out_text += process_file(in_file, list(file), deps) return out_text -def process_file(in_file: str, lines) -> str: - out_text = '' + +def process_file(in_file: str, lines: List[str], deps: List[str]) -> str: + out_text = "" for idx, line in enumerate(lines): - guard_match = guard_pattern.match(line.strip()) - if idx == 0: - if guard_match: - if guard_match[1] in defines: - break - defines.add(guard_match[1]) - print("Processing file", in_file) - include_match = include_pattern.match(line.strip()) - if include_match and not include_match[1].endswith(".s"): - out_text += f"/* \"{in_file}\" line {idx} \"{include_match[1]}\" */\n" - out_text += import_h_file(include_match[1], os.path.dirname(in_file)) - out_text += f"/* end \"{include_match[1]}\" */\n" - else: - out_text += line + guard_match = guard_pattern.match(line.strip()) + if idx == 0: + if guard_match: + if guard_match[1] in defines: + break + defines.add(guard_match[1]) + print("Processing file", in_file) + include_match = include_pattern.match(line.strip()) + if include_match and not include_match[1].endswith(".s"): + out_text += f'/* "{in_file}" line {idx} "{include_match[1]}" */\n' + out_text += import_h_file(include_match[1], os.path.dirname(in_file), deps) + out_text += f'/* end "{include_match[1]}" */\n' + else: + out_text += line return out_text + +def sanitize_path(path: str) -> str: + return path.replace("\\", "/").replace(" ", "\\ ") + + def main(): parser = argparse.ArgumentParser( description="""Create a context file which can be used for decomp.me""" @@ -75,13 +89,32 @@ def main(): "c_file", help="""File from which to create context""", ) + parser.add_argument( + "-o", + "--output", + help="""Output file""", + default="ctx.c", + ) + parser.add_argument( + "-d", + "--depfile", + help="""Dependency file""", + ) args = parser.parse_args() - output = import_c_file(args.c_file) + deps = [] + output = import_c_file(args.c_file, deps) - with open(os.path.join(root_dir, "ctx.c"), "w", encoding="utf-8") as f: + with open(os.path.join(root_dir, args.output), "w", encoding="utf-8") as f: f.write(output) + if args.depfile: + with open(os.path.join(root_dir, args.depfile), "w", encoding="utf-8") as f: + f.write(sanitize_path(args.output) + ":") + for dep in deps: + path = sanitize_path(dep) + f.write(f" \\\n\t{path}") + if __name__ == "__main__": - main() \ No newline at end of file + main() diff --git a/tools/download_tool.py b/tools/download_tool.py index 681c65b5..7b386a4b 100644 --- a/tools/download_tool.py +++ b/tools/download_tool.py @@ -16,14 +16,31 @@ import os import platform import shutil import stat -import sys import urllib.request import zipfile - +from typing import Callable, Dict from pathlib import Path -def dtk_url(tag): +def binutils_url(tag): + uname = platform.uname() + system = uname.system.lower() + arch = uname.machine.lower() + if system == "darwin": + system = "macos" + arch = "universal" + elif arch == "amd64": + arch = "x86_64" + + repo = "https://github.com/encounter/gc-wii-binutils" + return f"{repo}/releases/download/{tag}/{system}-{arch}.zip" + + +def compilers_url(tag: str) -> str: + return f"https://files.decomp.dev/compilers_{tag}.zip" + + +def dtk_url(tag: str) -> str: uname = platform.uname() suffix = "" system = uname.system.lower() @@ -39,28 +56,26 @@ def dtk_url(tag): return f"{repo}/releases/download/{tag}/dtk-{system}-{arch}{suffix}" -def sjiswrap_url(tag): +def sjiswrap_url(tag: str) -> str: repo = "https://github.com/encounter/sjiswrap" return f"{repo}/releases/download/{tag}/sjiswrap-windows-x86.exe" -def wibo_url(tag): +def wibo_url(tag: str) -> str: repo = "https://github.com/decompals/wibo" return f"{repo}/releases/download/{tag}/wibo" -def compilers_url(tag): - return f"https://files.decomp.dev/compilers_{tag}.zip" - -TOOLS = { +TOOLS: Dict[str, Callable[[str], str]] = { + "binutils": binutils_url, + "compilers": compilers_url, "dtk": dtk_url, "sjiswrap": sjiswrap_url, "wibo": wibo_url, - "compilers": compilers_url, } -def main(): +def main() -> None: parser = argparse.ArgumentParser() parser.add_argument("tool", help="Tool name") parser.add_argument("output", type=Path, help="output file path") @@ -77,7 +92,11 @@ def main(): data = io.BytesIO(response.read()) with zipfile.ZipFile(data) as f: f.extractall(output) - output.touch(mode=0o755) + # Make all files executable + for root, _, files in os.walk(output): + for name in files: + os.chmod(os.path.join(root, name), 0o755) + output.touch(mode=0o755) # Update dir modtime else: with open(output, "wb") as f: shutil.copyfileobj(response, f) @@ -86,4 +105,4 @@ def main(): if __name__ == "__main__": - main() \ No newline at end of file + main() diff --git a/tools/ninja_syntax.py b/tools/ninja_syntax.py index ffd88a01..97e71f26 100644 --- a/tools/ninja_syntax.py +++ b/tools/ninja_syntax.py @@ -21,50 +21,66 @@ use Python. import re import textwrap +from io import StringIO +from pathlib import Path +from typing import Dict, List, Match, Optional, Tuple, Union + +NinjaPath = Union[str, Path] +NinjaPaths = Union[ + List[str], + List[Path], + List[NinjaPath], + List[Optional[str]], + List[Optional[Path]], + List[Optional[NinjaPath]], +] +NinjaPathOrPaths = Union[NinjaPath, NinjaPaths] -def escape_path(word): +def escape_path(word: str) -> str: return word.replace("$ ", "$$ ").replace(" ", "$ ").replace(":", "$:") class Writer(object): - def __init__(self, output, width=78): + def __init__(self, output: StringIO, width: int = 78) -> None: self.output = output self.width = width - def newline(self): + def newline(self) -> None: self.output.write("\n") - def comment(self, text): + def comment(self, text: str) -> None: for line in textwrap.wrap( text, self.width - 2, break_long_words=False, break_on_hyphens=False ): self.output.write("# " + line + "\n") - def variable(self, key, value, indent=0): - if value is None: - return - if isinstance(value, list): - value = " ".join(filter(None, value)) # Filter out empty strings. + def variable( + self, + key: str, + value: Optional[NinjaPathOrPaths], + indent: int = 0, + ) -> None: + value = " ".join(serialize_paths(value)) self._line("%s = %s" % (key, value), indent) - def pool(self, name, depth): + def pool(self, name: str, depth: int) -> None: self._line("pool %s" % name) - self.variable("depth", depth, indent=1) + self.variable("depth", str(depth), indent=1) def rule( self, - name, - command, - description=None, - depfile=None, - generator=False, - pool=None, - restat=False, - rspfile=None, - rspfile_content=None, - deps=None, - ): + name: str, + command: str, + description: Optional[str] = None, + depfile: Optional[NinjaPath] = None, + generator: bool = False, + pool: Optional[str] = None, + restat: bool = False, + rspfile: Optional[NinjaPath] = None, + rspfile_content: Optional[NinjaPath] = None, + deps: Optional[NinjaPathOrPaths] = None, + ) -> None: self._line("rule %s" % name) self.variable("command", command, indent=1) if description: @@ -86,32 +102,39 @@ class Writer(object): def build( self, - outputs, - rule, - inputs=None, - implicit=None, - order_only=None, - variables=None, - implicit_outputs=None, - pool=None, - dyndep=None, - ): - outputs = as_list(outputs) + outputs: NinjaPathOrPaths, + rule: str, + inputs: Optional[NinjaPathOrPaths] = None, + implicit: Optional[NinjaPathOrPaths] = None, + order_only: Optional[NinjaPathOrPaths] = None, + variables: Optional[ + Union[ + List[Tuple[str, Optional[NinjaPathOrPaths]]], + Dict[str, Optional[NinjaPathOrPaths]], + ] + ] = None, + implicit_outputs: Optional[NinjaPathOrPaths] = None, + pool: Optional[str] = None, + dyndep: Optional[NinjaPath] = None, + ) -> List[str]: + outputs = serialize_paths(outputs) out_outputs = [escape_path(x) for x in outputs] - all_inputs = [escape_path(x) for x in as_list(inputs)] + all_inputs = [escape_path(x) for x in serialize_paths(inputs)] if implicit: - implicit = [escape_path(x) for x in as_list(implicit)] + implicit = [escape_path(x) for x in serialize_paths(implicit)] all_inputs.append("|") - all_inputs.extend(implicit) + all_inputs.extend(map(str, implicit)) if order_only: - order_only = [escape_path(x) for x in as_list(order_only)] + order_only = [escape_path(x) for x in serialize_paths(order_only)] all_inputs.append("||") - all_inputs.extend(order_only) + all_inputs.extend(map(str, order_only)) if implicit_outputs: - implicit_outputs = [escape_path(x) for x in as_list(implicit_outputs)] + implicit_outputs = [ + escape_path(x) for x in serialize_paths(implicit_outputs) + ] out_outputs.append("|") - out_outputs.extend(implicit_outputs) + out_outputs.extend(map(str, implicit_outputs)) self._line( "build %s: %s" % (" ".join(out_outputs), " ".join([rule] + all_inputs)) @@ -119,7 +142,7 @@ class Writer(object): if pool is not None: self._line(" pool = %s" % pool) if dyndep is not None: - self._line(" dyndep = %s" % dyndep) + self._line(" dyndep = %s" % serialize_path(dyndep)) if variables: if isinstance(variables, dict): @@ -132,16 +155,16 @@ class Writer(object): return outputs - def include(self, path): + def include(self, path: str) -> None: self._line("include %s" % path) - def subninja(self, path): + def subninja(self, path: str) -> None: self._line("subninja %s" % path) - def default(self, paths): - self._line("default %s" % " ".join(as_list(paths))) + def default(self, paths: NinjaPathOrPaths) -> None: + self._line("default %s" % " ".join(serialize_paths(paths))) - def _count_dollars_before_index(self, s, i): + def _count_dollars_before_index(self, s: str, i: int) -> int: """Returns the number of '$' characters right in front of s[i].""" dollar_count = 0 dollar_index = i - 1 @@ -150,7 +173,7 @@ class Writer(object): dollar_index -= 1 return dollar_count - def _line(self, text, indent=0): + def _line(self, text: str, indent: int = 0) -> None: """Write 'text' word-wrapped at self.width characters.""" leading_space = " " * indent while len(leading_space) + len(text) > self.width: @@ -187,19 +210,21 @@ class Writer(object): self.output.write(leading_space + text + "\n") - def close(self): + def close(self) -> None: self.output.close() -def as_list(input): - if input is None: - return [] +def serialize_path(input: Optional[NinjaPath]) -> str: + return str(input).replace("\\", "/") if input else "" + + +def serialize_paths(input: Optional[NinjaPathOrPaths]) -> List[str]: if isinstance(input, list): - return input - return [input] + return [serialize_path(path) for path in input if path] + return [serialize_path(input)] if input else [] -def escape(string): +def escape(string: str) -> str: """Escape a string such that it can be embedded into a Ninja file without further interpretation.""" assert "\n" not in string, "Ninja syntax does not allow newlines" @@ -207,17 +232,17 @@ def escape(string): return string.replace("$", "$$") -def expand(string, vars, local_vars={}): +def expand(string: str, vars: Dict[str, str], local_vars: Dict[str, str] = {}) -> str: """Expand a string containing $vars as Ninja would. Note: doesn't handle the full Ninja variable syntax, but it's enough to make configure.py's use of it work. """ - def exp(m): + def exp(m: Match[str]) -> str: var = m.group(1) if var == "$": return "$" return local_vars.get(var, vars.get(var, "")) - return re.sub(r"\$(\$|\w*)", exp, string) \ No newline at end of file + return re.sub(r"\$(\$|\w*)", exp, string) diff --git a/tools/project.py b/tools/project.py index 0f5ad89b..991b94a9 100644 --- a/tools/project.py +++ b/tools/project.py @@ -12,11 +12,13 @@ import io import json +import math import os import platform import sys - from pathlib import Path +from typing import Any, Dict, List, Optional, Set, Tuple, Union + from . import ninja_syntax if sys.platform == "cygwin": @@ -27,46 +29,85 @@ if sys.platform == "cygwin": ) +class Object: + def __init__(self, completed: bool, name: str, **options: Any) -> None: + self.name = name + self.base_name = Path(name).with_suffix("") + self.completed = completed + self.options: Dict[str, Any] = { + "add_to_all": True, + "asflags": None, + "extra_asflags": None, + "cflags": None, + "extra_cflags": None, + "mw_version": None, + "shift_jis": None, + "source": name, + } + self.options.update(options) + + +PathLike = Union[str, os.PathLike] + + class ProjectConfig: - def __init__(self): + def __init__(self) -> None: # Paths - self.build_dir = Path("build") - self.src_dir = Path("src") - self.tools_dir = Path("tools") + self.build_dir: PathLike = Path("build") # Output build files + self.src_dir: PathLike = Path("src") # C/C++/asm source files + self.tools_dir: PathLike = Path("tools") # Python scripts + self.asm_dir: PathLike = Path( + "asm" + ) # Override incomplete objects (for modding) # Tooling - self.dtk_tag = None # Git tag - self.build_dtk_path = None # If None, download - self.compilers_tag = None # 1 - self.compilers_path = None # If None, download - self.wibo_tag = None # Git tag - self.wrapper = None # If None, download wibo on Linux - self.sjiswrap_tag = None # Git tag - self.sjiswrap_path = None # If None, download + self.binutils_tag: Optional[str] = None # Git tag + self.binutils_path: Optional[PathLike] = None # If None, download + self.dtk_tag: Optional[str] = None # Git tag + self.build_dtk_path: Optional[PathLike] = None # If None, download + self.compilers_tag: Optional[str] = None # 1 + self.compilers_path: Optional[PathLike] = None # If None, download + self.wibo_tag: Optional[str] = None # Git tag + self.wrapper: Optional[PathLike] = None # If None, download wibo on Linux + self.sjiswrap_tag: Optional[str] = None # Git tag + self.sjiswrap_path: Optional[PathLike] = None # If None, download # Project config - self.build_rels = True # Build REL files - self.check_sha_path = None # Path to version.sha1 - self.config_path = None # Path to config.yml - self.debug = False # Build with debug info - self.generate_map = False # Generate map file(s) - self.ldflags = None # Linker flags - self.libs = None # List of libraries - self.linker_version = None # mwld version - self.version = None # Version name - self.warn_missing_config = False # Warn on missing unit configuration - self.warn_missing_source = False # Warn on missing source file - self.rel_strip_partial = True # Generate PLFs with -strip_partial - self.rel_empty_file = None # Path to empty.c for generating empty RELs + self.build_rels: bool = True # Build REL files + self.check_sha_path: Optional[PathLike] = None # Path to version.sha1 + self.config_path: Optional[PathLike] = None # Path to config.yml + self.debug: bool = False # Build with debug info + self.generate_map: bool = False # Generate map file(s) + self.asflags: Optional[List[str]] = None # Assembler flags + self.ldflags: Optional[List[str]] = None # Linker flags + self.libs: Optional[List[Dict[str, Any]]] = None # List of libraries + self.linker_version: Optional[str] = None # mwld version + self.version: Optional[str] = None # Version name + self.warn_missing_config: bool = False # Warn on missing unit configuration + self.warn_missing_source: bool = False # Warn on missing source file + self.rel_strip_partial: bool = True # Generate PLFs with -strip_partial + self.rel_empty_file: Optional[PathLike] = ( + None # Path to empty.c for generating empty RELs + ) + self.shift_jis = ( + True # Convert source files from UTF-8 to Shift JIS automatically + ) # Progress output and progress.json config - self.progress_all = True # Include combined "all" category - self.progress_modules = True # Include combined "modules" category - self.progress_each_module = ( + self.progress_all: bool = True # Include combined "all" category + self.progress_modules: bool = True # Include combined "modules" category + self.progress_each_module: bool = ( True # Include individual modules, disable for large numbers of modules ) - def validate(self): + # Progress fancy printing + self.progress_use_fancy: bool = False + self.progress_code_fancy_frac: int = 0 + self.progress_code_fancy_item: str = "" + self.progress_data_fancy_frac: int = 0 + self.progress_data_fancy_item: str = "" + + def validate(self) -> None: required_attrs = [ "build_dir", "src_dir", @@ -82,33 +123,18 @@ class ProjectConfig: if getattr(self, attr) is None: sys.exit(f"ProjectConfig.{attr} missing") - def find_object(self, name): - for lib in self.libs: + def find_object(self, name: str) -> Optional[Tuple[Dict[str, Any], Object]]: + for lib in self.libs or {}: for obj in lib["objects"]: if obj.name == name: - return [lib, obj] + return lib, obj return None - def out_path(self): - return self.build_dir / self.version + def out_path(self) -> Path: + return self.build_dir / str(self.version) -class Object: - def __init__(self, completed, name, **options): - self.name = name - self.completed = completed - self.options = { - "add_to_all": True, - "cflags": None, - "extra_cflags": None, - "mw_version": None, - "shiftjis": True, - "source": name, - } - self.options.update(options) - - -def is_windows(): +def is_windows() -> bool: return os.name == "nt" @@ -118,36 +144,25 @@ CHAIN = "cmd /c " if is_windows() else "" EXE = ".exe" if is_windows() else "" -# Replace forward slashes with backslashes on Windows -def os_str(value): - return str(value).replace("/", os.sep) - - -# Replace backslashes with forward slashes on Windows -def unix_str(value): - return str(value).replace(os.sep, "/") - - -# Stringify paths for ninja_syntax -def path(value): - if value is None: - return None - elif isinstance(value, list): - return list(map(os_str, filter(lambda x: x is not None, value))) +def make_flags_str(cflags: Union[str, List[str]]) -> str: + if isinstance(cflags, list): + return " ".join(cflags) else: - return [os_str(value)] + return cflags # Load decomp-toolkit generated config.json -def load_build_config(config, build_config_path): +def load_build_config( + config: ProjectConfig, build_config_path: Path +) -> Optional[Dict[str, Any]]: if not build_config_path.is_file(): return None - def versiontuple(v): + def versiontuple(v: str) -> Tuple[int, ...]: return tuple(map(int, (v.split(".")))) f = open(build_config_path, "r", encoding="utf-8") - build_config = json.load(f) + build_config: Dict[str, Any] = json.load(f) config_version = build_config.get("version") if not config_version: # Invalid config.json @@ -155,7 +170,7 @@ def load_build_config(config, build_config_path): os.remove(build_config_path) return None - dtk_version = config.dtk_tag[1:] # Strip v + dtk_version = str(config.dtk_tag)[1:] # Strip v if versiontuple(config_version) < versiontuple(dtk_version): # Outdated config.json f.close() @@ -167,14 +182,16 @@ def load_build_config(config, build_config_path): # Generate build.ninja and objdiff.json -def generate_build(config): +def generate_build(config: ProjectConfig) -> None: build_config = load_build_config(config, config.out_path() / "config.json") generate_build_ninja(config, build_config) generate_objdiff_config(config, build_config) # Generate build.ninja -def generate_build_ninja(config, build_config): +def generate_build_ninja( + config: ProjectConfig, build_config: Optional[Dict[str, Any]] +) -> None: config.validate() out = io.StringIO() @@ -182,9 +199,9 @@ def generate_build_ninja(config, build_config): n.variable("ninja_required_version", "1.3") n.newline() - configure_script = os.path.relpath(os.path.abspath(sys.argv[0])) - python_lib = os.path.relpath(__file__) - python_lib_dir = os.path.dirname(python_lib) + configure_script = Path(os.path.relpath(os.path.abspath(sys.argv[0]))) + python_lib = Path(os.path.relpath(__file__)) + python_lib_dir = python_lib.parent n.comment("The arguments passed to configure.py, for rerunning it.") n.variable("configure_args", sys.argv[1:]) n.variable("python", f'"{sys.executable}"') @@ -194,7 +211,7 @@ def generate_build_ninja(config, build_config): # Variables ### n.comment("Variables") - ldflags = " ".join(config.ldflags) + ldflags = " ".join(config.ldflags or []) if config.generate_map: ldflags += " -mapunused" if config.debug: @@ -209,6 +226,7 @@ def generate_build_ninja(config, build_config): n.comment("Tooling") build_path = config.out_path() + progress_path = build_path / "progress.json" build_tools_path = config.build_dir / "tools" download_tool = config.tools_dir / "download_tool.py" n.rule( @@ -217,20 +235,29 @@ def generate_build_ninja(config, build_config): description="TOOL $out", ) + decompctx = config.tools_dir / "decompctx.py" + n.rule( + name="decompctx", + command=f"$python {decompctx} $in -o $out -d $out.d", + description="CTX $in", + depfile="$out.d", + deps="gcc", + ) + if config.build_dtk_path: dtk = build_tools_path / "release" / f"dtk{EXE}" n.rule( name="cargo", command="cargo build --release --manifest-path $in --bin $bin --target-dir $target", description="CARGO $bin", - depfile=path(Path("$target") / "release" / "$bin.d"), + depfile=Path("$target") / "release" / "$bin.d", deps="gcc", ) n.build( - outputs=path(dtk), + outputs=dtk, rule="cargo", - inputs=path(config.build_dtk_path / "Cargo.toml"), - implicit=path(config.build_dtk_path / "Cargo.lock"), + inputs=config.build_dtk_path / "Cargo.toml", + implicit=config.build_dtk_path / "Cargo.lock", variables={ "bin": "dtk", "target": build_tools_path, @@ -239,9 +266,9 @@ def generate_build_ninja(config, build_config): elif config.dtk_tag: dtk = build_tools_path / f"dtk{EXE}" n.build( - outputs=path(dtk), + outputs=dtk, rule="download_tool", - implicit=path(download_tool), + implicit=download_tool, variables={ "tool": "dtk", "tag": config.dtk_tag, @@ -255,9 +282,9 @@ def generate_build_ninja(config, build_config): elif config.sjiswrap_tag: sjiswrap = build_tools_path / "sjiswrap.exe" n.build( - outputs=path(sjiswrap), + outputs=sjiswrap, rule="download_tool", - implicit=path(download_tool), + implicit=download_tool, variables={ "tool": "sjiswrap", "tag": config.sjiswrap_tag, @@ -268,7 +295,7 @@ def generate_build_ninja(config, build_config): # Only add an implicit dependency on wibo if we download it wrapper = config.wrapper - wrapper_implicit = None + wrapper_implicit: Optional[Path] = None if ( config.wibo_tag is not None and sys.platform == "linux" @@ -278,33 +305,53 @@ def generate_build_ninja(config, build_config): wrapper = build_tools_path / "wibo" wrapper_implicit = wrapper n.build( - outputs=path(wrapper), + outputs=wrapper, rule="download_tool", - implicit=path(download_tool), + implicit=download_tool, variables={ "tool": "wibo", "tag": config.wibo_tag, }, ) if not is_windows() and wrapper is None: - wrapper = "wine" + wrapper = Path("wine") wrapper_cmd = f"{wrapper} " if wrapper else "" - compilers_implicit = None + compilers_implicit: Optional[Path] = None if config.compilers_path: compilers = config.compilers_path elif config.compilers_tag: compilers = config.build_dir / "compilers" compilers_implicit = compilers n.build( - outputs=path(compilers), + outputs=compilers, rule="download_tool", - implicit=path(download_tool), + implicit=download_tool, variables={ "tool": "compilers", "tag": config.compilers_tag, }, ) + else: + sys.exit("ProjectConfig.compilers_tag missing") + + binutils_implicit = None + if config.binutils_path: + binutils = config.binutils_path + elif config.binutils_tag: + binutils = config.build_dir / "binutils" + binutils_implicit = binutils + n.build( + outputs=binutils, + rule="download_tool", + implicit=download_tool, + variables={ + "tool": "binutils", + "tag": config.binutils_tag, + }, + ) + else: + sys.exit("ProjectConfig.binutils_tag missing") n.newline() @@ -316,16 +363,24 @@ def generate_build_ninja(config, build_config): # MWCC mwcc = compiler_path / "mwcceppc.exe" mwcc_cmd = f"{wrapper_cmd}{mwcc} $cflags -MMD -c $in -o $basedir" - mwcc_implicit = [compilers_implicit or mwcc, wrapper_implicit] + mwcc_implicit: List[Optional[Path]] = [compilers_implicit or mwcc, wrapper_implicit] # MWCC with UTF-8 to Shift JIS wrapper mwcc_sjis_cmd = f"{wrapper_cmd}{sjiswrap} {mwcc} $cflags -MMD -c $in -o $basedir" - mwcc_sjis_implicit = [*mwcc_implicit, sjiswrap] + mwcc_sjis_implicit: List[Optional[Path]] = [*mwcc_implicit, sjiswrap] # MWLD mwld = compiler_path / "mwldeppc.exe" mwld_cmd = f"{wrapper_cmd}{mwld} $ldflags -o $out @$out.rsp" - mwld_implicit = [compilers_implicit or mwld, wrapper_implicit] + mwld_implicit: List[Optional[Path]] = [compilers_implicit or mwld, wrapper_implicit] + + # GNU as + gnu_as = binutils / f"powerpc-eabi-as{EXE}" + gnu_as_cmd = ( + f"{CHAIN}{gnu_as} $asflags -o $out $in -MD $out.d" + + f" && {dtk} elf fixup $out $out" + ) + gnu_as_implicit = [binutils_implicit or gnu_as, dtk] if os.name != "nt": transform_dep = config.tools_dir / "transform_dep.py" @@ -372,6 +427,16 @@ def generate_build_ninja(config, build_config): ) n.newline() + n.comment("Assemble asm") + n.rule( + name="as", + command=gnu_as_cmd, + description="AS $out", + depfile="$out.d", + deps="gcc", + ) + n.newline() + n.comment("Host build") n.variable("host_cflags", "-I include -Wno-trigraphs") n.variable( @@ -394,37 +459,38 @@ def generate_build_ninja(config, build_config): # Source files ### n.comment("Source files") + build_asm_path = build_path / "mod" build_src_path = build_path / "src" build_host_path = build_path / "host" build_config_path = build_path / "config.json" - def map_path(path): + def map_path(path: Path) -> Path: return path.parent / (path.name + ".MAP") class LinkStep: - def __init__(self, config): - self.name = config["name"] - self.module_id = config["module_id"] - self.ldscript = config["ldscript"] + def __init__(self, config: Dict[str, Any]) -> None: + self.name: str = config["name"] + self.module_id: int = config["module_id"] + self.ldscript: Optional[Path] = config["ldscript"] self.entry = config["entry"] - self.inputs = [] + self.inputs: List[str] = [] - def add(self, obj): - self.inputs.append(obj) + def add(self, obj: os.PathLike) -> None: + self.inputs.append(str(obj)) - def output(self): + def output(self) -> Path: if self.module_id == 0: return build_path / f"{self.name}.dol" else: return build_path / self.name / f"{self.name}.rel" - def partial_output(self): + def partial_output(self) -> Path: if self.module_id == 0: return build_path / f"{self.name}.elf" else: return build_path / self.name / f"{self.name}.plf" - def write(self, n): + def write(self, n: ninja_syntax.Writer) -> None: n.comment(f"Link {self.name}") if self.module_id == 0: elf_path = build_path / f"{self.name}.elf" @@ -436,23 +502,23 @@ def generate_build_ninja(config, build_config): else: elf_map = None n.build( - outputs=path(elf_path), + outputs=elf_path, rule="link", - inputs=path(self.inputs), - implicit=path([self.ldscript, *mwld_implicit]), - implicit_outputs=path(elf_map), + inputs=self.inputs, + implicit=[self.ldscript, *mwld_implicit], + implicit_outputs=elf_map, variables={"ldflags": elf_ldflags}, ) n.build( - outputs=path(dol_path), + outputs=dol_path, rule="elf2dol", - inputs=path(elf_path), - implicit=path(dtk), + inputs=elf_path, + implicit=dtk, ) else: preplf_path = build_path / self.name / f"{self.name}.preplf" plf_path = build_path / self.name / f"{self.name}.plf" - preplf_ldflags = f"$ldflags -sdata 0 -sdata2 0 -r" + preplf_ldflags = "$ldflags -sdata 0 -sdata2 0 -r" plf_ldflags = f"$ldflags -sdata 0 -sdata2 0 -r1 -lcf {self.ldscript}" if self.entry: plf_ldflags += f" -m {self.entry}" @@ -468,37 +534,133 @@ def generate_build_ninja(config, build_config): preplf_map = None plf_map = None n.build( - outputs=path(preplf_path), + outputs=preplf_path, rule="link", - inputs=path(self.inputs), - implicit=path(mwld_implicit), - implicit_outputs=path(preplf_map), + inputs=self.inputs, + implicit=mwld_implicit, + implicit_outputs=preplf_map, variables={"ldflags": preplf_ldflags}, ) n.build( - outputs=path(plf_path), + outputs=plf_path, rule="link", - inputs=path(self.inputs), - implicit=path([self.ldscript, preplf_path, *mwld_implicit]), - implicit_outputs=path(plf_map), + inputs=self.inputs, + implicit=[self.ldscript, preplf_path, *mwld_implicit], + implicit_outputs=plf_map, variables={"ldflags": plf_ldflags}, ) n.newline() if build_config: - link_steps = [] - used_compiler_versions = set() - source_inputs = [] - host_source_inputs = [] - source_added = set() + link_steps: List[LinkStep] = [] + used_compiler_versions: Set[str] = set() + source_inputs: List[Path] = [] + host_source_inputs: List[Path] = [] + source_added: Set[Path] = set() - def make_cflags_str(cflags): - if isinstance(cflags, list): - return " ".join(cflags) - else: - return cflags + def c_build( + obj: Object, options: Dict[str, Any], lib_name: str, src_path: Path + ) -> Optional[Path]: - def add_unit(build_obj, link_step): + cflags_str = make_flags_str(options["cflags"]) + if options["extra_cflags"] is not None: + extra_cflags_str = make_flags_str(options["extra_cflags"]) + cflags_str += " " + extra_cflags_str + used_compiler_versions.add(options["mw_version"]) + + src_obj_path = build_src_path / f"{obj.base_name}.o" + src_base_path = build_src_path / obj.base_name + + # Avoid creating duplicate build rules + if src_obj_path in source_added: + return src_obj_path + source_added.add(src_obj_path) + + shift_jis = options["shift_jis"] + if shift_jis is None: + shift_jis = config.shift_jis + + # Add MWCC build rule + n.comment(f"{obj.name}: {lib_name} (linked {obj.completed})") + n.build( + outputs=src_obj_path, + rule="mwcc_sjis" if shift_jis else "mwcc", + inputs=src_path, + variables={ + "mw_version": Path(options["mw_version"]), + "cflags": cflags_str, + "basedir": os.path.dirname(src_base_path), + "basefile": src_base_path, + }, + implicit=mwcc_sjis_implicit if shift_jis else mwcc_implicit, + ) + + # Add ctx build rule + ctx_path = build_src_path / f"{obj.base_name}.ctx" + n.build( + outputs=ctx_path, + rule="decompctx", + inputs=src_path, + implicit=decompctx, + ) + + # Add host build rule + if options.get("host", False): + host_obj_path = build_host_path / f"{obj.base_name}.o" + host_base_path = build_host_path / obj.base_name + n.build( + outputs=host_obj_path, + rule="host_cc" if src_path.suffix == ".c" else "host_cpp", + inputs=src_path, + variables={ + "basedir": os.path.dirname(host_base_path), + "basefile": host_base_path, + }, + ) + if options["add_to_all"]: + host_source_inputs.append(host_obj_path) + n.newline() + + if options["add_to_all"]: + source_inputs.append(src_obj_path) + + return src_obj_path + + def asm_build( + obj: Object, options: Dict[str, Any], lib_name: str, src_path: Path + ) -> Optional[Path]: + asflags = options["asflags"] or config.asflags + if asflags is None: + sys.exit("ProjectConfig.asflags missing") + asflags_str = make_flags_str(asflags) + if options["extra_asflags"] is not None: + extra_asflags_str = make_flags_str(options["extra_asflags"]) + asflags_str += " " + extra_asflags_str + + asm_obj_path = build_asm_path / f"{obj.base_name}.o" + + # Avoid creating duplicate build rules + if asm_obj_path in source_added: + return asm_obj_path + source_added.add(asm_obj_path) + + # Add assembler build rule + n.comment(f"{obj.name}: {lib_name} (linked {obj.completed})") + n.build( + outputs=asm_obj_path, + rule="as", + inputs=src_path, + variables={"asflags": asflags_str}, + implicit=gnu_as_implicit, + ) + n.newline() + + if options["add_to_all"]: + source_inputs.append(asm_obj_path) + + return asm_obj_path + + def add_unit(build_obj, link_step: LinkStep): obj_path, obj_name = build_obj["object"], build_obj["name"] result = config.find_object(obj_name) if not result: @@ -509,70 +671,47 @@ def generate_build_ninja(config, build_config): lib, obj = result lib_name = lib["lib"] - src_dir = Path(lib.get("src_dir", config.src_dir)) - options = obj.options - completed = obj.completed + # Use object options, then library options + options = lib.copy() + for key, value in obj.options.items(): + if value is not None or key not in options: + options[key] = value - unit_src_path = src_dir / options["source"] - if not unit_src_path.exists(): - if config.warn_missing_source or completed: + unit_src_path = Path(lib.get("src_dir", config.src_dir)) / options["source"] + if config.asm_dir is not None: + unit_asm_path = ( + Path(lib.get("asm_dir", config.asm_dir)) / options["source"] + ).with_suffix(".s") + + link_built_obj = obj.completed + if unit_src_path.exists(): + if unit_src_path.suffix in (".c", ".cp", ".cpp"): + # Add MWCC & host build rules + built_obj_path = c_build(obj, options, lib_name, unit_src_path) + elif unit_src_path.suffix == ".s": + # Add assembler build rule + built_obj_path = asm_build(obj, options, lib_name, unit_src_path) + else: + sys.exit(f"Unknown source file type {unit_src_path}") + else: + if config.warn_missing_source or obj.completed: print(f"Missing source file {unit_src_path}") + link_built_obj = False + + # Assembly overrides + if unit_asm_path is not None and unit_asm_path.exists(): + link_built_obj = True + built_obj_path = asm_build(obj, options, lib_name, unit_asm_path) + + if link_built_obj and built_obj_path is not None: + # Use the source-built object + link_step.add(built_obj_path) + elif obj_path is not None: + # Use the original (extracted) object link_step.add(obj_path) - return - - mw_version = options["mw_version"] or lib["mw_version"] - cflags_str = make_cflags_str(options["cflags"] or lib["cflags"]) - if options["extra_cflags"] is not None: - extra_cflags_str = make_cflags_str(options["extra_cflags"]) - cflags_str += " " + extra_cflags_str - used_compiler_versions.add(mw_version) - - base_object = Path(obj.name).with_suffix("") - src_obj_path = build_src_path / f"{base_object}.o" - src_base_path = build_src_path / base_object - - if src_obj_path not in source_added: - source_added.add(src_obj_path) - - n.comment(f"{obj_name}: {lib_name} (linked {completed})") - n.build( - outputs=path(src_obj_path), - rule="mwcc_sjis" if options["shiftjis"] else "mwcc", - inputs=path(unit_src_path), - variables={ - "mw_version": path(Path(mw_version)), - "cflags": cflags_str, - "basedir": os.path.dirname(src_base_path), - "basefile": path(src_base_path), - }, - implicit=path( - mwcc_sjis_implicit if options["shiftjis"] else mwcc_implicit - ), - ) - - if lib["host"]: - host_obj_path = build_host_path / f"{base_object}.o" - host_base_path = build_host_path / base_object - n.build( - outputs=path(host_obj_path), - rule="host_cc" if unit_src_path.suffix == ".c" else "host_cpp", - inputs=path(unit_src_path), - variables={ - "basedir": os.path.dirname(host_base_path), - "basefile": path(host_base_path), - }, - ) - if options["add_to_all"]: - host_source_inputs.append(host_obj_path) - n.newline() - - if options["add_to_all"]: - source_inputs.append(src_obj_path) - - if completed: - obj_path = src_obj_path - link_step.add(obj_path) + else: + sys.exit(f"Missing object for {obj_name}: {unit_src_path} {lib} {obj}") # Add DOL link step link_step = LinkStep(build_config) @@ -608,7 +747,7 @@ def generate_build_ninja(config, build_config): sys.exit(f"Compiler {mw_path} does not exist") # Check if linker exists - mw_path = compilers / config.linker_version / "mwldeppc.exe" + mw_path = compilers / str(config.linker_version) / "mwldeppc.exe" if config.compilers_path and not os.path.exists(mw_path): sys.exit(f"Linker {mw_path} does not exist") @@ -648,7 +787,7 @@ def generate_build_ninja(config, build_config): rels_to_generate = list( filter( lambda step: step.module_id != 0 - and not step.name in generated_rels, + and step.name not in generated_rels, link_steps_local, ) ) @@ -669,15 +808,13 @@ def generate_build_ninja(config, build_config): ) rel_names_arg = " ".join(map(lambda name: f"-n {name}", rel_names)) n.build( - outputs=path(rel_outputs), + outputs=rel_outputs, rule="makerel", - inputs=path( - list(map(lambda step: step.partial_output(), link_steps_local)) - ), - implicit=path([dtk, config.config_path]), + inputs=list(map(lambda step: step.partial_output(), link_steps_local)), + implicit=[dtk, config.config_path], variables={ - "config": path(config.config_path), - "rspfile": path(config.out_path() / f"rel{idx}.rsp"), + "config": config.config_path, + "rspfile": config.out_path() / f"rel{idx}.rsp", "names": rel_names_arg, }, ) @@ -690,7 +827,7 @@ def generate_build_ninja(config, build_config): n.build( outputs="all_source", rule="phony", - inputs=path(source_inputs), + inputs=source_inputs, ) n.newline() @@ -701,7 +838,7 @@ def generate_build_ninja(config, build_config): n.build( outputs="all_source_host", rule="phony", - inputs=path(host_source_inputs), + inputs=host_source_inputs, ) n.newline() @@ -717,10 +854,10 @@ def generate_build_ninja(config, build_config): description="CHECK $in", ) n.build( - outputs=path(ok_path), + outputs=ok_path, rule="check", - inputs=path(config.check_sha_path), - implicit=path([dtk, *map(lambda step: step.output(), link_steps)]), + inputs=config.check_sha_path, + implicit=[dtk, *map(lambda step: step.output(), link_steps)], ) n.newline() @@ -728,16 +865,15 @@ def generate_build_ninja(config, build_config): # Calculate progress ### n.comment("Calculate progress") - progress_path = build_path / "progress.json" n.rule( name="progress", command=f"$python {configure_script} $configure_args progress", description="PROGRESS", ) n.build( - outputs=path(progress_path), + outputs=progress_path, rule="progress", - implicit=path([ok_path, configure_script, python_lib, config.config_path]), + implicit=[ok_path, configure_script, python_lib, config.config_path], ) ### @@ -753,7 +889,7 @@ def generate_build_ninja(config, build_config): description=f"DIFF {dol_elf_path}", ) n.build( - inputs=path([config.config_path, dol_elf_path]), + inputs=[config.config_path, dol_elf_path], outputs="dol_diff", rule="dol_diff", ) @@ -771,10 +907,10 @@ def generate_build_ninja(config, build_config): description=f"APPLY {dol_elf_path}", ) n.build( - inputs=path([config.config_path, dol_elf_path]), + inputs=[config.config_path, dol_elf_path], outputs="dol_apply", rule="dol_apply", - implicit=path([ok_path]), + implicit=[ok_path], ) n.build( outputs="apply", @@ -795,11 +931,11 @@ def generate_build_ninja(config, build_config): deps="gcc", ) n.build( - inputs=path(config.config_path), - outputs=path(build_config_path), + inputs=config.config_path, + outputs=build_config_path, rule="split", - implicit=path(dtk), - variables={"out_dir": path(build_path)}, + implicit=dtk, + variables={"out_dir": build_path}, ) n.newline() @@ -816,14 +952,12 @@ def generate_build_ninja(config, build_config): n.build( outputs="build.ninja", rule="configure", - implicit=path( - [ - build_config_path, - configure_script, - python_lib, - Path(python_lib_dir) / "ninja_syntax.py", - ] - ), + implicit=[ + build_config_path, + configure_script, + python_lib, + python_lib_dir / "ninja_syntax.py", + ], ) n.newline() @@ -832,9 +966,9 @@ def generate_build_ninja(config, build_config): ### n.comment("Default rule") if build_config: - n.default(path(progress_path)) + n.default(progress_path) else: - n.default(path(build_config_path)) + n.default(build_config_path) # Write build.ninja with open("build.ninja", "w", encoding="utf-8") as f: @@ -843,12 +977,14 @@ def generate_build_ninja(config, build_config): # Generate objdiff.json -def generate_objdiff_config(config, build_config): +def generate_objdiff_config( + config: ProjectConfig, build_config: Optional[Dict[str, Any]] +) -> None: if not build_config: return - objdiff_config = { - "min_version": "0.4.3", + objdiff_config: Dict[str, Any] = { + "min_version": "1.0.0", "custom_make": "ninja", "build_target": False, "watch_patterns": [ @@ -866,18 +1002,50 @@ def generate_objdiff_config(config, build_config): "units": [], } + # decomp.me compiler name mapping + # Commented out versions have not been added to decomp.me yet + COMPILER_MAP = { + "GC/1.0": "mwcc_233_144", + "GC/1.1": "mwcc_233_159", + "GC/1.2.5": "mwcc_233_163", + "GC/1.2.5e": "mwcc_233_163e", + "GC/1.2.5n": "mwcc_233_163n", + "GC/1.3.2": "mwcc_242_81", + "GC/1.3.2r": "mwcc_242_81r", + "GC/2.0": "mwcc_247_92", + "GC/2.5": "mwcc_247_105", + "GC/2.6": "mwcc_247_107", + "GC/2.7": "mwcc_247_108", + "GC/3.0": "mwcc_41_60831", + # "GC/3.0a3": "mwcc_41_51213", + "GC/3.0a3.2": "mwcc_41_60126", + # "GC/3.0a3.3": "mwcc_41_60209", + # "GC/3.0a3.4": "mwcc_42_60308", + # "GC/3.0a5": "mwcc_42_60422", + "GC/3.0a5.2": "mwcc_41_60831", + "Wii/0x4201_127": "mwcc_42_142", + # "Wii/1.0": "mwcc_43_145", + # "Wii/1.0RC1": "mwcc_42_140", + "Wii/1.0a": "mwcc_42_142", + "Wii/1.1": "mwcc_43_151", + "Wii/1.3": "mwcc_43_172", + # "Wii/1.5": "mwcc_43_188", + "Wii/1.6": "mwcc_43_202", + "Wii/1.7": "mwcc_43_213", + } + build_path = config.out_path() - def add_unit(build_obj, module_name): + def add_unit(build_obj: Dict[str, Any], module_name: str) -> None: if build_obj["autogenerated"]: # Skip autogenerated objects return obj_path, obj_name = build_obj["object"], build_obj["name"] base_object = Path(obj_name).with_suffix("") - unit_config = { - "name": unix_str(Path(module_name) / base_object), - "target_path": unix_str(obj_path), + unit_config: Dict[str, Any] = { + "name": Path(module_name) / base_object, + "target_path": obj_path, } result = config.find_object(obj_name) @@ -888,13 +1056,21 @@ def generate_objdiff_config(config, build_config): lib, obj = result src_dir = Path(lib.get("src_dir", config.src_dir)) - unit_src_path = src_dir / obj.options["source"] + # Use object options, then library options + options = lib.copy() + for key, value in obj.options.items(): + if value is not None or key not in options: + options[key] = value + + unit_src_path = src_dir / str(options["source"]) + if not unit_src_path.exists(): objdiff_config["units"].append(unit_config) return - cflags = obj.options["cflags"] or lib["cflags"] - src_obj_path = build_path / "src" / f"{base_object}.o" + cflags = options["cflags"] + src_obj_path = build_path / "src" / f"{obj.base_name}.o" + src_ctx_path = build_path / "src" / f"{obj.base_name}.ctx" reverse_fn_order = False if type(cflags) is list: @@ -907,9 +1083,32 @@ def generate_objdiff_config(config, build_config): elif value == "nodeferred": reverse_fn_order = False - unit_config["base_path"] = unix_str(src_obj_path) + # Filter out include directories + def keep_flag(flag): + return not flag.startswith("-i ") and not flag.startswith("-I ") + + cflags = list(filter(keep_flag, cflags)) + + # Add appropriate lang flag + if unit_src_path.suffix in (".cp", ".cpp"): + cflags.insert(0, "-lang=c++") + else: + cflags.insert(0, "-lang=c") + + unit_config["base_path"] = src_obj_path unit_config["reverse_fn_order"] = reverse_fn_order unit_config["complete"] = obj.completed + compiler_version = COMPILER_MAP.get(options["mw_version"]) + if compiler_version is None: + print(f"Missing scratch compiler mapping for {options['mw_version']}") + else: + unit_config["scratch"] = { + "platform": "gc_wii", + "compiler": compiler_version, + "c_flags": make_flags_str(cflags), + "ctx_path": src_ctx_path, + "build_ctx": True, + } objdiff_config["units"].append(unit_config) # Add DOL units @@ -923,28 +1122,34 @@ def generate_objdiff_config(config, build_config): # Write objdiff.json with open("objdiff.json", "w", encoding="utf-8") as w: - json.dump(objdiff_config, w, indent=4) + from .ninja_syntax import serialize_path + + json.dump(objdiff_config, w, indent=4, default=serialize_path) # Calculate, print and write progress to progress.json -def calculate_progress(config): +def calculate_progress(config: ProjectConfig) -> None: out_path = config.out_path() build_config = load_build_config(config, out_path / "config.json") if not build_config: return class ProgressUnit: - def __init__(self, name): - self.name = name - self.code_total = 0 - self.code_progress = 0 - self.data_total = 0 - self.data_progress = 0 - self.objects_progress = 0 - self.objects_total = 0 - self.objects = set() + def __init__(self, name: str) -> None: + self.name: str = name + self.code_total: int = 0 + self.code_fancy_frac: int = config.progress_code_fancy_frac + self.code_fancy_item: str = config.progress_code_fancy_item + self.code_progress: int = 0 + self.data_total: int = 0 + self.data_fancy_frac: int = config.progress_data_fancy_frac + self.data_fancy_item: str = config.progress_data_fancy_item + self.data_progress: int = 0 + self.objects_progress: int = 0 + self.objects_total: int = 0 + self.objects: Set[Object] = set() - def add(self, build_obj): + def add(self, build_obj: Dict[str, Any]) -> None: self.code_total += build_obj["code_size"] self.data_total += build_obj["data_size"] @@ -971,10 +1176,10 @@ def calculate_progress(config): if include_object: self.objects_progress += 1 - def code_frac(self): + def code_frac(self) -> float: return self.code_progress / self.code_total - def data_frac(self): + def data_frac(self) -> float: return self.data_progress / self.data_total # Add DOL units @@ -987,7 +1192,7 @@ def calculate_progress(config): # Add REL units rels_progress = ProgressUnit("Modules") if config.progress_modules else None - modules_progress = [] + modules_progress: List[ProgressUnit] = [] for module in build_config["modules"]: progress = ProgressUnit(module["name"]) modules_progress.append(progress) @@ -1001,7 +1206,10 @@ def calculate_progress(config): # Print human-readable progress print("Progress:") - def print_category(unit): + def print_category(unit: Optional[ProgressUnit]) -> None: + if unit is None: + return + code_frac = unit.code_frac() data_frac = unit.data_frac() print( @@ -1009,6 +1217,17 @@ def calculate_progress(config): ) print(f" Code: {unit.code_progress} / {unit.code_total} bytes") print(f" Data: {unit.data_progress} / {unit.data_total} bytes") + if config.progress_use_fancy: + print( + "\nYou have {} out of {} {} and collected {} out of {} {}.".format( + math.floor(code_frac * unit.code_fancy_frac), + unit.code_fancy_frac, + unit.code_fancy_item, + math.floor(data_frac * unit.data_fancy_frac), + unit.data_fancy_frac, + unit.data_fancy_item, + ) + ) if all_progress: print_category(all_progress) @@ -1021,9 +1240,9 @@ def calculate_progress(config): print_category(progress) # Generate and write progress.json - progress_json = {} + progress_json: Dict[str, Any] = {} - def add_category(category, unit): + def add_category(category: str, unit: ProgressUnit) -> None: progress_json[category] = { "code": unit.code_progress, "code/total": unit.code_total, diff --git a/tools/transform_dep.py b/tools/transform_dep.py index 86bd2ecb..124de04b 100644 --- a/tools/transform_dep.py +++ b/tools/transform_dep.py @@ -25,7 +25,7 @@ def in_wsl() -> bool: return "microsoft-standard" in uname().release -def import_d_file(in_file) -> str: +def import_d_file(in_file: str) -> str: out_text = "" with open(in_file) as file: @@ -60,7 +60,7 @@ def import_d_file(in_file) -> str: return out_text -def main(): +def main() -> None: parser = argparse.ArgumentParser( description="""Transform a .d file from Wine paths to normal paths""" ) @@ -81,4 +81,4 @@ def main(): if __name__ == "__main__": - main() \ No newline at end of file + main() diff --git a/tools/upload_progress.py b/tools/upload_progress.py index 673bb3d5..dc61d156 100644 --- a/tools/upload_progress.py +++ b/tools/upload_progress.py @@ -51,7 +51,7 @@ if __name__ == "__main__": args = parser.parse_args() api_key = args.api_key or os.environ.get("PROGRESS_API_KEY") if not api_key: - raise "API key required" + raise KeyError("API key required") url = generate_url(args) entries = [] @@ -68,9 +68,12 @@ if __name__ == "__main__": print("Publishing entry to", url) json.dump(entries[0], sys.stdout, indent=4) print() - r = requests.post(url, json={ - "api_key": api_key, - "entries": entries, - }) + r = requests.post( + url, + json={ + "api_key": api_key, + "entries": entries, + }, + ) r.raise_for_status() - print("Done!") \ No newline at end of file + print("Done!")