diff --git a/.gitattributes b/.gitattributes index 9545db5f..47d43017 100644 --- a/.gitattributes +++ b/.gitattributes @@ -9,5 +9,5 @@ *.sh text eol=lf *.sha1 text eol=lf -# DTK keeps these files with LF +# decomp-toolkit writes files with LF config/**/*.txt text eol=lf diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 7c5f3917..4dfcd1c6 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -6,15 +6,12 @@ on: jobs: build: - # This is a *private* build container. - # See docs/github_actions.md for more information. container: ghcr.io/rainchus/marioparty4-build:main runs-on: ubuntu-latest strategy: fail-fast: false matrix: - # Add game versions here version: [GMPE01_00] steps: @@ -38,17 +35,15 @@ jobs: run: | python configure.py --map --version ${{ matrix.version }} \ --binutils /binutils --compilers /compilers - ninja all_source build/${{ matrix.version }}/progress.json + ninja all_source build/${{ matrix.version }}/progress.json \ + build/${{ matrix.version }}/report.json # Upload progress if we're on the main branch - name: Upload progress - # If you're using a different branch, change this to match if: github.ref == 'refs/heads/main' continue-on-error: true env: - # Replace with your project slug PROGRESS_SLUG: marioparty4 - # Set the API key in your repository secrets PROGRESS_API_KEY: ${{ secrets.PROGRESS_API_KEY }} run: | python tools/upload_progress.py -b https://progress.decomp.club/ \ @@ -57,7 +52,14 @@ jobs: # Upload map files - name: Upload map - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: ${{ matrix.version }}_maps - path: build/${{ matrix.version }}/**/*.MAP \ No newline at end of file + path: build/${{ matrix.version }}/**/*.MAP + + # Upload progress report + - name: Upload report + uses: actions/upload-artifact@v4 + with: + name: ${{ matrix.version }}_report + path: build/${{ matrix.version }}/report.json diff --git a/README.md b/README.md index 53e82df1..bca1a5fb 100644 --- a/README.md +++ b/README.md @@ -2,32 +2,11 @@ Mario Party 4 [![Build Status]][actions] ![Progress] ![DOL Progress] ![RELs Progress] [![Discord Badge]][discord] ============= - [Build Status]: https://github.com/Rainchus/marioparty4/actions/workflows/build.yml/badge.svg [actions]: https://github.com/Rainchus/marioparty4/actions/workflows/build.yml - [Progress]: https://img.shields.io/endpoint?label=Code&url=https%3A%2F%2Fprogress.decomp.club%2Fdata%2Fmarioparty4%2FGMPE01_00%2Fall%2F%3Fmode%3Dshield%26measure%3Dcode - [DOL Progress]: https://img.shields.io/endpoint?label=DOL&url=https%3A%2F%2Fprogress.decomp.club%2Fdata%2Fmarioparty4%2FGMPE01_00%2Fdol%2F%3Fmode%3Dshield%26measure%3Dcode - [RELs Progress]: https://img.shields.io/endpoint?label=RELs&url=https%3A%2F%2Fprogress.decomp.club%2Fdata%2Fmarioparty4%2FGMPE01_00%2Fmodules%2F%3Fmode%3Dshield%26measure%3Dcode - [Discord Badge]: https://img.shields.io/discord/994839212618690590?color=%237289DA&logo=discord&logoColor=%23FFFFFF [discord]: https://discord.gg/T4faGveujK @@ -37,12 +16,12 @@ This repository does **not** contain any game assets or assembly whatsoever. An Supported versions: -- `GAMEID`: Rev 0 (USA) +- `GMPE01_00`: Rev 0 (USA) Dependencies ============ -Windows: +Windows -------- On Windows, it's **highly recommended** to use native tooling. WSL or msys2 are **not** required. @@ -53,7 +32,7 @@ When running under WSL, [objdiff](#diffing) is unable to get filesystem notifica - Download [ninja](https://github.com/ninja-build/ninja/releases) and add it to `%PATH%`. - Quick install via pip: `pip install ninja` -macOS: +macOS ------ - Install [ninja](https://github.com/ninja-build/ninja/wiki/Pre-built-Ninja-packages): ``` @@ -69,29 +48,29 @@ After OS upgrades, if macOS complains about `Wine Crossover.app` being unverifie sudo xattr -rd com.apple.quarantine '/Applications/Wine Crossover.app' ``` -Linux: +Linux ------ - Install [ninja](https://github.com/ninja-build/ninja/wiki/Pre-built-Ninja-packages). - For non-x86(_64) platforms: Install wine from your package manager. - - For x86(_64), [WiBo](https://github.com/decompals/WiBo), a minimal 32-bit Windows binary wrapper, will be automatically downloaded and used. + - For x86(_64), [wibo](https://github.com/decompals/wibo), a minimal 32-bit Windows binary wrapper, will be automatically downloaded and used. Building ======== - Clone the repository: ``` - git clone https://github.com/Rainchus/mp4-dtk.git + git clone https://github.com/Rainchus/marioparty4.git ``` -- Using [Dolphin Emulator](https://dolphin-emu.org/), extract your game to `orig/GMPE01`. +- Using [Dolphin Emulator](https://dolphin-emu.org/), extract your game to `orig/GMPE01_00`. ![](assets/dolphin-extract.png) - To save space, the only necessary files are the following. Any others can be deleted. - `sys/main.dol` - - `files/rels/*.rel` + - `files/dll/*.rel` - Configure: ``` python configure.py ``` - To use a version other than `GMPE01` (USA), specify it with `--version`. + To use a version other than `GMPE01_00` (USA), specify it with `--version`. - Build: ``` ninja diff --git a/config/GMPE01_00/splits.txt b/config/GMPE01_00/splits.txt index 47bd92ab..7eefd3c3 100644 --- a/config/GMPE01_00/splits.txt +++ b/config/GMPE01_00/splits.txt @@ -1,17 +1,17 @@ Sections: - .init type:code - extab type:rodata - extabindex type:rodata - .text type:code - .ctors type:rodata - .dtors type:rodata - .rodata type:rodata - .data type:data - .bss type:bss - .sdata type:data - .sbss type:bss - .sdata2 type:rodata - .sbss2 type:bss + .init type:code align:4 + extab type:rodata align:32 + extabindex type:rodata align:32 + .text type:code align:8 + .ctors type:rodata align:32 + .dtors type:rodata align:32 + .rodata type:rodata align:32 + .data type:data align:32 + .bss type:bss align:32 + .sdata type:data align:4 + .sbss type:bss align:8 + .sdata2 type:rodata align:32 + .sbss2 type:bss align:16 game/main.c: .text start:0x800057C0 end:0x80005A5C diff --git a/configure.py b/configure.py index bbd9b2a0..4f226ebb 100644 --- a/configure.py +++ b/configure.py @@ -12,17 +12,11 @@ # Append --help to see available options. ### -import sys import argparse - +import sys from pathlib import Path -from tools.project import ( - Object, - ProjectConfig, - calculate_progress, - generate_build, - is_windows, -) +from typing import Any, Dict, List +from tools.project import * # Game versions DEFAULT_VERSION = 0 @@ -30,27 +24,25 @@ VERSIONS = [ "GMPE01_00", # USA 1.0 ] -if len(VERSIONS) > 1: - versions_str = ", ".join(VERSIONS[:-1]) + f" or {VERSIONS[-1]}" -else: - versions_str = VERSIONS[0] - parser = argparse.ArgumentParser() parser.add_argument( "mode", + choices=["configure", "progress"], default="configure", - help="configure or progress (default: configure)", + help="script mode (default: configure)", nargs="?", ) parser.add_argument( + "-v", "--version", - dest="version", + choices=VERSIONS, + type=str.upper, default=VERSIONS[DEFAULT_VERSION], - help=f"version to build ({versions_str})", + help="version to build", ) parser.add_argument( "--build-dir", - dest="build_dir", + metavar="DIR", type=Path, default=Path("build"), help="base build directory (default: build)", @@ -63,31 +55,24 @@ parser.add_argument( ) parser.add_argument( "--compilers", - dest="compilers", + metavar="DIR", type=Path, help="path to compilers (optional)", ) parser.add_argument( "--map", - dest="map", action="store_true", help="generate map file(s)", ) -parser.add_argument( - "--no-asm", - action="store_true", - help="don't incorporate .s files from asm directory", -) parser.add_argument( "--debug", - dest="debug", action="store_true", help="build with debug info (non-matching)", ) if not is_windows(): parser.add_argument( "--wrapper", - dest="wrapper", + metavar="BINARY", type=Path, help="path to wibo or wine (optional)", ) @@ -97,15 +82,20 @@ parser.add_argument( type=Path, help="path to decomp-toolkit binary or source (optional)", ) +parser.add_argument( + "--objdiff", + metavar="BINARY | DIR", + type=Path, + help="path to objdiff-cli binary or source (optional)", +) parser.add_argument( "--sjiswrap", - dest="sjiswrap", + metavar="EXE", type=Path, help="path to sjiswrap.exe (optional)", ) parser.add_argument( "--verbose", - dest="verbose", action="store_true", help="print verbose output", ) @@ -118,14 +108,13 @@ parser.add_argument( args = parser.parse_args() config = ProjectConfig() -config.version = args.version.upper() -if config.version not in VERSIONS: - sys.exit(f"Invalid version '{config.version}', expected {versions_str}") +config.version = str(args.version) version_num = VERSIONS.index(config.version) # Apply arguments config.build_dir = args.build_dir config.dtk_path = args.dtk +config.objdiff_path = args.objdiff config.binutils_path = args.binutils config.compilers_path = args.compilers config.debug = args.debug @@ -134,13 +123,15 @@ config.non_matching = args.non_matching config.sjiswrap_path = args.sjiswrap if not is_windows(): config.wrapper = args.wrapper -if args.no_asm: +# Don't build asm unless we're --non-matching +if not config.non_matching: config.asm_dir = None # Tool versions config.binutils_tag = "2.42-1" -config.compilers_tag = "20231018" -config.dtk_tag = "v0.7.4" +config.compilers_tag = "20240706" +config.dtk_tag = "v0.9.6" +config.objdiff_tag = "v2.0.0-beta.6" config.sjiswrap_tag = "v1.1.1" config.wibo_tag = "0.6.11" @@ -329,8 +320,9 @@ def Rel(lib_name, objects): } -Matching = True -NonMatching = False +Matching = True # Object matches and should be linked +NonMatching = False # Object does not match and should not be linked +Equivalent = config.non_matching # Object should be linked when configured with --non-matching config.warn_missing_config = True config.warn_missing_source = False @@ -788,19 +780,18 @@ config.libs = [ "objects": [ Object(Matching, "REL/executor.c"), Object(Matching, "REL/empty.c"), # Must be marked as matching + Object(Matching, "REL/board_executor.c"), ], }, Rel( "_minigameDLL", objects={ - Object(Matching, "REL/executor.c"), Object(Matching, "REL/_minigameDLL/_minigameDLL.c"), }, ), Rel( "bootDll", objects={ - Object(Matching, "REL/executor.c"), Object(Matching, "REL/bootDll/main.c"), Object(Matching, "REL/bootDll/nintendo_data.c"), }, @@ -809,14 +800,12 @@ config.libs = [ "E3setupDLL", objects={ Object(Matching, "REL/E3setupDLL/mgselect.c"), - Object(Matching, "REL/executor.c"), Object(Matching, "REL/E3setupDLL/main.c"), }, ), Rel( "instDll", objects={ - Object(Matching, "REL/executor.c"), Object(Matching, "REL/instDll/main.c"), Object(NonMatching, "REL/instDll/font.c"), }, @@ -824,7 +813,6 @@ config.libs = [ Rel( "m401Dll", # Manta Rings objects={ - Object(Matching, "REL/executor.c"), Object(Matching, "REL/m401Dll/main.c"), Object(Matching, "REL/m401Dll/main_ex.c"), }, @@ -838,7 +826,6 @@ config.libs = [ Rel( "m403Dll", # Booksquirm objects={ - Object(Matching, "REL/executor.c"), Object(Matching, "REL/m403Dll/main.c"), Object(Matching, "REL/m403Dll/scene.c"), }, @@ -846,7 +833,6 @@ config.libs = [ Rel( "m404Dll", # Trace Race objects={ - Object(Matching, "REL/executor.c"), Object(Matching, "REL/m404Dll/main.c"), }, ), @@ -859,7 +845,6 @@ config.libs = [ Rel( "m406Dll", # Avalanche! objects={ - Object(Matching, "REL/executor.c"), Object(Matching, "REL/m406Dll/main.c"), Object(Matching, "REL/m406Dll/map.c"), Object(Matching, "REL/m406Dll/player.c"), @@ -868,7 +853,6 @@ config.libs = [ Rel( "m407dll", # Domination objects={ - Object(Matching, "REL/executor.c"), Object(Matching, "REL/m407dll/player.c"), Object(Matching, "REL/m407dll/map.c"), Object(Matching, "REL/m407dll/camera.c"), @@ -882,7 +866,6 @@ config.libs = [ Rel( "m408Dll", # Paratrooper Plunge objects={ - Object(Matching, "REL/executor.c"), Object(Matching, "REL/m408Dll/main.c"), Object(Matching, "REL/m408Dll/camera.c"), Object(Matching, "REL/m408Dll/stage.c"), @@ -892,7 +875,6 @@ config.libs = [ Rel( "m409Dll", # Toad's Quick Draw objects={ - Object(Matching, "REL/executor.c"), Object(Matching, "REL/m409Dll/main.c"), Object(Matching, "REL/m409Dll/player.c"), Object(Matching, "REL/m409Dll/cursor.c"), @@ -901,7 +883,6 @@ config.libs = [ Rel( "m410Dll", # Three Throw objects={ - Object(Matching, "REL/executor.c"), Object(Matching, "REL/m410Dll/main.c"), Object(Matching, "REL/m410Dll/stage.c"), Object(Matching, "REL/m410Dll/game.c"), @@ -911,35 +892,30 @@ config.libs = [ Rel( "m411Dll", # Photo Finish objects={ - Object(Matching, "REL/executor.c"), Object(Matching, "REL/m411Dll/main.c"), }, ), Rel( "m412Dll", # Mr. Blizzard's Brigade objects={ - Object(Matching, "REL/executor.c"), Object(Matching, "REL/m412Dll/main.c"), }, ), Rel( "m413Dll", # Bob-omb Breakers objects={ - Object(Matching, "REL/executor.c"), Object(NonMatching, "REL/m413Dll/main.c"), }, ), Rel( "m414Dll", # Long Claw of the Law objects={ - Object(Matching, "REL/executor.c"), Object(Matching, "REL/m414Dll/main.c"), }, ), Rel( "m415Dll", # Stamp Out! objects={ - Object(Matching, "REL/executor.c"), Object(Matching, "REL/m415Dll/main.c"), Object(NonMatching, "REL/m415Dll/map.c"), }, @@ -947,7 +923,6 @@ config.libs = [ Rel( "m416Dll", # Candlelight Flight objects={ - Object(Matching, "REL/executor.c"), Object(Matching, "REL/m416Dll/main.c"), Object(Matching, "REL/m416Dll/map.c"), }, @@ -955,7 +930,6 @@ config.libs = [ Rel( "m417Dll", # Makin' Waves objects={ - Object(Matching, "REL/executor.c"), Object(Matching, "REL/m417Dll/main.c"), Object(Matching, "REL/m417Dll/water.c"), Object(Matching, "REL/m417Dll/player.c"), @@ -972,14 +946,12 @@ config.libs = [ Rel( "m419Dll", # Tree Stomp objects={ - Object(Matching, "REL/executor.c"), Object(NonMatching, "REL/m419Dll/main.c"), }, ), Rel( "m420dll", # Fish n' Drips objects={ - Object(Matching, "REL/executor.c"), Object(NonMatching, "REL/m420dll/main.c"), Object(NonMatching, "REL/m420dll/player.c"), Object(NonMatching, "REL/m420dll/map.c"), @@ -989,7 +961,6 @@ config.libs = [ Rel( "m421Dll", # Hop or Pop objects={ - Object(Matching, "REL/executor.c"), Object(NonMatching, "REL/m421Dll/main.c"), Object(NonMatching, "REL/m421Dll/player.c"), Object(NonMatching, "REL/m421Dll/map.c"), @@ -998,21 +969,18 @@ config.libs = [ Rel( "m422Dll", # Money Belts objects={ - Object(Matching, "REL/executor.c"), Object(NonMatching, "REL/m422Dll/main.c"), }, ), Rel( "m423Dll", # GOOOOOOOAL!! objects={ - Object(Matching, "REL/executor.c"), Object(NonMatching, "REL/m423Dll/main.c"), }, ), Rel( "m424Dll", # Blame it on the Crane objects={ - Object(Matching, "REL/executor.c"), Object(NonMatching, "REL/m424Dll/main.c"), Object(NonMatching, "REL/m424Dll/map.c"), Object(NonMatching, "REL/m424Dll/ball.c"), @@ -1022,7 +990,6 @@ config.libs = [ Rel( "m425Dll", # The Great Deflate objects={ - Object(Matching, "REL/executor.c"), Object(NonMatching, "REL/m425Dll/main.c"), Object(NonMatching, "REL/m425Dll/thwomp.c"), }, @@ -1030,14 +997,12 @@ config.libs = [ Rel( "m426Dll", # Revers-a-Bomb objects={ - Object(Matching, "REL/executor.c"), Object(NonMatching, "REL/m426Dll/main.c"), }, ), Rel( "m427Dll", # Right Oar Left? objects={ - Object(Matching, "REL/executor.c"), Object(NonMatching, "REL/m427Dll/main.c"), Object(NonMatching, "REL/m427Dll/map.c"), Object(NonMatching, "REL/m427Dll/player.c"), @@ -1046,7 +1011,6 @@ config.libs = [ Rel( "m428Dll", # Cliffhangers objects={ - Object(Matching, "REL/executor.c"), Object(NonMatching, "REL/m428Dll/main.c"), Object(NonMatching, "REL/m428Dll/map.c"), Object(NonMatching, "REL/m428Dll/player.c"), @@ -1061,7 +1025,6 @@ config.libs = [ Rel( "m430Dll", # Pair-a-sailing objects={ - Object(Matching, "REL/executor.c"), Object(NonMatching, "REL/m430Dll/main.c"), Object(NonMatching, "REL/m430Dll/water.c"), Object(NonMatching, "REL/m430Dll/player.c"), @@ -1070,7 +1033,6 @@ config.libs = [ Rel( "m431Dll", # Order Up objects={ - Object(Matching, "REL/executor.c"), Object(Matching, "REL/m431Dll/main.c"), Object(Matching, "REL/m431Dll/object.c"), }, @@ -1084,7 +1046,6 @@ config.libs = [ Rel( "m433Dll", # Beach Volley Folly objects={ - Object(Matching, "REL/executor.c"), Object(NonMatching, "REL/m433Dll/main.c"), Object(NonMatching, "REL/m433Dll/map.c"), Object(NonMatching, "REL/m433Dll/player.c"), @@ -1093,7 +1054,6 @@ config.libs = [ Rel( "m434Dll", # Cheep Cheep Sweep objects={ - Object(Matching, "REL/executor.c"), Object(Matching, "REL/m434Dll/main.c"), Object(Matching, "REL/m434Dll/map.c"), Object(NonMatching, "REL/m434Dll/player.c"), @@ -1124,7 +1084,6 @@ config.libs = [ Rel( "m438Dll", # Chain Chomp Fever objects={ - Object(Matching, "REL/executor.c"), Object(NonMatching, "REL/m438Dll/main.c"), Object(NonMatching, "REL/m438Dll/map.c"), Object(NonMatching, "REL/m438Dll/fire.c"), @@ -1133,14 +1092,12 @@ config.libs = [ Rel( "m439Dll", # Paths of Peril objects={ - Object(Matching, "REL/executor.c"), Object(Matching, "REL/m439Dll/main.c"), }, ), Rel( "m440Dll", # Bowser's Bigger Blast objects={ - Object(Matching, "REL/executor.c"), Object(Matching, "REL/m440Dll/main.c"), Object(Matching, "REL/m440Dll/object.c"), }, @@ -1154,7 +1111,6 @@ config.libs = [ Rel( "m442Dll", # Barrel Baron objects={ - Object(Matching, "REL/executor.c"), Object(Matching, "REL/m442Dll/main.c"), Object(Matching, "REL/m442Dll/score.c"), }, @@ -1162,7 +1118,6 @@ config.libs = [ Rel( "m443Dll", # Mario Speedwagons objects={ - Object(Matching, "REL/executor.c"), Object(NonMatching, "REL/m443Dll/main.c"), Object(NonMatching, "REL/m443Dll/map.c"), Object(NonMatching, "REL/m443Dll/player.c"), @@ -1171,7 +1126,6 @@ config.libs = [ Rel( "m444dll", # Reversal of Fortune objects={ - Object(Matching, "REL/executor.c"), Object(NonMatching, "REL/m444dll/main.c"), Object(Matching, "REL/m444dll/pinball.c"), Object(Matching, "REL/m444dll/datalist.c"), @@ -1181,14 +1135,12 @@ config.libs = [ Rel( "m445Dll", # Bowser Bop objects={ - Object(Matching, "REL/executor.c"), Object(Matching, "REL/m445Dll/main.c"), }, ), Rel( "m446dll", # Mystic Match 'Em objects={ - Object(Matching, "REL/executor.c"), Object(Matching, "REL/m446Dll/main.c"), Object(Matching, "REL/m446Dll/card.c"), Object(Matching, "REL/m446Dll/deck.c"), @@ -1202,7 +1154,6 @@ config.libs = [ Rel( "m447dll", # Archaeologuess objects={ - Object(Matching, "REL/executor.c"), Object(Matching, "REL/m447dll/main.c"), Object(Matching, "REL/m447dll/stage.c"), Object(Matching, "REL/m447dll/camera.c"), @@ -1214,14 +1165,12 @@ config.libs = [ Rel( "m448Dll", # Goomba's Chip Flip objects={ - Object(Matching, "REL/executor.c"), Object(Matching, "REL/m448Dll/main.c"), }, ), Rel( "m449Dll", # Kareening Koopa objects={ - Object(Matching, "REL/executor.c"), Object(NonMatching, "REL/m449Dll/main.c"), }, ), @@ -1240,7 +1189,6 @@ config.libs = [ Rel( "m453Dll", # Challenge Booksquirm objects={ - Object(Matching, "REL/executor.c"), Object(NonMatching, "REL/m453Dll/main.c"), Object(NonMatching, "REL/m453Dll/map.c"), Object(NonMatching, "REL/m453Dll/score.c"), @@ -1249,7 +1197,6 @@ config.libs = [ Rel( "m455Dll", # Rumble Fishing objects={ - Object(Matching, "REL/executor.c"), Object(Matching, "REL/m455Dll/main.c"), Object(NonMatching, "REL/m455Dll/stage.c"), }, @@ -1257,7 +1204,6 @@ config.libs = [ Rel( "m456Dll", # Take a Breather objects={ - Object(Matching, "REL/executor.c"), Object(NonMatching, "REL/m456Dll/main.c"), Object(NonMatching, "REL/m456Dll/stage.c"), }, @@ -1265,28 +1211,24 @@ config.libs = [ Rel( "m457Dll", # Bowser Wrestling objects={ - Object(Matching, "REL/executor.c"), Object(NonMatching, "REL/m457Dll/main.c"), }, ), Rel( "m458Dll", # Panels of Doom objects={ - Object(Matching, "REL/executor.c"), Object(NonMatching, "REL/m458Dll/main.c"), }, ), Rel( "m459dll", # Mushroom Medic objects={ - Object(Matching, "REL/executor.c"), Object(NonMatching, "REL/m459dll/main.c"), }, ), Rel( "m460Dll", # Doors of Doom objects={ - Object(Matching, "REL/executor.c"), Object(NonMatching, "REL/m460Dll/main.c"), Object(NonMatching, "REL/m460Dll/player.c"), Object(NonMatching, "REL/m460Dll/map.c"), @@ -1302,7 +1244,6 @@ config.libs = [ Rel( "m462Dll", # Goomba Stomp objects={ - Object(Matching, "REL/executor.c"), Object(NonMatching, "REL/m462Dll/main.c"), }, ), @@ -1322,14 +1263,12 @@ config.libs = [ Rel( "messDll", objects={ - Object(Matching, "REL/executor.c"), Object(Matching, "REL/messDll/main.c"), }, ), Rel( "mgmodedll", objects={ - Object(Matching, "REL/executor.c"), Object(Matching, "REL/mgmodedll/mgmode.c"), Object(Matching, "REL/mgmodedll/free_play.c"), Object(Matching, "REL/mgmodedll/record.c"), @@ -1343,7 +1282,6 @@ config.libs = [ Rel( "modeltestDll", objects={ - Object(Matching, "REL/executor.c"), Object(Matching, "REL/modeltestDll/main.c"), Object(Matching, "REL/modeltestDll/modeltest00.c"), Object(Matching, "REL/modeltestDll/modeltest01.c"), @@ -1352,7 +1290,6 @@ config.libs = [ Rel( "modeseldll", objects={ - Object(Matching, "REL/executor.c"), Object(Matching, "REL/modeseldll/main.c"), Object(NonMatching, "REL/modeseldll/modesel.c"), Object(Matching, "REL/modeseldll/filesel.c"), @@ -1410,7 +1347,6 @@ config.libs = [ Rel( "option", objects={ - Object(Matching, "REL/executor.c"), Object(Matching, "REL/option/scene.c"), Object(Matching, "REL/option/camera.c"), Object(Matching, "REL/option/room.c"), @@ -1425,7 +1361,6 @@ config.libs = [ Rel( "present", objects={ - Object(Matching, "REL/executor.c"), Object(Matching, "REL/present/init.c"), Object(Matching, "REL/present/camera.c"), Object(Matching, "REL/present/present.c"), @@ -1436,7 +1371,6 @@ config.libs = [ Rel( "resultDll", objects={ - Object(Matching, "REL/executor.c"), Object(Matching, "REL/resultDll/main.c"), Object(Matching, "REL/resultDll/battle.c"), Object(Matching, "REL/resultDll/datalist.c"), @@ -1451,28 +1385,24 @@ config.libs = [ Rel( "selmenuDll", objects={ - Object(Matching, "REL/executor.c"), Object(Matching, "REL/selmenuDll/main.c"), }, ), Rel( "staffDll", objects={ - Object(Matching, "REL/executor.c"), Object(Matching, "REL/staffDll/main.c"), }, ), Rel( "subchrselDll", objects={ - Object(Matching, "REL/executor.c"), Object(Matching, "REL/subchrselDll/main.c"), }, ), Rel( "w01Dll", # Toad's Midway Madness objects={ - Object(Matching, "REL/board_executor.c"), Object(Matching, "REL/w01Dll/main.c"), Object(Matching, "REL/w01Dll/mg_coin.c"), Object(Matching, "REL/w01Dll/mg_item.c"), @@ -1481,7 +1411,6 @@ config.libs = [ Rel( "w02Dll", # Goomba's Greedy Gala objects={ - Object(Matching, "REL/board_executor.c"), Object(Matching, "REL/w02Dll/main.c"), Object(Matching, "REL/w02Dll/gendice.c"), Object(Matching, "REL/w02Dll/gamble.c"), @@ -1494,7 +1423,6 @@ config.libs = [ Rel( "w03Dll", # Shy Guy's Jungle Jam objects={ - Object(Matching, "REL/board_executor.c"), Object(Matching, "REL/w03Dll/main.c"), Object(Matching, "REL/w03Dll/statue.c"), Object(Matching, "REL/w03Dll/condor.c"), @@ -1507,7 +1435,6 @@ config.libs = [ Rel( "w04Dll", # Boo's Haunted Bash objects={ - Object(Matching, "REL/board_executor.c"), Object(Matching, "REL/w04Dll/main.c"), Object(Matching, "REL/w04Dll/bridge.c"), Object(Matching, "REL/w04Dll/boo_event.c"), @@ -1519,7 +1446,6 @@ config.libs = [ Rel( "w05Dll", # Koopa's Seaside Soiree objects={ - Object(Matching, "REL/board_executor.c"), Object(Matching, "REL/w05Dll/main.c"), Object(Matching, "REL/w05Dll/hotel.c"), Object(Matching, "REL/w05Dll/monkey.c"), @@ -1531,7 +1457,6 @@ config.libs = [ Rel( "w06Dll", # Bowser's Gnarly Party objects={ - Object(Matching, "REL/board_executor.c"), Object(Matching, "REL/w06Dll/main.c"), Object(Matching, "REL/w06Dll/mg_item.c"), Object(Matching, "REL/w06Dll/mg_coin.c"), @@ -1543,7 +1468,6 @@ config.libs = [ Rel( "w10Dll", # Tutorial board objects={ - Object(Matching, "REL/board_executor.c"), Object(Matching, "REL/w10Dll/main.c"), Object(Matching, "REL/w10Dll/host.c"), Object(Matching, "REL/w10Dll/scene.c"), @@ -1553,21 +1477,18 @@ config.libs = [ Rel( "w20Dll", # Mega Board Mayhem objects={ - Object(Matching, "REL/board_executor.c"), Object(Matching, "REL/w20Dll/main.c"), }, ), Rel( "w21Dll", # Mini Board Mad Dash objects={ - Object(Matching, "REL/board_executor.c"), Object(Matching, "REL/w21Dll/main.c"), }, ), Rel( "ztardll", objects={ - Object(Matching, "REL/executor.c"), Object(Matching, "REL/ztardll/main.c"), Object(Matching, "REL/ztardll/font.c"), Object(Matching, "REL/ztardll/select.c"), @@ -1575,12 +1496,15 @@ config.libs = [ ), ] +# Optional extra categories for progress tracking +config.progress_categories = [] +config.progress_each_module = args.verbose + if args.mode == "configure": # Write build.ninja and objdiff.json generate_build(config) elif args.mode == "progress": # Print progress and write progress.json - config.progress_each_module = args.verbose calculate_progress(config) else: sys.exit("Unknown mode: " + args.mode) diff --git a/tools/decompctx.py b/tools/decompctx.py index e86d5ef3..290946f1 100644 --- a/tools/decompctx.py +++ b/tools/decompctx.py @@ -23,8 +23,8 @@ include_dirs = [ # Add additional include directories here ] -include_pattern = re.compile(r'^#include\s*[<"](.+?)[>"]$') -guard_pattern = re.compile(r"^#ifndef\s+(.*)$") +include_pattern = re.compile(r'^#\s*include\s*[<"](.+?)[>"]$') +guard_pattern = re.compile(r"^#\s*ifndef\s+(.*)$") defines = set() diff --git a/tools/download_tool.py b/tools/download_tool.py index 7b386a4b..69ef96a7 100644 --- a/tools/download_tool.py +++ b/tools/download_tool.py @@ -55,6 +55,21 @@ def dtk_url(tag: str) -> str: repo = "https://github.com/encounter/decomp-toolkit" return f"{repo}/releases/download/{tag}/dtk-{system}-{arch}{suffix}" +def objdiff_cli_url(tag: str) -> str: + uname = platform.uname() + suffix = "" + system = uname.system.lower() + if system == "darwin": + system = "macos" + elif system == "windows": + suffix = ".exe" + arch = uname.machine.lower() + if arch == "amd64": + arch = "x86_64" + + repo = "https://github.com/encounter/objdiff" + return f"{repo}/releases/download/{tag}/objdiff-cli-{system}-{arch}{suffix}" + def sjiswrap_url(tag: str) -> str: repo = "https://github.com/encounter/sjiswrap" @@ -70,6 +85,7 @@ TOOLS: Dict[str, Callable[[str], str]] = { "binutils": binutils_url, "compilers": compilers_url, "dtk": dtk_url, + "objdiff-cli": objdiff_cli_url, "sjiswrap": sjiswrap_url, "wibo": wibo_url, } diff --git a/tools/project.py b/tools/project.py index 8f8c6e07..639f5a13 100644 --- a/tools/project.py +++ b/tools/project.py @@ -17,7 +17,7 @@ import os import platform import sys from pathlib import Path -from typing import Any, Dict, List, Optional, Set, Tuple, Union +from typing import Any, Dict, List, Optional, Set, Tuple, Union, cast from . import ninja_syntax from .ninja_syntax import serialize_path @@ -29,24 +29,78 @@ if sys.platform == "cygwin": f"\n(Current path: {sys.executable})" ) +Library = Dict[str, Any] + class Object: def __init__(self, completed: bool, name: str, **options: Any) -> None: self.name = name - self.base_name = Path(name).with_suffix("") self.completed = completed self.options: Dict[str, Any] = { - "add_to_all": True, + "add_to_all": None, "asflags": None, - "extra_asflags": None, + "asm_dir": None, "cflags": None, + "extra_asflags": None, "extra_cflags": None, + "host": None, + "lib": None, "mw_version": None, + "progress_category": None, "shift_jis": None, "source": name, + "src_dir": None, } self.options.update(options) + # Internal + self.src_path: Optional[Path] = None + self.asm_path: Optional[Path] = None + self.src_obj_path: Optional[Path] = None + self.asm_obj_path: Optional[Path] = None + self.host_obj_path: Optional[Path] = None + self.ctx_path: Optional[Path] = None + + def resolve(self, config: "ProjectConfig", lib: Library) -> "Object": + # Use object options, then library options + obj = Object(self.completed, self.name, **lib) + for key, value in self.options.items(): + if value is not None or key not in obj.options: + obj.options[key] = value + + # Use default options from config + def set_default(key: str, value: Any) -> None: + if obj.options[key] is None: + obj.options[key] = value + + set_default("add_to_all", True) + set_default("asflags", config.asflags) + set_default("asm_dir", config.asm_dir) + set_default("host", False) + set_default("mw_version", config.linker_version) + set_default("shift_jis", config.shift_jis) + set_default("src_dir", config.src_dir) + + # Resolve paths + build_dir = config.out_path() + obj.src_path = Path(obj.options["src_dir"]) / obj.options["source"] + if obj.options["asm_dir"] is not None: + obj.asm_path = ( + Path(obj.options["asm_dir"]) / obj.options["source"] + ).with_suffix(".s") + base_name = Path(self.name).with_suffix("") + obj.src_obj_path = build_dir / "src" / f"{base_name}.o" + obj.asm_obj_path = build_dir / "mod" / f"{base_name}.o" + obj.host_obj_path = build_dir / "host" / f"{base_name}.o" + obj.ctx_path = build_dir / "src" / f"{base_name}.ctx" + return obj + + +class ProgressCategory: + def __init__(self, id: str, name: str) -> None: + self.id = id + self.name = name + class ProjectConfig: def __init__(self) -> None: @@ -54,21 +108,26 @@ class ProjectConfig: self.build_dir: Path = Path("build") # Output build files self.src_dir: Path = Path("src") # C/C++/asm source files self.tools_dir: Path = Path("tools") # Python scripts - self.asm_dir: Path = Path("asm") # Override incomplete objects (for modding) + self.asm_dir: Optional[Path] = Path( + "asm" + ) # Override incomplete objects (for modding) # Tooling self.binutils_tag: Optional[str] = None # Git tag self.binutils_path: Optional[Path] = None # If None, download self.dtk_tag: Optional[str] = None # Git tag - self.build_dtk_path: Optional[Path] = None # If None, download + self.dtk_path: Optional[Path] = None # If None, download self.compilers_tag: Optional[str] = None # 1 self.compilers_path: Optional[Path] = None # If None, download self.wibo_tag: Optional[str] = None # Git tag self.wrapper: Optional[Path] = None # If None, download wibo on Linux self.sjiswrap_tag: Optional[str] = None # Git tag self.sjiswrap_path: Optional[Path] = None # If None, download + self.objdiff_tag: Optional[str] = None # Git tag + self.objdiff_path: Optional[Path] = None # If None, download # Project config + self.non_matching: bool = False self.build_rels: bool = True # Build REL files self.check_sha_path: Optional[Path] = None # Path to version.sha1 self.config_path: Optional[Path] = None # Path to config.yml @@ -76,7 +135,7 @@ class ProjectConfig: self.generate_map: bool = False # Generate map file(s) self.asflags: Optional[List[str]] = None # Assembler flags self.ldflags: Optional[List[str]] = None # Linker flags - self.libs: Optional[List[Dict[str, Any]]] = None # List of libraries + self.libs: Optional[List[Library]] = None # List of libraries self.linker_version: Optional[str] = None # mwld version self.version: Optional[str] = None # Version name self.warn_missing_config: bool = False # Warn on missing unit configuration @@ -88,13 +147,23 @@ class ProjectConfig: self.shift_jis = ( True # Convert source files from UTF-8 to Shift JIS automatically ) + self.reconfig_deps: Optional[List[Path]] = ( + None # Additional re-configuration dependency files + ) + self.custom_build_rules: Optional[List[Dict[str, Any]]] = ( + None # Custom ninja build rules + ) + self.custom_build_steps: Optional[Dict[str, List[Dict[str, Any]]]] = ( + None # Custom build steps, types are ["pre-compile", "post-compile", "post-link", "post-build"] + ) - # Progress output and progress.json config + # Progress output, progress.json and report.json config self.progress_all: bool = True # Include combined "all" category self.progress_modules: bool = True # Include combined "modules" category self.progress_each_module: bool = ( - True # Include individual modules, disable for large numbers of modules + False # Include individual modules, disable for large numbers of modules ) + self.progress_categories: List[ProgressCategory] = [] # Additional categories # Progress fancy printing self.progress_use_fancy: bool = False @@ -119,12 +188,17 @@ class ProjectConfig: if getattr(self, attr) is None: sys.exit(f"ProjectConfig.{attr} missing") - def find_object(self, name: str) -> Optional[Tuple[Dict[str, Any], Object]]: + # Creates a map of object names to Object instances + # Options are fully resolved from the library and object + def objects(self) -> Dict[str, Object]: + out = {} for lib in self.libs or {}: - for obj in lib["objects"]: - if obj.name == name: - return lib, obj - return None + objects: List[Object] = lib["objects"] + for obj in objects: + if obj.name in out: + sys.exit(f"Duplicate object name {obj.name}") + out[obj.name] = obj.resolve(self, lib) + return out def out_path(self) -> Path: return self.build_dir / str(self.version) @@ -160,7 +234,7 @@ def load_build_config( f = open(build_config_path, "r", encoding="utf-8") build_config: Dict[str, Any] = json.load(f) config_version = build_config.get("version") - if not config_version: + if config_version is None: # Invalid config.json f.close() os.remove(build_config_path) @@ -179,17 +253,19 @@ def load_build_config( # Generate build.ninja and objdiff.json def generate_build(config: ProjectConfig) -> None: + config.validate() + objects = config.objects() build_config = load_build_config(config, config.out_path() / "config.json") - generate_build_ninja(config, build_config) - generate_objdiff_config(config, build_config) + generate_build_ninja(config, objects, build_config) + generate_objdiff_config(config, objects, build_config) # Generate build.ninja def generate_build_ninja( - config: ProjectConfig, build_config: Optional[Dict[str, Any]] + config: ProjectConfig, + objects: Dict[str, Object], + build_config: Optional[Dict[str, Any]], ) -> None: - config.validate() - out = io.StringIO() n = ninja_syntax.Writer(out) n.variable("ninja_required_version", "1.3") @@ -213,7 +289,7 @@ def generate_build_ninja( if config.debug: ldflags += " -g" n.variable("ldflags", ldflags) - if not config.linker_version: + if config.linker_version is None: sys.exit("ProjectConfig.linker_version missing") n.variable("mw_version", Path(config.linker_version)) n.newline() @@ -225,6 +301,7 @@ def generate_build_ninja( build_path = config.out_path() progress_path = build_path / "progress.json" + report_path = build_path / "report.json" build_tools_path = config.build_dir / "tools" download_tool = config.tools_dir / "download_tool.py" n.rule( @@ -242,20 +319,32 @@ def generate_build_ninja( deps="gcc", ) - if config.build_dtk_path: + cargo_rule_written = False + + def write_cargo_rule(): + nonlocal cargo_rule_written + if not cargo_rule_written: + n.pool("cargo", 1) + n.rule( + name="cargo", + command="cargo build --release --manifest-path $in --bin $bin --target-dir $target", + description="CARGO $bin", + pool="cargo", + depfile=Path("$target") / "release" / "$bin.d", + deps="gcc", + ) + cargo_rule_written = True + + if config.dtk_path is not None and config.dtk_path.is_file(): + dtk = config.dtk_path + elif config.dtk_path is not None: dtk = build_tools_path / "release" / f"dtk{EXE}" - n.rule( - name="cargo", - command="cargo build --release --manifest-path $in --bin $bin --target-dir $target", - description="CARGO $bin", - depfile=Path("$target") / "release" / "$bin.d", - deps="gcc", - ) + write_cargo_rule() n.build( outputs=dtk, rule="cargo", - inputs=config.build_dtk_path / "Cargo.toml", - implicit=config.build_dtk_path / "Cargo.lock", + inputs=config.dtk_path / "Cargo.toml", + implicit=config.dtk_path / "Cargo.lock", variables={ "bin": "dtk", "target": build_tools_path, @@ -275,6 +364,35 @@ def generate_build_ninja( else: sys.exit("ProjectConfig.dtk_tag missing") + if config.objdiff_path is not None and config.objdiff_path.is_file(): + objdiff = config.objdiff_path + elif config.objdiff_path is not None: + objdiff = build_tools_path / "release" / f"objdiff-cli{EXE}" + write_cargo_rule() + n.build( + outputs=objdiff, + rule="cargo", + inputs=config.objdiff_path / "Cargo.toml", + implicit=config.objdiff_path / "Cargo.lock", + variables={ + "bin": "objdiff-cli", + "target": build_tools_path, + }, + ) + elif config.objdiff_tag: + objdiff = build_tools_path / f"objdiff-cli{EXE}" + n.build( + outputs=objdiff, + rule="download_tool", + implicit=download_tool, + variables={ + "tool": "objdiff-cli", + "tag": config.objdiff_tag, + }, + ) + else: + sys.exit("ProjectConfig.objdiff_tag missing") + if config.sjiswrap_path: sjiswrap = config.sjiswrap_path elif config.sjiswrap_tag: @@ -353,6 +471,17 @@ def generate_build_ninja( n.newline() + ### + # Helper rule for downloading all tools + ### + n.comment("Download all tools") + n.build( + outputs="tools", + rule="phony", + inputs=[dtk, sjiswrap, wrapper, compilers, binutils, objdiff], + ) + n.newline() + ### # Build rules ### @@ -435,6 +564,49 @@ def generate_build_ninja( ) n.newline() + if len(config.custom_build_rules or {}) > 0: + n.comment("Custom project build rules (pre/post-processing)") + for rule in config.custom_build_rules or {}: + n.rule( + name=cast(str, rule.get("name")), + command=cast(str, rule.get("command")), + description=rule.get("description", None), + depfile=rule.get("depfile", None), + generator=rule.get("generator", False), + pool=rule.get("pool", None), + restat=rule.get("restat", False), + rspfile=rule.get("rspfile", None), + rspfile_content=rule.get("rspfile_content", None), + deps=rule.get("deps", None), + ) + n.newline() + + def write_custom_step(step: str) -> List[str | Path]: + implicit: List[str | Path] = [] + if config.custom_build_steps and step in config.custom_build_steps: + n.comment(f"Custom build steps ({step})") + for custom_step in config.custom_build_steps[step]: + outputs = cast(List[str | Path], custom_step.get("outputs")) + + if isinstance(outputs, list): + implicit.extend(outputs) + else: + implicit.append(outputs) + + n.build( + outputs=outputs, + rule=cast(str, custom_step.get("rule")), + inputs=custom_step.get("inputs", None), + implicit=custom_step.get("implicit", None), + order_only=custom_step.get("order_only", None), + variables=custom_step.get("variables", None), + implicit_outputs=custom_step.get("implicit_outputs", None), + pool=custom_step.get("pool", None), + dyndep=custom_step.get("dyndep", None), + ) + n.newline() + return implicit + n.comment("Host build") n.variable("host_cflags", "-I include -Wno-trigraphs") n.variable( @@ -453,14 +625,13 @@ def generate_build_ninja( ) n.newline() + # Add all build steps needed before we compile (e.g. processing assets) + precompile_implicit = write_custom_step("pre-compile") + ### # Source files ### n.comment("Source files") - build_asm_path = build_path / "mod" - build_src_path = build_path / "src" - build_host_path = build_path / "host" - build_config_path = build_path / "config.json" def map_path(path: Path) -> Path: return path.parent / (path.name + ".MAP") @@ -503,16 +674,15 @@ def generate_build_ninja( outputs=elf_path, rule="link", inputs=self.inputs, - implicit=[self.ldscript, *mwld_implicit], + implicit=[ + *precompile_implicit, + self.ldscript, + *mwld_implicit, + *postcompile_implicit, + ], implicit_outputs=elf_map, variables={"ldflags": elf_ldflags}, ) - n.build( - outputs=dol_path, - rule="elf2dol", - inputs=elf_path, - implicit=dtk, - ) else: preplf_path = build_path / self.name / f"{self.name}.preplf" plf_path = build_path / self.name / f"{self.name}.plf" @@ -549,6 +719,7 @@ def generate_build_ninja( ) n.newline() + link_outputs: List[Path] = [] if build_config: link_steps: List[LinkStep] = [] used_compiler_versions: Set[str] = set() @@ -556,96 +727,85 @@ def generate_build_ninja( host_source_inputs: List[Path] = [] source_added: Set[Path] = set() - def c_build( - obj: Object, options: Dict[str, Any], lib_name: str, src_path: Path - ) -> Optional[Path]: - - cflags_str = make_flags_str(options["cflags"]) - if options["extra_cflags"] is not None: - extra_cflags_str = make_flags_str(options["extra_cflags"]) + def c_build(obj: Object, src_path: Path) -> Optional[Path]: + cflags_str = make_flags_str(obj.options["cflags"]) + if obj.options["extra_cflags"] is not None: + extra_cflags_str = make_flags_str(obj.options["extra_cflags"]) cflags_str += " " + extra_cflags_str - used_compiler_versions.add(options["mw_version"]) - - src_obj_path = build_src_path / f"{obj.base_name}.o" - src_base_path = build_src_path / obj.base_name + used_compiler_versions.add(obj.options["mw_version"]) # Avoid creating duplicate build rules - if src_obj_path in source_added: - return src_obj_path - source_added.add(src_obj_path) - - shift_jis = options["shift_jis"] - if shift_jis is None: - shift_jis = config.shift_jis + if obj.src_obj_path is None or obj.src_obj_path in source_added: + return obj.src_obj_path + source_added.add(obj.src_obj_path) # Add MWCC build rule + lib_name = obj.options["lib"] n.comment(f"{obj.name}: {lib_name} (linked {obj.completed})") n.build( - outputs=src_obj_path, - rule="mwcc_sjis" if shift_jis else "mwcc", + outputs=obj.src_obj_path, + rule="mwcc_sjis" if obj.options["shift_jis"] else "mwcc", inputs=src_path, variables={ - "mw_version": Path(options["mw_version"]), + "mw_version": Path(obj.options["mw_version"]), "cflags": cflags_str, - "basedir": os.path.dirname(src_base_path), - "basefile": src_base_path, + "basedir": os.path.dirname(obj.src_obj_path), + "basefile": obj.src_obj_path.with_suffix(""), }, - implicit=mwcc_sjis_implicit if shift_jis else mwcc_implicit, + implicit=( + mwcc_sjis_implicit if obj.options["shift_jis"] else mwcc_implicit + ), ) # Add ctx build rule - ctx_path = build_src_path / f"{obj.base_name}.ctx" - n.build( - outputs=ctx_path, - rule="decompctx", - inputs=src_path, - implicit=decompctx, - ) + if obj.ctx_path is not None: + n.build( + outputs=obj.ctx_path, + rule="decompctx", + inputs=src_path, + implicit=decompctx, + ) # Add host build rule - if options.get("host", False): - host_obj_path = build_host_path / f"{obj.base_name}.o" - host_base_path = build_host_path / obj.base_name + if obj.options["host"] and obj.host_obj_path is not None: n.build( - outputs=host_obj_path, + outputs=obj.host_obj_path, rule="host_cc" if src_path.suffix == ".c" else "host_cpp", inputs=src_path, variables={ - "basedir": os.path.dirname(host_base_path), - "basefile": host_base_path, + "basedir": os.path.dirname(obj.host_obj_path), + "basefile": obj.host_obj_path.with_suffix(""), }, ) - if options["add_to_all"]: - host_source_inputs.append(host_obj_path) + if obj.options["add_to_all"]: + host_source_inputs.append(obj.host_obj_path) n.newline() - if options["add_to_all"]: - source_inputs.append(src_obj_path) + if obj.options["add_to_all"]: + source_inputs.append(obj.src_obj_path) - return src_obj_path + return obj.src_obj_path def asm_build( - obj: Object, options: Dict[str, Any], lib_name: str, src_path: Path + obj: Object, src_path: Path, obj_path: Optional[Path] ) -> Optional[Path]: - asflags = options["asflags"] or config.asflags - if asflags is None: + if obj.options["asflags"] is None: sys.exit("ProjectConfig.asflags missing") - asflags_str = make_flags_str(asflags) - if options["extra_asflags"] is not None: - extra_asflags_str = make_flags_str(options["extra_asflags"]) + asflags_str = make_flags_str(obj.options["asflags"]) + if obj.options["extra_asflags"] is not None: + extra_asflags_str = make_flags_str(obj.options["extra_asflags"]) asflags_str += " " + extra_asflags_str - asm_obj_path = build_asm_path / f"{obj.base_name}.o" - # Avoid creating duplicate build rules - if asm_obj_path in source_added: - return asm_obj_path - source_added.add(asm_obj_path) + if obj_path is None or obj_path in source_added: + return obj_path + source_added.add(obj_path) # Add assembler build rule + lib_name = obj.options["lib"] n.comment(f"{obj.name}: {lib_name} (linked {obj.completed})") n.build( - outputs=asm_obj_path, + outputs=obj_path, rule="as", inputs=src_path, variables={"asflags": asflags_str}, @@ -653,54 +813,40 @@ def generate_build_ninja( ) n.newline() - if options["add_to_all"]: - source_inputs.append(asm_obj_path) + if obj.options["add_to_all"]: + source_inputs.append(obj_path) - return asm_obj_path + return obj_path def add_unit(build_obj, link_step: LinkStep): obj_path, obj_name = build_obj["object"], build_obj["name"] - result = config.find_object(obj_name) - if not result: + obj = objects.get(obj_name) + if obj is None: if config.warn_missing_config and not build_obj["autogenerated"]: print(f"Missing configuration for {obj_name}") link_step.add(obj_path) return - lib, obj = result - lib_name = lib["lib"] - - # Use object options, then library options - options = lib.copy() - for key, value in obj.options.items(): - if value is not None or key not in options: - options[key] = value - - unit_src_path = Path(lib.get("src_dir", config.src_dir)) / options["source"] - if config.asm_dir is not None: - unit_asm_path = ( - Path(lib.get("asm_dir", config.asm_dir)) / options["source"] - ).with_suffix(".s") - link_built_obj = obj.completed - if unit_src_path.exists(): - if unit_src_path.suffix in (".c", ".cp", ".cpp"): + built_obj_path: Optional[Path] = None + if obj.src_path is not None and obj.src_path.exists(): + if obj.src_path.suffix in (".c", ".cp", ".cpp"): # Add MWCC & host build rules - built_obj_path = c_build(obj, options, lib_name, unit_src_path) - elif unit_src_path.suffix == ".s": + built_obj_path = c_build(obj, obj.src_path) + elif obj.src_path.suffix == ".s": # Add assembler build rule - built_obj_path = asm_build(obj, options, lib_name, unit_src_path) + built_obj_path = asm_build(obj, obj.src_path, obj.src_obj_path) else: - sys.exit(f"Unknown source file type {unit_src_path}") + sys.exit(f"Unknown source file type {obj.src_path}") else: if config.warn_missing_source or obj.completed: - print(f"Missing source file {unit_src_path}") + print(f"Missing source file {obj.src_path}") link_built_obj = False # Assembly overrides - if unit_asm_path is not None and unit_asm_path.exists(): + if obj.asm_path is not None and obj.asm_path.exists(): link_built_obj = True - built_obj_path = asm_build(obj, options, lib_name, unit_asm_path) + built_obj_path = asm_build(obj, obj.asm_path, obj.asm_obj_path) if link_built_obj and built_obj_path is not None: # Use the source-built object @@ -709,7 +855,10 @@ def generate_build_ninja( # Use the original (extracted) object link_step.add(obj_path) else: - sys.exit(f"Missing object for {obj_name}: {unit_src_path} {lib} {obj}") + lib_name = obj.options["lib"] + sys.exit( + f"Missing object for {obj_name}: {obj.src_path} {lib_name} {obj}" + ) # Add DOL link step link_step = LinkStep(build_config) @@ -725,7 +874,7 @@ def generate_build_ninja( add_unit(unit, module_link_step) # Add empty object to empty RELs if len(module_link_step.inputs) == 0: - if not config.rel_empty_file: + if config.rel_empty_file is None: sys.exit("ProjectConfig.rel_empty_file missing") add_unit( { @@ -749,13 +898,30 @@ def generate_build_ninja( if config.compilers_path and not os.path.exists(mw_path): sys.exit(f"Linker {mw_path} does not exist") + # Add all build steps needed before we link and after compiling objects + postcompile_implicit = write_custom_step("post-compile") + ### # Link ### for step in link_steps: step.write(n) + link_outputs.append(step.output()) n.newline() + # Add all build steps needed after linking and before GC/Wii native format generation + postlink_implicit = write_custom_step("post-link") + + ### + # Generate DOL + ### + n.build( + outputs=link_steps[0].output(), + rule="elf2dol", + inputs=link_steps[0].partial_output(), + implicit=[*postlink_implicit, dtk], + ) + ### # Generate RELs ### @@ -770,7 +936,7 @@ def generate_build_ninja( rspfile="$rspfile", rspfile_content="$in_newline", ) - generated_rels = [] + generated_rels: List[str] = [] for idx, link in enumerate(build_config["links"]): # Map module names to link steps link_steps_local = list( @@ -818,6 +984,9 @@ def generate_build_ninja( ) n.newline() + # Add all build steps needed post-build (re-building archives and such) + postbuild_implicit = write_custom_step("post-build") + ### # Helper rule for building all source files ### @@ -855,7 +1024,7 @@ def generate_build_ninja( outputs=ok_path, rule="check", inputs=config.check_sha_path, - implicit=[dtk, *map(lambda step: step.output(), link_steps)], + implicit=[dtk, *link_outputs, *postbuild_implicit], ) n.newline() @@ -874,6 +1043,22 @@ def generate_build_ninja( implicit=[ok_path, configure_script, python_lib, config.config_path], ) + ### + # Generate progress report + ### + n.comment("Generate progress report") + n.rule( + name="report", + command=f"{objdiff} report generate -o $out", + description="REPORT", + ) + report_implicit: List[str | Path] = [objdiff, "all_source"] + n.build( + outputs=report_path, + rule="report", + implicit=report_implicit, + ) + ### # Helper tools ### @@ -920,6 +1105,7 @@ def generate_build_ninja( ### # Split DOL ### + build_config_path = build_path / "config.json" n.comment("Split DOL into relocatable objects") n.rule( name="split", @@ -955,6 +1141,7 @@ def generate_build_ninja( configure_script, python_lib, python_lib_dir / "ninja_syntax.py", + *(config.reconfig_deps or []), ], ) n.newline() @@ -964,7 +1151,10 @@ def generate_build_ninja( ### n.comment("Default rule") if build_config: - n.default(progress_path) + if config.non_matching: + n.default(link_outputs) + else: + n.default(progress_path) else: n.default(build_config_path) @@ -976,13 +1166,15 @@ def generate_build_ninja( # Generate objdiff.json def generate_objdiff_config( - config: ProjectConfig, build_config: Optional[Dict[str, Any]] + config: ProjectConfig, + objects: Dict[str, Object], + build_config: Optional[Dict[str, Any]], ) -> None: - if not build_config: + if build_config is None: return objdiff_config: Dict[str, Any] = { - "min_version": "1.0.0", + "min_version": "2.0.0-beta.5", "custom_make": "ninja", "build_target": False, "watch_patterns": [ @@ -998,6 +1190,7 @@ def generate_objdiff_config( "*.json", ], "units": [], + "progress_categories": [], } # decomp.me compiler name mapping @@ -1008,68 +1201,55 @@ def generate_objdiff_config( "GC/1.2.5": "mwcc_233_163", "GC/1.2.5e": "mwcc_233_163e", "GC/1.2.5n": "mwcc_233_163n", + "GC/1.3": "mwcc_242_53", "GC/1.3.2": "mwcc_242_81", "GC/1.3.2r": "mwcc_242_81r", "GC/2.0": "mwcc_247_92", "GC/2.5": "mwcc_247_105", "GC/2.6": "mwcc_247_107", "GC/2.7": "mwcc_247_108", - "GC/3.0": "mwcc_41_60831", - # "GC/3.0a3": "mwcc_41_51213", + "GC/3.0a3": "mwcc_41_51213", "GC/3.0a3.2": "mwcc_41_60126", - # "GC/3.0a3.3": "mwcc_41_60209", - # "GC/3.0a3.4": "mwcc_42_60308", - # "GC/3.0a5": "mwcc_42_60422", + "GC/3.0a3.3": "mwcc_41_60209", + "GC/3.0a3.4": "mwcc_42_60308", + "GC/3.0a5": "mwcc_42_60422", "GC/3.0a5.2": "mwcc_41_60831", + "GC/3.0": "mwcc_41_60831", + "Wii/1.0RC1": "mwcc_42_140", "Wii/0x4201_127": "mwcc_42_142", - # "Wii/1.0": "mwcc_43_145", - # "Wii/1.0RC1": "mwcc_42_140", "Wii/1.0a": "mwcc_42_142", + "Wii/1.0": "mwcc_43_145", "Wii/1.1": "mwcc_43_151", "Wii/1.3": "mwcc_43_172", - # "Wii/1.5": "mwcc_43_188", + "Wii/1.5": "mwcc_43_188", "Wii/1.6": "mwcc_43_202", "Wii/1.7": "mwcc_43_213", } - build_path = config.out_path() - - def add_unit(build_obj: Dict[str, Any], module_name: str) -> None: - if build_obj["autogenerated"]: - # Skip autogenerated objects - return - + def add_unit( + build_obj: Dict[str, Any], module_name: str, progress_categories: List[str] + ) -> None: obj_path, obj_name = build_obj["object"], build_obj["name"] base_object = Path(obj_name).with_suffix("") unit_config: Dict[str, Any] = { "name": Path(module_name) / base_object, "target_path": obj_path, + "metadata": { + "auto_generated": build_obj["autogenerated"], + "progress_categories": progress_categories, + }, } - result = config.find_object(obj_name) - if not result: + obj = objects.get(obj_name) + if obj is None: objdiff_config["units"].append(unit_config) return - lib, obj = result - src_dir = Path(lib.get("src_dir", config.src_dir)) - - # Use object options, then library options - options = lib.copy() - for key, value in obj.options.items(): - if value is not None or key not in options: - options[key] = value - - unit_src_path = src_dir / str(options["source"]) - - if not unit_src_path.exists(): - objdiff_config["units"].append(unit_config) - return - - cflags = options["cflags"] - src_obj_path = build_path / "src" / f"{obj.base_name}.o" - src_ctx_path = build_path / "src" / f"{obj.base_name}.ctx" + src_exists = obj.src_path is not None and obj.src_path.exists() + if src_exists: + unit_config["base_path"] = obj.src_obj_path + cflags = obj.options["cflags"] reverse_fn_order = False if type(cflags) is list: for flag in cflags: @@ -1088,35 +1268,86 @@ def generate_objdiff_config( cflags = list(filter(keep_flag, cflags)) # Add appropriate lang flag - if unit_src_path.suffix in (".cp", ".cpp"): - cflags.insert(0, "-lang=c++") - else: - cflags.insert(0, "-lang=c") + if obj.src_path is not None and not any( + flag.startswith("-lang") for flag in cflags + ): + if obj.src_path.suffix in (".cp", ".cpp"): + cflags.insert(0, "-lang=c++") + else: + cflags.insert(0, "-lang=c") - unit_config["base_path"] = src_obj_path - unit_config["reverse_fn_order"] = reverse_fn_order - unit_config["complete"] = obj.completed - compiler_version = COMPILER_MAP.get(options["mw_version"]) + compiler_version = COMPILER_MAP.get(obj.options["mw_version"]) if compiler_version is None: - print(f"Missing scratch compiler mapping for {options['mw_version']}") + print(f"Missing scratch compiler mapping for {obj.options['mw_version']}") else: + cflags_str = make_flags_str(cflags) + if obj.options["extra_cflags"] is not None: + extra_cflags_str = make_flags_str(obj.options["extra_cflags"]) + cflags_str += " " + extra_cflags_str unit_config["scratch"] = { "platform": "gc_wii", "compiler": compiler_version, - "c_flags": make_flags_str(cflags), - "ctx_path": src_ctx_path, - "build_ctx": True, + "c_flags": cflags_str, } + if src_exists: + unit_config["scratch"].update( + { + "ctx_path": obj.ctx_path, + "build_ctx": True, + } + ) + category_opt: List[str] | str = obj.options["progress_category"] + if isinstance(category_opt, list): + progress_categories.extend(category_opt) + elif category_opt is not None: + progress_categories.append(category_opt) + unit_config["metadata"].update( + { + "complete": obj.completed, + "reverse_fn_order": reverse_fn_order, + "source_path": obj.src_path, + "progress_categories": progress_categories, + } + ) objdiff_config["units"].append(unit_config) # Add DOL units for unit in build_config["units"]: - add_unit(unit, build_config["name"]) + progress_categories = [] + # Only include a "dol" category if there are any modules + # Otherwise it's redundant with the global report measures + if len(build_config["modules"]) > 0: + progress_categories.append("dol") + add_unit(unit, build_config["name"], progress_categories) # Add REL units for module in build_config["modules"]: for unit in module["units"]: - add_unit(unit, module["name"]) + progress_categories = [] + if config.progress_modules: + progress_categories.append("modules") + if config.progress_each_module: + progress_categories.append(module["name"]) + add_unit(unit, module["name"], progress_categories) + + # Add progress categories + def add_category(id: str, name: str): + objdiff_config["progress_categories"].append( + { + "id": id, + "name": name, + } + ) + + if len(build_config["modules"]) > 0: + add_category("dol", "DOL") + if config.progress_modules: + add_category("modules", "Modules") + if config.progress_each_module: + for module in build_config["modules"]: + add_category(module["name"], module["name"]) + for category in config.progress_categories: + add_category(category.id, category.name) # Write objdiff.json with open("objdiff.json", "w", encoding="utf-8") as w: @@ -1129,9 +1360,11 @@ def generate_objdiff_config( # Calculate, print and write progress to progress.json def calculate_progress(config: ProjectConfig) -> None: + config.validate() + objects = config.objects() out_path = config.out_path() build_config = load_build_config(config, out_path / "config.json") - if not build_config: + if build_config is None: return class ProgressUnit: @@ -1163,12 +1396,8 @@ def calculate_progress(config: ProjectConfig) -> None: # Skip autogenerated objects return - result = config.find_object(build_obj["name"]) - if not result: - return - - _, obj = result - if not obj.completed: + obj = objects.get(build_obj["name"]) + if obj is None or not obj.completed: return self.code_progress += build_obj["code_size"] @@ -1182,26 +1411,52 @@ def calculate_progress(config: ProjectConfig) -> None: def data_frac(self) -> float: return self.data_progress / self.data_total + progress_units: Dict[str, ProgressUnit] = {} + if config.progress_all: + progress_units["all"] = ProgressUnit("All") + progress_units["dol"] = ProgressUnit("DOL") + if len(build_config["modules"]) > 0: + if config.progress_modules: + progress_units["modules"] = ProgressUnit("Modules") + if len(config.progress_categories) > 0: + for category in config.progress_categories: + progress_units[category.id] = ProgressUnit(category.name) + if config.progress_each_module: + for module in build_config["modules"]: + progress_units[module["name"]] = ProgressUnit(module["name"]) + + def add_unit(id: str, unit: Dict[str, Any]) -> None: + progress = progress_units.get(id) + if progress is not None: + progress.add(unit) + # Add DOL units - all_progress = ProgressUnit("All") if config.progress_all else None - dol_progress = ProgressUnit("DOL") for unit in build_config["units"]: - if all_progress: - all_progress.add(unit) - dol_progress.add(unit) + add_unit("all", unit) + add_unit("dol", unit) + obj = objects.get(unit["name"]) + if obj is not None: + category_opt = obj.options["progress_category"] + if isinstance(category_opt, list): + for id in category_opt: + add_unit(id, unit) + elif category_opt is not None: + add_unit(category_opt, unit) # Add REL units - rels_progress = ProgressUnit("Modules") if config.progress_modules else None - modules_progress: List[ProgressUnit] = [] for module in build_config["modules"]: - progress = ProgressUnit(module["name"]) - modules_progress.append(progress) for unit in module["units"]: - if all_progress: - all_progress.add(unit) - if rels_progress: - rels_progress.add(unit) - progress.add(unit) + add_unit("all", unit) + add_unit("modules", unit) + add_unit(module["name"], unit) + obj = objects.get(unit["name"]) + if obj is not None: + category_opt = obj.options["progress_category"] + if isinstance(category_opt, list): + for id in category_opt: + add_unit(id, unit) + elif category_opt is not None: + add_unit(category_opt, unit) # Print human-readable progress print("Progress:") @@ -1219,7 +1474,7 @@ def calculate_progress(config: ProjectConfig) -> None: print(f" Data: {unit.data_progress} / {unit.data_total} bytes") if config.progress_use_fancy: print( - "\nYou have {} out of {} {} and collected {} out of {} {}.".format( + "\nYou have {} out of {} {} and {} out of {} {}.".format( math.floor(code_frac * unit.code_fancy_frac), unit.code_fancy_frac, unit.code_fancy_item, @@ -1229,15 +1484,8 @@ def calculate_progress(config: ProjectConfig) -> None: ) ) - if all_progress: - print_category(all_progress) - print_category(dol_progress) - module_count = len(build_config["modules"]) - if module_count > 0: - print_category(rels_progress) - if config.progress_each_module: - for progress in modules_progress: - print_category(progress) + for progress in progress_units.values(): + print_category(progress) # Generate and write progress.json progress_json: Dict[str, Any] = {} @@ -1250,14 +1498,7 @@ def calculate_progress(config: ProjectConfig) -> None: "data/total": unit.data_total, } - if all_progress: - add_category("all", all_progress) - add_category("dol", dol_progress) - if len(build_config["modules"]) > 0: - if rels_progress: - add_category("modules", rels_progress) - if config.progress_each_module: - for progress in modules_progress: - add_category(progress.name, progress) + for id, progress in progress_units.items(): + add_category(id, progress) with open(out_path / "progress.json", "w", encoding="utf-8") as w: json.dump(progress_json, w, indent=4)