Add SI Loader extension (#664)

* SiLoader draft

* Fixes

* Fixes

* Fix asset build on Windows

* Remove whitespace

* Package assets in CI

* Disable clang-tidy

* Fix NCC

* Try this

* Disable extensions on Xbox for now

* Update bitmap

* Update 3rdparty/CMakeLists.txt

Co-authored-by: Anonymous Maarten <madebr@users.noreply.github.com>

* Add DEPFILE for asset command

* Use assets.d instead of .d

---------

Co-authored-by: Anonymous Maarten <madebr@users.noreply.github.com>

.github/workflows/ci.yml

@@ -34,7 +34,7 @@ jobs:
       fail-fast: false
       matrix:
         include:
-          - { name: 'Linux',               os: 'ubuntu-latest',  generator: 'Ninja',                 dx5: false, config: true,  linux: true, werror: true,   clang-tidy: true }
+          - { name: 'Linux',               os: 'ubuntu-latest',  generator: 'Ninja',                 dx5: false, config: true,  linux: true, werror: true,   clang-tidy: true, build-assets: true }
           - { name: 'Linux (Debug)',       os: 'ubuntu-latest',  generator: 'Ninja',                 dx5: false, config: true,  linux: true, werror: true,   clang-tidy: true,   debug: true }
           - { name: 'MSVC (x86)',          os: 'windows-latest', generator: 'Ninja',                 dx5: true,  config: false, msvc: true,  werror: false,  clang-tidy: false,  vc-arch: 'amd64_x86' }
           - { name: 'MSVC (x64)',          os: 'windows-latest', generator: 'Ninja',                 dx5: false, config: true,  msvc: true,  werror: false,  clang-tidy: false,   vc-arch: 'amd64' }
@@ -113,6 +113,7 @@ jobs:
             -DCMAKE_BUILD_TYPE=Release \
             -DISLE_USE_DX5=${{ !!matrix.dx5 }} \
             -DISLE_BUILD_CONFIG=${{ !!matrix.config }} \
+            -DISLE_BUILD_ASSETS=${{ !!matrix.build-assets }} \
             -DENABLE_CLANG_TIDY=${{ !!matrix.clang-tidy  }} \
             -DISLE_WERROR=${{ !!matrix.werror }} \
             -DISLE_DEBUG=${{ matrix.debug || 'OFF' }} \
@@ -121,6 +122,10 @@ jobs:
       - name: Build (CMake)
         run: cmake --build build --verbose --config Release
 
+      - name: Package Assets Separately
+        if: matrix.build-assets
+        run: (cd build/assets && zip -r ../../isle-assets.zip .)
+
       - name: Package (CPack)
         if: ${{ !matrix.n3ds }}
         run: |
@@ -180,6 +185,8 @@ jobs:
             build/dist/*.AppImage
             build/dist/*.3dsx
             build/dist/*.cia
+            isle-assets.zip
+          if-no-files-found: ignore
 
   flatpak:
     name: "Flatpak (${{ matrix.arch }})"

.gitmodules

@@ -7,3 +7,6 @@
 [submodule "imgui"]
 	path = 3rdparty/imgui
 	url = https://github.com/ocornut/imgui
+[submodule "3rdparty/libweaver"]
+	path = 3rdparty/libweaver
+	url = https://github.com/isledecomp/SIEdit

3rdparty/CMakeLists.txt

@@ -1,4 +1,5 @@
 set(CMAKE_C_CLANG_TIDY)
+set(CMAKE_CXX_CLANG_TIDY)
 
 if(DOWNLOAD_DEPENDENCIES)
     include(FetchContent)
@@ -72,3 +73,23 @@ target_include_directories(imgui PUBLIC ${imgui_SOURCE_DIR})
 target_link_libraries(imgui PUBLIC SDL3::Headers)
 target_link_libraries(imgui PRIVATE SDL3::SDL3)
 set_property(TARGET imgui PROPERTY CXX_CLANG_TIDY "")
+
+if(DOWNLOAD_DEPENDENCIES)
+    include(FetchContent)
+    FetchContent_Populate(
+        libweaver
+        URL https://github.com/isledecomp/SIEdit/archive/6da93b2072c41c41d526b8b9df7d4292be1f0f55.tar.gz
+        URL_MD5 ae59007fcb9efadc06c67621e1e107cb
+    )
+else()
+    set(libweaver_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/libweaver")
+endif()
+
+add_library(libweaver STATIC
+    ${libweaver_SOURCE_DIR}/lib/core.cpp
+    ${libweaver_SOURCE_DIR}/lib/file.cpp
+    ${libweaver_SOURCE_DIR}/lib/interleaf.cpp
+    ${libweaver_SOURCE_DIR}/lib/object.cpp
+    ${libweaver_SOURCE_DIR}/lib/sitypes.cpp
+)
+target_include_directories(libweaver PUBLIC ${libweaver_SOURCE_DIR}/lib)

3rdparty/libweaver

@@ -0,0 +1 @@
+Subproject commit 6da93b2072c41c41d526b8b9df7d4292be1f0f55

CMakeLists.txt

@@ -42,13 +42,14 @@ find_program(SDL_SHADERCROSS_BIN NAMES "shadercross")
 find_package(Python3 3.12 COMPONENTS Interpreter)
 
 option(ISLE_BUILD_APP "Build isle application" ON)
+option(ISLE_BUILD_ASSETS "Build assets from the /assets directory" OFF)
 option(ISLE_ASAN "Enable Address Sanitizer" OFF)
 option(ISLE_UBSAN "Enable Undefined Behavior Sanitizer" OFF)
 option(ISLE_WERROR "Treat warnings as errors" OFF)
 option(ISLE_DEBUG "Enable imgui debug" ON)
 cmake_dependent_option(ISLE_USE_DX5 "Build with internal DirectX 5 SDK" "${NOT_MINGW}" "WIN32;CMAKE_SIZEOF_VOID_P EQUAL 4" OFF)
 cmake_dependent_option(ISLE_MINIWIN "Use miniwin" ON "NOT ISLE_USE_DX5" OFF)
-cmake_dependent_option(ISLE_EXTENSIONS "Use extensions" ON "NOT ISLE_USE_DX5" OFF)
+cmake_dependent_option(ISLE_EXTENSIONS "Use extensions" ON "NOT ISLE_USE_DX5;NOT WINDOWS_STORE" OFF)
 cmake_dependent_option(ISLE_BUILD_CONFIG "Build CONFIG.EXE application" ON "MSVC OR ISLE_MINIWIN;NOT NINTENDO_3DS;NOT WINDOWS_STORE" OFF)
 cmake_dependent_option(ISLE_COMPILE_SHADERS "Compile shaders" ON "SDL_SHADERCROSS_BIN;TARGET Python3::Interpreter" OFF)
 option(CMAKE_POSITION_INDEPENDENT_CODE "Build with -fPIC" ON)
@@ -490,9 +491,11 @@ if (NOT ISLE_MINIWIN)
 endif()
 
 if (ISLE_EXTENSIONS)
+  target_link_libraries(lego1 PRIVATE libweaver)
   target_compile_definitions(lego1 PUBLIC EXTENSIONS)
   target_sources(lego1 PRIVATE
     extensions/src/extensions.cpp
+    extensions/src/siloader.cpp
     extensions/src/textureloader.cpp
   )
 endif()
@@ -644,6 +647,38 @@ if (ISLE_BUILD_CONFIG)
   endif()
 endif()
 
+if(ISLE_BUILD_ASSETS)
+  message(STATUS "Asset building is enabled")
+  set(GENERATED_ASSETS_DIR "${CMAKE_BINARY_DIR}/assets")
+  set(GENERATED_ASSETS_DEPFILE "${GENERATED_ASSETS_DIR}/assets.d")
+
+  add_executable(asset_generator EXCLUDE_FROM_ALL
+    assets/main.cpp
+  )
+  target_link_libraries(asset_generator PRIVATE libweaver)
+  target_include_directories(asset_generator PRIVATE "${CMAKE_SOURCE_DIR}/util" "${CMAKE_SOURCE_DIR}/LEGO1/omni/include" "${CMAKE_SOURCE_DIR}/LEGO1" "${CMAKE_SOURCE_DIR}/LEGO1/lego/sources")
+
+  add_custom_command(
+    OUTPUT  ${GENERATED_ASSETS_DIR}/.stamp
+    DEPFILE ${GENERATED_ASSETS_DEPFILE}
+    COMMAND ${CMAKE_COMMAND} -E make_directory ${GENERATED_ASSETS_DIR}
+    COMMAND $<TARGET_FILE:asset_generator> ${GENERATED_ASSETS_DIR} ${GENERATED_ASSETS_DEPFILE}
+    COMMAND ${CMAKE_COMMAND} -E touch ${GENERATED_ASSETS_DIR}/.stamp
+    WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/assets
+    DEPENDS asset_generator
+    COMMENT "Generating assets into ${GENERATED_ASSETS_DIR}/"
+    VERBATIM
+  )
+
+  add_custom_target(build_assets ALL
+    DEPENDS ${GENERATED_ASSETS_DIR}/.stamp
+  )
+
+  install(DIRECTORY ${GENERATED_ASSETS_DIR}/
+    DESTINATION assets
+  )
+endif()
+
 if (ISLE_MINIWIN)
   set_property(TARGET ${isle_targets} APPEND PROPERTY LINK_LIBRARIES "miniwin")
 endif()
@@ -673,6 +708,9 @@ if (MSVC)
     if (TARGET isle-config)
       target_compile_options(isle-config PRIVATE "-Zc:__cplusplus")
     endif()
+    if (TARGET asset_generator)
+      target_compile_options(asset_generator PRIVATE "-Zc:__cplusplus")
+    endif()
   endif()
 endif()
 

LEGO1/lego/legoomni/src/common/legoutils.cpp

@@ -2,6 +2,7 @@
 
 #include "3dmanager/lego3dmanager.h"
 #include "anim/legoanim.h"
+#include "extensions/siloader.h"
 #include "isle.h"
 #include "isle_actions.h"
 #include "islepathactor.h"
@@ -37,6 +38,8 @@
 #include <string.h>
 #include <vec.h>
 
+using namespace Extensions;
+
 // FUNCTION: LEGO1 0x1003dd70
 // FUNCTION: BETA10 0x100d3410
 LegoROI* PickROI(MxLong p_x, MxLong p_y)
@@ -515,6 +518,8 @@ MxBool RemoveFromCurrentWorld(const MxAtomId& p_atomId, MxS32 p_id)
 				}
 
 				((MxPresenter*) object)->EndAction();
+
+				Extension<SiLoader>::Call(RemoveWith, SiLoader::StreamObject{p_atomId, p_id}, world);
 			}
 
 			return TRUE;
@@ -549,6 +554,8 @@ MxBool RemoveFromWorld(
 				}
 
 				((MxPresenter*) object)->EndAction();
+
+				Extension<SiLoader>::Call(RemoveWith, SiLoader::StreamObject{p_entityAtom, p_entityId}, world);
 			}
 
 			return TRUE;

LEGO1/lego/legoomni/src/main/legomain.cpp

@@ -1,6 +1,7 @@
 #include "legomain.h"
 
 #include "3dmanager/lego3dmanager.h"
+#include "extensions/siloader.h"
 #include "islepathactor.h"
 #include "legoanimationmanager.h"
 #include "legobuildingmanager.h"
@@ -40,6 +41,8 @@ DECOMP_SIZE_ASSERT(LegoOmni::WorldContainer, 0x1c)
 DECOMP_SIZE_ASSERT(LegoWorldList, 0x18)
 DECOMP_SIZE_ASSERT(LegoWorldListCursor, 0x10)
 
+using namespace Extensions;
+
 // GLOBAL: LEGO1 0x100f6718
 // GLOBAL: BETA10 0x101ee748
 // STRING: LEGO1 0x100f6710
@@ -351,6 +354,9 @@ MxResult LegoOmni::Create(MxOmniCreateParam& p_param)
 	else {
 		SDL_LogError(SDL_LOG_CATEGORY_APPLICATION, "Failed to create MxTransitionManager");
 	}
+
+	Extension<SiLoader>::Call(Load);
+
 done:
 	return result;
 	// LINE: BETA10 0x1008e35d
@@ -670,6 +676,14 @@ MxResult LegoOmni::Start(MxDSAction* p_dsAction)
 	this->m_action.SetObjectId(p_dsAction->GetObjectId());
 	this->m_action.SetUnknown24(p_dsAction->GetUnknown24());
 #endif
+
+	if (result == SUCCESS) {
+		Extension<SiLoader>::Call(
+			StartWith,
+			SiLoader::StreamObject{p_dsAction->GetAtomId(), p_dsAction->GetObjectId()}
+		);
+	}
+
 	return result;
 }
 

LEGO1/omni/src/video/mxvideopresenter.cpp

@@ -200,22 +200,6 @@ inline MxS32 MxVideoPresenter::PrepareRects(RECT& p_rectDest, RECT& p_rectSrc)
 		return -1;
 	}
 
-	if (p_rectDest.bottom > 480) {
-		p_rectDest.bottom = 480;
-	}
-
-	if (p_rectDest.right > 640) {
-		p_rectDest.right = 640;
-	}
-
-	if (p_rectSrc.bottom > 480) {
-		p_rectSrc.bottom = 480;
-	}
-
-	if (p_rectSrc.right > 640) {
-		p_rectSrc.right = 640;
-	}
-
 	int height, width;
 	if ((height = (p_rectDest.bottom - p_rectDest.top) + 1) <= 1 ||
 		(width = (p_rectDest.right - p_rectDest.left) + 1) <= 1) {

assets/main.cpp

@@ -0,0 +1,58 @@
+#include "mxdsaction.h"
+
+#include <file.h>
+#include <filesystem>
+#include <fstream>
+#include <interleaf.h>
+#include <object.h>
+
+si::Interleaf::Version version = si::Interleaf::Version2_2;
+uint32_t bufferSize = 65536;
+uint32_t bufferCount = 8;
+
+std::string out;
+std::ofstream depfile;
+si::MemoryBuffer mxHd;
+
+void CreateWidescreen()
+{
+	si::Interleaf si;
+	std::string file = out + "/widescreen.si";
+	mxHd.seek(0, si::MemoryBuffer::SeekStart);
+	si.Read(&mxHd);
+
+	si::Object GaraDoor_Wide;
+	const char extra[] =
+		"World:current, StartWith:\\Lego\\Scripts\\Isle\\Isle;1160, RemoveWith:\\Lego\\Scripts\\Isle\\Isle;1161";
+	GaraDoor_Wide.type_ = si::MxOb::Bitmap;
+	GaraDoor_Wide.flags_ = MxDSAction::c_enabled | MxDSAction::c_bit4;
+	GaraDoor_Wide.duration_ = -1;
+	GaraDoor_Wide.loops_ = 1;
+	GaraDoor_Wide.extra_ = si::bytearray(extra, sizeof(extra));
+	GaraDoor_Wide.presenter_ = "MxStillPresenter";
+	GaraDoor_Wide.name_ = "GaraDoor_Wide";
+	GaraDoor_Wide.filetype_ = si::MxOb::STL;
+	GaraDoor_Wide.location_.x = -240.0;
+	GaraDoor_Wide.location_.z = -1.0;
+	GaraDoor_Wide.up_.y = 1.0;
+	GaraDoor_Wide.ReplaceWithFile("widescreen/garadoor.bmp");
+	si.AppendChild(&GaraDoor_Wide);
+	depfile << file << ": " << (std::filesystem::current_path() / "widescreen/garadoor.bmp").string() << std::endl;
+
+	si.Write(file.c_str());
+}
+
+int main(int argc, char* argv[])
+{
+	out = argv[1];
+	depfile = std::ofstream(argv[2]);
+
+	mxHd.WriteU32(si::RIFF::MxHd);
+	mxHd.WriteU32(3 * sizeof(uint32_t));
+	mxHd.WriteU32(version);
+	mxHd.WriteU32(bufferSize);
+	mxHd.WriteU32(bufferCount);
+
+	CreateWidescreen();
+	return 0;
+}

extensions/include/extensions/extensions.h

@@ -9,7 +9,7 @@
 
 namespace Extensions
 {
-constexpr const char* availableExtensions[] = {"extensions:texture loader"};
+constexpr const char* availableExtensions[] = {"extensions:texture loader", "extensions:si loader"};
 
 LEGO1_EXPORT void Enable(const char* p_key, std::map<std::string, std::string> p_options);
 

extensions/include/extensions/siloader.h

@@ -0,0 +1,41 @@
+#pragma once
+
+#include "extensions/extensions.h"
+#include "legoworld.h"
+#include "mxatom.h"
+
+#include <map>
+#include <vector>
+
+namespace Extensions
+{
+class SiLoader {
+public:
+	typedef std::pair<MxAtomId, MxU32> StreamObject;
+
+	static void Initialize();
+	static bool Load();
+	static bool StartWith(StreamObject p_object);
+	static bool RemoveWith(StreamObject p_object, LegoWorld* world);
+
+	static std::map<std::string, std::string> options;
+	static bool enabled;
+
+private:
+	static std::vector<std::string> files;
+	static std::vector<std::pair<StreamObject, StreamObject>> startWith;
+	static std::vector<std::pair<StreamObject, StreamObject>> removeWith;
+
+	static bool LoadFile(const char* p_file);
+};
+
+#ifdef EXTENSIONS
+constexpr auto Load = &SiLoader::Load;
+constexpr auto StartWith = &SiLoader::StartWith;
+constexpr auto RemoveWith = &SiLoader::RemoveWith;
+#else
+constexpr decltype(&SiLoader::Load) Load = nullptr;
+constexpr decltype(&SiLoader::StartWith) StartWith = nullptr;
+constexpr decltype(&SiLoader::RemoveWith) RemoveWith = nullptr;
+#endif
+}; // namespace Extensions

extensions/src/extensions.cpp

@@ -1,5 +1,6 @@
 #include "extensions/extensions.h"
 
+#include "extensions/siloader.h"
 #include "extensions/textureloader.h"
 
 #include <SDL3/SDL_log.h>
@@ -13,6 +14,11 @@ void Extensions::Enable(const char* p_key, std::map<std::string, std::string> p_
 				TextureLoader::enabled = true;
 				TextureLoader::Initialize();
 			}
+			else if (!SDL_strcasecmp(p_key, "extensions:si loader")) {
+				SiLoader::options = std::move(p_options);
+				SiLoader::enabled = true;
+				SiLoader::Initialize();
+			}
 
 			SDL_Log("Enabled extension: %s", p_key);
 			break;

extensions/src/siloader.cpp

@@ -0,0 +1,110 @@
+#include "extensions/siloader.h"
+
+#include "mxdsaction.h"
+#include "mxmisc.h"
+#include "mxstreamer.h"
+
+#include <SDL3/SDL.h>
+#include <interleaf.h>
+
+using namespace Extensions;
+
+std::map<std::string, std::string> SiLoader::options;
+std::vector<std::pair<SiLoader::StreamObject, SiLoader::StreamObject>> SiLoader::startWith;
+std::vector<std::pair<SiLoader::StreamObject, SiLoader::StreamObject>> SiLoader::removeWith;
+bool SiLoader::enabled = false;
+
+void SiLoader::Initialize()
+{
+}
+
+bool SiLoader::Load()
+{
+	char* files = SDL_strdup(options["si loader:files"].c_str());
+	char* saveptr;
+
+	for (char* file = SDL_strtok_r(files, ",\n", &saveptr); file; file = SDL_strtok_r(NULL, ",\n", &saveptr)) {
+		LoadFile(file);
+	}
+
+	SDL_free(files);
+	return true;
+}
+
+bool SiLoader::StartWith(StreamObject p_object)
+{
+	for (const auto& key : startWith) {
+		if (key.first == p_object) {
+			MxDSAction action;
+			action.SetAtomId(key.second.first);
+			action.SetObjectId(key.second.second);
+			Start(&action);
+		}
+	}
+
+	return true;
+}
+
+bool SiLoader::RemoveWith(StreamObject p_object, LegoWorld* world)
+{
+	for (const auto& key : removeWith) {
+		if (key.first == p_object) {
+			RemoveFromWorld(key.second.first, key.second.second, world->GetAtomId(), world->GetEntityId());
+		}
+	}
+
+	return true;
+}
+
+bool SiLoader::LoadFile(const char* p_file)
+{
+	si::Interleaf si;
+	MxStreamController* controller;
+
+	MxString path = MxString(MxOmni::GetHD()) + p_file;
+	path.MapPathToFilesystem();
+	if (si.Read(path.GetData()) != si::Interleaf::ERROR_SUCCESS) {
+		path = MxString(MxOmni::GetCD()) + p_file;
+		path.MapPathToFilesystem();
+		if (si.Read(path.GetData()) != si::Interleaf::ERROR_SUCCESS) {
+			SDL_Log("Could not parse SI file %s", p_file);
+			return false;
+		}
+	}
+
+	if (!(controller = Streamer()->Open(p_file, MxStreamer::e_diskStream))) {
+		SDL_Log("Could not load SI file %s", p_file);
+		return false;
+	}
+
+	for (si::Core* child : si.GetChildren()) {
+		if (si::Object* object = dynamic_cast<si::Object*>(child)) {
+			if (object->type() != si::MxOb::Null) {
+				std::string extra(object->extra_.data(), object->extra_.size());
+				const char* directive;
+				char atom[256];
+				uint32_t id;
+
+				if ((directive = SDL_strstr(extra.c_str(), "StartWith:"))) {
+					if (SDL_sscanf(directive, "StartWith:%255[^;];%d", atom, &id) == 2) {
+						startWith.emplace_back(
+							StreamObject{MxAtomId{atom, e_lowerCase2}, id},
+							StreamObject{controller->GetAtom(), object->id_}
+						);
+					}
+				}
+
+				if ((directive = SDL_strstr(extra.c_str(), "RemoveWith:"))) {
+					if (SDL_sscanf(directive, "RemoveWith:%255[^;];%d", atom, &id) == 2) {
+						removeWith.emplace_back(
+							StreamObject{MxAtomId{atom, e_lowerCase2}, id},
+							StreamObject{controller->GetAtom(), object->id_}
+						);
+					}
+				}
+			}
+		}
+	}
+
+	return true;
+}

tools/ncc/skip.yml

@@ -78,3 +78,4 @@ SDL_KeyboardID_v: "SDL-based name"
 SDL_MouseID_v: "SDL-based name"
 SDL_JoystickID_v: "SDL-based name"
 SDL_TouchID_v: "SDL-based name"
+Load: "Not a variable but function name"