loader: add external CXI loading.

When "load external firms and modules" is enabled, Loader will load the
sysmodule from /luma/sysmodule/<titleid>.cxi (all uppercase, and with
the N3DS title ID bit if relevant) and skip patching. Note that this is
a title ID here, not a process name (unlike what we do for KIPs).

While this is aimed at enabling people to easily load replacements for
official sysmodules, you can load your own custom sysmodules that don't
correspond to anything installed. You can use gdb to do so:

  set remote exec-file <tid>
  run

sysmodules/loader/source/3dsx.c

@@ -54,15 +54,6 @@ static inline u32 TranslateAddr(u32 off, _3DSX_LoadInfo* d, u32* offsets)
     return d->segAddrs[2] + off - offsets[1];
 }
 
-u32 IFile_Read2(IFile *file, void* buffer, u32 size, u32 offset)
-{
-    Result res;
-    u64 total = 0;
-    file->pos = offset;
-    res = IFile_Read(file, &total, buffer, size);
-    return R_SUCCEEDED(res) ? total : 0;
-}
-
 bool Ldr_Get3dsxSize(u32* pSize, IFile *file)
 {
     _3DSX_Header hdr;

sysmodules/loader/source/ifile.c

@@ -121,3 +121,17 @@ Result IFile_Write(IFile *file, u64 *total, const void *buffer, u32 len, u32 fla
   *total = cur;
   return res;
 }
+
+Result IFile_ReadAt(IFile *file, u64 *total, void *buffer, u32 offset, u32 len)
+{
+    *total = 0;
+    file->pos = offset;
+    return IFile_Read(file, total, buffer, len);
+}
+
+u32 IFile_Read2(IFile *file, void *buffer, u32 size, u32 offset)
+{
+    u64 total = 0;
+    Result res = IFile_ReadAt(file, &total, buffer, offset, size);
+    return R_SUCCEEDED(res) ? (u32)total : 0;
+}

sysmodules/loader/source/ifile.h

@@ -18,3 +18,6 @@ Result IFile_GetSize(IFile *file, u64 *size);
 Result IFile_SetSize(IFile *file, u64 size);
 Result IFile_Read(IFile *file, u64 *total, void *buffer, u32 len);
 Result IFile_Write(IFile *file, u64 *total, const void *buffer, u32 len, u32 flags);
+
+Result IFile_ReadAt(IFile *file, u64 *total, void *buffer, u32 offset, u32 len);
+u32 IFile_Read2(IFile *file, void *buffer, u32 size, u32 offset);

sysmodules/loader/source/loader.c

@@ -6,12 +6,18 @@
 #include "util.h"
 #include "hbldr.h"
 
+#define SYSMODULE_CXI_COOKIE_MASK 0xEEEE000000000000ull
+
 extern u32 config, multiConfig, bootConfig;
 extern bool isN3DS, isSdMode;
 
-static u64 g_cached_programHandle;
+static u64 g_cached_programHandle; // for exheader info only
 static ExHeader_Info g_exheaderInfo;
 
+static IFile g_cached_sysmoduleCxiFile;
+static u64 g_cached_sysmoduleCxiCookie;
+static Ncch g_cached_sysmoduleCxiNcch;
+
 typedef struct ContentPath {
     u32 contentType;
     char fileName[8]; // for exefs
@@ -99,6 +105,25 @@ static int lzss_decompress(u8 *end)
     return ret;
 }
 
+static inline bool IsSysmoduleId(u64 tid)
+{
+    return (tid >> 32) == 0x00040130;
+}
+
+static inline bool IsSysmoduleCxiCookie(u64 programHandle)
+{
+    return (programHandle >> 32) == (SYSMODULE_CXI_COOKIE_MASK >> 32);
+}
+
+static void InvalidateCachedCxiFile(void)
+{
+    if (g_cached_sysmoduleCxiFile.handle != 0)
+        IFile_Close(&g_cached_sysmoduleCxiFile);
+    memset(&g_cached_sysmoduleCxiFile, 0, sizeof(IFile));
+
+    g_cached_sysmoduleCxiCookie = 0;
+}
+
 static Result allocateProgramMemoryWrapper(prog_addrs_t *mapped, const ExHeader_Info *exhi, const prog_addrs_t *vaddr)
 {
     memcpy(mapped, vaddr, sizeof(prog_addrs_t));
@@ -108,11 +133,6 @@ static Result allocateProgramMemoryWrapper(prog_addrs_t *mapped, const ExHeader_
     return allocateProgramMemory(exhi, mapped->text_addr, mapped->total_size << 12);
 }
 
-static inline bool IsSysmoduleId(u64 tid)
-{
-    return (tid >> 32) == 0x00040130;
-}
-
 static Result loadCode(const ExHeader_Info *exhi, u64 programHandle, const prog_addrs_t *mapped)
 {
     IFile file;
@@ -125,6 +145,25 @@ static Result loadCode(const ExHeader_Info *exhi, u64 programHandle, const prog_
     const ExHeader_CodeSetInfo *csi = &exhi->sci.codeset_info;
     bool isCompressed = csi->flags.compress_exefs_code;
 
+    // Load from CXI, and skip patches, if we were opened from it
+    if (IsSysmoduleCxiCookie(programHandle))
+    {
+        u32 sz_ = 0;
+        bool ok = readSysmoduleCxiCode((u8 *)mapped->text_addr, &sz_, (u64)mapped->total_size << 12, &g_cached_sysmoduleCxiFile, &g_cached_sysmoduleCxiNcch);
+        size = sz_;
+
+        if (!ok)
+            return (Result)-2;
+
+        // Decompress
+        if (isCompressed)
+            lzss_decompress((u8 *)mapped->text_addr + size);
+
+        // No need to keep the file open at this point
+        InvalidateCachedCxiFile();
+        return 0;
+    }
+
     bool codeLoadedExternally = false;
     if (CONFIG(PATCHGAMES))
     {
@@ -186,6 +225,29 @@ static inline bool IsHioId(u64 id)
 static Result GetProgramInfoImpl(ExHeader_Info *exheaderInfo, u64 programHandle)
 {
     Result res;
+
+    // Load from CXI, and skip patches, if we were opened from it
+    if (IsSysmoduleCxiCookie(programHandle))
+    {
+        u64 titleId = 0x0004013000000000ull | (u32)programHandle;
+        if (g_cached_sysmoduleCxiCookie != programHandle)
+        {
+            InvalidateCachedCxiFile();
+            res = openSysmoduleCxi(&g_cached_sysmoduleCxiFile, titleId);
+            if (R_FAILED(res))
+                return res;
+            g_cached_sysmoduleCxiCookie = programHandle;
+        }
+
+        bool ok = readSysmoduleCxiNcchHeader(&g_cached_sysmoduleCxiNcch, &g_cached_sysmoduleCxiFile);
+        if (!ok)
+            return (Result)-2;
+        ok = readSysmoduleCxiExHeaderInfo(exheaderInfo, &g_cached_sysmoduleCxiNcch, &g_cached_sysmoduleCxiFile);
+        if (!ok)
+            return (Result)-2;
+        return 0;
+    }
+
     TRY(IsHioId(programHandle) ? FSREG_GetProgramInfo(exheaderInfo, 1, programHandle) : PXIPM_GetProgramInfo(exheaderInfo, programHandle));
 
     // Tweak 3dsx placeholder title exheaderInfo
@@ -251,7 +313,7 @@ static Result LoadProcessImpl(Handle *outProcessHandle, const ExHeader_Info *exh
             region = desc & 0xF00;
     }
     if (region == 0)
-            return MAKERESULT(RL_PERMANENT, RS_INVALIDARG, 1, 2);
+            return MAKERESULT(RL_PERMANENT, RS_INVALIDARG, RM_KERNEL, 2);
 
     // allocate process memory
     vaddr.text_addr = csi->text.address;
@@ -322,9 +384,30 @@ static Result RegisterProgram(u64 *programHandle, FS_ProgramInfo *title, FS_Prog
     }
     else
     {
-        TRY(PXIPM_RegisterProgram(programHandle, title, update));
+        bool loadedCxiFromStorage = false;
-        if (IsHioId(*programHandle)) // double check this is indeed *not* HIO
+        if (IsSysmoduleId(titleId) && CONFIG(LOADEXTFIRMSANDMODULES))
-            panic(0);
+        {
+            // Forbid having two such file handles being open at the same time
+            // Also reload the file even if already cached.
+            InvalidateCachedCxiFile();
+
+            res = openSysmoduleCxi(&g_cached_sysmoduleCxiFile, titleId);
+            if (R_SUCCEEDED(res))
+            {
+                // A .cxi with the correct name in /luma/sysmodule exists, proceed
+                *programHandle = SYSMODULE_CXI_COOKIE_MASK | (u32)titleId;
+                g_cached_sysmoduleCxiCookie = *programHandle;
+                loadedCxiFromStorage = true;
+            }
+        }
+
+        if (!loadedCxiFromStorage)
+        {
+            // Otherwise, just load as normal
+            TRY(PXIPM_RegisterProgram(programHandle, title, update));
+            if (IsHioId(*programHandle)) // double check this is indeed *not* HIO
+                panic(0);
+        }
     }
 
     return res;
@@ -332,7 +415,17 @@ static Result RegisterProgram(u64 *programHandle, FS_ProgramInfo *title, FS_Prog
 
 static Result UnregisterProgram(u64 programHandle)
 {
-    return IsHioId(programHandle) ? FSREG_UnloadProgram(programHandle) : PXIPM_UnregisterProgram(programHandle);
+    if (g_cached_programHandle == programHandle)
+        g_cached_programHandle = 0;
+
+    if (IsSysmoduleCxiCookie(programHandle))
+    {
+        if (programHandle == g_cached_sysmoduleCxiCookie)
+            InvalidateCachedCxiFile();
+        return 0;
+    }
+    else
+        return IsHioId(programHandle) ? FSREG_UnloadProgram(programHandle) : PXIPM_UnregisterProgram(programHandle);
 }
 
 void loaderHandleCommands(void *ctx)
@@ -368,9 +461,6 @@ void loaderHandleCommands(void *ctx)
             break;
         case 3: // UnregisterProgram
             memcpy(&programHandle, &cmdbuf[1], 8);
-
-            if (g_cached_programHandle == programHandle)
-                g_cached_programHandle = 0;
             cmdbuf[1] = UnregisterProgram(programHandle);
             cmdbuf[0] = IPC_MakeHeader(3, 1, 0);
             break;

sysmodules/loader/source/patcher.c

@@ -4,6 +4,7 @@
 #include "memory.h"
 #include "strings.h"
 #include "romfsredir.h"
+#include "util.h"
 
 static u32 patchMemory(u8 *start, u32 size, const void *pattern, u32 patSize, s32 offset, const void *replace, u32 repSize, u32 count)
 {
@@ -318,6 +319,69 @@ exit:
     return ret;
 }
 
+Result openSysmoduleCxi(IFile *outFile, u64 progId)
+{
+    char path[] = "/luma/sysmodules/0000000000000000.cxi";
+    progIdToStr(path + sizeof("/luma/sysmodules/0000000000000000") - 2, progId);
+
+    FS_ArchiveID archiveId = isSdMode ? ARCHIVE_SDMC : ARCHIVE_NAND_RW;
+    return fileOpen(outFile, archiveId, path, FS_OPEN_READ);
+}
+
+bool readSysmoduleCxiNcchHeader(Ncch *outNcchHeader, IFile *file)
+{
+    u64 total = 0;
+    return R_SUCCEEDED(IFile_ReadAt(file, &total, outNcchHeader, 0, sizeof(Ncch))) && total == sizeof(Ncch);
+}
+
+bool readSysmoduleCxiExHeaderInfo(ExHeader_Info *outExhi, const Ncch *ncchHeader, IFile *file)
+{
+    u64 total = 0;
+    u32 size = ncchHeader->exHeaderSize;
+    if (size < sizeof(ExHeader_Info))
+        return false;
+    return R_SUCCEEDED(IFile_ReadAt(file, &total, outExhi, sizeof(Ncch), size)) && total == size;
+}
+
+bool readSysmoduleCxiCode(u8 *outCode, u32 *outSize, u32 maxSize, IFile *file, const Ncch *ncchHeader)
+{
+    u32 contentUnitShift = 9 + ncchHeader->flags[6];
+    u32 exeFsOffset = ncchHeader->exeFsOffset << contentUnitShift;
+    u32 exeFsSize = ncchHeader->exeFsSize << contentUnitShift;
+
+    if (exeFsSize < sizeof(ExeFsHeader) || exeFsOffset < 0x200 + ncchHeader->exHeaderSize)
+        return false;
+
+    // Read ExeFs header
+    ExeFsHeader hdr;
+    u64 total = 0;
+    u32 size = sizeof(ExeFsHeader);
+    Result res = IFile_ReadAt(file, &total, &hdr, exeFsOffset, size);
+
+    if (R_FAILED(res) || total != size)
+        return false;
+
+    // Get .code section info
+    ExeFsFileHeader *codeHdr = NULL;
+    for (u32 i = 0; i < 8; i++)
+    {
+        ExeFsFileHeader *fileHdr = &hdr.fileHeaders[i];
+        if (strncmp(fileHdr->name, ".code", 8) == 0)
+            codeHdr = fileHdr;
+    }
+
+    if (codeHdr == NULL)
+        return false;
+
+    size = codeHdr->size;
+    *outSize = size;
+    if (size > maxSize)
+        return false;
+
+    res = IFile_ReadAt(file, &total, outCode, exeFsOffset + sizeof(ExeFsHeader) + codeHdr->offset, size);
+    return R_SUCCEEDED(res) && total == size;
+}
+
 bool loadTitleCodeSection(u64 progId, u8 *code, u32 size)
 {
     /* Here we look for "/luma/titles/[u64 titleID in hex, uppercase]/code.bin"

sysmodules/loader/source/patcher.h

@@ -3,6 +3,7 @@
 #include <3ds/types.h>
 #include <3ds/exheader.h>
 #include "ifile.h"
+#include "util.h"
 
 #define MAKE_BRANCH(src,dst)      (0xEA000000 | ((u32)((((u8 *)(dst) - (u8 *)(src)) >> 2) - 2) & 0xFFFFFF))
 #define MAKE_BRANCH_LINK(src,dst) (0xEB000000 | ((u32)((((u8 *)(dst) - (u8 *)(src)) >> 2) - 2) & 0xFFFFFF))
@@ -45,3 +46,8 @@ extern bool isN3DS, isSdMode;
 void patchCode(u64 progId, u16 progVer, u8 *code, u32 size, u32 textSize, u32 roSize, u32 dataSize, u32 roAddress, u32 dataAddress);
 bool loadTitleCodeSection(u64 progId, u8 *code, u32 size);
 bool loadTitleExheaderInfo(u64 progId, ExHeader_Info *exheaderInfo);
+
+Result openSysmoduleCxi(IFile *outFile, u64 progId);
+bool readSysmoduleCxiNcchHeader(Ncch *outNcchHeader, IFile *file);
+bool readSysmoduleCxiExHeaderInfo(ExHeader_Info *outExhi, const Ncch *ncchHeader, IFile *file);
+bool readSysmoduleCxiCode(u8 *outCode, u32 *outSize, u32 maxSize, IFile *file, const Ncch *ncchHeader);

sysmodules/loader/source/util.h

@@ -11,6 +11,50 @@
 #define TRY(expr)               if(R_FAILED(res = (expr))) return res;
 #define TRYG(expr, label)       if(R_FAILED(res = (expr))) goto label;
 
+typedef struct Ncch {
+    u8 sig[0x100]; //RSA-2048 signature of the NCCH header, using SHA-256
+    char magic[4]; //NCCH
+    u32 contentSize; //Media unit
+    u8 partitionId[8];
+    u8 makerCode[2];
+    u16 version;
+    u8 reserved1[4];
+    u8 programID[8];
+    u8 reserved2[0x10];
+    u8 logoHash[0x20]; //Logo Region SHA-256 hash
+    char productCode[0x10];
+    u8 exHeaderHash[0x20]; //Extended header SHA-256 hash
+    u32 exHeaderSize; //Extended header size
+    u32 reserved3;
+    u8 flags[8];
+    u32 plainOffset; //Media unit
+    u32 plainSize; //Media unit
+    u32 logoOffset; //Media unit
+    u32 logoSize; //Media unit
+    u32 exeFsOffset; //Media unit
+    u32 exeFsSize; //Media unit
+    u32 exeFsHashSize; //Media unit
+    u32 reserved4;
+    u32 romFsOffset; //Media unit
+    u32 romFsSize; //Media unit
+    u32 romFsHashSize; //Media unit
+    u32 reserved5;
+    u8 exeFsHash[0x20]; //ExeFS superblock SHA-256 hash
+    u8 romFsHash[0x20]; //RomFS superblock SHA-256 hash
+} Ncch;
+
+typedef struct ExeFsFileHeader {
+    char name[8];
+    u32 offset;
+    u32 size;
+} ExeFsFileHeader;
+
+typedef struct ExeFsHeader {
+    ExeFsFileHeader fileHeaders[10];
+    u8 _reserved_0xa0[0xC0 - 0xA0];
+    u8 fileHashes[10][32];
+} ExeFsHeader;
+
 #ifdef XDS
 static void hexItoa(u64 number, char *out, u32 digits, bool uppercase)
 {