From a5127470d72be60cf79e4e7ae397745d9a5f815e Mon Sep 17 00:00:00 2001 From: Sewer56 Date: Tue, 23 Apr 2024 06:42:36 +0100 Subject: [PATCH 1/6] WIP: VFS Docs (Migrated from R2 Mod) via LLM --- .../Essentials/Virtual-FileSystem/About.md | 94 ++++++ .../Implementation-Details.md | 275 ++++++++++++++++++ .../Implementing-Redirection.md | 107 +++++++ .../Virtual-FileSystem/Performance.md | 73 +++++ .../Virtual-FileSystem/Programmer-Usage.md | 63 ++++ .../Essentials/Virtual-FileSystem/Usage.md | 62 ++++ mkdocs.yml | 5 + 7 files changed, 679 insertions(+) create mode 100644 docs/Mods/Essentials/Virtual-FileSystem/Implementation-Details.md create mode 100644 docs/Mods/Essentials/Virtual-FileSystem/Implementing-Redirection.md create mode 100644 docs/Mods/Essentials/Virtual-FileSystem/Performance.md create mode 100644 docs/Mods/Essentials/Virtual-FileSystem/Programmer-Usage.md create mode 100644 docs/Mods/Essentials/Virtual-FileSystem/Usage.md diff --git a/docs/Mods/Essentials/Virtual-FileSystem/About.md b/docs/Mods/Essentials/Virtual-FileSystem/About.md index e69de29..8cbb103 100644 --- a/docs/Mods/Essentials/Virtual-FileSystem/About.md +++ b/docs/Mods/Essentials/Virtual-FileSystem/About.md @@ -0,0 +1,94 @@ +# About the Reloaded Virtual FileSystem + +The Reloaded Virtual File System (VFS) is an invisible helper that sits between your games and the files they use. It allows your games to 'see' and open files that aren't really 'there', keeping your game folder unmodified. + +```mermaid +flowchart LR + + p[Game] -- Open File --> vfs[Reloaded VFS] + vfs -- Open Different File --> of[Operating System] +``` + +The VFS sits in the middle and does some magic 😇. + +```mermaid +classDiagram + + class `Mod Folder` + `Mod Folder` : data3.pak + + class `Mod 2 Folder` + `Mod 2 Folder` : data4.pak + + class `Real Game Folder` + `Real Game Folder` : data1.pak + `Real Game Folder` : data2.pak + `Real Game Folder` : game.exe + + class `Virtual Game Folder [What Game Sees]` + `Virtual Game Folder [What Game Sees]` : data1.pak + `Virtual Game Folder [What Game Sees]` : data2.pak + `Virtual Game Folder [What Game Sees]` : data3.pak + `Virtual Game Folder [What Game Sees]` : data4.pak + `Virtual Game Folder [What Game Sees]` : game.exe + + `Mod Folder` --|> `Virtual Game Folder [What Game Sees]` + `Mod 2 Folder` --|> `Virtual Game Folder [What Game Sees]` + `Real Game Folder` --|> `Virtual Game Folder [What Game Sees]` +``` + +## Characteristics + +Compared to Windows symlinks/hardlinks: + +- Links are only visible to the current application. +- Write access to game folder is not needed. Can even link new content into read-only folders. +- Administrator rights are not needed. +- Can overlay multiple directories on top of the destination. + +And with the following benefits: + +- Easy to use API for programmers. +- Practically zero overhead. +- Can add/remove and remap files on the fly (without making changes on disk). +- Supports Wine on Linux. + +## Limitations + +The Reloaded VFS only implements *well defined* functionality, effectively becoming a ***read-only*** VFS. Write operations, such as creating a file are unaffected. + +If a game wants to write a new file (such as a savefile), no action will be taken and the file will be written to the game folder. If a native DLL plugin wants to write a config file, it will write it to the game folder, as normal. + +### Warning + +Proceed with care if any of the following applies: + +- If your game's modding tools operate on a modified game directory (e.g. Skyrim xEdit), using VFS is not recommended as new files might be written to the game folder. + +- Do not use VFS to redirect files deleted and then recreated by games; ***you will lose the files from inside your mod***. + +### Error Cases + +Using this VFS is not appropriate for your game if any of the following is true. + +- This VFS does not handle child processes. Do not use VFS for games that can run external tools with virtualized files. + +### Additional Limitations + +The following limitations should not cause concern. + +- Reloaded VFS does not support Reparse Point Tags. + - However, this shouldn't cause issues with mods stored on cloud/OneDrive/etc. +- Reloaded VFS does not return 8.3 DOS file names for virtualized files. + +## File Write Behaviours + +Reloaded VFS is a read-only VFS, so what happens when you try editing files? + +| Description | Action Performed | +|------------------------------------|------------------------------------------------------------------------------------------| +| File Deletion | Delete the mod file instead of the original file | +| New File Creation | Create new files in the original game folder | +| File Editing | Edits the redirected file | +| File Delete & Recreate (New) | Delete the overwritten file and place the new file in game folder | +| Renaming Folders to Other Location | Either move the original folder or files in original and overlaid folders (depends on how API is used) | \ No newline at end of file diff --git a/docs/Mods/Essentials/Virtual-FileSystem/Implementation-Details.md b/docs/Mods/Essentials/Virtual-FileSystem/Implementation-Details.md new file mode 100644 index 0000000..c2988fa --- /dev/null +++ b/docs/Mods/Essentials/Virtual-FileSystem/Implementation-Details.md @@ -0,0 +1,275 @@ +# Implementation Details + +This section dives deeper into some of the implementation details of the Reloaded VFS. + +## Used Hooks + +The VFS hooks several Win32 and NT File APIs to intercept file operations. The goal is to handle every API which: + +- Accepts a File Path: In this case we set a new path to our redirected file. + +- Returns Files at Given Path: In this case we inject new files into the result. + +Here is a flowchart of the hooked APIs: + +```mermaid +flowchart LR + subgraph Win32 + + %% Definitions + FindFirstFileA + FindFirstFileExA + FindFirstFileW + FindFirstFileExW + FindFirstFileExFromAppW + FindNextFileA + FindNextFileW + + CreateDirectoryA + CreateDirectoryW + CreateFileA + CreateFileW + CreateFile2 + CreateFile2FromAppW + CreateFileFromAppW + CreateDirectoryExW + CreateDirectoryFromAppW + DeleteFileA + DeleteFileW + DeleteFileFromAppW + GetCompressedFileSizeA + GetCompressedFileSizeW + CloseHandle + + GetFileAttributesA + GetFileAttributesExA + GetFileAttributesExFromAppW + GetFileAttributesExW + GetFileAttributesW + SetFileAttributesA + SetFileAttributesFromAppW + SetFileAttributesW + + RemoveDirectoryA + RemoveDirectoryFromAppW + RemoveDirectoryW + + %%% Win32 Internal Redirects + FindFirstFileA --> FindFirstFileExW + FindFirstFileExA --> FindFirstFileExW + FindFirstFileExFromAppW --> FindFirstFileExW + FindNextFileA --> FindNextFileW + CreateDirectoryA --> CreateDirectoryW + CreateFile2FromAppW --> CreateFile2 + CreateDirectoryFromAppW --> CreateDirectoryExW + CreateFileFromAppW --> CreateFile2FromAppW + DeleteFileFromAppW --> DeleteFileW + DeleteFileA --> DeleteFileW + GetCompressedFileSizeA --> GetCompressedFileSizeW + GetFileAttributesA --> GetFileAttributesW + GetFileAttributesExA --> GetFileAttributesExW + GetFileAttributesExFromAppW --> GetFileAttributesExW + RemoveDirectoryA --> RemoveDirectoryW + RemoveDirectoryFromAppW --> RemoveDirectoryW + SetFileAttributesFromAppW --> SetFileAttributesW + SetFileAttributesA --> SetFileAttributesW + end + + subgraph NT API + %% Definitions + NtCreateFile + NtOpenFile + NtQueryDirectoryFile + NtQueryDirectoryFileEx + NtDeleteFile + NtQueryAttributesFile + NtQueryFullAttributesFile + NtClose + + %%% Win32 -> NT API + FindFirstFileExW --> NtOpenFile + FindFirstFileExW --> NtQueryDirectoryFileEx + FindFirstFileW --> NtOpenFile + FindFirstFileW --> NtQueryDirectoryFileEx + FindNextFileW --> NtQueryDirectoryFileEx + CreateFileA --> NtCreateFile + CreateFileW --> NtCreateFile + CreateFile2 --> NtCreateFile + CreateDirectoryW --> NtCreateFile + CreateDirectoryExW --> NtOpenFile + CreateDirectoryExW --> NtCreateFile + DeleteFileW --> NtOpenFile + RemoveDirectoryW --> NtOpenFile + GetCompressedFileSizeW --> NtOpenFile + CloseHandle --> NtClose + GetFileAttributesExW --> NtQueryFullAttributesFile + GetFileAttributesW --> NtQueryAttributesFile + SetFileAttributesW --> NtOpenFile + end + + %%% Hooks + subgraph Hooks + NtCreateFile_Hook + NtOpenFile_Hook + NtQueryDirectoryFileEx_Hook + NtDeleteFile_Hook + NtQueryAttributesFile_Hook + NtQueryFullAttributesFile_Hook + NtClose_Hook + + %% NT API -> Hooks + NtCreateFile --> NtCreateFile_Hook + NtOpenFile --> NtOpenFile_Hook + NtQueryDirectoryFileEx --> NtQueryDirectoryFileEx_Hook + NtQueryDirectoryFile --> NtQueryDirectoryFile_Hook + + NtDeleteFile --> NtDeleteFile_Hook + NtQueryAttributesFile --> NtQueryAttributesFile_Hook + NtQueryFullAttributesFile --> NtQueryFullAttributesFile_Hook + NtClose --> NtClose_Hook + end +``` + +On Windows 10+, `NtQueryDirectoryFileEx` API becomes available and `NtQueryDirectoryFile` acts as a wrapper around it. On Wine and earlier Windows, only `NtQueryDirectoryFile` exists. + +In this VFS we hook both, and detect if one recurses to the other using a semaphore. If we're recursing from `NtQueryDirectoryFile` to `NtQueryDirectoryFileEx`, we skip the hook code. + +## Lookup Tree + +The `LookupTree` is a visualization of the data structure used to map paths of old files to new files ***after*** all mods are loaded during startup. +When all mods are loaded, this structure is generated from the `RedirectionTree`. + +It uses a strategy of: + +1. Check common prefix. +2. Check remaining path in dictionary. +3. Check file name in dictionary. + +The prefix is based on the idea that a game will have all of its files stored under a common folder path. +We use this to save memory in potentially huge games. + +```mermaid +flowchart LR + subgraph LookupTree + subgraph Common Prefix + C[C:/SteamLibrary/steamapps/common/Game] + end + subgraph Dictionary + D[.] + M[music] + end + C --> D + D --> data_1.pak + D --> data_2.pak + D --> data_3.pak + M --> jingle.wav + M --> ocean.wav + end + data_2.pak --> f[FULL_PATH_TO_NEW_FILE 'isDirectory: false'] +``` + +### In Code + +```csharp +/// +/// A version of optimised for faster lookups in the scenario of use with game folders. +/// +public struct LookupTree +{ + /// + /// Prefix of all paths. + /// Stored in upper case for faster performance. + /// + public string Prefix { get; private set; } + + /// + /// Dictionary that maps individual subfolders to map of files. + /// + public SpanOfCharDict> SubfolderToFiles { get; private set; } +} +``` + +## Redirection Tree + +The `RedirectionTree` is a visualization of the data structure used to map paths of old files to new files as the mods are loading during startup. + +It uses an O(N) lookup time (where N is the number of components separated by '/') that make up the final file path. The resolution steps are: + +1. Start at tree root. +2. Split the input path on '/' character. +3. Traverse the tree one level at a time, using each split component to move down next level. +4. At each level check if there's a child node corresponding to current path component. + - If there is no child node, lookup has failed and path is not in the tree. +5. When all components have been consumed, check the `Items` dictionary of the final node reached to see if the path is present. +6. If it is, the lookup succeeds and the corresponding value is returned. If it is not, the lookup fails and the path is not found in the tree. + +When all mods are loaded, this trie-like structure is converted to a `LookupTree`. + +```mermaid +flowchart LR + subgraph RedirectionTree + C: --> SteamLibrary --> steamapps --> common --> Game + Game --> data_1.pak + Game --> data_2.pak + Game --> data_3.pak + end + data_2.pak --> f[FULL_PATH_TO_NEW_FILE 'isDirectory: false'] +``` + +### In Code + +```csharp +/// +/// Represents that will be used for performing redirections. +/// +public struct RedirectionTree +{ + /// + /// Root nodes, e.g. would store drive: C:/D:/E: etc. + /// In most cases there is only one. + /// + public RedirectionTreeNode RootNode { get; private set; } +} +``` + +```csharp +/// +/// Individual node in the redirection tree. +/// +public struct RedirectionTreeNode +{ + /// + /// Child nodes of this nodes. + /// i.e. Maps 'folder' to next child. + /// + public SpanOfCharDict> Children; + + /// + /// Files present at this level of the tree. + /// + public SpanOfCharDict Items; +} +``` + +```csharp +/// +/// Target for a file covered by the redirection tree. +/// +public struct RedirectionTreeTarget +{ + /// + /// Path to the directory storing the file. + /// + public string Directory; // (This is deduplicated, saving memory) + + /// + /// Name of the file in the directory. + /// + public string FileName; + + /// + /// True if this is a directory, else false. + /// + public bool IsDirectory; +} +``` \ No newline at end of file diff --git a/docs/Mods/Essentials/Virtual-FileSystem/Implementing-Redirection.md b/docs/Mods/Essentials/Virtual-FileSystem/Implementing-Redirection.md new file mode 100644 index 0000000..a5b5db3 --- /dev/null +++ b/docs/Mods/Essentials/Virtual-FileSystem/Implementing-Redirection.md @@ -0,0 +1,107 @@ +## Implementing Redirection + +The virtual file system uses two main data structures: + +1. The `RedirectionTree` which maps old file paths to new paths as mods are loading. It uses an O(N) trie-like structure. + +2. The `LookupTree` which is an optimized version of the RedirectionTree used after all mods are loaded. It uses an O(3) structure for fast lookups. + +Here is how you could implement these structures in Rust using the `hashbrown` crate: + +```rust +use hashbrown::HashMap; + +pub struct RedirectionTree { + pub root: RedirectionNode +} + +pub struct RedirectionNode { + pub children: HashMap, + pub items: HashMap, +} + +impl RedirectionNode { + pub fn new() -> Self { + RedirectionNode { + children: HashMap::new(), + items: HashMap::new(), + } + } +} + +impl RedirectionTree { + pub fn add_path(&mut self, old_path: &str, new_path: &str) { + let mut current = &mut self.root; + for part in old_path.split('/') { + current = current.children.entry(part.to_string()).or_insert(RedirectionNode::new()); + } + current.items.insert(old_path.to_string(), new_path.to_string()); + } +} +``` + +```rust +use hashbrown::HashMap; + +pub struct LookupTree { + pub prefix: String, + pub subdir_to_files: HashMap>, +} + +impl LookupTree { + pub fn from_redirection_tree(tree: &RedirectionTree) -> Self { + let mut lookup = LookupTree { + prefix: String::new(), + subdir_to_files: HashMap::new(), + }; + + lookup.build_prefix(tree); + lookup.build_subdir_to_files(&tree.root, ""); + + lookup + } + + fn build_prefix(&mut self, tree: &RedirectionTree) { + // Find common prefix of all paths + // ... + } + + fn build_subdir_to_files(&mut self, node: &RedirectionNode, path: &str) { + if !node.items.is_empty() { + self.subdir_to_files.insert(path.to_string(), node.items.clone()); + } + + for (name, child) in &node.children { + let sub_path = format!("{}/{}", path, name); + self.build_subdir_to_files(child, &sub_path); + } + } +} +``` + +The `LookupTree` is built from the `RedirectionTree` when all mods are done loading. It extracts a common prefix from all paths to save memory, and builds a flat map of subdirectories to files. + +To detect file changes efficiently, you can use the `notify` crate to watch the redirect folders for changes and rebuild the trees as needed. Here's a sketch: + +```rust +use notify::{RecommendedWatcher, Watcher, RecursiveMode}; +use std::sync::mpsc::channel; +use std::time::Duration; + +fn main() { + let (tx, rx) = channel(); + let mut watcher: RecommendedWatcher = Watcher::new(tx, Duration::from_secs(1)).unwrap(); + + watcher.watch("path/to/redirector", RecursiveMode::Recursive).unwrap(); + + loop { + match rx.recv() { + // Rebuild trees on file change events + Ok(event) => println!("event: {:?}", event), + Err(e) => println!("watch error: {:?}", e), + } + } +} +``` + +This sets up a watcher on the Redirector folder that will receive events when files change. You can then trigger a rebuild of the `RedirectionTree` and `LookupTree` to pick up those changes. \ No newline at end of file diff --git a/docs/Mods/Essentials/Virtual-FileSystem/Performance.md b/docs/Mods/Essentials/Virtual-FileSystem/Performance.md new file mode 100644 index 0000000..b4f7e96 --- /dev/null +++ b/docs/Mods/Essentials/Virtual-FileSystem/Performance.md @@ -0,0 +1,73 @@ +# Performance Characteristics + +All numbers were obtained on a stock clock AMD Ryzen 5900X and 3000MHz CL16 RAM. + +The Reloaded VFS is heavily optimized for performance. A lot of micro-optimizations were done to squeeze every bit out of making opening files faster... + +- All strings stored as Wide Strings. + - Windows APIs use Wide Strings under the hood, even for ANSI APIs. + - Therefore we save time by not having to widen them again. + +- Custom string Hash Function for file paths. + - With AVX and SSE implementations; as well as unrolled `nint` as backup. + +- Custom Vectorized `ToUpper` for Strings. + - Modified backport from .NET 8. + - Super fast for 99% of the paths that are pure ASCII. + - Non-ASCII paths use slower fallback since they can't be vectorized. + - Partial ASCII paths have the first ASCII part vectorized and rest handled in fallback. + +- Custom Dictionary (HashMap) that can query string slices (to avoid copying/realloc). + +- Uses a custom ['LookupTree'](./implementationdetails/lookup-tree.md). + - Provides lookup for resolving file paths in O(3) time. + +## File Mapping Performance & Memory Usage + +This section describes how long it takes to create a file map. A file map is a structure that helps redirect original files to their new versions found in mod folders. + +Whenever changes are made to mod folders, the file map needs to be rebuilt. + +Two types of maps exist, [RedirectionTree](./implementationdetails/redirection-tree.md) and [LookupTree](./implementationdetails/lookup-tree.md). The latter, `LookupTree` is optimized for performance, but takes roughly twice as long to build as it is built from the former `RedirectionTree`. + +| Folder Type | Directories | Total Items | RedirectionTree (Time) | RedirectionTree (Memory) | LookupTree (Time) | LookupTree (Memory) | +|------------------------------|-------------|-------------|------------------------|-------------------------|-------------------|---------------------| +| Windows Folder | 40,796 | 170,438 | 43ms | 27MB | 32ms | 25MB | +| Steam Folder
(65 games) | 9,318 | 172,896 | 18ms | 12MB | 20ms | 11MB | + +The performance of mapping operations mainly depends on the directory count. The table above shows the time and memory usage for building the `RedirectionTree` and `LookupTree` for both Windows and Steam folders. The `LookupTree` memory usage should be approximately equal to the total runtime memory usage. + +For a typical game (based on the median of a Steam library), building the `RedirectionTree` should take around `0.017ms` and allocate `48KB`. +Creating the optimized `LookupTree` takes about `0.012ms` and allocates `47KB`. + +In other words, you can assume remapping files is basically real-time. + +### Fast Append + +Both `LookupTree` and `RedirectionTree` support `'fast append'` operations. + +If a file is added to the mod folder while the game is running and isn't previously mapped, it can be added to the tree directly without a full rebuild. + +However, if the currently mapped file's source mod cannot be determined, the entire tree must be rebuilt. + +This process doesn't require scanning mod folders again for files when not necessary. Each folder mapping has a cache of subdirectories and files, and the same string instances are reused between the trees and cache to save memory. + +## File Open Overhead + +File open has negligible performance difference compared to not using VFS. +In a test with opening+closing 21,000 files (+70,000 virtualized), the difference was only ~41ms (~3%) or less than 2 microseconds per file. + +``` +// All tests done in separate processes for accuracy. +| Method | Mean | Error | StdDev | Ratio | +|--------------------------------- |--------:|---------:|---------:|------:| +| OpenAllHandles_WithVfs | 1.650 s | 0.0102 s | 0.0095 s | 1.03 | +| OpenAllHandles_WithVfs_Optimized | 1.643 s | 0.0145 s | 0.0135 s | 1.03 | +| OpenAllHandles_WithoutVfs | 1.602 s | 0.0128 s | 0.0120 s | 1.00 | +``` + +In real-world `"cold-start"` scenarios (e.g. after a machine reboot), opening these many files would take around 80 seconds, making this difference effectively margin of error (~0%). + +## Built-in Benchmarks + +If you're a programmer, a lot of microbenchmarks are available in the `Reloaded.Universal.Redirector.Benchmarks` project; have a look! \ No newline at end of file diff --git a/docs/Mods/Essentials/Virtual-FileSystem/Programmer-Usage.md b/docs/Mods/Essentials/Virtual-FileSystem/Programmer-Usage.md new file mode 100644 index 0000000..e2b6388 --- /dev/null +++ b/docs/Mods/Essentials/Virtual-FileSystem/Programmer-Usage.md @@ -0,0 +1,63 @@ + +# Programmer Usage + +The Redirector uses [Reloaded Dependency Injection](https://reloaded-project.github.io/Reloaded-II/DependencyInjection_HowItWork/) to expose an API. + +To use the Redirector API: + +1. Add the `Reloaded.Universal.Redirector.Interfaces` NuGet package to your project. + +2. Add the dependency `reloaded.universal.redirector` to `ModDependencies` in your `ModConfig.json`. + +3. In your `Mod()` entry point, acquire the Controller: + +```csharp +IRedirectorController _redirectorController; + +public void Start(IModLoaderV1 loader) +{ + _redirectorController = _modLoader.GetController(); +} +``` + +## IRedirectorController API + +The `IRedirectorController` interface provides the following methods: + +- `AddRedirect(string oldPath, string newPath)`: Redirects an individual file path. + +- `RemoveRedirect(string oldPath)`: Removes redirection for an individual file path. + +- `AddRedirectFolder(string folderPath)`: Adds a new redirect folder. Files in this folder will overlay files in the game directory. + +- `AddRedirectFolder(string folderPath, string sourceFolder)`: Adds a new redirect folder with a custom source folder. Files in `folderPath` will overlay files in `sourceFolder`. + +- `RemoveRedirectFolder(string folderPath)`: Removes a redirect folder. + +- `RemoveRedirectFolder(string folderPath, string sourceFolder)`: Removes a redirect folder with a specific source folder. + +- `GetRedirectorSetting(RedirectorSettings setting)`: Gets the current value of a redirector setting. See RedirectorSettings enum for options. + +- `SetRedirectorSetting(bool enable, RedirectorSettings setting)`: Enables or disables a specific redirector setting. + +- `Enable()` / `Disable()`: Enables or disables the redirector entirely. + +## Examples + +Redirect an individual file: + +```csharp +_redirectorController.AddRedirect(@"dvdroot\bgm\SNG_STG26.adx", @"mods\mybgm.adx"); +``` + +Add a new redirect folder: + +```csharp +_redirectorController.AddRedirectFolder(@"mods\mymod"); +``` + +Print file redirections to console: + +```csharp +_redirectorController.SetRedirectorSetting(true, RedirectorSettings.PrintRedirect); +``` \ No newline at end of file diff --git a/docs/Mods/Essentials/Virtual-FileSystem/Usage.md b/docs/Mods/Essentials/Virtual-FileSystem/Usage.md new file mode 100644 index 0000000..3a851af --- /dev/null +++ b/docs/Mods/Essentials/Virtual-FileSystem/Usage.md @@ -0,0 +1,62 @@ + +# Basic Usage + +## Download the Mod + +First, download the `Reloaded File Redirector` mod which provides the virtual filesystem functionality. + +![DownloadMod](https://raw.githubusercontent.com/Reloaded-Project/reloaded.universal.redirector/master/docs/images/DownloadMod.png) + +## Add Dependency to Redirector + +In the `Edit Mod` menu, add `Reloaded File Redirector` as a dependency to your mod. + +![AddDependency](https://raw.githubusercontent.com/Reloaded-Project/reloaded.universal.redirector/master/docs/images/AddDependency.png) + +This will ensure the `Reloaded File Redirector` mod is always loaded when your mod is loaded. + +### Opening the Mod Folder + +![OpenModFolder](https://raw.githubusercontent.com/Reloaded-Project/reloaded.universal.redirector/master/docs/images/OpenModFolder.png) + +Go to the folder where your mod is stored by clicking the `Open Folder` button. + +### Add Some Files + +Make a folder called `Redirector`. Inside it, place files that you want to be replaced. + +![FileRedirectorFolder](https://raw.githubusercontent.com/Reloaded-Project/reloaded.universal.redirector/master/docs/images/FileRedirectorFolder.png) + +Files are mapped by their location relative to the EXE of the application. + +For example, if the game is at `E:/SonicHeroes/TSonic_win.exe`, the paths are relative to: `E:/SonicHeroes/`. + +To replace a music file at `E:/SonicHeroes/dvdroot/bgm/SNG_STG26.adx`, your mod should place the file at `Redirector/dvdroot/bgm/SNG_STG26.adx`. + +The contents of the mod folder should now look as follows: + +``` +// Mod Contents +ModConfig.json +Preview.png +Redirector +└─dvdroot + ├─advertise + │ adv_pl_rouge.one + └─playmodel + ro.txd + ro_dff.one +``` + +The connectors `└─` represent folders. + +## Debugging + +![DebugMod](https://raw.githubusercontent.com/Reloaded-Project/reloaded.universal.redirector/master/docs/images/DebugMod.png) + +To debug the mod, highlight the `Reloaded File Redirector` mod in your mod manager and click `Configure Mod`. + +The following settings are available: +- `Log Open Files`: Prints a message to `Console` when a new file is being opened. +- `Log Redirections`: Prints a message when a custom file is loaded from your or another mod. +- `Log Attribute Fetches`: Prints a message when game gets file properties such as file size. diff --git a/mkdocs.yml b/mkdocs.yml index a203b2a..8a8bc5b 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -106,6 +106,11 @@ nav: - Essentials: - Virtual FileSystem: - About: Mods/Essentials/Virtual-FileSystem/About.md + - Usage: Mods/Essentials/Virtual-FileSystem/Usage.md + - Programmer Usage: Mods/Essentials/Virtual-FileSystem/Programmer-Usage.md + - Implementing Redirection: Mods/Essentials/Virtual-FileSystem/Implementing-Redirection.md + - Implementation Details: Mods/Essentials/Virtual-FileSystem/Implementation-Details.md + - Performance: Mods/Essentials/Virtual-FileSystem/Performance.md - Essentials (Libraries): - Merged File Cache: - About: Mods/Libraries/Merged-File-Cache/About.md From e6fb8fcf95220250ba41b1537b681b194845546e Mon Sep 17 00:00:00 2001 From: Sewer56 Date: Sat, 27 Apr 2024 09:05:54 +0100 Subject: [PATCH 2/6] Added: Further Progress as of 27/04/24 --- .../Essentials/Virtual-FileSystem/About.md | 44 +-- .../Behaviours-Limitations.md | 47 +++ .../Implementation-Details.md | 273 ------------------ .../Implementation-Details/Hooks.md | 137 +++++++++ .../Implementation-Details/Optimizations.md | 158 ++++++++++ .../Implementation-Details/Trees.md | 156 ++++++++++ .../Implementing-Redirection.md | 107 ------- .../Virtual-FileSystem/Performance.md | 93 +++--- .../Virtual-FileSystem/Programmer-Usage.md | 7 +- .../Essentials/Virtual-FileSystem/Usage.md | 24 +- mkdocs.yml | 9 +- 11 files changed, 587 insertions(+), 468 deletions(-) create mode 100644 docs/Mods/Essentials/Virtual-FileSystem/Behaviours-Limitations.md create mode 100644 docs/Mods/Essentials/Virtual-FileSystem/Implementation-Details/Hooks.md create mode 100644 docs/Mods/Essentials/Virtual-FileSystem/Implementation-Details/Optimizations.md create mode 100644 docs/Mods/Essentials/Virtual-FileSystem/Implementation-Details/Trees.md delete mode 100644 docs/Mods/Essentials/Virtual-FileSystem/Implementing-Redirection.md diff --git a/docs/Mods/Essentials/Virtual-FileSystem/About.md b/docs/Mods/Essentials/Virtual-FileSystem/About.md index 8cbb103..aebca08 100644 --- a/docs/Mods/Essentials/Virtual-FileSystem/About.md +++ b/docs/Mods/Essentials/Virtual-FileSystem/About.md @@ -1,6 +1,8 @@ # About the Reloaded Virtual FileSystem -The Reloaded Virtual File System (VFS) is an invisible helper that sits between your games and the files they use. It allows your games to 'see' and open files that aren't really 'there', keeping your game folder unmodified. +The Reloaded Virtual File System (VFS) is an invisible helper that sits between your games and the +files they use. It allows your games to 'see' and open files that aren't really 'there', keeping +your game folder unmodified. ```mermaid flowchart LR @@ -52,43 +54,3 @@ And with the following benefits: - Practically zero overhead. - Can add/remove and remap files on the fly (without making changes on disk). - Supports Wine on Linux. - -## Limitations - -The Reloaded VFS only implements *well defined* functionality, effectively becoming a ***read-only*** VFS. Write operations, such as creating a file are unaffected. - -If a game wants to write a new file (such as a savefile), no action will be taken and the file will be written to the game folder. If a native DLL plugin wants to write a config file, it will write it to the game folder, as normal. - -### Warning - -Proceed with care if any of the following applies: - -- If your game's modding tools operate on a modified game directory (e.g. Skyrim xEdit), using VFS is not recommended as new files might be written to the game folder. - -- Do not use VFS to redirect files deleted and then recreated by games; ***you will lose the files from inside your mod***. - -### Error Cases - -Using this VFS is not appropriate for your game if any of the following is true. - -- This VFS does not handle child processes. Do not use VFS for games that can run external tools with virtualized files. - -### Additional Limitations - -The following limitations should not cause concern. - -- Reloaded VFS does not support Reparse Point Tags. - - However, this shouldn't cause issues with mods stored on cloud/OneDrive/etc. -- Reloaded VFS does not return 8.3 DOS file names for virtualized files. - -## File Write Behaviours - -Reloaded VFS is a read-only VFS, so what happens when you try editing files? - -| Description | Action Performed | -|------------------------------------|------------------------------------------------------------------------------------------| -| File Deletion | Delete the mod file instead of the original file | -| New File Creation | Create new files in the original game folder | -| File Editing | Edits the redirected file | -| File Delete & Recreate (New) | Delete the overwritten file and place the new file in game folder | -| Renaming Folders to Other Location | Either move the original folder or files in original and overlaid folders (depends on how API is used) | \ No newline at end of file diff --git a/docs/Mods/Essentials/Virtual-FileSystem/Behaviours-Limitations.md b/docs/Mods/Essentials/Virtual-FileSystem/Behaviours-Limitations.md new file mode 100644 index 0000000..4176646 --- /dev/null +++ b/docs/Mods/Essentials/Virtual-FileSystem/Behaviours-Limitations.md @@ -0,0 +1,47 @@ +# Behaviours & Limitations + +!!! info "The Reloaded VFS is only intended to be used for '*well defined*' functionality." + +For now, this means the VFS is 'read only'. Write operations, such as creating a file are unaffected. + +If a game wants to write a new file (such as a savefile), no action will be taken and the file will +be written to the game folder. If a native DLL plugin wants to write a config file, it will write it +to the game folder, as normal. + +## Warnings + +!!! warning "Proceed with care if any of the following applies" + +- If your game's modding tools operate on a modified game directory (e.g. Skyrim xEdit), + using VFS is not recommended as new files might be written to the game folder. + +- Do not use VFS to redirect files deleted and then recreated by games; ***you will lose the files from inside your mod***. + +## Error Cases + +!!! error "Using this VFS is not appropriate for your game if any of the following is true" + +- This VFS does not handle child processes. Do not use VFS for games that can run + external tools with virtualized files. + - Will be implemented in the future if mods ever will end up opening external tools. + - However that workflow is not recommended... (e.g. might be problematic for Linux users) + +## Additional Limitations + +!!! note "The following limitations also exist but should not cause concern." + +- Reloaded VFS does not support Reparse Point Tags. + - However, this shouldn't cause issues with mods stored on cloud/OneDrive/etc. +- Reloaded VFS does not return 8.3 DOS file names for virtualized files. + +## File Write Behaviours + +!!! warning "What happens when you try editing files in a 'read-only' VFS?" + +| Description | Action Performed | +| ---------------------------------- | ------------------------------------------------------------------------------------------------------------- | +| File Deletion | Delete the mod file instead of the original file | +| New File Creation | Create new files in the original game folder | +| File Editing (in Place) | Edits the redirected file | +| File Delete & Recreate (New) | Delete the overwritten file and place the new file in game folder | +| Renaming Folders to Other Location | Either move the original folder or files in original folder and overlaid folders (depends on how API is used) | \ No newline at end of file diff --git a/docs/Mods/Essentials/Virtual-FileSystem/Implementation-Details.md b/docs/Mods/Essentials/Virtual-FileSystem/Implementation-Details.md index c2988fa..d4c8081 100644 --- a/docs/Mods/Essentials/Virtual-FileSystem/Implementation-Details.md +++ b/docs/Mods/Essentials/Virtual-FileSystem/Implementation-Details.md @@ -1,275 +1,2 @@ # Implementation Details -This section dives deeper into some of the implementation details of the Reloaded VFS. - -## Used Hooks - -The VFS hooks several Win32 and NT File APIs to intercept file operations. The goal is to handle every API which: - -- Accepts a File Path: In this case we set a new path to our redirected file. - -- Returns Files at Given Path: In this case we inject new files into the result. - -Here is a flowchart of the hooked APIs: - -```mermaid -flowchart LR - subgraph Win32 - - %% Definitions - FindFirstFileA - FindFirstFileExA - FindFirstFileW - FindFirstFileExW - FindFirstFileExFromAppW - FindNextFileA - FindNextFileW - - CreateDirectoryA - CreateDirectoryW - CreateFileA - CreateFileW - CreateFile2 - CreateFile2FromAppW - CreateFileFromAppW - CreateDirectoryExW - CreateDirectoryFromAppW - DeleteFileA - DeleteFileW - DeleteFileFromAppW - GetCompressedFileSizeA - GetCompressedFileSizeW - CloseHandle - - GetFileAttributesA - GetFileAttributesExA - GetFileAttributesExFromAppW - GetFileAttributesExW - GetFileAttributesW - SetFileAttributesA - SetFileAttributesFromAppW - SetFileAttributesW - - RemoveDirectoryA - RemoveDirectoryFromAppW - RemoveDirectoryW - - %%% Win32 Internal Redirects - FindFirstFileA --> FindFirstFileExW - FindFirstFileExA --> FindFirstFileExW - FindFirstFileExFromAppW --> FindFirstFileExW - FindNextFileA --> FindNextFileW - CreateDirectoryA --> CreateDirectoryW - CreateFile2FromAppW --> CreateFile2 - CreateDirectoryFromAppW --> CreateDirectoryExW - CreateFileFromAppW --> CreateFile2FromAppW - DeleteFileFromAppW --> DeleteFileW - DeleteFileA --> DeleteFileW - GetCompressedFileSizeA --> GetCompressedFileSizeW - GetFileAttributesA --> GetFileAttributesW - GetFileAttributesExA --> GetFileAttributesExW - GetFileAttributesExFromAppW --> GetFileAttributesExW - RemoveDirectoryA --> RemoveDirectoryW - RemoveDirectoryFromAppW --> RemoveDirectoryW - SetFileAttributesFromAppW --> SetFileAttributesW - SetFileAttributesA --> SetFileAttributesW - end - - subgraph NT API - %% Definitions - NtCreateFile - NtOpenFile - NtQueryDirectoryFile - NtQueryDirectoryFileEx - NtDeleteFile - NtQueryAttributesFile - NtQueryFullAttributesFile - NtClose - - %%% Win32 -> NT API - FindFirstFileExW --> NtOpenFile - FindFirstFileExW --> NtQueryDirectoryFileEx - FindFirstFileW --> NtOpenFile - FindFirstFileW --> NtQueryDirectoryFileEx - FindNextFileW --> NtQueryDirectoryFileEx - CreateFileA --> NtCreateFile - CreateFileW --> NtCreateFile - CreateFile2 --> NtCreateFile - CreateDirectoryW --> NtCreateFile - CreateDirectoryExW --> NtOpenFile - CreateDirectoryExW --> NtCreateFile - DeleteFileW --> NtOpenFile - RemoveDirectoryW --> NtOpenFile - GetCompressedFileSizeW --> NtOpenFile - CloseHandle --> NtClose - GetFileAttributesExW --> NtQueryFullAttributesFile - GetFileAttributesW --> NtQueryAttributesFile - SetFileAttributesW --> NtOpenFile - end - - %%% Hooks - subgraph Hooks - NtCreateFile_Hook - NtOpenFile_Hook - NtQueryDirectoryFileEx_Hook - NtDeleteFile_Hook - NtQueryAttributesFile_Hook - NtQueryFullAttributesFile_Hook - NtClose_Hook - - %% NT API -> Hooks - NtCreateFile --> NtCreateFile_Hook - NtOpenFile --> NtOpenFile_Hook - NtQueryDirectoryFileEx --> NtQueryDirectoryFileEx_Hook - NtQueryDirectoryFile --> NtQueryDirectoryFile_Hook - - NtDeleteFile --> NtDeleteFile_Hook - NtQueryAttributesFile --> NtQueryAttributesFile_Hook - NtQueryFullAttributesFile --> NtQueryFullAttributesFile_Hook - NtClose --> NtClose_Hook - end -``` - -On Windows 10+, `NtQueryDirectoryFileEx` API becomes available and `NtQueryDirectoryFile` acts as a wrapper around it. On Wine and earlier Windows, only `NtQueryDirectoryFile` exists. - -In this VFS we hook both, and detect if one recurses to the other using a semaphore. If we're recursing from `NtQueryDirectoryFile` to `NtQueryDirectoryFileEx`, we skip the hook code. - -## Lookup Tree - -The `LookupTree` is a visualization of the data structure used to map paths of old files to new files ***after*** all mods are loaded during startup. -When all mods are loaded, this structure is generated from the `RedirectionTree`. - -It uses a strategy of: - -1. Check common prefix. -2. Check remaining path in dictionary. -3. Check file name in dictionary. - -The prefix is based on the idea that a game will have all of its files stored under a common folder path. -We use this to save memory in potentially huge games. - -```mermaid -flowchart LR - subgraph LookupTree - subgraph Common Prefix - C[C:/SteamLibrary/steamapps/common/Game] - end - subgraph Dictionary - D[.] - M[music] - end - C --> D - D --> data_1.pak - D --> data_2.pak - D --> data_3.pak - M --> jingle.wav - M --> ocean.wav - end - data_2.pak --> f[FULL_PATH_TO_NEW_FILE 'isDirectory: false'] -``` - -### In Code - -```csharp -/// -/// A version of optimised for faster lookups in the scenario of use with game folders. -/// -public struct LookupTree -{ - /// - /// Prefix of all paths. - /// Stored in upper case for faster performance. - /// - public string Prefix { get; private set; } - - /// - /// Dictionary that maps individual subfolders to map of files. - /// - public SpanOfCharDict> SubfolderToFiles { get; private set; } -} -``` - -## Redirection Tree - -The `RedirectionTree` is a visualization of the data structure used to map paths of old files to new files as the mods are loading during startup. - -It uses an O(N) lookup time (where N is the number of components separated by '/') that make up the final file path. The resolution steps are: - -1. Start at tree root. -2. Split the input path on '/' character. -3. Traverse the tree one level at a time, using each split component to move down next level. -4. At each level check if there's a child node corresponding to current path component. - - If there is no child node, lookup has failed and path is not in the tree. -5. When all components have been consumed, check the `Items` dictionary of the final node reached to see if the path is present. -6. If it is, the lookup succeeds and the corresponding value is returned. If it is not, the lookup fails and the path is not found in the tree. - -When all mods are loaded, this trie-like structure is converted to a `LookupTree`. - -```mermaid -flowchart LR - subgraph RedirectionTree - C: --> SteamLibrary --> steamapps --> common --> Game - Game --> data_1.pak - Game --> data_2.pak - Game --> data_3.pak - end - data_2.pak --> f[FULL_PATH_TO_NEW_FILE 'isDirectory: false'] -``` - -### In Code - -```csharp -/// -/// Represents that will be used for performing redirections. -/// -public struct RedirectionTree -{ - /// - /// Root nodes, e.g. would store drive: C:/D:/E: etc. - /// In most cases there is only one. - /// - public RedirectionTreeNode RootNode { get; private set; } -} -``` - -```csharp -/// -/// Individual node in the redirection tree. -/// -public struct RedirectionTreeNode -{ - /// - /// Child nodes of this nodes. - /// i.e. Maps 'folder' to next child. - /// - public SpanOfCharDict> Children; - - /// - /// Files present at this level of the tree. - /// - public SpanOfCharDict Items; -} -``` - -```csharp -/// -/// Target for a file covered by the redirection tree. -/// -public struct RedirectionTreeTarget -{ - /// - /// Path to the directory storing the file. - /// - public string Directory; // (This is deduplicated, saving memory) - - /// - /// Name of the file in the directory. - /// - public string FileName; - - /// - /// True if this is a directory, else false. - /// - public bool IsDirectory; -} -``` \ No newline at end of file diff --git a/docs/Mods/Essentials/Virtual-FileSystem/Implementation-Details/Hooks.md b/docs/Mods/Essentials/Virtual-FileSystem/Implementation-Details/Hooks.md new file mode 100644 index 0000000..ffffcaa --- /dev/null +++ b/docs/Mods/Essentials/Virtual-FileSystem/Implementation-Details/Hooks.md @@ -0,0 +1,137 @@ +!!! info "This currently only contains information for Windows." + + Native support for other OSes will be added in the future. + +## Used Hooks + +The VFS hooks several Win32 and NT File APIs to intercept file operations. The goal is to handle every API which: + +- Accepts a File Path: In this case we set a new path to our redirected file. + +- Returns Files at Given Path: In this case we inject new files into the result. + +Here is a flowchart of the hooked APIs: + +```mermaid +flowchart LR + subgraph Win32 + + %% Definitions + FindFirstFileA + FindFirstFileExA + FindFirstFileW + FindFirstFileExW + FindFirstFileExFromAppW + FindNextFileA + FindNextFileW + + CreateDirectoryA + CreateDirectoryW + CreateFileA + CreateFileW + CreateFile2 + CreateFile2FromAppW + CreateFileFromAppW + CreateDirectoryExW + CreateDirectoryFromAppW + DeleteFileA + DeleteFileW + DeleteFileFromAppW + GetCompressedFileSizeA + GetCompressedFileSizeW + CloseHandle + + GetFileAttributesA + GetFileAttributesExA + GetFileAttributesExFromAppW + GetFileAttributesExW + GetFileAttributesW + SetFileAttributesA + SetFileAttributesFromAppW + SetFileAttributesW + + RemoveDirectoryA + RemoveDirectoryFromAppW + RemoveDirectoryW + + %%% Win32 Internal Redirects + FindFirstFileA --> FindFirstFileExW + FindFirstFileExA --> FindFirstFileExW + FindFirstFileExFromAppW --> FindFirstFileExW + FindNextFileA --> FindNextFileW + CreateDirectoryA --> CreateDirectoryW + CreateFile2FromAppW --> CreateFile2 + CreateDirectoryFromAppW --> CreateDirectoryExW + CreateFileFromAppW --> CreateFile2FromAppW + DeleteFileFromAppW --> DeleteFileW + DeleteFileA --> DeleteFileW + GetCompressedFileSizeA --> GetCompressedFileSizeW + GetFileAttributesA --> GetFileAttributesW + GetFileAttributesExA --> GetFileAttributesExW + GetFileAttributesExFromAppW --> GetFileAttributesExW + RemoveDirectoryA --> RemoveDirectoryW + RemoveDirectoryFromAppW --> RemoveDirectoryW + SetFileAttributesFromAppW --> SetFileAttributesW + SetFileAttributesA --> SetFileAttributesW + end + + subgraph NT API + %% Definitions + NtCreateFile + NtOpenFile + NtQueryDirectoryFile + NtQueryDirectoryFileEx + NtDeleteFile + NtQueryAttributesFile + NtQueryFullAttributesFile + NtClose + + %%% Win32 -> NT API + FindFirstFileExW --> NtOpenFile + FindFirstFileExW --> NtQueryDirectoryFileEx + FindFirstFileW --> NtOpenFile + FindFirstFileW --> NtQueryDirectoryFileEx + FindNextFileW --> NtQueryDirectoryFileEx + CreateFileA --> NtCreateFile + CreateFileW --> NtCreateFile + CreateFile2 --> NtCreateFile + CreateDirectoryW --> NtCreateFile + CreateDirectoryExW --> NtOpenFile + CreateDirectoryExW --> NtCreateFile + DeleteFileW --> NtOpenFile + RemoveDirectoryW --> NtOpenFile + GetCompressedFileSizeW --> NtOpenFile + CloseHandle --> NtClose + GetFileAttributesExW --> NtQueryFullAttributesFile + GetFileAttributesW --> NtQueryAttributesFile + SetFileAttributesW --> NtOpenFile + end + + %%% Hooks + subgraph Hooks + NtCreateFile_Hook + NtOpenFile_Hook + NtQueryDirectoryFileEx_Hook + NtDeleteFile_Hook + NtQueryAttributesFile_Hook + NtQueryFullAttributesFile_Hook + NtClose_Hook + + %% NT API -> Hooks + NtCreateFile --> NtCreateFile_Hook + NtOpenFile --> NtOpenFile_Hook + NtQueryDirectoryFileEx --> NtQueryDirectoryFileEx_Hook + NtQueryDirectoryFile --> NtQueryDirectoryFile_Hook + + NtDeleteFile --> NtDeleteFile_Hook + NtQueryAttributesFile --> NtQueryAttributesFile_Hook + NtQueryFullAttributesFile --> NtQueryFullAttributesFile_Hook + NtClose --> NtClose_Hook + end +``` + +On Windows 10+, `NtQueryDirectoryFileEx` API becomes available and `NtQueryDirectoryFile` acts as +a wrapper around it. On Wine and earlier Windows, only `NtQueryDirectoryFile` exists. + +In this VFS we hook both, and detect if one recurses to the other using a semaphore. If we're +recursing from `NtQueryDirectoryFile` to `NtQueryDirectoryFileEx`, we skip the hook code. \ No newline at end of file diff --git a/docs/Mods/Essentials/Virtual-FileSystem/Implementation-Details/Optimizations.md b/docs/Mods/Essentials/Virtual-FileSystem/Implementation-Details/Optimizations.md new file mode 100644 index 0000000..26044c7 --- /dev/null +++ b/docs/Mods/Essentials/Virtual-FileSystem/Implementation-Details/Optimizations.md @@ -0,0 +1,158 @@ +# Optimizations + +!!! note "Some parts of this section will compare against the [original unreleased C# implementation][original-impl]" + + This is to give an idea of where things can be improved while translating some existing code. + +## Strings + +!!! info "All strings internally are stored in the platform's native string format." + +### Encoding + +#### Unix-based Systems + +!!! info "On Linux and other unix-based systems, a file or folder is usually represented by a sequence of bytes." + +We'll assume UTF-8, as that's the case on pretty much all modern distros. + +#### Windows + +!!! info "On Windows, that is a sequence of 16-bit characters (UTF-16)." + +Although Windows has ANSI (`A`) and Unicode (`W`) APIs side by side, under the hood, Windows +converts everything to UTF-16. + +In order to speed things up, it is planned to use UTF-16 strings internally for non-ASCII input to +avoid re-encoding them like in the original C# implementation. + +!!! note "This may change, in Rust version, we may investigate the tradeoff." + +### Case Conversion + +!!! info "In Windows, expect case insensitive paths." + +We have to convert paths to uppercase inside various hooks. + +!!! note "We choose uppercase because some languages e.g. greek have special rules for lowercase" + + This saves us from having to deal with those rules, improving performance. + +#### Previous Implementation + +In the C# implementation, we backported a `ToUpper` function from one of the .NET 8 Previews. +[Implementation Here][reloaded-memory-toupper]. (This function is faster than the one which +shipped in .NET 8 proper) + +In said implementation, the code is optimized for 99% of the cases where the paths are pure +ASCII; with rare other cases falling to a slower implementation. + +#### Rust Implementation + +In Rust we can use the [make_ascii_uppercase][make-ascii-uppercase] function if we can trust +the paths to be ASCII for comparisons. Alternatively there's also functions like `to_uppercase`. + +!!! note "We may want to copy out the implementation of `to_uppercase` to return result if string is ASCII" + + Rather than explicitly checking in another step, which is expensive. + +These are not vectorised in the Rust source, but thanks to LLVM magic, they do get vectorised, +so there is generally no need to write our own. The `to_uppercase` function is generally limited +to unrolled SSE2, while `make_ascii_uppercase` will use rolled AVX2 if possible on CPU target. + +### Hashing + +!!! info "A hardware accelerated string hash function is used for file paths." + +In the original C# implementation, this was a vectorized function which now lives here +[Reloaded.Memory String Hash][reloaded-memory-hash]. + +For the Rust implementation, it's recommended to use [AHash][ahash], it is the top performer +in the [smhasher][smhasher] benchmark suite for hashtables. + +- All strings stored as Wide Strings. + - Windows APIs use Wide Strings under the hood, even for ANSI APIs. + - Therefore we save time by not having to widen them again. + +### HashMaps + +!!! info "HashMaps need to be able to query string slices." + +In original C# implementation, it was necessary to use a custom dictionary that could query string +slices as keys, due to limitation of the built in one. + +In Rust, this won't be needed; as thanks to the [Equivalent][equivalent] trait. +Hashbrown (Swisstable) is also faster. + +### Memory Storage + +!!! info "In the Rust implementation, we will pack strings together for efficiency." + +In C#, some code was limited to the standard `String` type due to interoperability with +existing APIs. With some additional foresight and experience, we can do a bit better in Rust. + +The idea is to use `string pooling` and custom string headers. Rather than having an allocation for +each string, we can perform a single memory allocation and then store all of the strings inline in +a single buffer. + +```rust +// Pseudocode +pub struct StringEntry +{ + /// Truth Table for char_count_and_flag: + /// + /// | Bit Position (from LSB) | 0 (IsAscii flag) | 1-15 (char_count) | + /// |--------------------------------|------------------|----------------------| + /// | Value when ASCII | 1 | ASCII char count | + /// | Value when non-ASCII (Windows) | 0 | UTF-16 char count | + /// | Value when non-ASCII (Unix) | 0 | UTF-8 byte count | + /// + /// - The least significant bit (LSB, position 0) is used as the `IsAscii` flag. + /// If it's 1, the string is ASCII. If it's 0, the string is non-ASCII. + /// + /// - The remaining 15 bits (positions 1-15) are used to store the character count. + /// If the string is ASCII, this is the count of ASCII characters. + /// If the string is non-ASCII, on Windows this is the count of UTF-16 characters, + /// and on Unix this is the count of UTF-8 bytes. + char_count_and_flag: u16, + + // Inline in memory, right after header + // Or u8 if using ANSI strings. + data: u16[length] +} +``` + +String Widen is Cheap !! + +### awa + +!!! tip "And for `Hashbrown`'s low level [HashTable][hashtable] primitive." + +```rust +pub struct TableEntry +{ + pub key: u64, + pub value: &str +} + +// To search (hash at minimum, as shown here) +table.find(KEY_HASH, |&val| val.key == KEY_HASH); +``` + +## Lookup Tree + +!!! info "After creating the 'Redirection Tree' we create a 'Lookup Tree' for translating file paths" + +[More details on both structures can be found in 'Implementation Details' section.][lookup-tree] + +The `LookupTree` allows us to resolve file paths in O(3) time, at expense of some RAM. + +[ahash]: https://github.com/tkaitchuck/aHash +[equivalent]: https://docs.rs/hashbrown/latest/hashbrown/trait.Equivalent.html +[lookup-tree]: ./Trees.md#lookup-tree +[make-ascii-uppercase]: https://github.com/rust-lang/rust/blob/80d1c8349ab7f1281b9e2f559067380549e2a4e6/library/core/src/num/mod.rs#L627 +[original-impl]: https://github.com/Reloaded-Project/reloaded.universal.redirector/tree/rewrite-usvfs-read-features +[redirection-tree]: ./Trees.md#redirection-tree +[reloaded-memory-hash]: https://github.com/Reloaded-Project/Reloaded.Memory/blob/5d13b256c89ffa2b18bf430b6ef39925e4324412/src/Reloaded.Memory/Internals/Algorithms/UnstableStringHash.cs#L16 +[smhasher]: https://github.com/rurban/smhasher +[reloaded-memory-toupper]: https://github.com/Reloaded-Project/Reloaded.Memory/blob/5d13b256c89ffa2b18bf430b6ef39925e4324412/src/Reloaded.Memory/Internals/Backports/System/Globalization/TextInfo.cs#L79 diff --git a/docs/Mods/Essentials/Virtual-FileSystem/Implementation-Details/Trees.md b/docs/Mods/Essentials/Virtual-FileSystem/Implementation-Details/Trees.md new file mode 100644 index 0000000..dc4ccb9 --- /dev/null +++ b/docs/Mods/Essentials/Virtual-FileSystem/Implementation-Details/Trees.md @@ -0,0 +1,156 @@ +!!! info "This page detail the 'Tree' implementations used in the VFS." + +The actual struct definitions here are valid for the original C# implementation, but will differ a +tiny bit in the Rust implementation. We still show the C# ones for simplicity, to help understanding. + +## Redirection Tree + +!!! info "About the 'Redirection Tree'" + + The `RedirectionTree` is a visualization of the data structure used to map paths of old files + to new files as the mods are loading during startup. + + It uses an O(N) lookup time (where N is the number of components separated by '/') that make up + the final file path. + +The resolution steps are: + +1. Start at tree root. +2. Split the input path on '/' character. +3. Traverse the tree one level at a time, using each split component to move down next level. +4. At each level check if there's a child node corresponding to current path component. + - If there is no child node, lookup has failed and path is not in the tree. +5. When all components have been consumed, check the `Items` dictionary of the final node reached to see if the path is present. +6. If it is, the lookup succeeds and the corresponding value is returned. If it is not, the lookup fails and the path is not found in the tree. + +When all mods are loaded, this trie-like structure is converted to a `LookupTree`. + +```mermaid +flowchart LR + subgraph RedirectionTree + C: --> SteamLibrary --> steamapps --> common --> Game + Game --> data_1.pak + Game --> data_2.pak + Game --> data_3.pak + end + data_2.pak --> f[FULL_PATH_TO_NEW_FILE 'isDirectory: false'] +``` + +### In Code + +```csharp +/// +/// Represents that will be used for performing redirections. +/// +public struct RedirectionTree +{ + /// + /// Root nodes, e.g. would store drive: C:/D:/E: etc. + /// In most cases there is only one. + /// + public RedirectionTreeNode RootNode { get; private set; } +} +``` + +```csharp +/// +/// Individual node in the redirection tree. +/// +public struct RedirectionTreeNode +{ + /// + /// Child nodes of this nodes. + /// i.e. Maps 'folder' to next child. + /// + public SpanOfCharDict> Children; + + /// + /// Files present at this level of the tree. + /// + public SpanOfCharDict Items; +} +``` + +```csharp +/// +/// Target for a file covered by the redirection tree. +/// +public struct RedirectionTreeTarget +{ + /// + /// Path to the directory storing the file. + /// + public string Directory; // (This is deduplicated, saving memory) + + /// + /// Name of the file in the directory. + /// + public string FileName; + + /// + /// True if this is a directory, else false. + /// + public bool IsDirectory; +} +``` + +## Lookup Tree + +!!! info "About the 'Lookup Tree'" + + The `LookupTree` is a visualization of the data structure used to map paths of old files to new + files ***after*** all mods are loaded during startup. + + When all mods are loaded, this structure is generated from the `RedirectionTree`. + +The idea is that usually the VFS will be used to redirect game files only. + +It uses a strategy of: + +1. Check common prefix. +2. Check remaining path in dictionary. +3. Check file name in dictionary. + +The prefix is based on the idea that a game will have all of its files stored under a common folder path. +We use this to save memory in potentially huge games. + +```mermaid +flowchart LR + subgraph LookupTree + subgraph Common Prefix + C[C:/SteamLibrary/steamapps/common/Game] + end + subgraph Dictionary + D[.] + M[music] + end + C --> D + D --> data_1.pak + D --> data_2.pak + D --> data_3.pak + M --> jingle.wav + M --> ocean.wav + end + data_2.pak --> f[FULL_PATH_TO_NEW_FILE 'isDirectory: false'] +``` + +### In Code + +```csharp +/// +/// A version of optimised for faster lookups in the scenario of use with game folders. +/// +public struct LookupTree +{ + /// + /// Prefix of all paths. + /// Stored in upper case for faster performance. + /// + public string Prefix { get; private set; } + + /// + /// Dictionary that maps individual subfolders to map of files. + /// + public SpanOfCharDict> SubfolderToFiles { get; private set; } +} +``` \ No newline at end of file diff --git a/docs/Mods/Essentials/Virtual-FileSystem/Implementing-Redirection.md b/docs/Mods/Essentials/Virtual-FileSystem/Implementing-Redirection.md deleted file mode 100644 index a5b5db3..0000000 --- a/docs/Mods/Essentials/Virtual-FileSystem/Implementing-Redirection.md +++ /dev/null @@ -1,107 +0,0 @@ -## Implementing Redirection - -The virtual file system uses two main data structures: - -1. The `RedirectionTree` which maps old file paths to new paths as mods are loading. It uses an O(N) trie-like structure. - -2. The `LookupTree` which is an optimized version of the RedirectionTree used after all mods are loaded. It uses an O(3) structure for fast lookups. - -Here is how you could implement these structures in Rust using the `hashbrown` crate: - -```rust -use hashbrown::HashMap; - -pub struct RedirectionTree { - pub root: RedirectionNode -} - -pub struct RedirectionNode { - pub children: HashMap, - pub items: HashMap, -} - -impl RedirectionNode { - pub fn new() -> Self { - RedirectionNode { - children: HashMap::new(), - items: HashMap::new(), - } - } -} - -impl RedirectionTree { - pub fn add_path(&mut self, old_path: &str, new_path: &str) { - let mut current = &mut self.root; - for part in old_path.split('/') { - current = current.children.entry(part.to_string()).or_insert(RedirectionNode::new()); - } - current.items.insert(old_path.to_string(), new_path.to_string()); - } -} -``` - -```rust -use hashbrown::HashMap; - -pub struct LookupTree { - pub prefix: String, - pub subdir_to_files: HashMap>, -} - -impl LookupTree { - pub fn from_redirection_tree(tree: &RedirectionTree) -> Self { - let mut lookup = LookupTree { - prefix: String::new(), - subdir_to_files: HashMap::new(), - }; - - lookup.build_prefix(tree); - lookup.build_subdir_to_files(&tree.root, ""); - - lookup - } - - fn build_prefix(&mut self, tree: &RedirectionTree) { - // Find common prefix of all paths - // ... - } - - fn build_subdir_to_files(&mut self, node: &RedirectionNode, path: &str) { - if !node.items.is_empty() { - self.subdir_to_files.insert(path.to_string(), node.items.clone()); - } - - for (name, child) in &node.children { - let sub_path = format!("{}/{}", path, name); - self.build_subdir_to_files(child, &sub_path); - } - } -} -``` - -The `LookupTree` is built from the `RedirectionTree` when all mods are done loading. It extracts a common prefix from all paths to save memory, and builds a flat map of subdirectories to files. - -To detect file changes efficiently, you can use the `notify` crate to watch the redirect folders for changes and rebuild the trees as needed. Here's a sketch: - -```rust -use notify::{RecommendedWatcher, Watcher, RecursiveMode}; -use std::sync::mpsc::channel; -use std::time::Duration; - -fn main() { - let (tx, rx) = channel(); - let mut watcher: RecommendedWatcher = Watcher::new(tx, Duration::from_secs(1)).unwrap(); - - watcher.watch("path/to/redirector", RecursiveMode::Recursive).unwrap(); - - loop { - match rx.recv() { - // Rebuild trees on file change events - Ok(event) => println!("event: {:?}", event), - Err(e) => println!("watch error: {:?}", e), - } - } -} -``` - -This sets up a watcher on the Redirector folder that will receive events when files change. You can then trigger a rebuild of the `RedirectionTree` and `LookupTree` to pick up those changes. \ No newline at end of file diff --git a/docs/Mods/Essentials/Virtual-FileSystem/Performance.md b/docs/Mods/Essentials/Virtual-FileSystem/Performance.md index b4f7e96..7017a6e 100644 --- a/docs/Mods/Essentials/Virtual-FileSystem/Performance.md +++ b/docs/Mods/Essentials/Virtual-FileSystem/Performance.md @@ -1,73 +1,92 @@ # Performance Characteristics -All numbers were obtained on a stock clock AMD Ryzen 5900X and 3000MHz CL16 RAM. - -The Reloaded VFS is heavily optimized for performance. A lot of micro-optimizations were done to squeeze every bit out of making opening files faster... +!!! note "The numbers here are expected worst-case numbers." -- All strings stored as Wide Strings. - - Windows APIs use Wide Strings under the hood, even for ANSI APIs. - - Therefore we save time by not having to widen them again. + These were gathered from the original heavily optimized C# implementation of the VFS. -- Custom string Hash Function for file paths. - - With AVX and SSE implementations; as well as unrolled `nint` as backup. + Some of these may improve a bit, others, not so much. -- Custom Vectorized `ToUpper` for Strings. - - Modified backport from .NET 8. - - Super fast for 99% of the paths that are pure ASCII. - - Non-ASCII paths use slower fallback since they can't be vectorized. - - Partial ASCII paths have the first ASCII part vectorized and rest handled in fallback. +All numbers were obtained on a stock clock AMD Ryzen 5900X and 3000MHz CL16 RAM. -- Custom Dictionary (HashMap) that can query string slices (to avoid copying/realloc). +## File Mapping Performance & Memory Usage -- Uses a custom ['LookupTree'](./implementationdetails/lookup-tree.md). - - Provides lookup for resolving file paths in O(3) time. +!!! note "These are numbers from the original C# version." -## File Mapping Performance & Memory Usage + We can save some additional memory in Rust with clever use of custom string packing.
+ AHash is also a better hash algorithm. Expect both memory and speed numbers to improve a tiny bit. -This section describes how long it takes to create a file map. A file map is a structure that helps redirect original files to their new versions found in mod folders. +!!! info "This section describes how long it takes to create a file map." -Whenever changes are made to mod folders, the file map needs to be rebuilt. + A file map is used to redirect original files to their new versions found in mod folders.
+ Whenever changes are made to mod folders, a 'rebuild' of the map is done. -Two types of maps exist, [RedirectionTree](./implementationdetails/redirection-tree.md) and [LookupTree](./implementationdetails/lookup-tree.md). The latter, `LookupTree` is optimized for performance, but takes roughly twice as long to build as it is built from the former `RedirectionTree`. +Two types of maps exist, [RedirectionTree][redirection-tree] and [LookupTree][lookup-tree]. +The latter, `LookupTree` is optimized for performance, but takes roughly twice as long to build as +it is built from the former `RedirectionTree`. | Folder Type | Directories | Total Items | RedirectionTree (Time) | RedirectionTree (Memory) | LookupTree (Time) | LookupTree (Memory) | -|------------------------------|-------------|-------------|------------------------|-------------------------|-------------------|---------------------| -| Windows Folder | 40,796 | 170,438 | 43ms | 27MB | 32ms | 25MB | -| Steam Folder
(65 games) | 9,318 | 172,896 | 18ms | 12MB | 20ms | 11MB | +| ---------------------------- | ----------- | ----------- | ---------------------- | ------------------------ | ----------------- | ------------------- | +| Windows Folder | 40,796 | 170,438 | 43ms | 27MB | 32ms | 25MB | +| Steam Folder
(65 games) | 9,318 | 172,896 | 18ms | 12MB | 20ms | 11MB | -The performance of mapping operations mainly depends on the directory count. The table above shows the time and memory usage for building the `RedirectionTree` and `LookupTree` for both Windows and Steam folders. The `LookupTree` memory usage should be approximately equal to the total runtime memory usage. +The performance of mapping operations mainly depends on the directory count. The table above shows +the time and memory usage for building the `RedirectionTree` and `LookupTree` for both Windows and +Steam folders. The `LookupTree` memory usage should be approximately equal to the total runtime +memory usage. -For a typical game (based on the median of a Steam library), building the `RedirectionTree` should take around `0.017ms` and allocate `48KB`. -Creating the optimized `LookupTree` takes about `0.012ms` and allocates `47KB`. +For a typical game (based on the median of a Steam library), building the `RedirectionTree` should +take around `0.017ms` and allocate `48KB`. Creating the optimized `LookupTree` takes about `0.012ms` +(+ time taken for `RedirectionTree`) and allocates `47KB`. In other words, you can assume remapping files is basically real-time. ### Fast Append -Both `LookupTree` and `RedirectionTree` support `'fast append'` operations. +!!! tip "It's possible to skip a 'rebuild' in some situations." + +Both `LookupTree` and `RedirectionTree` can have `'fast append'` operations. +(Just like in the original C# implementation) -If a file is added to the mod folder while the game is running and isn't previously mapped, it can be added to the tree directly without a full rebuild. +If a file is added to the mod folder while the game is running and isn't previously mapped, +it can be added to the tree directly without a full rebuild. However, if the currently mapped file's source mod cannot be determined, the entire tree must be rebuilt. -This process doesn't require scanning mod folders again for files when not necessary. Each folder mapping has a cache of subdirectories and files, and the same string instances are reused between the trees and cache to save memory. +This process doesn't require scanning mod folders again for files when not necessary. +Each folder mapping can have a cache of subdirectories and files, and the same string instances can be +reused between the trees and cache to save memory. ## File Open Overhead +!!! info "These are numbers for original C# implementation" + File open has negligible performance difference compared to not using VFS. -In a test with opening+closing 21,000 files (+70,000 virtualized), the difference was only ~41ms (~3%) or less than 2 microseconds per file. +In a test with opening+closing 21,000 files (+70,000 virtualized), the difference was +only ~41ms (~3%) or less than 2 microseconds per file. ``` // All tests done in separate processes for accuracy. -| Method | Mean | Error | StdDev | Ratio | -|--------------------------------- |--------:|---------:|---------:|------:| -| OpenAllHandles_WithVfs | 1.650 s | 0.0102 s | 0.0095 s | 1.03 | +| Method | Mean | Error | StdDev | Ratio | +| -------------------------------- | ------: | -------: | -------: | ----: | +| OpenAllHandles_WithVfs | 1.650 s | 0.0102 s | 0.0095 s | 1.03 | | OpenAllHandles_WithVfs_Optimized | 1.643 s | 0.0145 s | 0.0135 s | 1.03 | -| OpenAllHandles_WithoutVfs | 1.602 s | 0.0128 s | 0.0120 s | 1.00 | +| OpenAllHandles_WithoutVfs | 1.602 s | 0.0128 s | 0.0120 s | 1.00 | ``` -In real-world `"cold-start"` scenarios (e.g. after a machine reboot), opening these many files would take around 80 seconds, making this difference effectively margin of error (~0%). +In real-world `"cold-start"` scenarios (e.g. after a machine reboot), opening these many files +would take around 80 seconds, making this difference effectively margin of error (~0%). + +## Existing Benchmarks -## Built-in Benchmarks +A lot of microbenchmarks are available in the original C# project under the +[Reloaded.Universal.Redirector.Benchmarks][microbenchmarks] project; have a look! -If you're a programmer, a lot of microbenchmarks are available in the `Reloaded.Universal.Redirector.Benchmarks` project; have a look! \ No newline at end of file +[ahash]: https://github.com/tkaitchuck/aHash +[equivalent]: https://docs.rs/hashbrown/latest/hashbrown/trait.Equivalent.html +[lookup-tree]: ./Implementation-Details.md#lookup-tree +[make-ascii-uppercase]: https://github.com/rust-lang/rust/blob/80d1c8349ab7f1281b9e2f559067380549e2a4e6/library/core/src/num/mod.rs#L627 +[microbenchmarks]: https://github.com/Reloaded-Project/reloaded.universal.redirector/tree/rewrite-usvfs-read-features/Reloaded.Universal.Redirector.Benchmarks +[redirection-tree]: ./Implementation-Details.md#redirection-tree +[reloaded-memory-hash]: https://github.com/Reloaded-Project/Reloaded.Memory/blob/5d13b256c89ffa2b18bf430b6ef39925e4324412/src/Reloaded.Memory/Internals/Algorithms/UnstableStringHash.cs#L16 +[smhasher]: https://github.com/rurban/smhasher +[reloaded-memory-toupper]: https://github.com/Reloaded-Project/Reloaded.Memory/blob/5d13b256c89ffa2b18bf430b6ef39925e4324412/src/Reloaded.Memory/Internals/Backports/System/Globalization/TextInfo.cs#L79 diff --git a/docs/Mods/Essentials/Virtual-FileSystem/Programmer-Usage.md b/docs/Mods/Essentials/Virtual-FileSystem/Programmer-Usage.md index e2b6388..6dba38b 100644 --- a/docs/Mods/Essentials/Virtual-FileSystem/Programmer-Usage.md +++ b/docs/Mods/Essentials/Virtual-FileSystem/Programmer-Usage.md @@ -1,6 +1,11 @@ - # Programmer Usage +!!! warning "The presented API is just for reference." + + It may be modified ahead of release. + +## Using the API + The Redirector uses [Reloaded Dependency Injection](https://reloaded-project.github.io/Reloaded-II/DependencyInjection_HowItWork/) to expose an API. To use the Redirector API: diff --git a/docs/Mods/Essentials/Virtual-FileSystem/Usage.md b/docs/Mods/Essentials/Virtual-FileSystem/Usage.md index 3a851af..46948c1 100644 --- a/docs/Mods/Essentials/Virtual-FileSystem/Usage.md +++ b/docs/Mods/Essentials/Virtual-FileSystem/Usage.md @@ -1,23 +1,29 @@ - # Basic Usage +!!! info "This will nee updating." + + This is the original Reloaded-II documentation from the scrapped C# version which couldn't be + released due to teechnical runtime limitations. + + This guide is just for reference and will need to be rewritten. + ## Download the Mod First, download the `Reloaded File Redirector` mod which provides the virtual filesystem functionality. -![DownloadMod](https://raw.githubusercontent.com/Reloaded-Project/reloaded.universal.redirector/master/docs/images/DownloadMod.png) +![DownloadMod][download-mod] ## Add Dependency to Redirector In the `Edit Mod` menu, add `Reloaded File Redirector` as a dependency to your mod. -![AddDependency](https://raw.githubusercontent.com/Reloaded-Project/reloaded.universal.redirector/master/docs/images/AddDependency.png) +![AddDependency][add-dependency] This will ensure the `Reloaded File Redirector` mod is always loaded when your mod is loaded. ### Opening the Mod Folder -![OpenModFolder](https://raw.githubusercontent.com/Reloaded-Project/reloaded.universal.redirector/master/docs/images/OpenModFolder.png) +![OpenModFolder][open-mod-folder] Go to the folder where your mod is stored by clicking the `Open Folder` button. @@ -25,7 +31,7 @@ Go to the folder where your mod is stored by clicking the `Open Folder` button. Make a folder called `Redirector`. Inside it, place files that you want to be replaced. -![FileRedirectorFolder](https://raw.githubusercontent.com/Reloaded-Project/reloaded.universal.redirector/master/docs/images/FileRedirectorFolder.png) +![FileRedirectorFolder][file-redirector-folder] Files are mapped by their location relative to the EXE of the application. @@ -52,7 +58,7 @@ The connectors `└─` represent folders. ## Debugging -![DebugMod](https://raw.githubusercontent.com/Reloaded-Project/reloaded.universal.redirector/master/docs/images/DebugMod.png) +![DebugMod](https://raw.githubusercontent.com/Reloaded-Project/reloaded.universal.redirector/rewrite-usvfs-read-features/docs/images/DebugMod.png) To debug the mod, highlight the `Reloaded File Redirector` mod in your mod manager and click `Configure Mod`. @@ -60,3 +66,9 @@ The following settings are available: - `Log Open Files`: Prints a message to `Console` when a new file is being opened. - `Log Redirections`: Prints a message when a custom file is loaded from your or another mod. - `Log Attribute Fetches`: Prints a message when game gets file properties such as file size. + + +[add-dependency]: https://raw.githubusercontent.com/Reloaded-Project/reloaded.universal.redirector/master/docs/images/AddDependency.png +[download-mod]: https://raw.githubusercontent.com/Reloaded-Project/reloaded.universal.redirector/master/docs/images/DownloadMod.png +[file-redirector-folder]: https://raw.githubusercontent.com/Reloaded-Project/reloaded.universal.redirector/master/docs/images/FileRedirectorFolder.png +[open-mod-folder]: https://raw.githubusercontent.com/Reloaded-Project/reloaded.universal.redirector/master/docs/images/OpenModFolder.png \ No newline at end of file diff --git a/mkdocs.yml b/mkdocs.yml index 8a8bc5b..3aa24d6 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -106,11 +106,14 @@ nav: - Essentials: - Virtual FileSystem: - About: Mods/Essentials/Virtual-FileSystem/About.md + - Behaviours & Limitations: Mods/Essentials/Virtual-FileSystem/Behaviours-Limitations.md - Usage: Mods/Essentials/Virtual-FileSystem/Usage.md - Programmer Usage: Mods/Essentials/Virtual-FileSystem/Programmer-Usage.md - - Implementing Redirection: Mods/Essentials/Virtual-FileSystem/Implementing-Redirection.md - - Implementation Details: Mods/Essentials/Virtual-FileSystem/Implementation-Details.md - - Performance: Mods/Essentials/Virtual-FileSystem/Performance.md + - Implementation Details: + - Hooks: Mods/Essentials/Virtual-FileSystem/Implementation-Details/Hooks.md + - Optimizations: Mods/Essentials/Virtual-FileSystem/Implementation-Details/Optimizations.md + - Trees: Mods/Essentials/Virtual-FileSystem/Implementation-Details/Trees.md + - Reference Performance Numbers: Mods/Essentials/Virtual-FileSystem/Performance.md - Essentials (Libraries): - Merged File Cache: - About: Mods/Libraries/Merged-File-Cache/About.md From dc886ea4296c7e6f4384375cf31400f6199c979f Mon Sep 17 00:00:00 2001 From: Sewer56 Date: Sat, 27 Apr 2024 21:28:10 +0100 Subject: [PATCH 3/6] Added: Remaining Performance Related Documentation --- .../Implementation-Details.md | 2 - .../Implementation-Details/Optimizations.md | 51 ++++++++++++++++--- .../Implementation-Details/Trees.md | 11 +++- .../Virtual-FileSystem/Performance.md | 9 ++-- 4 files changed, 60 insertions(+), 13 deletions(-) delete mode 100644 docs/Mods/Essentials/Virtual-FileSystem/Implementation-Details.md diff --git a/docs/Mods/Essentials/Virtual-FileSystem/Implementation-Details.md b/docs/Mods/Essentials/Virtual-FileSystem/Implementation-Details.md deleted file mode 100644 index d4c8081..0000000 --- a/docs/Mods/Essentials/Virtual-FileSystem/Implementation-Details.md +++ /dev/null @@ -1,2 +0,0 @@ -# Implementation Details - diff --git a/docs/Mods/Essentials/Virtual-FileSystem/Implementation-Details/Optimizations.md b/docs/Mods/Essentials/Virtual-FileSystem/Implementation-Details/Optimizations.md index 26044c7..534fd69 100644 --- a/docs/Mods/Essentials/Virtual-FileSystem/Implementation-Details/Optimizations.md +++ b/docs/Mods/Essentials/Virtual-FileSystem/Implementation-Details/Optimizations.md @@ -81,7 +81,9 @@ in the [smhasher][smhasher] benchmark suite for hashtables. In original C# implementation, it was necessary to use a custom dictionary that could query string slices as keys, due to limitation of the built in one. -In Rust, this won't be needed; as thanks to the [Equivalent][equivalent] trait. +In Rust, this won't be needed; as thanks to the [Equivalent][equivalent] trait. Or using the +`HashTable` API raw. + Hashbrown (Swisstable) is also faster. ### Memory Storage @@ -122,21 +124,57 @@ pub struct StringEntry } ``` -String Widen is Cheap !! +In 99% of cases, file paths will map to ASCII, meaning we only need 1 byte per character. + +On Windows, where paths are internally UTF-16, we can then dynamically widen these paths to +UTF-16 when needed, as that operation is very cheap for ASCII strings. + +[Godbolt example][godbolt-string-widen]. + +```rust +#[no_mangle] +pub fn ascii_to_utf16(ascii_str: &str, buffer: &mut [u16]) { + let bytes = ascii_str.as_bytes(); + let len = bytes.len(); + for i in 0..len { + buffer[i] = bytes[i] as u16; + } +} +``` + +Auto-vectorizes nicely, thanks to LLVM. -### awa +#### Pooled Strings in HashTables !!! tip "And for `Hashbrown`'s low level [HashTable][hashtable] primitive." ```rust +// 16 bytes, nicely aligns with cache lines pub struct TableEntry { + // 64-bit key pub key: u64, - pub value: &str + + // Offset for directory string in pool. + // We deduplicate strings by directory, reducing memory usage further. + // On average there's typically 7 files per 1 folder in Sewer's Steam folder. + pub dir: u32, + + // Offset for file string in the pool. + pub file: u32, } +// If additional metadata is required, trim `dir` and `file` to 30 bits, +// and use remaining space for metadata. +``` + +And to search, we should compare key at minimum + +```rust // To search (hash at minimum, as shown here) table.find(KEY_HASH, |&val| val.key == KEY_HASH); + +// TODO: Add string comparison here. ``` ## Lookup Tree @@ -149,10 +187,11 @@ The `LookupTree` allows us to resolve file paths in O(3) time, at expense of som [ahash]: https://github.com/tkaitchuck/aHash [equivalent]: https://docs.rs/hashbrown/latest/hashbrown/trait.Equivalent.html +[godbolt-string-widen]: https://godbolt.org/z/K18b81rE8 +[hashtable]: https://docs.rs/hashbrown/latest/hashbrown/struct.HashTable.html#method.find [lookup-tree]: ./Trees.md#lookup-tree [make-ascii-uppercase]: https://github.com/rust-lang/rust/blob/80d1c8349ab7f1281b9e2f559067380549e2a4e6/library/core/src/num/mod.rs#L627 [original-impl]: https://github.com/Reloaded-Project/reloaded.universal.redirector/tree/rewrite-usvfs-read-features -[redirection-tree]: ./Trees.md#redirection-tree [reloaded-memory-hash]: https://github.com/Reloaded-Project/Reloaded.Memory/blob/5d13b256c89ffa2b18bf430b6ef39925e4324412/src/Reloaded.Memory/Internals/Algorithms/UnstableStringHash.cs#L16 -[smhasher]: https://github.com/rurban/smhasher [reloaded-memory-toupper]: https://github.com/Reloaded-Project/Reloaded.Memory/blob/5d13b256c89ffa2b18bf430b6ef39925e4324412/src/Reloaded.Memory/Internals/Backports/System/Globalization/TextInfo.cs#L79 +[smhasher]: https://github.com/rurban/smhasher \ No newline at end of file diff --git a/docs/Mods/Essentials/Virtual-FileSystem/Implementation-Details/Trees.md b/docs/Mods/Essentials/Virtual-FileSystem/Implementation-Details/Trees.md index dc4ccb9..64bc62f 100644 --- a/docs/Mods/Essentials/Virtual-FileSystem/Implementation-Details/Trees.md +++ b/docs/Mods/Essentials/Virtual-FileSystem/Implementation-Details/Trees.md @@ -3,6 +3,8 @@ The actual struct definitions here are valid for the original C# implementation, but will differ a tiny bit in the Rust implementation. We still show the C# ones for simplicity, to help understanding. +For more information on Rust strings, see [String Pooling][string-pooling]. + ## Redirection Tree !!! info "About the 'Redirection Tree'" @@ -134,6 +136,11 @@ flowchart LR data_2.pak --> f[FULL_PATH_TO_NEW_FILE 'isDirectory: false'] ``` +!!! note "There may (rarely) sometimes be more than 1 common prefix." + + Sometimes there may be 2/3 if we're redirecting things like saves, + or doing emulation within mod folders. + ### In Code ```csharp @@ -153,4 +160,6 @@ public struct LookupTree /// public SpanOfCharDict> SubfolderToFiles { get; private set; } } -``` \ No newline at end of file +``` + +[string-pooling]: ./Optimizations.md#pooled-strings-in-hashtables \ No newline at end of file diff --git a/docs/Mods/Essentials/Virtual-FileSystem/Performance.md b/docs/Mods/Essentials/Virtual-FileSystem/Performance.md index 7017a6e..a6b529f 100644 --- a/docs/Mods/Essentials/Virtual-FileSystem/Performance.md +++ b/docs/Mods/Essentials/Virtual-FileSystem/Performance.md @@ -12,7 +12,7 @@ All numbers were obtained on a stock clock AMD Ryzen 5900X and 3000MHz CL16 RAM. !!! note "These are numbers from the original C# version." - We can save some additional memory in Rust with clever use of custom string packing.
+ We can save some additional memory in Rust with clever use of [custom string pooling][string-pooling].
AHash is also a better hash algorithm. Expect both memory and speed numbers to improve a tiny bit. !!! info "This section describes how long it takes to create a file map." @@ -83,10 +83,11 @@ A lot of microbenchmarks are available in the original C# project under the [ahash]: https://github.com/tkaitchuck/aHash [equivalent]: https://docs.rs/hashbrown/latest/hashbrown/trait.Equivalent.html -[lookup-tree]: ./Implementation-Details.md#lookup-tree +[lookup-tree]: ./Implementation-Details/Trees.md#lookup-tree [make-ascii-uppercase]: https://github.com/rust-lang/rust/blob/80d1c8349ab7f1281b9e2f559067380549e2a4e6/library/core/src/num/mod.rs#L627 [microbenchmarks]: https://github.com/Reloaded-Project/reloaded.universal.redirector/tree/rewrite-usvfs-read-features/Reloaded.Universal.Redirector.Benchmarks -[redirection-tree]: ./Implementation-Details.md#redirection-tree +[redirection-tree]: ./Implementation-Details/Trees.md#redirection-tree [reloaded-memory-hash]: https://github.com/Reloaded-Project/Reloaded.Memory/blob/5d13b256c89ffa2b18bf430b6ef39925e4324412/src/Reloaded.Memory/Internals/Algorithms/UnstableStringHash.cs#L16 -[smhasher]: https://github.com/rurban/smhasher [reloaded-memory-toupper]: https://github.com/Reloaded-Project/Reloaded.Memory/blob/5d13b256c89ffa2b18bf430b6ef39925e4324412/src/Reloaded.Memory/Internals/Backports/System/Globalization/TextInfo.cs#L79 +[string-pooling]: ./Implementation-Details/Optimizations.md#string-pooling +[smhasher]: https://github.com/rurban/smhasher From 8fa466b4d494b7ad8631a4acc18429e13dd4304e Mon Sep 17 00:00:00 2001 From: Sewer56 Date: Sat, 27 Apr 2024 23:03:56 +0100 Subject: [PATCH 4/6] Updated: Programmer notes for VFS Usage --- .../Virtual-FileSystem/Programmer-Usage.md | 221 +++++++++++++++--- 1 file changed, 187 insertions(+), 34 deletions(-) diff --git a/docs/Mods/Essentials/Virtual-FileSystem/Programmer-Usage.md b/docs/Mods/Essentials/Virtual-FileSystem/Programmer-Usage.md index 6dba38b..a7406aa 100644 --- a/docs/Mods/Essentials/Virtual-FileSystem/Programmer-Usage.md +++ b/docs/Mods/Essentials/Virtual-FileSystem/Programmer-Usage.md @@ -1,68 +1,221 @@ # Programmer Usage -!!! warning "The presented API is just for reference." +!!! warning "All APIs shown here are for reference only, not final." - It may be modified ahead of release. +The Redirector uses the Reloaded Dependency Injection [TODO: Link Pending] system to expose an API. -## Using the API +To use the Redirector API: -The Redirector uses [Reloaded Dependency Injection](https://reloaded-project.github.io/Reloaded-II/DependencyInjection_HowItWork/) to expose an API. +1. Add the `reloaded3.api.windows.vfs.interfaces.s56` (or equivalent for your language) package to your project. + +2. Add the dependency `reloaded3.api.windows.vfs.s56` to your mod's dependencies. + +3. In your mod's entry point, acquire the Controller: + +=== "C#" + ```csharp + IRedirectorController _redirectorController; + + public void Start(IModLoaderV1 loader) + { + _redirectorController = _modLoader.GetService(); + } + ``` + +=== "Rust" + ```rust + struct MyMod { + redirector_controller: Option, + } + + impl MyMod { + fn new(loader: &mut IModLoader) -> Self { + Self { + redirector_controller: loader.get_service::().ok(), + } + } + } + ``` + +=== "C++" + ```cpp + class MyMod { + private: + IRedirectorController* _redirectorController; + + public: + MyMod(IModLoader* loader) + { + _redirectorController = loader->GetService(); + } + }; + ``` -To use the Redirector API: +## IRedirectorController API -1. Add the `Reloaded.Universal.Redirector.Interfaces` NuGet package to your project. +!!! info "Using basic C# types for easier understanding. Actual types may vary." -2. Add the dependency `reloaded.universal.redirector` to `ModDependencies` in your `ModConfig.json`. -3. In your `Mod()` entry point, acquire the Controller: +### Redirecting Individual Files -```csharp -IRedirectorController _redirectorController; +- `AddRedirect(string sourcePath, string targetPath)`: Redirects an individual file path from + `sourcePath` (original game path) to `targetPath` (mod file path). + +- `RemoveRedirect(string sourcePath, string targetPath)`: Removes redirection for an individual file + path from `sourcePath` to `targetPath`. + +### Redirecting All Files in Folder + +- `AddRedirectFolder(string sourceFolder, string targetFolder)`: Adds a new redirect folder. + Files in `targetFolder` will overlay files in `sourceFolder`. + +- `RemoveRedirectFolder(string sourceFolder, string targetFolder)`: Removes a redirect folder where + files in `targetFolder` were overlaying `sourceFolder`. + +### Registering Virtual Files + +!!! info "For file emulation framework. [TODO: Link Pending]" + + These APIs allow you to inject virtual files into search results, such that they appear + alongside real files when game folders are being searched. + +- `RegisterVirtualFile(string filePath, VirtualFileMetadata metadata)`: Registers a new virtual + file at `filePath` with the provided metadata. This allows the virtual file to be seen during file searches. + +- `UnregisterVirtualFile(string filePath)`: Unregisters a previously registered virtual file at `filePath`. + +The `VirtualFileMetadata` structure should look something like: -public void Start(IModLoaderV1 loader) +```csharp +// This may differ for Unix. That's to be determined. +public struct VirtualFileMetadata { - _redirectorController = _modLoader.GetController(); + public long CreationTime; + public long LastAccessTime; + public long LastWriteTime; + public long ChangeTime; + public long EndOfFile; + public long AllocationSize; + public FileAttributes FileAttributes; } ``` -## IRedirectorController API - -The `IRedirectorController` interface provides the following methods: - -- `AddRedirect(string oldPath, string newPath)`: Redirects an individual file path. +Actually reading the files etc. is handled by the file emulation framework itself. -- `RemoveRedirect(string oldPath)`: Removes redirection for an individual file path. +!!! note "This API is intended to be called by the framework" -- `AddRedirectFolder(string folderPath)`: Adds a new redirect folder. Files in this folder will overlay files in the game directory. + And not by individual 'File Emulators' using the framework. + i.e. The end user of the framework should not be calling this API. -- `AddRedirectFolder(string folderPath, string sourceFolder)`: Adds a new redirect folder with a custom source folder. Files in `folderPath` will overlay files in `sourceFolder`. +### Setting Adjustment -- `RemoveRedirectFolder(string folderPath)`: Removes a redirect folder. +!!! info "Toggling settings flags." -- `RemoveRedirectFolder(string folderPath, string sourceFolder)`: Removes a redirect folder with a specific source folder. +- `GetRedirectorSetting(VfsSettings setting)`: Gets the current value of a redirector setting. + See `VfsSettings` enum for options. -- `GetRedirectorSetting(RedirectorSettings setting)`: Gets the current value of a redirector setting. See RedirectorSettings enum for options. +- `SetRedirectorSetting(bool enable, VfsSettings setting)`: Enables or disables a specific redirector setting. -- `SetRedirectorSetting(bool enable, RedirectorSettings setting)`: Enables or disables a specific redirector setting. +### Debugging -- `Enable()` / `Disable()`: Enables or disables the redirector entirely. +- `Enable()` / `Disable()`: Enables or disables the VFS entirely. ## Examples Redirect an individual file: -```csharp -_redirectorController.AddRedirect(@"dvdroot\bgm\SNG_STG26.adx", @"mods\mybgm.adx"); -``` +=== "C#" + ```csharp + _redirectorController.AddRedirect(@"dvdroot\bgm\SNG_STG26.adx", @"mods\mybgm.adx"); + ``` + +=== "Rust" + ```rust + redirector_controller.add_redirect(r"dvdroot\bgm\SNG_STG26.adx", r"mods\mybgm.adx"); + ``` + +=== "C++" + ```cpp + _redirectorController->AddRedirect(R"(dvdroot\bgm\SNG_STG26.adx)", R"(mods\mybgm.adx)"); + ``` Add a new redirect folder: -```csharp -_redirectorController.AddRedirectFolder(@"mods\mymod"); -``` +=== "C#" + ```csharp + _redirectorController.AddRedirectFolder(@"game\data", @"mods\mymod\data"); + ``` + +=== "Rust" + ```rust + redirector_controller.add_redirect_folder(r"game\data", r"mods\mymod\data"); + ``` + +=== "C++" + ```cpp + _redirectorController->AddRedirectFolder(R"(game\data)", R"(mods\mymod\data)"); + ``` + +Register a virtual file (dummy example): + +=== "C#" + ```csharp + var metadata = new VirtualFileMetadata + { + CreationTime = DateTime.Now.Ticks, + LastAccessTime = DateTime.Now.Ticks, + LastWriteTime = DateTime.Now.Ticks, + ChangeTime = DateTime.Now.Ticks, + EndOfFile = 1024, + AllocationSize = 1024, + FileAttributes = FileAttributes.Normal + }; + + _redirectorController.RegisterVirtualFile(@"game\virtualfile.txt", metadata); + ``` + +=== "Rust" + ```rust + let metadata = VirtualFileMetadata { + creation_time: SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_nanos() as i64, + last_access_time: SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_nanos() as i64, + last_write_time: SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_nanos() as i64, + change_time: SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_nanos() as i64, + end_of_file: 1024, + allocation_size: 1024, + file_attributes: FileAttributes::Normal, + }; + + redirector_controller.register_virtual_file(r"game\virtualfile.txt", metadata); + ``` + +=== "C++" + ```cpp + VirtualFileMetadata metadata; + metadata.CreationTime = std::chrono::system_clock::now().time_since_epoch().count(); + metadata.LastAccessTime = std::chrono::system_clock::now().time_since_epoch().count(); + metadata.LastWriteTime = std::chrono::system_clock::now().time_since_epoch().count(); + metadata.ChangeTime = std::chrono::system_clock::now().time_since_epoch().count(); + metadata.EndOfFile = 1024; + metadata.AllocationSize = 1024; + metadata.FileAttributes = FileAttributes::Normal; + + _redirectorController->RegisterVirtualFile(R"(game\virtualfile.txt)", metadata); + ``` Print file redirections to console: -```csharp -_redirectorController.SetRedirectorSetting(true, RedirectorSettings.PrintRedirect); -``` \ No newline at end of file +=== "C#" + ```csharp + _redirectorController.SetRedirectorSetting(true, VfsSettings.PrintRedirect); + ``` + +=== "Rust" + ```rust + redirector_controller.set_redirector_setting(true, VfsSettings::PrintRedirect); + ``` + +=== "C++" + ```cpp + _redirectorController->SetRedirectorSetting(true, VfsSettings::PrintRedirect); + ``` \ No newline at end of file From af8bd884fc19c487afca31cfae2b04990f06ac23 Mon Sep 17 00:00:00 2001 From: Sewer56 Date: Sun, 28 Apr 2024 02:32:38 +0100 Subject: [PATCH 5/6] Added: Minor typo fixes --- .../Virtual-FileSystem/Programmer-Usage.md | 95 +++++++++++++------ .../Implementation-Details.md | 2 +- 2 files changed, 67 insertions(+), 30 deletions(-) diff --git a/docs/Mods/Essentials/Virtual-FileSystem/Programmer-Usage.md b/docs/Mods/Essentials/Virtual-FileSystem/Programmer-Usage.md index a7406aa..c565183 100644 --- a/docs/Mods/Essentials/Virtual-FileSystem/Programmer-Usage.md +++ b/docs/Mods/Essentials/Virtual-FileSystem/Programmer-Usage.md @@ -10,15 +10,17 @@ To use the Redirector API: 2. Add the dependency `reloaded3.api.windows.vfs.s56` to your mod's dependencies. -3. In your mod's entry point, acquire the Controller: +3. In your mod's entry point, acquire the necessary services: === "C#" ```csharp IRedirectorController _redirectorController; + IVirtualFileSystem _vfsController; public void Start(IModLoaderV1 loader) { _redirectorController = _modLoader.GetService(); + _vfsController = _modLoader.GetService(); } ``` @@ -26,12 +28,14 @@ To use the Redirector API: ```rust struct MyMod { redirector_controller: Option, + vfs_controller: Option, } impl MyMod { fn new(loader: &mut IModLoader) -> Self { Self { redirector_controller: loader.get_service::().ok(), + vfs_controller: loader.get_service::().ok(), } } } @@ -42,11 +46,13 @@ To use the Redirector API: class MyMod { private: IRedirectorController* _redirectorController; + IVirtualFileSystem* _vfsController; public: MyMod(IModLoader* loader) { _redirectorController = loader->GetService(); + _vfsController = loader->GetService(); } }; ``` @@ -55,22 +61,23 @@ To use the Redirector API: !!! info "Using basic C# types for easier understanding. Actual types may vary." - ### Redirecting Individual Files -- `AddRedirect(string sourcePath, string targetPath)`: Redirects an individual file path from - `sourcePath` (original game path) to `targetPath` (mod file path). +- `RedirectHandle AddRedirect(string sourcePath, string targetPath)`: Redirects an individual file path from + `sourcePath` (original game path) to `targetPath` (mod file path). Returns a handle to the redirection. -- `RemoveRedirect(string sourcePath, string targetPath)`: Removes redirection for an individual file - path from `sourcePath` to `targetPath`. +- `void RemoveRedirect(RedirectHandle handle)`: Removes the redirection associated with the given `handle`. ### Redirecting All Files in Folder -- `AddRedirectFolder(string sourceFolder, string targetFolder)`: Adds a new redirect folder. - Files in `targetFolder` will overlay files in `sourceFolder`. +- `RedirectFolderHandle AddRedirectFolder(string sourceFolder, string targetFolder)`: Adds a new redirect folder. + Files in `targetFolder` will overlay files in `sourceFolder`. Returns a handle to the redirect folder. + +- `void RemoveRedirectFolder(RedirectFolderHandle handle)`: Removes the redirect folder associated with the given `handle`. + +## IVirtualFileSystem API -- `RemoveRedirectFolder(string sourceFolder, string targetFolder)`: Removes a redirect folder where - files in `targetFolder` were overlaying `sourceFolder`. +!!! info "Using basic C# types for easier understanding. Actual types may vary." ### Registering Virtual Files @@ -79,10 +86,11 @@ To use the Redirector API: These APIs allow you to inject virtual files into search results, such that they appear alongside real files when game folders are being searched. -- `RegisterVirtualFile(string filePath, VirtualFileMetadata metadata)`: Registers a new virtual +- `VirtualFileHandle RegisterVirtualFile(string filePath, VirtualFileMetadata metadata)`: Registers a new virtual file at `filePath` with the provided metadata. This allows the virtual file to be seen during file searches. + Returns a handle to the virtual file. -- `UnregisterVirtualFile(string filePath)`: Unregisters a previously registered virtual file at `filePath`. +- `void UnregisterVirtualFile(VirtualFileHandle handle)`: Unregisters the virtual file associated with the given `handle`. The `VirtualFileMetadata` structure should look something like: @@ -107,14 +115,25 @@ Actually reading the files etc. is handled by the file emulation framework itsel And not by individual 'File Emulators' using the framework. i.e. The end user of the framework should not be calling this API. -### Setting Adjustment - -!!! info "Toggling settings flags." +### VFS Settings -- `GetRedirectorSetting(VfsSettings setting)`: Gets the current value of a redirector setting. +- `GetVfsSetting(VfsSettings setting)`: Gets the current value of a VFS setting. See `VfsSettings` enum for options. -- `SetRedirectorSetting(bool enable, VfsSettings setting)`: Enables or disables a specific redirector setting. +- `SetVfsSetting(bool enable, VfsSettings setting)`: Enables or disables a specific VFS setting. + +The `VfsSettings` enum provides the following options: + +```csharp +public enum VfsSettings +{ + None = 0, // Default value. + PrintRedirect = 1 << 0, // Prints when a file redirect is performed. + PrintOpen = 1 << 1, // Prints file open operations. (debug setting) + DontPrintNonFiles = 1 << 2, // Skips printing non-files to the console. + PrintGetAttributes = 1 << 3 // Prints operations that get file attributes (debug setting) +} +``` ### Debugging @@ -126,34 +145,46 @@ Redirect an individual file: === "C#" ```csharp - _redirectorController.AddRedirect(@"dvdroot\bgm\SNG_STG26.adx", @"mods\mybgm.adx"); + var handle = _redirectorController.AddRedirect(@"dvdroot\bgm\SNG_STG26.adx", @"mods\mybgm.adx"); + // ... + _redirectorController.RemoveRedirect(handle); ``` === "Rust" ```rust - redirector_controller.add_redirect(r"dvdroot\bgm\SNG_STG26.adx", r"mods\mybgm.adx"); + let handle = redirector_controller.add_redirect(r"dvdroot\bgm\SNG_STG26.adx", r"mods\mybgm.adx"); + // ... + redirector_controller.remove_redirect(handle); ``` === "C++" ```cpp - _redirectorController->AddRedirect(R"(dvdroot\bgm\SNG_STG26.adx)", R"(mods\mybgm.adx)"); + auto handle = _redirectorController->AddRedirect(R"(dvdroot\bgm\SNG_STG26.adx)", R"(mods\mybgm.adx)"); + // ... + _redirectorController->RemoveRedirect(handle); ``` Add a new redirect folder: === "C#" ```csharp - _redirectorController.AddRedirectFolder(@"game\data", @"mods\mymod\data"); + var handle = _redirectorController.AddRedirectFolder(@"game\data", @"mods\mymod\data"); + // ... + _redirectorController.RemoveRedirectFolder(handle); ``` === "Rust" ```rust - redirector_controller.add_redirect_folder(r"game\data", r"mods\mymod\data"); + let handle = redirector_controller.add_redirect_folder(r"game\data", r"mods\mymod\data"); + // ... + redirector_controller.remove_redirect_folder(handle); ``` === "C++" ```cpp - _redirectorController->AddRedirectFolder(R"(game\data)", R"(mods\mymod\data)"); + auto handle = _redirectorController->AddRedirectFolder(R"(game\data)", R"(mods\mymod\data)"); + // ... + _redirectorController->RemoveRedirectFolder(handle); ``` Register a virtual file (dummy example): @@ -171,7 +202,9 @@ Register a virtual file (dummy example): FileAttributes = FileAttributes.Normal }; - _redirectorController.RegisterVirtualFile(@"game\virtualfile.txt", metadata); + var handle = _vfsController.RegisterVirtualFile(@"game\virtualfile.txt", metadata); + // ... + _vfsController.UnregisterVirtualFile(handle); ``` === "Rust" @@ -186,7 +219,9 @@ Register a virtual file (dummy example): file_attributes: FileAttributes::Normal, }; - redirector_controller.register_virtual_file(r"game\virtualfile.txt", metadata); + let handle = vfs_controller.register_virtual_file(r"game\virtualfile.txt", metadata); + // ... + vfs_controller.unregister_virtual_file(handle); ``` === "C++" @@ -200,22 +235,24 @@ Register a virtual file (dummy example): metadata.AllocationSize = 1024; metadata.FileAttributes = FileAttributes::Normal; - _redirectorController->RegisterVirtualFile(R"(game\virtualfile.txt)", metadata); + auto handle = _vfsController->RegisterVirtualFile(R"(game\virtualfile.txt)", metadata); + // ... + _vfsController->UnregisterVirtualFile(handle); ``` Print file redirections to console: === "C#" ```csharp - _redirectorController.SetRedirectorSetting(true, VfsSettings.PrintRedirect); + _vfsController.SetVfsSetting(true, VfsSettings.PrintRedirect); ``` === "Rust" ```rust - redirector_controller.set_redirector_setting(true, VfsSettings::PrintRedirect); + vfs_controller.set_vfs_setting(true, VfsSettings::PrintRedirect); ``` === "C++" ```cpp - _redirectorController->SetRedirectorSetting(true, VfsSettings::PrintRedirect); + _vfsController->SetVfsSetting(true, VfsSettings::PrintRedirect); ``` \ No newline at end of file diff --git a/docs/Mods/Libraries/Merged-File-Cache/Implementation-Details.md b/docs/Mods/Libraries/Merged-File-Cache/Implementation-Details.md index b334b93..d396975 100644 --- a/docs/Mods/Libraries/Merged-File-Cache/Implementation-Details.md +++ b/docs/Mods/Libraries/Merged-File-Cache/Implementation-Details.md @@ -140,7 +140,7 @@ This creates an [IMergedFileCache](#imergedfilecache-interface). ### `IMergedFileCache` Interface -!!! info "This is an abstraction over [Caches.bin](#cachebin)" +!!! info "This is an abstraction over [Cache.bin](#cachebin)" !!! info "Returned from [ICacheFactory](#icachefactory-interface)" From ce83e59d3238aea1fff130f50b4160e684ea9083 Mon Sep 17 00:00:00 2001 From: Sewer56 Date: Sun, 28 Apr 2024 04:17:34 +0100 Subject: [PATCH 6/6] APIs: Don't use central 'locators', instead use direct interfaces. --- .../Virtual-FileSystem/Programmer-Usage.md | 99 ++++--- .../Scanning-For-Signatures.md | 269 ++++++------------ 2 files changed, 153 insertions(+), 215 deletions(-) diff --git a/docs/Mods/Essentials/Virtual-FileSystem/Programmer-Usage.md b/docs/Mods/Essentials/Virtual-FileSystem/Programmer-Usage.md index c565183..ab63a9a 100644 --- a/docs/Mods/Essentials/Virtual-FileSystem/Programmer-Usage.md +++ b/docs/Mods/Essentials/Virtual-FileSystem/Programmer-Usage.md @@ -14,28 +14,32 @@ To use the Redirector API: === "C#" ```csharp - IRedirectorController _redirectorController; - IVirtualFileSystem _vfsController; + IRedirectorService _redirectorService; + IVfsService _vfsService; + IVfsSettingsService _vfsSettingsService; public void Start(IModLoaderV1 loader) { - _redirectorController = _modLoader.GetService(); - _vfsController = _modLoader.GetService(); + _redirectorService = _modLoader.GetService(); + _vfsService = _modLoader.GetService(); + _vfsSettingsService = _modLoader.GetService(); } ``` === "Rust" ```rust struct MyMod { - redirector_controller: Option, - vfs_controller: Option, + redirector_service: Option, + vfs_service: Option, + vfs_settings_service: Option, } impl MyMod { fn new(loader: &mut IModLoader) -> Self { Self { - redirector_controller: loader.get_service::().ok(), - vfs_controller: loader.get_service::().ok(), + redirector_service: loader.get_service::().ok(), + vfs_service: loader.get_service::().ok(), + vfs_settings_service: loader.get_service::().ok(), } } } @@ -45,19 +49,21 @@ To use the Redirector API: ```cpp class MyMod { private: - IRedirectorController* _redirectorController; - IVirtualFileSystem* _vfsController; + IRedirectorService* _redirectorService; + IVfsService* _vfsService; + IVfsSettingsService* _vfsSettingsService; public: MyMod(IModLoader* loader) { - _redirectorController = loader->GetService(); - _vfsController = loader->GetService(); + _redirectorService = loader->GetService(); + _vfsService = loader->GetService(); + _vfsSettingsService = loader->GetService(); } }; ``` -## IRedirectorController API +## IRedirectorService API !!! info "Using basic C# types for easier understanding. Actual types may vary." @@ -75,7 +81,7 @@ To use the Redirector API: - `void RemoveRedirectFolder(RedirectFolderHandle handle)`: Removes the redirect folder associated with the given `handle`. -## IVirtualFileSystem API +## IVfsService API !!! info "Using basic C# types for easier understanding. Actual types may vary." @@ -115,6 +121,10 @@ Actually reading the files etc. is handled by the file emulation framework itsel And not by individual 'File Emulators' using the framework. i.e. The end user of the framework should not be calling this API. +## IVfsSettingsService API + +!!! info "Using basic C# types for easier understanding. Actual types may vary." + ### VFS Settings - `GetVfsSetting(VfsSettings setting)`: Gets the current value of a VFS setting. @@ -125,6 +135,7 @@ Actually reading the files etc. is handled by the file emulation framework itsel The `VfsSettings` enum provides the following options: ```csharp +[Flags] public enum VfsSettings { None = 0, // Default value. @@ -139,52 +150,56 @@ public enum VfsSettings - `Enable()` / `Disable()`: Enables or disables the VFS entirely. +Sure, here are the updated examples sections for both the VFS and Signature Scanner documentation: + +Examples for VFS: + ## Examples Redirect an individual file: === "C#" ```csharp - var handle = _redirectorController.AddRedirect(@"dvdroot\bgm\SNG_STG26.adx", @"mods\mybgm.adx"); + var handle = _redirectorService.AddRedirect(@"dvdroot\bgm\SNG_STG26.adx", @"mods\mybgm.adx"); // ... - _redirectorController.RemoveRedirect(handle); + _redirectorService.RemoveRedirect(handle); ``` === "Rust" ```rust - let handle = redirector_controller.add_redirect(r"dvdroot\bgm\SNG_STG26.adx", r"mods\mybgm.adx"); + let handle = redirector_service.add_redirect(r"dvdroot\bgm\SNG_STG26.adx", r"mods\mybgm.adx"); // ... - redirector_controller.remove_redirect(handle); + redirector_service.remove_redirect(handle); ``` === "C++" ```cpp - auto handle = _redirectorController->AddRedirect(R"(dvdroot\bgm\SNG_STG26.adx)", R"(mods\mybgm.adx)"); + auto handle = _redirectorService->AddRedirect(R"(dvdroot\bgm\SNG_STG26.adx)", R"(mods\mybgm.adx)"); // ... - _redirectorController->RemoveRedirect(handle); + _redirectorService->RemoveRedirect(handle); ``` Add a new redirect folder: === "C#" ```csharp - var handle = _redirectorController.AddRedirectFolder(@"game\data", @"mods\mymod\data"); + var handle = _redirectorService.AddRedirectFolder(@"game\data", @"mods\mymod\data"); // ... - _redirectorController.RemoveRedirectFolder(handle); + _redirectorService.RemoveRedirectFolder(handle); ``` === "Rust" ```rust - let handle = redirector_controller.add_redirect_folder(r"game\data", r"mods\mymod\data"); + let handle = redirector_service.add_redirect_folder(r"game\data", r"mods\mymod\data"); // ... - redirector_controller.remove_redirect_folder(handle); + redirector_service.remove_redirect_folder(handle); ``` === "C++" ```cpp - auto handle = _redirectorController->AddRedirectFolder(R"(game\data)", R"(mods\mymod\data)"); + auto handle = _redirectorService->AddRedirectFolder(R"(game\data)", R"(mods\mymod\data)"); // ... - _redirectorController->RemoveRedirectFolder(handle); + _redirectorService->RemoveRedirectFolder(handle); ``` Register a virtual file (dummy example): @@ -202,9 +217,9 @@ Register a virtual file (dummy example): FileAttributes = FileAttributes.Normal }; - var handle = _vfsController.RegisterVirtualFile(@"game\virtualfile.txt", metadata); + var handle = _vfsService.RegisterVirtualFile(@"game\virtualfile.txt", metadata); // ... - _vfsController.UnregisterVirtualFile(handle); + _vfsService.UnregisterVirtualFile(handle); ``` === "Rust" @@ -219,9 +234,9 @@ Register a virtual file (dummy example): file_attributes: FileAttributes::Normal, }; - let handle = vfs_controller.register_virtual_file(r"game\virtualfile.txt", metadata); + let handle = vfs_service.register_virtual_file(r"game\virtualfile.txt", metadata); // ... - vfs_controller.unregister_virtual_file(handle); + vfs_service.unregister_virtual_file(handle); ``` === "C++" @@ -235,24 +250,36 @@ Register a virtual file (dummy example): metadata.AllocationSize = 1024; metadata.FileAttributes = FileAttributes::Normal; - auto handle = _vfsController->RegisterVirtualFile(R"(game\virtualfile.txt)", metadata); + auto handle = _vfsService->RegisterVirtualFile(R"(game\virtualfile.txt)", metadata); // ... - _vfsController->UnregisterVirtualFile(handle); + _vfsService->UnregisterVirtualFile(handle); ``` -Print file redirections to console: +Change VFS settings: === "C#" ```csharp - _vfsController.SetVfsSetting(true, VfsSettings.PrintRedirect); + // Enable printing of file redirects + _vfsSettingsService.SetVfsSetting(true, VfsSettings.PrintRedirect); + + // Disable the VFS entirely + _vfsSettingsService.Disable(); ``` === "Rust" ```rust - vfs_controller.set_vfs_setting(true, VfsSettings::PrintRedirect); + // Enable printing of file redirects + vfs_settings_service.set_vfs_setting(true, VfsSettings::PrintRedirect); + + // Disable the VFS entirely + vfs_settings_service.disable(); ``` === "C++" ```cpp - _vfsController->SetVfsSetting(true, VfsSettings::PrintRedirect); + // Enable printing of file redirects + _vfsSettingsService->SetVfsSetting(true, VfsSettings::PrintRedirect); + + // Disable the VFS entirely + _vfsSettingsService->Disable(); ``` \ No newline at end of file diff --git a/docs/Mods/Libraries/Signature-Scanner/Scanning-For-Signatures.md b/docs/Mods/Libraries/Signature-Scanner/Scanning-For-Signatures.md index 0b79d41..5047edb 100644 --- a/docs/Mods/Libraries/Signature-Scanner/Scanning-For-Signatures.md +++ b/docs/Mods/Libraries/Signature-Scanner/Scanning-For-Signatures.md @@ -16,61 +16,39 @@ parallelism across mods, and provides a stable API. ## Common Setup 1. **Add the appropriate library or header file to your project**. - - C#: `Reloaded.Memory.SigScan.Reloaded3.Interfaces` NuGet package - - Rust: `reloaded_memory_sigscan_reloaded3` crate. - - C++: `reloaded_memory_sigscan_reloaded3.h` header file. + - C#: `reloaded3.utility.sigscan.interfaces.s56` NuGet package + - Rust: `reloaded3_utility_sigscan_interfaces_s56` crate. + - C++: `reloaded3_utility_sigscan_interfaces_s56.h` header file. -2. **Get an instance of `IScannerLibrary`**: Use the mod loader's `GetService()` - method to get an instance of `IScannerLibrary`. +2. **Get instances of the scanner services**: Use the mod loader's `GetService()` method to get instances of + `IStartupScanner` and `IScannerFactory`. -## IScannerLibrary Interface - -The `IScannerLibrary` interface provides access to the `IStartupScanner` and `IScannerFactory` -interfaces through the following methods: - -- `GetStartupScanner()`: Returns an instance of `IStartupScanner` for performing startup scans. -- `GetScannerFactory()`: Returns an instance of `IScannerFactory` for creating scanner instances - for arbitrary scans. - -Below is an example of how to obtain instances of `IStartupScanner` and `IScannerFactory` using -`IScannerLibrary`: +Below is an example of how to obtain instances of the scanner services: === "C#" ```csharp - private IScannerLibrary _scannerLibrary; private IStartupScanner _startupScanner; private IScannerFactory _scannerFactory; public void Start() { - // Get an instance of IScannerLibrary - _modLoader.GetService().TryGetTarget(out _scannerLibrary); - - // Get instances of IStartupScanner and IScannerFactory - _startupScanner = _scannerLibrary.GetStartupScanner(); - _scannerFactory = _scannerLibrary.GetScannerFactory(); + _startupScanner = _modLoader.GetService(); + _scannerFactory = _modLoader.GetService(); } ``` === "Rust" ```rust struct MyMod { - scanner_library: Option, - startup_scanner: Option, - scanner_factory: Option, + startup_scanner_service: Option, + scanner_factory_service: Option, // ... } impl MyMod { fn start(&mut self, mod_loader: &mut IModLoader) { - // Get an instance of IScannerLibrary - self.scanner_library = mod_loader.get_service::().ok(); - - // Get instances of IStartupScanner and IScannerFactory - if let Some(library) = &self.scanner_library { - self.startup_scanner = Some(library.get_startup_scanner()); - self.scanner_factory = Some(library.get_scanner_factory()); - } + self.startup_scanner_service = mod_loader.get_service::().ok(); + self.scanner_factory_service = mod_loader.get_service::().ok(); } } ``` @@ -79,19 +57,14 @@ Below is an example of how to obtain instances of `IStartupScanner` and `IScanne ```cpp class MyMod { private: - IScannerLibrary* _scannerLibrary; IStartupScanner* _startupScanner; IScannerFactory* _scannerFactory; public: void Start(IModLoader* modLoader) { - // Get an instance of IScannerLibrary - modLoader->GetService()->TryGetTarget(&_scannerLibrary); - - // Get instances of IStartupScanner and IScannerFactory - _startupScanner = _scannerLibrary->GetStartupScanner(); - _scannerFactory = _scannerLibrary->GetScannerFactory(); + _startupScanner = modLoader->GetService(); + _scannerFactory = modLoader->GetService(); } }; ``` @@ -116,13 +89,8 @@ Below is an example. === "C#" ```csharp - private IStartupScanner _startupScanner; - public void Start() { - // Get an instance of IStartupScanner - _startupScanner = _scannerLibrary.GetStartupScanner(); - // Queue a signature scan _startupScanner.AddMainModuleScan("89 15 ?? ?? ?? ?? EB ?? A3 ?? ?? ?? ??", OnLevelIdScan); @@ -154,88 +122,66 @@ Below is an example. === "Rust" ```rust - struct MyMod { - startup_scanner: Option, - // ... - } - - impl MyMod { - fn start(&mut self, mod_loader: &mut IModLoader) { - // Get an instance of IStartupScanner - if let Some(library) = &self.scanner_library { - self.startup_scanner = Some(library.get_startup_scanner()); - } - - // Queue a signature scan - if let Some(scanner) = &self.startup_scanner { - scanner.add_main_module_scan("89 15 ?? ?? ?? ?? EB ?? A3 ?? ?? ?? ??", Self::on_level_id_scan); + fn start(&mut self, mod_loader: &mut IModLoader) { + // Queue a signature scan + if let Some(scanner_service) = &self.startup_scanner_service { + scanner_service.add_main_module_scan("89 15 ?? ?? ?? ?? EB ?? A3 ?? ?? ?? ??", Self::on_level_id_scan); - // Queue a 'null' scan to execute a callback after previous scans - scanner.add_main_module_scan("", Self::on_scan_complete); - } + // Queue a 'null' scan to execute a callback after previous scans + scanner_service.add_main_module_scan("", Self::on_scan_complete); } + } - fn on_level_id_scan(result: PatternScanResult) { - if !result.found { - println!("Signature for getting LevelId not found."); - return; - } + fn on_level_id_scan(result: PatternScanResult) { + if !result.found { + println!("Signature for getting LevelId not found."); + return; + } - // Get the address of the levelId variable - let level_id_address = result.address + 2; + // Get the address of the levelId variable + let level_id_address = result.address + 2; - // Read the value of levelId - let level_id = unsafe { *(level_id_address as *const i32) }; - println!("LevelId: {}", level_id); - } + // Read the value of levelId + let level_id = unsafe { *(level_id_address as *const i32) }; + println!("LevelId: {}", level_id); + } - fn on_scan_complete(_result: PatternScanResult) { - println!("All scans completed."); - } + fn on_scan_complete(_result: PatternScanResult) { + println!("All scans completed."); } ``` === "C++" ```cpp - class MyMod { - private: - IStartupScanner* _startupScanner; - - public: - void Start(IModLoader* modLoader) - { - // Get an instance of IStartupScanner - _startupScanner = _scannerLibrary->GetStartupScanner(); - - // Queue a signature scan - _startupScanner->AddMainModuleScan("89 15 ?? ?? ?? ?? EB ?? A3 ?? ?? ?? ??", std::bind(&MyMod::OnLevelIdScan, this, std::placeholders::_1)); + void Start(IModLoader* modLoader) + { + // Queue a signature scan + _startupScanner->AddMainModuleScan("89 15 ?? ?? ?? ?? EB ?? A3 ?? ?? ?? ??", std::bind(&MyMod::OnLevelIdScan, this, std::placeholders::_1)); - // Queue a 'null' scan to execute a callback after previous scans - _startupScanner->AddMainModuleScan("", std::bind(&MyMod::OnScanComplete, this, std::placeholders::_1)); - } + // Queue a 'null' scan to execute a callback after previous scans + _startupScanner->AddMainModuleScan("", std::bind(&MyMod::OnScanComplete, this, std::placeholders::_1)); + } - private: - void OnLevelIdScan(const PatternScanResult& result) + void OnLevelIdScan(const PatternScanResult& result) + { + if (!result.Found) { - if (!result.Found) - { - std::cout << "Signature for getting LevelId not found." << std::endl; - return; - } + std::cout << "Signature for getting LevelId not found." << std::endl; + return; + } - // Get the address of the levelId variable - auto levelIdAddress = result.Address + 2; + // Get the address of the levelId variable + auto levelIdAddress = result.Address + 2; - // Read the value of levelId - auto levelId = *(int*)levelIdAddress; - std::cout << "LevelId: " << levelId << std::endl; - } + // Read the value of levelId + auto levelId = *(int*)levelIdAddress; + std::cout << "LevelId: " << levelId << std::endl; + } - void OnScanComplete(const PatternScanResult& result) - { - std::cout << "All scans completed." << std::endl; - } - }; + void OnScanComplete(const PatternScanResult& result) + { + std::cout << "All scans completed." << std::endl; + } ``` !!! warning "Using `IStartupScanner` is mandatory" @@ -263,19 +209,16 @@ need to worry about conflicts, load order or race conditions. !!! tip "To perform arbitrary scans using `IScannerFactory`, follow these steps:" -1. **Get an instance of `IScannerFactory`**: Use the `GetScannerFactory()` method of the - `IScannerLibrary` instance to get an instance of `IScannerFactory`. - -2. **Create a scanner instance**: Call one of the `CreateScanner` methods on the `IScannerFactory` +1. **Create a scanner instance**: Call one of the `CreateScanner` methods on the `IScannerFactory` instance, passing in the appropriate parameters based on your scanning needs. - `CreateScanner(data)` creates a scanner for an arbitrary slice of data (slice type native to the language). - `CreateScanner(data, length)` creates a scanner for a pointer to data and its length. -3. **Perform the scan**: Call the `FindPattern` method on the scanner instance, passing in your +2. **Perform the scan**: Call the `FindPattern` method on the scanner instance, passing in your signature as a string. -4. **Check the result**: The `FindPattern` method returns a `PatternScanResult` object, which +3. **Check the result**: The `FindPattern` method returns a `PatternScanResult` object, which contains a `Found` property indicating whether the pattern was found, and an `Address` property giving the memory address where the pattern was found. You can also use `Offset` property to calculate the offset from the start of the scan range. @@ -284,14 +227,6 @@ Below is an example. === "C#" ```csharp - private IScannerFactory _scannerFactory; - - public void Start() - { - // Get an instance of IScannerFactory - _scannerFactory = _scannerLibrary.GetScannerFactory(); - } - public void ScanForPattern(byte[] data) { // Create a scanner instance for the provided data @@ -315,74 +250,48 @@ Below is an example. === "Rust" ```rust - struct MyMod { - scanner_factory: Option, - // ... - } - - impl MyMod { - fn start(&mut self, mod_loader: &mut IModLoader) { - // Get an instance of IScannerFactory - if let Some(library) = &self.scanner_library { - self.scanner_factory = Some(library.get_scanner_factory()); - } - } - - fn scan_for_pattern(&self, data: &[u8]) { - if let Some(factory) = &self.scanner_factory { - // Create a scanner instance for the provided data - let scanner = factory.create_scanner(data); + fn scan_for_pattern(&self, data: &[u8]) { + if let Some(factory_service) = &self.scanner_factory_service { + // Create a scanner instance for the provided data + let scanner = factory_service.create_scanner(data); - // Perform the scan - let result = scanner.find_pattern("89 15 ?? ?? ?? ?? EB ?? A3 ?? ?? ?? ??"); + // Perform the scan + let result = scanner.find_pattern("89 15 ?? ?? ?? ?? EB ?? A3 ?? ?? ?? ??"); - if !result.found { - println!("Pattern not found."); - return; - } + if !result.found { + println!("Pattern not found."); + return; + } - // Get the address where the pattern was found - let pattern_address = result.address; + // Get the address where the pattern was found + let pattern_address = result.address; - println!("Pattern found at address: {:X}", pattern_address); - } + println!("Pattern found at address: {:X}", pattern_address); } } ``` === "C++" ```cpp - class MyMod { - private: - IScannerFactory* _scannerFactory; + void ScanForPattern(const uint8_t* data, size_t length) + { + // Create a scanner instance for the provided data + auto scanner = _scannerFactory->CreateScanner(data, length); - public: - void Start(IModLoader* modLoader) - { - // Get an instance of IScannerFactory - _scannerFactory = _scannerLibrary->GetScannerFactory(); - } + // Perform the scan + auto result = scanner->FindPattern("89 15 ?? ?? ?? ?? EB ?? A3 ?? ?? ?? ??"); - void ScanForPattern(const uint8_t* data, size_t length) + if (!result.Found) { - // Create a scanner instance for the provided data - auto scanner = _scannerFactory->CreateScanner(data, length); - - // Perform the scan - auto result = scanner->FindPattern("89 15 ?? ?? ?? ?? EB ?? A3 ?? ?? ?? ??"); - - if (!result.Found) - { - std::cout << "Pattern not found." << std::endl; - return; - } + std::cout << "Pattern not found." << std::endl; + return; + } - // Get the address where the pattern was found - auto patternAddress = result.Address; + // Get the address where the pattern was found + auto patternAddress = result.Address; - std::cout << "Pattern found at address: " << std::hex << patternAddress << std::endl; - } - }; + std::cout << "Pattern found at address: " << std::hex << patternAddress << std::endl; + } ``` !!! note "Arbitrary scans using `IScannerFactory` are not cached or parallelized." @@ -396,4 +305,6 @@ Below is an example. In the callback function or after performing a scan, you can check the `Found` property of the `PatternScanResult` to determine if the pattern was located, and log an error or take appropriate -action if it was not. \ No newline at end of file +action if it was not. + +That covers the key aspects of using the Signature Scanner services in Reloaded3 mods. Let me know if you have any further questions! \ No newline at end of file