From 4a7f4bde4a4384d92b79768cecaf147a7033e6f8 Mon Sep 17 00:00:00 2001 From: Jai A Date: Tue, 5 Oct 2021 22:52:17 -0700 Subject: [PATCH 01/85] Working mirroring of minecraft metadata --- .env | 11 ++ .gitignore | 121 ++++++++++++ .idea/.gitignore | 8 + .idea/daedalus.iml | 14 ++ .idea/modules.xml | 8 + .idea/runConfigurations.xml | 10 + .idea/vcs.xml | 6 + Cargo.toml | 6 + daedalus/Cargo.toml | 17 ++ daedalus/src/lib.rs | 99 ++++++++++ daedalus/src/minecraft.rs | 328 +++++++++++++++++++++++++++++++ daedalus_client/Cargo.toml | 21 ++ daedalus_client/src/main.rs | 125 ++++++++++++ daedalus_client/src/minecraft.rs | 172 ++++++++++++++++ 14 files changed, 946 insertions(+) create mode 100644 .env create mode 100644 .gitignore create mode 100644 .idea/.gitignore create mode 100644 .idea/daedalus.iml create mode 100644 .idea/modules.xml create mode 100644 .idea/runConfigurations.xml create mode 100644 .idea/vcs.xml create mode 100644 Cargo.toml create mode 100644 daedalus/Cargo.toml create mode 100644 daedalus/src/lib.rs create mode 100644 daedalus/src/minecraft.rs create mode 100644 daedalus_client/Cargo.toml create mode 100644 daedalus_client/src/main.rs create mode 100644 daedalus_client/src/minecraft.rs diff --git a/.env b/.env new file mode 100644 index 000000000..4e9b803ba --- /dev/null +++ b/.env @@ -0,0 +1,11 @@ +BASE_URL=https://modrinth-cdn-staging.nyc3.digitaloceanspaces.com +BASE_FOLDER=gamedata + +S3_ACCESS_TOKEN=none +S3_SECRET=none +S3_URL=none +S3_REGION=none +S3_BUCKET_NAME=none + +DO_INTEGRATION=false +DO_ACCESS_KEY=none \ No newline at end of file diff --git a/.gitignore b/.gitignore new file mode 100644 index 000000000..dae6eff4c --- /dev/null +++ b/.gitignore @@ -0,0 +1,121 @@ +### Intellij ### +# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio, WebStorm and Rider +# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 + +# User-specific stuff +.idea/**/workspace.xml +.idea/**/tasks.xml +.idea/**/usage.statistics.xml +.idea/**/dictionaries +.idea/**/shelf + +# AWS User-specific +.idea/**/aws.xml + +# Generated files +.idea/**/contentModel.xml + +# Sensitive or high-churn files +.idea/**/dataSources/ +.idea/**/dataSources.ids +.idea/**/dataSources.local.xml +.idea/**/sqlDataSources.xml +.idea/**/dynamic.xml +.idea/**/uiDesigner.xml +.idea/**/dbnavigator.xml + +# Gradle +.idea/**/gradle.xml +.idea/**/libraries + +# Gradle and Maven with auto-import +# When using Gradle or Maven with auto-import, you should exclude module files, +# since they will be recreated, and may cause churn. Uncomment if using +# auto-import. +# .idea/artifacts +# .idea/compiler.xml +# .idea/jarRepositories.xml +# .idea/modules.xml +# .idea/*.iml +# .idea/modules +# *.iml +# *.ipr + +# CMake +cmake-build-*/ + +# Mongo Explorer plugin +.idea/**/mongoSettings.xml + +# File-based project format +*.iws + +# IntelliJ +out/ + +# mpeltonen/sbt-idea plugin +.idea_modules/ + +# JIRA plugin +atlassian-ide-plugin.xml + +# Cursive Clojure plugin +.idea/replstate.xml + +# Crashlytics plugin (for Android Studio and IntelliJ) +com_crashlytics_export_strings.xml +crashlytics.properties +crashlytics-build.properties +fabric.properties + +# Editor-based Rest Client +.idea/httpRequests + +# Android studio 3.1+ serialized cache file +.idea/caches/build_file_checksums.ser + +### Intellij Patch ### +# Comment Reason: https://github.com/joeblau/gitignore.io/issues/186#issuecomment-215987721 + +# *.iml +# modules.xml +# .idea/misc.xml +# *.ipr + +# Sonarlint plugin +# https://plugins.jetbrains.com/plugin/7973-sonarlint +.idea/**/sonarlint/ + +# SonarQube Plugin +# https://plugins.jetbrains.com/plugin/7238-sonarqube-community-plugin +.idea/**/sonarIssues.xml + +# Markdown Navigator plugin +# https://plugins.jetbrains.com/plugin/7896-markdown-navigator-enhanced +.idea/**/markdown-navigator.xml +.idea/**/markdown-navigator-enh.xml +.idea/**/markdown-navigator/ + +# Cache file creation bug +# See https://youtrack.jetbrains.com/issue/JBR-2257 +.idea/$CACHE_FILE$ + +# CodeStream plugin +# https://plugins.jetbrains.com/plugin/12206-codestream +.idea/codestream.xml + +### Rust ### +# Generated by Cargo +# will have compiled files and executables +debug/ +target/ + +# Remove Cargo.lock from gitignore if creating an executable, leave it for libraries +# More information here https://doc.rust-lang.org/cargo/guide/cargo-toml-vs-cargo-lock.html +Cargo.lock + +# These are backup files generated by rustfmt +**/*.rs.bk + +# MSVC Windows builds of rustc generate these, which store debugging information +*.pdb diff --git a/.idea/.gitignore b/.idea/.gitignore new file mode 100644 index 000000000..73f69e095 --- /dev/null +++ b/.idea/.gitignore @@ -0,0 +1,8 @@ +# Default ignored files +/shelf/ +/workspace.xml +# Datasource local storage ignored files +/dataSources/ +/dataSources.local.xml +# Editor-based HTTP Client requests +/httpRequests/ diff --git a/.idea/daedalus.iml b/.idea/daedalus.iml new file mode 100644 index 000000000..ec7bb0139 --- /dev/null +++ b/.idea/daedalus.iml @@ -0,0 +1,14 @@ + + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/modules.xml b/.idea/modules.xml new file mode 100644 index 000000000..c73594817 --- /dev/null +++ b/.idea/modules.xml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/.idea/runConfigurations.xml b/.idea/runConfigurations.xml new file mode 100644 index 000000000..797acea53 --- /dev/null +++ b/.idea/runConfigurations.xml @@ -0,0 +1,10 @@ + + + + + + \ No newline at end of file diff --git a/.idea/vcs.xml b/.idea/vcs.xml new file mode 100644 index 000000000..35eb1ddfb --- /dev/null +++ b/.idea/vcs.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/Cargo.toml b/Cargo.toml new file mode 100644 index 000000000..dc360c2e5 --- /dev/null +++ b/Cargo.toml @@ -0,0 +1,6 @@ +[workspace] + +members = [ + "daedalus", + "daedalus_client" +] \ No newline at end of file diff --git a/daedalus/Cargo.toml b/daedalus/Cargo.toml new file mode 100644 index 000000000..a52c207f5 --- /dev/null +++ b/daedalus/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "daedalus" +version = "0.1.0" +authors = ["Jai A "] +edition = "2018" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +reqwest = { version = "0.11", features = ["json"] } +serde = { version = "1.0", features = ["derive"] } +serde_json = "1.0" +chrono = { version = "0.4", features = ["serde"] } +bytes = "1" +thiserror = "1.0" +tokio = { version = "1", features = ["full"] } +sha1 = { version = "0.6.0", features = ["std"]} \ No newline at end of file diff --git a/daedalus/src/lib.rs b/daedalus/src/lib.rs new file mode 100644 index 000000000..8aea3e1ec --- /dev/null +++ b/daedalus/src/lib.rs @@ -0,0 +1,99 @@ +//! # Daedalus +//! +//! Daedalus is a library which provides models and methods to fetch metadata about games + +#![warn(missing_docs, unused_import_braces, missing_debug_implementations)] + +/// Models and methods for fetching metadata for Minecraft +pub mod minecraft; + +#[derive(thiserror::Error, Debug)] +/// An error type representing possible errors when fetching metadata +pub enum Error { + #[error("Failed to validate file checksum at url {url} with hash {hash} after {tries} tries")] + /// A checksum was failed to validate for a file + ChecksumFailure { + /// The checksum's hash + hash: String, + /// The URL of the file attempted to be downloaded + url: String, + /// The amount of tries that the file was downloaded until failure + tries: u32, + }, + /// There was an error while deserializing metadata + #[error("Error while deserializing JSON")] + SerdeError(#[from] serde_json::Error), + /// There was a network error when fetching an object + #[error("Unable to fetch {item}")] + FetchError { + /// The internal reqwest error + inner: reqwest::Error, + /// The item that was failed to be fetched + item: String, + }, + /// There was an error when managing async tasks + #[error("Error while managing asynchronous tasks")] + TaskError(#[from] tokio::task::JoinError), +} + +/// Downloads a file with retry and checksum functionality +pub async fn download_file(url: &str, sha1: Option<&str>) -> Result { + let client = reqwest::Client::builder() + .tcp_keepalive(Some(std::time::Duration::from_secs(10))) + .build() + .map_err(|err| Error::FetchError { + inner: err, + item: url.to_string(), + })?; + + for attempt in 1..=4 { + let result = client.get(url).send().await; + + match result { + Ok(x) => { + let bytes = x.bytes().await; + + if let Ok(bytes) = bytes { + if let Some(sha1) = sha1 { + if &*get_hash(bytes.clone()).await? != sha1 { + if attempt <= 3 { + continue; + } else { + return Err(Error::ChecksumFailure { + hash: sha1.to_string(), + url: url.to_string(), + tries: attempt, + }); + } + } + } + + return Ok(bytes); + } else if attempt <= 3 { + continue; + } else if let Err(err) = bytes { + return Err(Error::FetchError { + inner: err, + item: url.to_string(), + }); + } + } + Err(_) if attempt <= 3 => continue, + Err(err) => { + return Err(Error::FetchError { + inner: err, + item: url.to_string(), + }) + } + } + } + + unreachable!() +} + +/// Computes a checksum of the input bytes +pub async fn get_hash(bytes: bytes::Bytes) -> Result { + let hash = tokio::task::spawn_blocking(|| sha1::Sha1::from(bytes).hexdigest()).await?; + + Ok(hash) +} diff --git a/daedalus/src/minecraft.rs b/daedalus/src/minecraft.rs new file mode 100644 index 000000000..89008da16 --- /dev/null +++ b/daedalus/src/minecraft.rs @@ -0,0 +1,328 @@ +use crate::{download_file, Error}; +use chrono::{DateTime, Utc}; +use serde::{Deserialize, Serialize}; +use std::collections::HashMap; + +/// The latest version of the format the model structs deserialize to +pub const CURRENT_FORMAT_VERSION: usize = 0; + +#[derive(Serialize, Deserialize, Debug, Clone)] +#[serde(rename_all = "snake_case")] +/// The version type +pub enum VersionType { + /// A major version, which is stable for all players to use + Release, + /// An experimental version, which is unstable and used for feature previews and beta testing + Snapshot, + /// The oldest versions before the game was released + OldAlpha, + /// Early versions of the game + OldBeta, +} + +impl VersionType { + /// Converts the version type to a string + pub fn as_str(&self) -> &'static str { + match self { + VersionType::Release => "release", + VersionType::Snapshot => "snapshot", + VersionType::OldAlpha => "old_alpha", + VersionType::OldBeta => "old_beta", + } + } +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +#[serde(rename_all = "camelCase")] +/// A game version of Minecraft +pub struct Version { + /// A unique identifier of the version + pub id: String, + #[serde(rename = "type")] + /// The release type of the version + pub type_: VersionType, + /// A link to additional information about the version + pub url: String, + /// The latest time a file in this version was updated + pub time: DateTime, + /// The time this version was released + pub release_time: DateTime, + /// The SHA1 hash of the additional information about the version + pub sha1: String, + /// Whether the version supports the latest player safety features + pub compliance_level: u32, + /// (Modrinth Provided) The link to the assets index for this version + /// This is only available when using the Modrinth mirror + pub assets_index_url: Option, + /// (Modrinth Provided) The SHA1 hash of the assets index for this version + /// This is only available when using the Modrinth mirror + pub assets_index_sha1: Option, +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +/// The latest snapshot and release of the game +pub struct LatestVersion { + /// The version id of the latest release + pub release: String, + /// The version id of the latest snapshot + pub snapshot: String, +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +/// Data of all game versions of Minecraft +pub struct VersionManifest { + /// A struct containing the latest snapshot and release of the game + pub latest: LatestVersion, + /// A list of game versions of Minecraft + pub versions: Vec, +} + +/// The URL to the version manifest +pub const VERSION_MANIFEST_URL: &str = + "https://launchermeta.mojang.com/mc/game/version_manifest_v2.json"; + +/// Fetches a version manifest from the specified URL. If no URL is specified, the default is used. +pub async fn fetch_version_manifest(url: Option<&str>) -> Result { + Ok(serde_json::from_slice( + &download_file(url.unwrap_or(VERSION_MANIFEST_URL), None).await?, + )?) +} + +#[derive(Serialize, Deserialize, Debug)] +#[serde(rename_all = "camelCase")] +/// Information about the assets of the game +pub struct AssetIndex { + /// The game version ID the assets are for + pub id: String, + /// The SHA1 hash of the assets index + pub sha1: String, + /// The size of the assets index + pub size: u32, + /// The size of the game version's assets + pub total_size: u32, + /// A URL to a file which contains information about the version's assets + pub url: String, +} + +#[derive(Serialize, Deserialize, Debug, Eq, PartialEq, Hash)] +#[serde(rename_all = "snake_case")] +/// The type of download +pub enum DownloadType { + /// The download is for the game client + Client, + /// The download is mappings for the game + ClientMappings, + /// The download is for the game server + Server, + /// The download is mappings for the game server + ServerMappings, + /// The download is for the windows server + WindowsServer, +} + +#[derive(Serialize, Deserialize, Debug)] +/// Download information of a file +pub struct Download { + /// The SHA1 hash of the file + pub sha1: String, + /// The size of the file + pub size: u32, + /// The URL where the file can be downloaded + pub url: String, +} + +#[derive(Serialize, Deserialize, Debug)] +/// Download information of a library +pub struct LibraryDownload { + /// The path that the library should be saved to + pub path: String, + /// The SHA1 hash of the library + pub sha1: String, + /// The size of the library + pub size: u32, + /// The URL where the library can be downloaded + pub url: String, +} + +#[derive(Serialize, Deserialize, Debug)] +/// A list of files that should be downloaded for libraries +pub struct LibraryDownloads { + /// The primary library artifact + pub artifact: Option, + /// Conditional files that may be needed to be downloaded alongside the library + /// The HashMap key specifies a classifier as additional information for downloading files + pub classifiers: Option>, +} + +#[derive(Serialize, Deserialize, Debug)] +#[serde(rename_all = "snake_case")] +/// The action a rule can follow +pub enum RuleAction { + /// The rule's status allows something to be done + Allow, + /// The rule's status disallows something to be done + Disallow, +} + +#[derive(Serialize, Deserialize, Debug, Eq, PartialEq, Hash)] +#[serde(rename_all = "snake_case")] +/// An enum representing the different types of operating systems +pub enum Os { + /// MacOS + Osx, + /// Windows + Windows, + /// Linux and its derivatives + Linux, + /// The OS is unknown + Unknown, +} + +#[derive(Serialize, Deserialize, Debug)] +/// A rule which depends on what OS the user is on +pub struct OsRule { + /// The name of the OS + pub name: Option, + /// The version of the OS. This is normally a RegEx + pub version: Option, + /// The architecture of the OS + pub arch: Option, +} + +#[derive(Serialize, Deserialize, Debug)] +/// A rule which depends on the toggled features of the launcher +pub struct FeatureRule { + /// Whether the user is in demo mode + pub is_demo_user: Option, + /// Whether the user is using the demo resolution + pub has_demo_resolution: Option, +} + +#[derive(Serialize, Deserialize, Debug)] +/// A rule deciding whether a file is downloaded, an argument is used, etc. +pub struct Rule { + /// The action the rule takes + pub action: RuleAction, + /// The OS rule + pub os: Option, + /// The feature rule + pub features: Option, +} + +#[derive(Serialize, Deserialize, Debug)] +/// Information delegating the extraction of the library +pub struct LibraryExtract { + /// Files/Folders to be excluded from the extraction of the library + pub exclude: Option>, +} + +#[derive(Serialize, Deserialize, Debug)] +/// A library which the game relies on to run +pub struct Library { + /// The files the library has + pub downloads: LibraryDownloads, + /// Rules of the extraction of the file + pub extract: Option, + /// The maven name of the library. The format is `groupId:artifactId:version` + pub name: String, + /// Native files that the library relies on + pub natives: Option>, + /// Rules deciding whether the library should be downloaded or not + pub rules: Option>, +} + +#[derive(Serialize, Deserialize, Debug)] +#[serde(untagged)] +/// A container for an argument or multiple arguments +pub enum ArgumentValue { + /// The container has one argument + Single(String), + /// The container has multiple arguments + Many(Vec), +} + +#[derive(Serialize, Deserialize, Debug)] +#[serde(untagged)] +/// A command line argument passed to a program +pub enum Argument { + /// An argument which is applied no matter what + Normal(String), + /// An argument which is only applied if certain conditions are met + Ruled { + /// The rules deciding whether the argument(s) is used or not + rules: Vec, + /// The container of the argument(s) that should be applied accordingly + value: ArgumentValue, + }, +} + +#[derive(Serialize, Deserialize, Debug, Eq, PartialEq, Hash)] +#[serde(rename_all = "snake_case")] +/// The type of argument +pub enum ArgumentType { + /// The argument is passed to the game + Game, + /// The argument is passed to the JVM + Jvm, +} + +#[derive(Serialize, Deserialize, Debug)] +#[serde(rename_all = "camelCase")] +/// Information about a version +pub struct VersionInfo { + /// Arguments passed to the game or JVM + pub arguments: Option>>, + /// Assets for the game + pub asset_index: AssetIndex, + /// The version ID of the assets + pub assets: String, + /// Game downloads of the version + pub downloads: HashMap, + /// The version ID of the version + pub id: String, + /// Libraries that the version depends on + pub libraries: Vec, + /// The classpath to the main class to launch the game + pub main_class: String, + /// (Legacy) Arguments passed to the game + pub minecraft_arguments: Option, + /// The minimum version of the Minecraft Launcher that can run this version of the game + pub minimum_launcher_version: u32, + /// The time that the version was released + pub release_time: DateTime, + /// The latest time a file in this version was updated + pub time: DateTime, + #[serde(rename = "type")] + /// The type of version + pub type_: VersionType, +} + +/// Fetches detailed information about a version from the manifest +pub async fn fetch_version_info(version: &Version) -> Result { + Ok(serde_json::from_slice( + &download_file(&version.url, Some(&version.sha1)).await?, + )?) +} + +#[derive(Serialize, Deserialize, Debug)] +/// An asset of the game +pub struct Asset { + /// The SHA1 hash of the asset file + pub hash: String, + /// The size of the asset file + pub size: u32, +} + +#[derive(Serialize, Deserialize, Debug)] +/// An index containing all assets the game needs +pub struct AssetsIndex { + /// A hashmap containing the filename (key) and asset (value) + pub objects: HashMap, +} + +/// Fetches the assets index from the version info +pub async fn fetch_assets_index(version: &VersionInfo) -> Result { + Ok(serde_json::from_slice( + &download_file(&version.asset_index.url, Some(&version.asset_index.sha1)).await?, + )?) +} diff --git a/daedalus_client/Cargo.toml b/daedalus_client/Cargo.toml new file mode 100644 index 000000000..4f2fa019e --- /dev/null +++ b/daedalus_client/Cargo.toml @@ -0,0 +1,21 @@ +[package] +name = "daedalus_client" +version = "0.1.0" +authors = ["Jai A "] +edition = "2018" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +daedalus = { path = "../daedalus" } +tokio = { version = "1", features = ["full"] } +futures = "0.3.17" +dotenv = "0.15.0" +log = "0.4.8" +serde_json = "1.0" +lazy_static = "1.4.0" +thiserror = "1.0" +reqwest = "0.11.4" + +rusoto_core = "0.47.0" +rusoto_s3 = "0.47.0" diff --git a/daedalus_client/src/main.rs b/daedalus_client/src/main.rs new file mode 100644 index 000000000..bb10303ac --- /dev/null +++ b/daedalus_client/src/main.rs @@ -0,0 +1,125 @@ +use log::{error, info, warn}; +use rusoto_core::credential::StaticProvider; +use rusoto_core::{HttpClient, Region, RusotoError}; +use rusoto_s3::{S3Client, PutObjectError}; +use rusoto_s3::{PutObjectRequest, S3}; + +mod minecraft; + +#[derive(thiserror::Error, Debug)] +pub enum Error { + #[error("{0}")] + DaedalusError(#[from] daedalus::Error), + #[error("Error while deserializing JSON")] + SerdeError(#[from] serde_json::Error), + #[error("Unable to fetch {item}")] + FetchError { + inner: reqwest::Error, + item: String, + }, + #[error("Error while managing asynchronous tasks")] + TaskError(#[from] tokio::task::JoinError), + #[error("Error while uploading file to S3")] + S3Error { + inner: RusotoError, + file: String, + }, +} + +#[tokio::main] +async fn main() { + if check_env_vars() { + error!("Some environment variables are missing!"); + + return; + } + + minecraft::retrieve_data().await.unwrap(); +} + +fn check_env_vars() -> bool { + let mut failed = false; + + fn check_var(var: &str) -> bool { + if dotenv::var(var) + .ok() + .and_then(|s| s.parse::().ok()) + .is_none() + { + warn!( + "Variable `{}` missing in dotenv or not of type `{}`", + var, + std::any::type_name::() + ); + true + } else { + false + } + } + + failed |= check_var::("BASE_URL"); + failed |= check_var::("BASE_FOLDER"); + + failed |= check_var::("S3_ACCESS_TOKEN"); + failed |= check_var::("S3_SECRET"); + failed |= check_var::("S3_URL"); + failed |= check_var::("S3_REGION"); + failed |= check_var::("S3_BUCKET_NAME"); + + failed |= check_var::("DO_INTEGRATION"); + + let do_integration = dotenv::var("DO_INTEGRATION") + .ok() + .map(|x| x.parse::().ok()) + .flatten(); + + if do_integration.unwrap_or(false) { + failed |= check_var::("DO_ACCESS_KEY"); + } + + failed +} + +lazy_static::lazy_static! { + static ref CLIENT : S3Client = S3Client::new_with( + HttpClient::new().unwrap(), + StaticProvider::new( + dotenv::var("S3_ACCESS_TOKEN").unwrap(), + dotenv::var("S3_SECRET").unwrap(), + None, + None, + ), + Region::Custom { + name: dotenv::var("S3_REGION").unwrap(), + endpoint: dotenv::var("S3_URL").unwrap(), + }, + ); +} + +pub async fn upload_file_to_bucket(path: String, bytes: Vec, content_type: Option) -> Result<(), Error> { + CLIENT + .put_object(PutObjectRequest { + bucket: dotenv::var("S3_BUCKET_NAME").unwrap(), + key: format!("{}/{}", &*dotenv::var("BASE_FOLDER").unwrap(), path), + body: Some(bytes.into()), + acl: Some("public-read".to_string()), + content_type, + ..Default::default() + }) + .await + .map_err(|err| Error::S3Error { + inner: err, + file: format!("{}/{}", &*dotenv::var("BASE_FOLDER").unwrap(), path) + })?; + + Ok(()) +} + +pub fn format_url(path: &str) -> String { + format!( + "{}/{}/{}", + &*dotenv::var("BASE_URL").unwrap(), + &*dotenv::var("BASE_FOLDER").unwrap(), + path + ) +} diff --git a/daedalus_client/src/minecraft.rs b/daedalus_client/src/minecraft.rs new file mode 100644 index 000000000..1336e9ffb --- /dev/null +++ b/daedalus_client/src/minecraft.rs @@ -0,0 +1,172 @@ +use crate::{format_url, upload_file_to_bucket, Error}; +use daedalus::download_file; +use std::sync::{Arc, Mutex}; +use std::time::{Duration, Instant}; + +pub async fn retrieve_data() -> Result<(), Error> { + let old_manifest = + daedalus::minecraft::fetch_version_manifest(Some(&*crate::format_url(&*format!( + "minecraft/v{}/version_manifest.json", + daedalus::minecraft::CURRENT_FORMAT_VERSION + )))) + .await + .ok(); + + let mut manifest = daedalus::minecraft::fetch_version_manifest(None) + .await?; + let cloned_manifest = Arc::new(Mutex::new(manifest.clone())); + + let visited_assets_mutex = Arc::new(Mutex::new(Vec::new())); + + let now = Instant::now(); + + let mut versions = manifest + .versions + .iter_mut() + .map(|version| async { + let old_version = if let Some(old_manifest) = &old_manifest { + old_manifest.versions.iter().find(|x| x.id == version.id) + } else { + None + }; + + if let Some(old_version) = old_version { + if old_version.sha1 == version.sha1 { + return Ok(()); + } + } + + let visited_assets_mutex = Arc::clone(&visited_assets_mutex); + let cloned_manifest_mutex = Arc::clone(&cloned_manifest); + + let assets_hash = old_version.map(|x| x.assets_index_sha1.clone()).flatten(); + + async move { + let mut upload_futures = Vec::new(); + + let now = Instant::now(); + let mut version_println = daedalus::minecraft::fetch_version_info(version) + .await + ?; + let elapsed = now.elapsed(); + println!("Version {} Elapsed: {:.2?}", version.id, elapsed); + + let version_path = format!( + "minecraft/v{}/versions/{}.json", + daedalus::minecraft::CURRENT_FORMAT_VERSION, + version.id + ); + let assets_path = format!( + "minecraft/v{}/assets/{}.json", + daedalus::minecraft::CURRENT_FORMAT_VERSION, + version_println.asset_index.id + ); + let assets_index_url = version_println.asset_index.url.clone(); + + { + let mut cloned_manifest = match cloned_manifest_mutex.lock() { + Ok(guard) => guard, + Err(poisoned) => poisoned.into_inner(), + }; + + let position = cloned_manifest + .versions + .iter() + .position(|x| version.id == x.id) + .unwrap(); + cloned_manifest.versions[position].url = format_url(&version_path); + cloned_manifest.versions[position].assets_index_sha1 = + Some(version_println.asset_index.sha1.clone()); + cloned_manifest.versions[position].assets_index_url = + Some(format_url(&assets_path)); + version_println.asset_index.url = format_url(&assets_path); + } + + let mut download_assets = false; + + { + let mut visited_assets = match visited_assets_mutex.lock() { + Ok(guard) => guard, + Err(poisoned) => poisoned.into_inner(), + }; + + if !visited_assets.contains(&version_println.asset_index.id) { + if let Some(assets_hash) = assets_hash { + if version_println.asset_index.sha1 != assets_hash { + download_assets = true; + } + } else { + download_assets = true; + } + } + + if download_assets { + visited_assets.push(version_println.asset_index.id.clone()); + } + } + + if download_assets { + let assets_index = + download_file(&assets_index_url, Some(&version_println.asset_index.sha1)) + .await?; + + { + upload_futures + .push(upload_file_to_bucket(assets_path, assets_index.to_vec(), Some("application/json".to_string()))); + } + } + + { + upload_futures.push(upload_file_to_bucket( + version_path, + serde_json::to_vec(&version_println)?, + Some("application/json".to_string()) + )); + } + + let now = Instant::now(); + futures::future::try_join_all(upload_futures).await?; + let elapsed = now.elapsed(); + println!("Spaces Upload {} Elapsed: {:.2?}", version.id, elapsed); + + Ok::<(), Error>(()) + } + .await?; + + Ok::<(), Error>(()) + }) + .peekable(); + + let mut chunk_index = 0; + while versions.peek().is_some() { + let now = Instant::now(); + + let chunk: Vec<_> = versions.by_ref().take(100).collect(); + futures::future::try_join_all(chunk).await?; + + std::thread::sleep(Duration::from_secs(1)); + + chunk_index += 1; + + let elapsed = now.elapsed(); + println!("Chunk {} Elapsed: {:.2?}", chunk_index, elapsed); + } + + upload_file_to_bucket( + format!( + "minecraft/v{}/version_manifest.json", + daedalus::minecraft::CURRENT_FORMAT_VERSION + ), + serde_json::to_vec(&*match cloned_manifest.lock() { + Ok(guard) => guard, + Err(poisoned) => poisoned.into_inner(), + })?, + Some("application/json".to_string()) + ) + .await?; + + let elapsed = now.elapsed(); + println!("Elapsed: {:.2?}", elapsed); + + Ok(()) +} From 32850f6770f13ff78dad6cb11ead9968e39b1c93 Mon Sep 17 00:00:00 2001 From: Jai A Date: Sat, 9 Oct 2021 14:04:45 -0700 Subject: [PATCH 02/85] Fabric support --- daedalus/src/fabric.rs | 132 ++++++++++++++++++++++ daedalus/src/lib.rs | 28 +++++ daedalus/src/minecraft.rs | 21 +++- daedalus_client/src/fabric.rs | 188 +++++++++++++++++++++++++++++++ daedalus_client/src/main.rs | 18 +-- daedalus_client/src/minecraft.rs | 43 ++++--- 6 files changed, 397 insertions(+), 33 deletions(-) create mode 100644 daedalus/src/fabric.rs create mode 100644 daedalus_client/src/fabric.rs diff --git a/daedalus/src/fabric.rs b/daedalus/src/fabric.rs new file mode 100644 index 000000000..052486ffa --- /dev/null +++ b/daedalus/src/fabric.rs @@ -0,0 +1,132 @@ +use crate::minecraft::{Argument, ArgumentType, Library, VersionInfo, VersionType}; +use crate::{download_file, Error}; +use chrono::{DateTime, Utc}; +use serde::{Deserialize, Serialize}; +use std::collections::HashMap; + +/// The latest version of the format the model structs deserialize to +pub const CURRENT_FORMAT_VERSION: usize = 0; + +#[derive(Serialize, Deserialize, Debug)] +#[serde(rename_all = "camelCase")] +/// A partial version returned by fabric meta +pub struct PartialVersionInfo { + /// The version ID of the version + pub id: String, + /// The version ID this partial version inherits from + pub inherits_from: String, + /// The time that the version was released + pub release_time: DateTime, + /// The latest time a file in this version was updated + pub time: DateTime, + /// The classpath to the main class to launch the game + pub main_class: String, + /// Arguments passed to the game or JVM + pub arguments: Option>>, + /// Libraries that the version depends on + pub libraries: Vec, + #[serde(rename = "type")] + /// The type of version + pub type_: VersionType, +} + +/// Merges a partial version into a complete one +pub fn merge_partial_version(partial: PartialVersionInfo, merge: VersionInfo) -> VersionInfo { + VersionInfo { + arguments: if let Some(partial_args) = partial.arguments { + if let Some(merge_args) = merge.arguments { + Some(partial_args.into_iter().chain(merge_args).collect()) + } else { + Some(partial_args) + } + } else { + merge.arguments + }, + asset_index: merge.asset_index, + assets: merge.assets, + downloads: merge.downloads, + id: merge.id, + libraries: partial + .libraries + .into_iter() + .chain(merge.libraries) + .collect::>(), + main_class: partial.main_class, + minecraft_arguments: merge.minecraft_arguments, + minimum_launcher_version: merge.minimum_launcher_version, + release_time: partial.release_time, + time: partial.time, + type_: partial.type_, + } +} + +/// The default servers for fabric meta +pub const FABRIC_META_URL: &str = "https://meta.fabricmc.net/v2"; + +/// Fetches the manifest of a fabric loader version and game version +pub async fn fetch_fabric_version( + version_number: &str, + loader_version: &str, +) -> Result { + Ok(serde_json::from_slice( + &download_file( + &*format!( + "{}/versions/loader/{}/{}/profile/json", + FABRIC_META_URL, version_number, loader_version + ), + None, + ) + .await?, + )?) +} + +/// Fetches the manifest of a game version's URL +pub async fn fetch_fabric_game_version(url: &str) -> Result { + Ok(serde_json::from_slice(&download_file(url, None).await?)?) +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +/// Versions of fabric components +pub struct FabricVersions { + /// Versions of Minecraft that fabric supports + pub game: Vec, + /// Available versions of the fabric loader + pub loader: Vec, +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +/// A version of Minecraft that fabric supports +pub struct FabricGameVersion { + /// The version number of the game + pub version: String, + /// Whether the Minecraft version is stable or not + pub stable: bool, + /// (Modrinth Provided) The URLs to download this version's profile with a loader + /// The key of the map is the loader version, and the value is the URL + pub urls: Option>, +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +/// A version of the fabric loader +pub struct FabricLoaderVersion { + /// The separator to get the build number + pub separator: String, + /// The build number + pub build: u32, + /// The maven artifact + pub maven: String, + /// The version number of the fabric loader + pub version: String, + /// Whether the loader is stable or not + pub stable: bool, +} +/// Fetches the list of fabric versions +pub async fn fetch_fabric_versions(url: Option<&str>) -> Result { + Ok(serde_json::from_slice( + &download_file( + url.unwrap_or(&*format!("{}/versions", FABRIC_META_URL)), + None, + ) + .await?, + )?) +} diff --git a/daedalus/src/lib.rs b/daedalus/src/lib.rs index 8aea3e1ec..92ff542f6 100644 --- a/daedalus/src/lib.rs +++ b/daedalus/src/lib.rs @@ -4,6 +4,8 @@ #![warn(missing_docs, unused_import_braces, missing_debug_implementations)] +/// Models and methods for fetching metadata for the Fabric mod loader +pub mod fabric; /// Models and methods for fetching metadata for Minecraft pub mod minecraft; @@ -34,6 +36,32 @@ pub enum Error { /// There was an error when managing async tasks #[error("Error while managing asynchronous tasks")] TaskError(#[from] tokio::task::JoinError), + /// Error while parsing input + #[error("{0}")] + ParseError(String), +} + +/// Converts a maven artifact to a path +pub fn get_path_from_artifact(artifact: &str) -> Result { + let name_items = artifact.split(':').collect::>(); + + let package = name_items.get(0).ok_or_else(|| { + Error::ParseError(format!("Unable to find package for library {}", &artifact)) + })?; + let name = name_items.get(1).ok_or_else(|| { + Error::ParseError(format!("Unable to find name for library {}", &artifact)) + })?; + let version = name_items.get(2).ok_or_else(|| { + Error::ParseError(format!("Unable to find version for library {}", &artifact)) + })?; + + Ok(format!( + "{}/{}/{}-{}.jar", + package.replace(".", "/"), + version, + name, + version + )) } /// Downloads a file with retry and checksum functionality diff --git a/daedalus/src/minecraft.rs b/daedalus/src/minecraft.rs index 89008da16..e90e8f4d4 100644 --- a/daedalus/src/minecraft.rs +++ b/daedalus/src/minecraft.rs @@ -69,7 +69,7 @@ pub struct LatestVersion { } #[derive(Serialize, Deserialize, Debug, Clone)] -/// Data of all game versions of Minecraft +/// Data of all game versions of Minecrafat pub struct VersionManifest { /// A struct containing the latest snapshot and release of the game pub latest: LatestVersion, @@ -181,10 +181,13 @@ pub enum Os { #[derive(Serialize, Deserialize, Debug)] /// A rule which depends on what OS the user is on pub struct OsRule { + #[serde(skip_serializing_if = "Option::is_none")] /// The name of the OS pub name: Option, + #[serde(skip_serializing_if = "Option::is_none")] /// The version of the OS. This is normally a RegEx pub version: Option, + #[serde(skip_serializing_if = "Option::is_none")] /// The architecture of the OS pub arch: Option, } @@ -192,8 +195,10 @@ pub struct OsRule { #[derive(Serialize, Deserialize, Debug)] /// A rule which depends on the toggled features of the launcher pub struct FeatureRule { + #[serde(skip_serializing_if = "Option::is_none")] /// Whether the user is in demo mode pub is_demo_user: Option, + #[serde(skip_serializing_if = "Option::is_none")] /// Whether the user is using the demo resolution pub has_demo_resolution: Option, } @@ -203,8 +208,10 @@ pub struct FeatureRule { pub struct Rule { /// The action the rule takes pub action: RuleAction, + #[serde(skip_serializing_if = "Option::is_none")] /// The OS rule pub os: Option, + #[serde(skip_serializing_if = "Option::is_none")] /// The feature rule pub features: Option, } @@ -212,6 +219,7 @@ pub struct Rule { #[derive(Serialize, Deserialize, Debug)] /// Information delegating the extraction of the library pub struct LibraryExtract { + #[serde(skip_serializing_if = "Option::is_none")] /// Files/Folders to be excluded from the extraction of the library pub exclude: Option>, } @@ -219,14 +227,21 @@ pub struct LibraryExtract { #[derive(Serialize, Deserialize, Debug)] /// A library which the game relies on to run pub struct Library { + #[serde(skip_serializing_if = "Option::is_none")] /// The files the library has - pub downloads: LibraryDownloads, + pub downloads: Option, + #[serde(skip_serializing_if = "Option::is_none")] /// Rules of the extraction of the file pub extract: Option, /// The maven name of the library. The format is `groupId:artifactId:version` pub name: String, + #[serde(skip_serializing_if = "Option::is_none")] + /// The URL to the repository where the library can be downloaded + pub url: Option, + #[serde(skip_serializing_if = "Option::is_none")] /// Native files that the library relies on pub natives: Option>, + #[serde(skip_serializing_if = "Option::is_none")] /// Rules deciding whether the library should be downloaded or not pub rules: Option>, } @@ -270,6 +285,7 @@ pub enum ArgumentType { #[serde(rename_all = "camelCase")] /// Information about a version pub struct VersionInfo { + #[serde(skip_serializing_if = "Option::is_none")] /// Arguments passed to the game or JVM pub arguments: Option>>, /// Assets for the game @@ -284,6 +300,7 @@ pub struct VersionInfo { pub libraries: Vec, /// The classpath to the main class to launch the game pub main_class: String, + #[serde(skip_serializing_if = "Option::is_none")] /// (Legacy) Arguments passed to the game pub minecraft_arguments: Option, /// The minimum version of the Minecraft Launcher that can run this version of the game diff --git a/daedalus_client/src/fabric.rs b/daedalus_client/src/fabric.rs new file mode 100644 index 000000000..890bd5be1 --- /dev/null +++ b/daedalus_client/src/fabric.rs @@ -0,0 +1,188 @@ +use crate::{format_url, upload_file_to_bucket, Error}; +use daedalus::fabric::PartialVersionInfo; +use daedalus::minecraft::Library; +use std::collections::HashMap; +use std::sync::{Arc, Mutex, RwLock}; +use std::time::{Duration, Instant}; + +pub async fn retrieve_data() -> Result<(), Error> { + let mut list = daedalus::fabric::fetch_fabric_versions(None).await?; + + let loaders = RwLock::new(Vec::new()); + let visited_artifacts_mutex = Arc::new(Mutex::new(Vec::new())); + + if let Some(latest) = list.loader.get(0) { + { + let mut loaders = match loaders.write() { + Ok(guard) => guard, + Err(poisoned) => poisoned.into_inner(), + }; + + loaders.push(latest.version.clone()); + + if !latest.stable { + if let Some(stable) = list.loader.iter().find(|x| x.stable) { + loaders.push(stable.version.clone()); + } + } + + list.loader = list + .loader + .into_iter() + .filter(|x| loaders.contains(&x.version)) + .collect(); + } + + let mut versions = list + .game + .iter_mut() + .map(|game_version| { + let loaders = match loaders.read() { + Ok(guard) => guard, + Err(poisoned) => poisoned.into_inner(), + }; + + let visited_artifacts_mutex = Arc::clone(&visited_artifacts_mutex); + let game_version_mutex = Mutex::new(HashMap::new()); + + async move { + let versions = futures::future::try_join_all(loaders.clone().into_iter().map( + |loader| async { + let version = daedalus::fabric::fetch_fabric_version( + &*game_version.version, + &*loader, + ) + .await + .expect(&*format!("{}, {}", game_version.version, loader)); + + Ok::<(String, PartialVersionInfo), Error>((loader, version)) + }, + )) + .await?; + + futures::future::try_join_all(versions.into_iter().map( + |(loader, version)| async { + let libs = futures::future::try_join_all( + version.libraries.into_iter().map(|mut lib| async { + { + let mut visited_assets = + match visited_artifacts_mutex.lock() { + Ok(guard) => guard, + Err(poisoned) => poisoned.into_inner(), + }; + + if visited_assets.contains(&lib.name) { + lib.url = Some(format_url("maven/")); + + return Ok(lib); + } else { + visited_assets.push(lib.name.clone()) + } + } + + let artifact_path = + daedalus::get_path_from_artifact(&*lib.name)?; + + let artifact = daedalus::download_file( + &*format!( + "{}{}", + lib.url.unwrap_or_else(|| { + "https://maven.fabricmc.net/".to_string() + }), + artifact_path + ), + None, + ) + .await?; + + lib.url = Some(format_url("maven/")); + + upload_file_to_bucket( + format!("{}/{}", "maven", artifact_path), + artifact.to_vec(), + Some("application/java-archive".to_string()), + ) + .await?; + + Ok::(lib) + }), + ) + .await?; + + let version_path = format!( + "fabric/v{}/versions/{}-{}.json", + daedalus::fabric::CURRENT_FORMAT_VERSION, + version.inherits_from, + loader + ); + + upload_file_to_bucket( + version_path.clone(), + serde_json::to_vec(&PartialVersionInfo { + arguments: version.arguments, + id: version.id, + main_class: version.main_class, + release_time: version.release_time, + time: version.time, + type_: version.type_, + inherits_from: version.inherits_from, + libraries: libs, + })?, + Some("application/json".to_string()), + ) + .await?; + + { + let mut game_version_map = match game_version_mutex.lock() { + Ok(guard) => guard, + Err(poisoned) => poisoned.into_inner(), + }; + game_version_map.insert(loader, format_url(&*version_path)); + } + + Ok::<(), Error>(()) + }, + )) + .await?; + + game_version.urls = Some( + match game_version_mutex.lock() { + Ok(guard) => guard, + Err(poisoned) => poisoned.into_inner(), + } + .clone(), + ); + + Ok::<(), Error>(()) + } + }) + .peekable(); + + let mut chunk_index = 0; + while versions.peek().is_some() { + let now = Instant::now(); + + let chunk: Vec<_> = versions.by_ref().take(10).collect(); + futures::future::try_join_all(chunk).await?; + + std::thread::sleep(Duration::from_secs(1)); + + chunk_index += 1; + + let elapsed = now.elapsed(); + println!("Chunk {} Elapsed: {:.2?}", chunk_index, elapsed); + } + } + + upload_file_to_bucket( + format!( + "fabric/v{}/manifest.json", + daedalus::fabric::CURRENT_FORMAT_VERSION, + ), + serde_json::to_vec(&list)?, + Some("application/json".to_string()), + ) + .await?; + + Ok(()) +} diff --git a/daedalus_client/src/main.rs b/daedalus_client/src/main.rs index bb10303ac..7ac9a1861 100644 --- a/daedalus_client/src/main.rs +++ b/daedalus_client/src/main.rs @@ -1,9 +1,10 @@ use log::{error, info, warn}; use rusoto_core::credential::StaticProvider; use rusoto_core::{HttpClient, Region, RusotoError}; -use rusoto_s3::{S3Client, PutObjectError}; +use rusoto_s3::{PutObjectError, S3Client}; use rusoto_s3::{PutObjectRequest, S3}; +mod fabric; mod minecraft; #[derive(thiserror::Error, Debug)] @@ -13,10 +14,7 @@ pub enum Error { #[error("Error while deserializing JSON")] SerdeError(#[from] serde_json::Error), #[error("Unable to fetch {item}")] - FetchError { - inner: reqwest::Error, - item: String, - }, + FetchError { inner: reqwest::Error, item: String }, #[error("Error while managing asynchronous tasks")] TaskError(#[from] tokio::task::JoinError), #[error("Error while uploading file to S3")] @@ -34,7 +32,7 @@ async fn main() { return; } - minecraft::retrieve_data().await.unwrap(); + fabric::retrieve_data().await.unwrap(); } fn check_env_vars() -> bool { @@ -96,7 +94,11 @@ lazy_static::lazy_static! { ); } -pub async fn upload_file_to_bucket(path: String, bytes: Vec, content_type: Option) -> Result<(), Error> { +pub async fn upload_file_to_bucket( + path: String, + bytes: Vec, + content_type: Option, +) -> Result<(), Error> { CLIENT .put_object(PutObjectRequest { bucket: dotenv::var("S3_BUCKET_NAME").unwrap(), @@ -109,7 +111,7 @@ pub async fn upload_file_to_bucket(path: String, bytes: Vec, content_type: O .await .map_err(|err| Error::S3Error { inner: err, - file: format!("{}/{}", &*dotenv::var("BASE_FOLDER").unwrap(), path) + file: format!("{}/{}", &*dotenv::var("BASE_FOLDER").unwrap(), path), })?; Ok(()) diff --git a/daedalus_client/src/minecraft.rs b/daedalus_client/src/minecraft.rs index 1336e9ffb..ddcf286e7 100644 --- a/daedalus_client/src/minecraft.rs +++ b/daedalus_client/src/minecraft.rs @@ -6,14 +6,13 @@ use std::time::{Duration, Instant}; pub async fn retrieve_data() -> Result<(), Error> { let old_manifest = daedalus::minecraft::fetch_version_manifest(Some(&*crate::format_url(&*format!( - "minecraft/v{}/version_manifest.json", + "minecraft/v{}/manifest.json", daedalus::minecraft::CURRENT_FORMAT_VERSION )))) .await .ok(); - let mut manifest = daedalus::minecraft::fetch_version_manifest(None) - .await?; + let mut manifest = daedalus::minecraft::fetch_version_manifest(None).await?; let cloned_manifest = Arc::new(Mutex::new(manifest.clone())); let visited_assets_mutex = Arc::new(Mutex::new(Vec::new())); @@ -44,12 +43,7 @@ pub async fn retrieve_data() -> Result<(), Error> { async move { let mut upload_futures = Vec::new(); - let now = Instant::now(); - let mut version_println = daedalus::minecraft::fetch_version_info(version) - .await - ?; - let elapsed = now.elapsed(); - println!("Version {} Elapsed: {:.2?}", version.id, elapsed); + let mut version_info = daedalus::minecraft::fetch_version_info(version).await?; let version_path = format!( "minecraft/v{}/versions/{}.json", @@ -59,9 +53,9 @@ pub async fn retrieve_data() -> Result<(), Error> { let assets_path = format!( "minecraft/v{}/assets/{}.json", daedalus::minecraft::CURRENT_FORMAT_VERSION, - version_println.asset_index.id + version_info.asset_index.id ); - let assets_index_url = version_println.asset_index.url.clone(); + let assets_index_url = version_info.asset_index.url.clone(); { let mut cloned_manifest = match cloned_manifest_mutex.lock() { @@ -76,10 +70,10 @@ pub async fn retrieve_data() -> Result<(), Error> { .unwrap(); cloned_manifest.versions[position].url = format_url(&version_path); cloned_manifest.versions[position].assets_index_sha1 = - Some(version_println.asset_index.sha1.clone()); + Some(version_info.asset_index.sha1.clone()); cloned_manifest.versions[position].assets_index_url = Some(format_url(&assets_path)); - version_println.asset_index.url = format_url(&assets_path); + version_info.asset_index.url = format_url(&assets_path); } let mut download_assets = false; @@ -90,9 +84,9 @@ pub async fn retrieve_data() -> Result<(), Error> { Err(poisoned) => poisoned.into_inner(), }; - if !visited_assets.contains(&version_println.asset_index.id) { + if !visited_assets.contains(&version_info.asset_index.id) { if let Some(assets_hash) = assets_hash { - if version_println.asset_index.sha1 != assets_hash { + if version_info.asset_index.sha1 != assets_hash { download_assets = true; } } else { @@ -101,26 +95,29 @@ pub async fn retrieve_data() -> Result<(), Error> { } if download_assets { - visited_assets.push(version_println.asset_index.id.clone()); + visited_assets.push(version_info.asset_index.id.clone()); } } if download_assets { let assets_index = - download_file(&assets_index_url, Some(&version_println.asset_index.sha1)) + download_file(&assets_index_url, Some(&version_info.asset_index.sha1)) .await?; { - upload_futures - .push(upload_file_to_bucket(assets_path, assets_index.to_vec(), Some("application/json".to_string()))); + upload_futures.push(upload_file_to_bucket( + assets_path, + assets_index.to_vec(), + Some("application/json".to_string()), + )); } } { upload_futures.push(upload_file_to_bucket( version_path, - serde_json::to_vec(&version_println)?, - Some("application/json".to_string()) + serde_json::to_vec(&version_info)?, + Some("application/json".to_string()), )); } @@ -154,14 +151,14 @@ pub async fn retrieve_data() -> Result<(), Error> { upload_file_to_bucket( format!( - "minecraft/v{}/version_manifest.json", + "minecraft/v{}/manifest.json", daedalus::minecraft::CURRENT_FORMAT_VERSION ), serde_json::to_vec(&*match cloned_manifest.lock() { Ok(guard) => guard, Err(poisoned) => poisoned.into_inner(), })?, - Some("application/json".to_string()) + Some("application/json".to_string()), ) .await?; From 16af479b833b86b1c0169085a56be7d252a767a9 Mon Sep 17 00:00:00 2001 From: Jai A Date: Sun, 17 Oct 2021 23:22:23 -0700 Subject: [PATCH 03/85] Legacy Forge Support --- LICENSE | 0 README.md | 0 daedalus/Cargo.toml | 9 ++++++++- daedalus/README.md | 6 ++++++ daedalus/src/fabric.rs | 8 ++++++-- daedalus/src/forge.rs | 21 +++++++++++++++++++++ daedalus/src/lib.rs | 27 ++++++++++++++++++++++++++- daedalus/src/minecraft.rs | 21 ++++++++++++--------- daedalus_client/src/forge.rs | 0 9 files changed, 79 insertions(+), 13 deletions(-) create mode 100644 LICENSE create mode 100644 README.md create mode 100644 daedalus/README.md create mode 100644 daedalus/src/forge.rs create mode 100644 daedalus_client/src/forge.rs diff --git a/LICENSE b/LICENSE new file mode 100644 index 000000000..e69de29bb diff --git a/README.md b/README.md new file mode 100644 index 000000000..e69de29bb diff --git a/daedalus/Cargo.toml b/daedalus/Cargo.toml index a52c207f5..def9e0dcc 100644 --- a/daedalus/Cargo.toml +++ b/daedalus/Cargo.toml @@ -3,6 +3,13 @@ name = "daedalus" version = "0.1.0" authors = ["Jai A "] edition = "2018" +license = "MIT" +description = "Utilities for querying and parsing Minecraft metadata" +repository = "https://github.com/modrinth/daedalus/" +include = ["Cargo.toml", "src/**/*.rs", "README.md", "LICENSE"] +keywords = ["minecraft", "launcher"] +categories = ["game-development", "api-bindings"] +readme = "README.md" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html @@ -14,4 +21,4 @@ chrono = { version = "0.4", features = ["serde"] } bytes = "1" thiserror = "1.0" tokio = { version = "1", features = ["full"] } -sha1 = { version = "0.6.0", features = ["std"]} \ No newline at end of file +sha1 = { version = "0.6.0", features = ["std"]} diff --git a/daedalus/README.md b/daedalus/README.md new file mode 100644 index 000000000..fb57d0f03 --- /dev/null +++ b/daedalus/README.md @@ -0,0 +1,6 @@ +# Daedalus + +Daedalus (the rust library) is a library providing model structs and methods for requesting and parsing things +from Minecraft and other mod loaders meta APIs. + +This is a work in progress! \ No newline at end of file diff --git a/daedalus/src/fabric.rs b/daedalus/src/fabric.rs index 052486ffa..d63705a73 100644 --- a/daedalus/src/fabric.rs +++ b/daedalus/src/fabric.rs @@ -20,7 +20,7 @@ pub struct PartialVersionInfo { /// The latest time a file in this version was updated pub time: DateTime, /// The classpath to the main class to launch the game - pub main_class: String, + pub main_class: Option, /// Arguments passed to the game or JVM pub arguments: Option>>, /// Libraries that the version depends on @@ -51,7 +51,11 @@ pub fn merge_partial_version(partial: PartialVersionInfo, merge: VersionInfo) -> .into_iter() .chain(merge.libraries) .collect::>(), - main_class: partial.main_class, + main_class: if let Some(main_class) = partial.main_class { + main_class + } else { + merge.main_class + }, minecraft_arguments: merge.minecraft_arguments, minimum_launcher_version: merge.minimum_launcher_version, release_time: partial.release_time, diff --git a/daedalus/src/forge.rs b/daedalus/src/forge.rs new file mode 100644 index 000000000..4381f0307 --- /dev/null +++ b/daedalus/src/forge.rs @@ -0,0 +1,21 @@ +use crate::{download_file, Error}; + +use std::collections::HashMap; + +/// The latest version of the format the model structs deserialize to +pub const CURRENT_FORMAT_VERSION: usize = 0; + +const DEFAULT_MAVEN_METADATA_URL: &str = + "https://files.minecraftforge.net/net/minecraftforge/forge/maven-metadata.json"; + +/// Fetches the forge maven metadata from the specified URL. If no URL is specified, the default is used. +/// Returns a hashmap specifying the versions of the forge mod loader +/// The hashmap key is a Minecraft version, and the value is the loader versions that work on +/// the specified Minecraft version +pub async fn fetch_maven_metadata( + url: Option<&str>, +) -> Result>, Error> { + Ok(serde_json::from_slice( + &download_file(url.unwrap_or(DEFAULT_MAVEN_METADATA_URL), None).await?, + )?) +} diff --git a/daedalus/src/lib.rs b/daedalus/src/lib.rs index 92ff542f6..4f3147553 100644 --- a/daedalus/src/lib.rs +++ b/daedalus/src/lib.rs @@ -6,6 +6,8 @@ /// Models and methods for fetching metadata for the Fabric mod loader pub mod fabric; +/// Models and methods for fetching metadata for the Forge mod loader +pub mod forge; /// Models and methods for fetching metadata for Minecraft pub mod minecraft; @@ -56,18 +58,41 @@ pub fn get_path_from_artifact(artifact: &str) -> Result { })?; Ok(format!( - "{}/{}/{}-{}.jar", + "{}/{}/{}/{}-{}.jar", package.replace(".", "/"), + name, version, name, version )) } +/// Downloads a file from specified mirrors +pub async fn download_file_mirrors( + base: &str, + mirrors: &[&str], + sha1: Option<&str>, +) -> Result { + if mirrors.is_empty() { + return Err(Error::ParseError("No mirrors provided!".to_string())); + } + + for (index, mirror) in mirrors.iter().enumerate() { + let result = download_file(&*format!("{}{}", mirror, base), sha1).await; + + if result.is_ok() || (result.is_err() && index == (mirrors.len() - 1)) { + return result; + } + } + + unreachable!() +} + /// Downloads a file with retry and checksum functionality pub async fn download_file(url: &str, sha1: Option<&str>) -> Result { let client = reqwest::Client::builder() .tcp_keepalive(Some(std::time::Duration::from_secs(10))) + .timeout(std::time::Duration::from_secs(30)) .build() .map_err(|err| Error::FetchError { inner: err, diff --git a/daedalus/src/minecraft.rs b/daedalus/src/minecraft.rs index e90e8f4d4..d345f04c3 100644 --- a/daedalus/src/minecraft.rs +++ b/daedalus/src/minecraft.rs @@ -69,7 +69,7 @@ pub struct LatestVersion { } #[derive(Serialize, Deserialize, Debug, Clone)] -/// Data of all game versions of Minecrafat +/// Data of all game versions of Minecraft pub struct VersionManifest { /// A struct containing the latest snapshot and release of the game pub latest: LatestVersion, @@ -154,7 +154,7 @@ pub struct LibraryDownloads { pub classifiers: Option>, } -#[derive(Serialize, Deserialize, Debug)] +#[derive(Serialize, Deserialize, Debug, Clone)] #[serde(rename_all = "snake_case")] /// The action a rule can follow pub enum RuleAction { @@ -164,7 +164,7 @@ pub enum RuleAction { Disallow, } -#[derive(Serialize, Deserialize, Debug, Eq, PartialEq, Hash)] +#[derive(Serialize, Deserialize, Debug, Eq, PartialEq, Hash, Clone)] #[serde(rename_all = "snake_case")] /// An enum representing the different types of operating systems pub enum Os { @@ -178,7 +178,7 @@ pub enum Os { Unknown, } -#[derive(Serialize, Deserialize, Debug)] +#[derive(Serialize, Deserialize, Debug, Clone)] /// A rule which depends on what OS the user is on pub struct OsRule { #[serde(skip_serializing_if = "Option::is_none")] @@ -192,7 +192,7 @@ pub struct OsRule { pub arch: Option, } -#[derive(Serialize, Deserialize, Debug)] +#[derive(Serialize, Deserialize, Debug, Clone)] /// A rule which depends on the toggled features of the launcher pub struct FeatureRule { #[serde(skip_serializing_if = "Option::is_none")] @@ -203,7 +203,7 @@ pub struct FeatureRule { pub has_demo_resolution: Option, } -#[derive(Serialize, Deserialize, Debug)] +#[derive(Serialize, Deserialize, Debug, Clone)] /// A rule deciding whether a file is downloaded, an argument is used, etc. pub struct Rule { /// The action the rule takes @@ -244,9 +244,12 @@ pub struct Library { #[serde(skip_serializing_if = "Option::is_none")] /// Rules deciding whether the library should be downloaded or not pub rules: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + /// SHA1 Checksums for validating the library's integrity. Only present for forge libraries + pub checksums: Option>, } -#[derive(Serialize, Deserialize, Debug)] +#[derive(Serialize, Deserialize, Debug, Clone)] #[serde(untagged)] /// A container for an argument or multiple arguments pub enum ArgumentValue { @@ -256,7 +259,7 @@ pub enum ArgumentValue { Many(Vec), } -#[derive(Serialize, Deserialize, Debug)] +#[derive(Serialize, Deserialize, Debug, Clone)] #[serde(untagged)] /// A command line argument passed to a program pub enum Argument { @@ -271,7 +274,7 @@ pub enum Argument { }, } -#[derive(Serialize, Deserialize, Debug, Eq, PartialEq, Hash)] +#[derive(Serialize, Deserialize, Debug, Eq, PartialEq, Hash, Clone, Copy)] #[serde(rename_all = "snake_case")] /// The type of argument pub enum ArgumentType { diff --git a/daedalus_client/src/forge.rs b/daedalus_client/src/forge.rs new file mode 100644 index 000000000..e69de29bb From 6528d3d7dac09b1f4e3ff42188d0790f826afe8b Mon Sep 17 00:00:00 2001 From: Jai A Date: Sun, 17 Oct 2021 23:23:27 -0700 Subject: [PATCH 04/85] Legacy Forge Support (forgot to git add) --- LICENSE | 7 + README.md | 12 ++ daedalus_client/Cargo.toml | 5 + daedalus_client/src/fabric.rs | 240 ++++++++++++++----------------- daedalus_client/src/forge.rs | 210 +++++++++++++++++++++++++++ daedalus_client/src/main.rs | 32 ++++- daedalus_client/src/minecraft.rs | 30 ++-- 7 files changed, 389 insertions(+), 147 deletions(-) diff --git a/LICENSE b/LICENSE index e69de29bb..6d21f198e 100644 --- a/LICENSE +++ b/LICENSE @@ -0,0 +1,7 @@ +Copyright © 2021 Guavy LLC + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. \ No newline at end of file diff --git a/README.md b/README.md index e69de29bb..f0d5bd82f 100644 --- a/README.md +++ b/README.md @@ -0,0 +1,12 @@ +# Daedalus + +Daedalus is a powerful tool which queries and generates metadata for the Minecraft (and other games in the future!) game +and mod loaders for: +- Performance (Serving static files can be easily cached and is extremely quick) +- Ease for Launcher Devs (Metadata is served in an easy to query and use format) +- Reliability (Provides a versioning system which ensures no breakage with updates) + +Daedalus is currently a work in progress, but will support the original Minecraft data and reposting for all Forge and +Fabric artifacts. + +Once Daedalus is done, Modrinth will provide full documentation for how to query from it and use it for your own launcher! \ No newline at end of file diff --git a/daedalus_client/Cargo.toml b/daedalus_client/Cargo.toml index 4f2fa019e..33f1c0638 100644 --- a/daedalus_client/Cargo.toml +++ b/daedalus_client/Cargo.toml @@ -12,10 +12,15 @@ tokio = { version = "1", features = ["full"] } futures = "0.3.17" dotenv = "0.15.0" log = "0.4.8" +serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" lazy_static = "1.4.0" thiserror = "1.0" reqwest = "0.11.4" +zip = "0.5.13" +semver = "1.0" +chrono = { version = "0.4", features = ["serde"] } +bytes = "1.1.0" rusoto_core = "0.47.0" rusoto_s3 = "0.47.0" diff --git a/daedalus_client/src/fabric.rs b/daedalus_client/src/fabric.rs index 890bd5be1..809ec832e 100644 --- a/daedalus_client/src/fabric.rs +++ b/daedalus_client/src/fabric.rs @@ -2,21 +2,19 @@ use crate::{format_url, upload_file_to_bucket, Error}; use daedalus::fabric::PartialVersionInfo; use daedalus::minecraft::Library; use std::collections::HashMap; -use std::sync::{Arc, Mutex, RwLock}; +use std::sync::{Arc}; use std::time::{Duration, Instant}; +use futures::lock::Mutex; pub async fn retrieve_data() -> Result<(), Error> { let mut list = daedalus::fabric::fetch_fabric_versions(None).await?; - let loaders = RwLock::new(Vec::new()); - let visited_artifacts_mutex = Arc::new(Mutex::new(Vec::new())); - if let Some(latest) = list.loader.get(0) { + let loaders_mutex = Arc::new(Mutex::new(Vec::new())); + let visited_artifacts_mutex = Arc::new(Mutex::new(Vec::new())); + { - let mut loaders = match loaders.write() { - Ok(guard) => guard, - Err(poisoned) => poisoned.into_inner(), - }; + let mut loaders = loaders_mutex.lock().await; loaders.push(latest.version.clone()); @@ -33,131 +31,115 @@ pub async fn retrieve_data() -> Result<(), Error> { .collect(); } - let mut versions = list - .game - .iter_mut() - .map(|game_version| { - let loaders = match loaders.read() { - Ok(guard) => guard, - Err(poisoned) => poisoned.into_inner(), - }; - - let visited_artifacts_mutex = Arc::clone(&visited_artifacts_mutex); - let game_version_mutex = Mutex::new(HashMap::new()); - - async move { - let versions = futures::future::try_join_all(loaders.clone().into_iter().map( - |loader| async { - let version = daedalus::fabric::fetch_fabric_version( - &*game_version.version, - &*loader, - ) - .await - .expect(&*format!("{}, {}", game_version.version, loader)); - - Ok::<(String, PartialVersionInfo), Error>((loader, version)) - }, - )) - .await?; - - futures::future::try_join_all(versions.into_iter().map( - |(loader, version)| async { - let libs = futures::future::try_join_all( - version.libraries.into_iter().map(|mut lib| async { - { - let mut visited_assets = - match visited_artifacts_mutex.lock() { - Ok(guard) => guard, - Err(poisoned) => poisoned.into_inner(), - }; - - if visited_assets.contains(&lib.name) { - lib.url = Some(format_url("maven/")); - - return Ok(lib); - } else { - visited_assets.push(lib.name.clone()) - } + let mut version_futures = Vec::new(); + + for game_version in list.game.iter_mut() { + let visited_artifacts_mutex = Arc::clone(&visited_artifacts_mutex); + let game_version_mutex = Mutex::new(HashMap::new()); + let loaders_mutex = Arc::clone(&loaders_mutex); + version_futures.push(async move { + let versions = futures::future::try_join_all(loaders_mutex.lock().await.clone().into_iter().map( + |loader| async { + let version = daedalus::fabric::fetch_fabric_version( + &*game_version.version, + &*loader, + ) + .await + .expect(&*format!("{}, {}", game_version.version, loader)); + + Ok::<(String, PartialVersionInfo), Error>((loader, version)) + }, + )) + .await?; + + futures::future::try_join_all(versions.into_iter().map( + |(loader, version)| async { + let libs = futures::future::try_join_all( + version.libraries.into_iter().map(|mut lib| async { + { + let mut visited_assets = visited_artifacts_mutex.lock().await; + + if visited_assets.contains(&lib.name) { + lib.url = Some(format_url("maven/")); + + return Ok(lib); + } else { + visited_assets.push(lib.name.clone()) } - - let artifact_path = - daedalus::get_path_from_artifact(&*lib.name)?; - - let artifact = daedalus::download_file( - &*format!( - "{}{}", - lib.url.unwrap_or_else(|| { - "https://maven.fabricmc.net/".to_string() - }), - artifact_path - ), - None, - ) - .await?; - - lib.url = Some(format_url("maven/")); - - upload_file_to_bucket( - format!("{}/{}", "maven", artifact_path), - artifact.to_vec(), - Some("application/java-archive".to_string()), - ) - .await?; - - Ok::(lib) - }), - ) - .await?; - - let version_path = format!( - "fabric/v{}/versions/{}-{}.json", - daedalus::fabric::CURRENT_FORMAT_VERSION, - version.inherits_from, - loader - ); - - upload_file_to_bucket( - version_path.clone(), - serde_json::to_vec(&PartialVersionInfo { - arguments: version.arguments, - id: version.id, - main_class: version.main_class, - release_time: version.release_time, - time: version.time, - type_: version.type_, - inherits_from: version.inherits_from, - libraries: libs, - })?, - Some("application/json".to_string()), - ) - .await?; - - { - let mut game_version_map = match game_version_mutex.lock() { - Ok(guard) => guard, - Err(poisoned) => poisoned.into_inner(), - }; - game_version_map.insert(loader, format_url(&*version_path)); - } - - Ok::<(), Error>(()) - }, - )) - .await?; - - game_version.urls = Some( - match game_version_mutex.lock() { - Ok(guard) => guard, - Err(poisoned) => poisoned.into_inner(), + } + + let artifact_path = + daedalus::get_path_from_artifact(&*lib.name)?; + + let artifact = daedalus::download_file( + &*format!( + "{}{}", + lib.url.unwrap_or_else(|| { + "https://maven.fabricmc.net/".to_string() + }), + artifact_path + ), + None, + ) + .await?; + + lib.url = Some(format_url("maven/")); + + upload_file_to_bucket( + format!("{}/{}", "maven", artifact_path), + artifact.to_vec(), + Some("application/java-archive".to_string()), + ) + .await?; + + Ok::(lib) + }), + ) + .await?; + + let version_path = format!( + "fabric/v{}/versions/{}-{}.json", + daedalus::fabric::CURRENT_FORMAT_VERSION, + version.inherits_from, + loader + ); + + upload_file_to_bucket( + version_path.clone(), + serde_json::to_vec(&PartialVersionInfo { + arguments: version.arguments, + id: version.id, + main_class: version.main_class, + release_time: version.release_time, + time: version.time, + type_: version.type_, + inherits_from: version.inherits_from, + libraries: libs, + })?, + Some("application/json".to_string()), + ) + .await?; + + { + let mut game_version_map = game_version_mutex.lock().await; + game_version_map.insert(loader, format_url(&*version_path)); } - .clone(), - ); - Ok::<(), Error>(()) - } - }) - .peekable(); + Ok::<(), Error>(()) + }, + )) + .await?; + + game_version.urls = Some( + game_version_mutex.lock().await + .clone(), + ); + + Ok::<(), Error>(()) + }); + } + let mut versions = version_futures.into_iter().peekable(); let mut chunk_index = 0; while versions.peek().is_some() { let now = Instant::now(); diff --git a/daedalus_client/src/forge.rs b/daedalus_client/src/forge.rs index e69de29bb..61dd1fc1b 100644 --- a/daedalus_client/src/forge.rs +++ b/daedalus_client/src/forge.rs @@ -0,0 +1,210 @@ +use crate::{format_url, upload_file_to_bucket, Error}; +use semver::{VersionReq, Version}; +use lazy_static::lazy_static; +use daedalus::download_file; +use std::io::Read; +use tokio::sync::{Mutex}; +use std::sync::{Arc}; +use daedalus::minecraft::{Library, VersionType, ArgumentType, Argument}; +use chrono::{DateTime, Utc}; +use serde::{Serialize, Deserialize}; +use daedalus::fabric::PartialVersionInfo; +use std::time::{Instant, Duration}; + +#[derive(Serialize, Deserialize, Debug)] +#[serde(rename_all = "camelCase")] +struct ForgeInstallerProfileInstallDataV1 { + pub mirror_list: String, + pub target: String, + /// Path to the Forge universal library + pub file_path: String, + pub logo: String, + pub welcome: String, + pub version: String, + /// Maven coordinates of the Forge universal library + pub path: String, + pub profile_name: String, + pub minecraft: String, +} + +#[derive(Serialize, Deserialize, Debug)] +#[serde(rename_all = "camelCase")] +struct ForgeInstallerProfileManifestV1 { + pub id: String, + pub libraries: Vec, + pub main_class: Option, + pub minecraft_arguments: Option, + pub release_time: DateTime, + pub time: DateTime, + pub type_: VersionType, + pub assets: Option, + pub inherits_from: Option, + pub jar: Option, +} + +#[derive(Serialize, Deserialize, Debug)] +#[serde(rename_all = "camelCase")] +struct ForgeInstallerProfileV1 { + pub install: ForgeInstallerProfileInstallDataV1, + pub version_info: ForgeInstallerProfileManifestV1, +} + +lazy_static! { + static ref FORGE_MANIFEST_V1_QUERY: VersionReq = VersionReq::parse(">=8.0.684, <23.5.2851").unwrap(); +} + +pub async fn retrieve_data() -> Result<(), Error> { + let maven_metadata = daedalus::forge::fetch_maven_metadata(None).await?; + + let visited_assets_mutex = Arc::new(Mutex::new(Vec::new())); + + let mut version_futures = Vec::new(); + + for (minecraft_version, loader_versions) in maven_metadata { + if let Some(loader_version_full) = loader_versions.into_iter().last() { + let loader_version = loader_version_full.split('-').into_iter().nth(1); + + if let Some(loader_version_raw) = loader_version { + // This is a dirty hack to get around Forge not complying with SemVer, but whatever + // Most of this is a hack anyways :( + // Works for all forge versions! + let split = loader_version_raw.split('.').collect::>(); + let loader_version =if split.len() >= 4 { + if split[0].parse::().unwrap() < 6 { + format!("{}.{}.{}", split[0], split[1], split[3]) + } else { + format!("{}.{}.{}", split[1], split[2], split[3]) + } + } else { + loader_version_raw.to_string() + }; + + if FORGE_MANIFEST_V1_QUERY.matches(&Version::parse(&*loader_version).unwrap()) { + version_futures.push(async { + let visited_assets = Arc::clone(&visited_assets_mutex); + async move { + println!("installer start {}", loader_version_full.clone()); + let bytes = download_file(&*format!("https://maven.minecraftforge.net/net/minecraftforge/forge/{0}/forge-{0}-installer.jar", loader_version_full), None).await.unwrap(); + + let reader = std::io::Cursor::new(&*bytes); + + if let Ok(mut archive) = zip::ZipArchive::new(reader) { + let install_profile = { + let mut install_profile = archive.by_name("install_profile.json").unwrap(); + + let mut contents = String::new(); + install_profile.read_to_string(&mut contents).unwrap(); + + contents + }; + + let profile = serde_json::from_str::(&*install_profile).unwrap(); + + let forge_universal_bytes = { + let mut forge_universal_file = archive.by_name(&*profile.install.file_path).unwrap(); + let mut forge_universal = Vec::new(); + forge_universal_file.read_to_end(&mut forge_universal).unwrap(); + + bytes::Bytes::from(forge_universal) + }; + let forge_universal_path = profile.install.file_path.clone(); + + let now = Instant::now(); + let libs = futures::future::try_join_all(profile.version_info.libraries.into_iter().map(|mut lib| async { + if let Some(url) = lib.url { + { + let mut visited_assets = visited_assets.lock().await; + + if visited_assets.contains(&lib.name) { + lib.url = Some(format_url("maven/")); + + return Ok::(lib); + } else { + visited_assets.push(lib.name.clone()) + } + } + + let artifact_path = + daedalus::get_path_from_artifact(&*lib.name)?; + + let artifact = if lib.name == forge_universal_path { + forge_universal_bytes.clone() + } else { + let mirrors = vec![&*url, "https://maven.creeperhost.net/", "https://libraries.minecraft.net/"]; + + daedalus::download_file_mirrors( + &*artifact_path, + &mirrors, + None, + ) + .await? + }; + + lib.url = Some(format_url("maven/")); + + upload_file_to_bucket( + format!("{}/{}", "maven", artifact_path), + artifact.to_vec(), + Some("application/java-archive".to_string()), + ).await?; + } + + Ok::(lib) + })).await?; + + let elapsed = now.elapsed(); + println!("Elapsed lib DL: {:.2?}", elapsed); + + let new_profile = PartialVersionInfo { + id: profile.version_info.id, + inherits_from: profile.install.minecraft, + release_time: profile.version_info.release_time, + time: profile.version_info.time, + main_class: profile.version_info.main_class, + arguments: profile.version_info.minecraft_arguments.map(|x| [(ArgumentType::Game, x.split(' ').map(|x| Argument::Normal(x.to_string())).collect())].iter().cloned().collect()), + libraries: libs, + type_: profile.version_info.type_, + }; + + let version_path = format!( + "forge/v{}/versions/{}.json", + daedalus::forge::CURRENT_FORMAT_VERSION, + new_profile.id + ); + + upload_file_to_bucket( + version_path.clone(), + serde_json::to_vec(&new_profile)?, + Some("application/json".to_string()), + ).await?; + } + + + Ok::<(), Error>(()) + }.await?; + + Ok::<(), Error>(()) + }); + } + } + } + } + + let mut versions = version_futures.into_iter().peekable(); + let mut chunk_index = 0; + while versions.peek().is_some() { + let now = Instant::now(); + + let chunk: Vec<_> = versions.by_ref().take(100).collect(); + futures::future::try_join_all(chunk).await?; + + std::thread::sleep(Duration::from_secs(1)); + + chunk_index += 1; + + let elapsed = now.elapsed(); + println!("Chunk {} Elapsed: {:.2?}", chunk_index, elapsed); + } + + Ok(()) +} \ No newline at end of file diff --git a/daedalus_client/src/main.rs b/daedalus_client/src/main.rs index 7ac9a1861..321ef10b3 100644 --- a/daedalus_client/src/main.rs +++ b/daedalus_client/src/main.rs @@ -3,9 +3,11 @@ use rusoto_core::credential::StaticProvider; use rusoto_core::{HttpClient, Region, RusotoError}; use rusoto_s3::{PutObjectError, S3Client}; use rusoto_s3::{PutObjectRequest, S3}; +use std::time::Duration; mod fabric; mod minecraft; +mod forge; #[derive(thiserror::Error, Debug)] pub enum Error { @@ -32,7 +34,35 @@ async fn main() { return; } - fabric::retrieve_data().await.unwrap(); + let mut timer = tokio::time::interval(Duration::from_secs(10 * 60)); + + loop { + timer.tick().await; + tokio::spawn( + async { + match fabric::retrieve_data().await { + Ok(..) => {} + Err(err) => error!("{:?}", err) + }; + } + ); + tokio::spawn( + async { + match minecraft::retrieve_data().await { + Ok(..) => {} + Err(err) => error!("{:?}", err) + }; + } + ); + tokio::spawn( + async { + match forge::retrieve_data().await { + Ok(..) => {} + Err(err) => error!("{:?}", err) + }; + } + ); + } } fn check_env_vars() -> bool { diff --git a/daedalus_client/src/minecraft.rs b/daedalus_client/src/minecraft.rs index ddcf286e7..90b58806d 100644 --- a/daedalus_client/src/minecraft.rs +++ b/daedalus_client/src/minecraft.rs @@ -1,7 +1,8 @@ use crate::{format_url, upload_file_to_bucket, Error}; use daedalus::download_file; -use std::sync::{Arc, Mutex}; +use std::sync::{Arc}; use std::time::{Duration, Instant}; +use futures::lock::Mutex; pub async fn retrieve_data() -> Result<(), Error> { let old_manifest = @@ -19,10 +20,13 @@ pub async fn retrieve_data() -> Result<(), Error> { let now = Instant::now(); - let mut versions = manifest + let mut version_futures = Vec::new(); + + for version in manifest .versions .iter_mut() - .map(|version| async { + { + version_futures.push(async { let old_version = if let Some(old_manifest) = &old_manifest { old_manifest.versions.iter().find(|x| x.id == version.id) } else { @@ -58,10 +62,7 @@ pub async fn retrieve_data() -> Result<(), Error> { let assets_index_url = version_info.asset_index.url.clone(); { - let mut cloned_manifest = match cloned_manifest_mutex.lock() { - Ok(guard) => guard, - Err(poisoned) => poisoned.into_inner(), - }; + let mut cloned_manifest = cloned_manifest_mutex.lock().await; let position = cloned_manifest .versions @@ -79,10 +80,7 @@ pub async fn retrieve_data() -> Result<(), Error> { let mut download_assets = false; { - let mut visited_assets = match visited_assets_mutex.lock() { - Ok(guard) => guard, - Err(poisoned) => poisoned.into_inner(), - }; + let mut visited_assets = visited_assets_mutex.lock().await; if !visited_assets.contains(&version_info.asset_index.id) { if let Some(assets_hash) = assets_hash { @@ -128,12 +126,13 @@ pub async fn retrieve_data() -> Result<(), Error> { Ok::<(), Error>(()) } - .await?; + .await?; Ok::<(), Error>(()) }) - .peekable(); + } + let mut versions = version_futures.into_iter().peekable(); let mut chunk_index = 0; while versions.peek().is_some() { let now = Instant::now(); @@ -154,10 +153,7 @@ pub async fn retrieve_data() -> Result<(), Error> { "minecraft/v{}/manifest.json", daedalus::minecraft::CURRENT_FORMAT_VERSION ), - serde_json::to_vec(&*match cloned_manifest.lock() { - Ok(guard) => guard, - Err(poisoned) => poisoned.into_inner(), - })?, + serde_json::to_vec(&*cloned_manifest.lock().await)?, Some("application/json".to_string()), ) .await?; From 673658dfd24facd6745258b84e34e5ae004e6e3a Mon Sep 17 00:00:00 2001 From: Jai A Date: Tue, 19 Oct 2021 23:08:44 -0700 Subject: [PATCH 05/85] Simplify mod loader manifests, start work on new forge profiles --- .idea/discord.xml | 7 + daedalus/src/fabric.rs | 136 -------------- daedalus/src/forge.rs | 21 --- daedalus/src/lib.rs | 6 +- daedalus/src/modded.rs | 115 ++++++++++++ daedalus_client/src/fabric.rs | 309 ++++++++++++++++++++----------- daedalus_client/src/forge.rs | 249 +++++++++++++++++-------- daedalus_client/src/main.rs | 46 +++-- daedalus_client/src/minecraft.rs | 11 +- 9 files changed, 522 insertions(+), 378 deletions(-) create mode 100644 .idea/discord.xml delete mode 100644 daedalus/src/fabric.rs delete mode 100644 daedalus/src/forge.rs create mode 100644 daedalus/src/modded.rs diff --git a/.idea/discord.xml b/.idea/discord.xml new file mode 100644 index 000000000..d8e956166 --- /dev/null +++ b/.idea/discord.xml @@ -0,0 +1,7 @@ + + + + + \ No newline at end of file diff --git a/daedalus/src/fabric.rs b/daedalus/src/fabric.rs deleted file mode 100644 index d63705a73..000000000 --- a/daedalus/src/fabric.rs +++ /dev/null @@ -1,136 +0,0 @@ -use crate::minecraft::{Argument, ArgumentType, Library, VersionInfo, VersionType}; -use crate::{download_file, Error}; -use chrono::{DateTime, Utc}; -use serde::{Deserialize, Serialize}; -use std::collections::HashMap; - -/// The latest version of the format the model structs deserialize to -pub const CURRENT_FORMAT_VERSION: usize = 0; - -#[derive(Serialize, Deserialize, Debug)] -#[serde(rename_all = "camelCase")] -/// A partial version returned by fabric meta -pub struct PartialVersionInfo { - /// The version ID of the version - pub id: String, - /// The version ID this partial version inherits from - pub inherits_from: String, - /// The time that the version was released - pub release_time: DateTime, - /// The latest time a file in this version was updated - pub time: DateTime, - /// The classpath to the main class to launch the game - pub main_class: Option, - /// Arguments passed to the game or JVM - pub arguments: Option>>, - /// Libraries that the version depends on - pub libraries: Vec, - #[serde(rename = "type")] - /// The type of version - pub type_: VersionType, -} - -/// Merges a partial version into a complete one -pub fn merge_partial_version(partial: PartialVersionInfo, merge: VersionInfo) -> VersionInfo { - VersionInfo { - arguments: if let Some(partial_args) = partial.arguments { - if let Some(merge_args) = merge.arguments { - Some(partial_args.into_iter().chain(merge_args).collect()) - } else { - Some(partial_args) - } - } else { - merge.arguments - }, - asset_index: merge.asset_index, - assets: merge.assets, - downloads: merge.downloads, - id: merge.id, - libraries: partial - .libraries - .into_iter() - .chain(merge.libraries) - .collect::>(), - main_class: if let Some(main_class) = partial.main_class { - main_class - } else { - merge.main_class - }, - minecraft_arguments: merge.minecraft_arguments, - minimum_launcher_version: merge.minimum_launcher_version, - release_time: partial.release_time, - time: partial.time, - type_: partial.type_, - } -} - -/// The default servers for fabric meta -pub const FABRIC_META_URL: &str = "https://meta.fabricmc.net/v2"; - -/// Fetches the manifest of a fabric loader version and game version -pub async fn fetch_fabric_version( - version_number: &str, - loader_version: &str, -) -> Result { - Ok(serde_json::from_slice( - &download_file( - &*format!( - "{}/versions/loader/{}/{}/profile/json", - FABRIC_META_URL, version_number, loader_version - ), - None, - ) - .await?, - )?) -} - -/// Fetches the manifest of a game version's URL -pub async fn fetch_fabric_game_version(url: &str) -> Result { - Ok(serde_json::from_slice(&download_file(url, None).await?)?) -} - -#[derive(Serialize, Deserialize, Debug, Clone)] -/// Versions of fabric components -pub struct FabricVersions { - /// Versions of Minecraft that fabric supports - pub game: Vec, - /// Available versions of the fabric loader - pub loader: Vec, -} - -#[derive(Serialize, Deserialize, Debug, Clone)] -/// A version of Minecraft that fabric supports -pub struct FabricGameVersion { - /// The version number of the game - pub version: String, - /// Whether the Minecraft version is stable or not - pub stable: bool, - /// (Modrinth Provided) The URLs to download this version's profile with a loader - /// The key of the map is the loader version, and the value is the URL - pub urls: Option>, -} - -#[derive(Serialize, Deserialize, Debug, Clone)] -/// A version of the fabric loader -pub struct FabricLoaderVersion { - /// The separator to get the build number - pub separator: String, - /// The build number - pub build: u32, - /// The maven artifact - pub maven: String, - /// The version number of the fabric loader - pub version: String, - /// Whether the loader is stable or not - pub stable: bool, -} -/// Fetches the list of fabric versions -pub async fn fetch_fabric_versions(url: Option<&str>) -> Result { - Ok(serde_json::from_slice( - &download_file( - url.unwrap_or(&*format!("{}/versions", FABRIC_META_URL)), - None, - ) - .await?, - )?) -} diff --git a/daedalus/src/forge.rs b/daedalus/src/forge.rs deleted file mode 100644 index 4381f0307..000000000 --- a/daedalus/src/forge.rs +++ /dev/null @@ -1,21 +0,0 @@ -use crate::{download_file, Error}; - -use std::collections::HashMap; - -/// The latest version of the format the model structs deserialize to -pub const CURRENT_FORMAT_VERSION: usize = 0; - -const DEFAULT_MAVEN_METADATA_URL: &str = - "https://files.minecraftforge.net/net/minecraftforge/forge/maven-metadata.json"; - -/// Fetches the forge maven metadata from the specified URL. If no URL is specified, the default is used. -/// Returns a hashmap specifying the versions of the forge mod loader -/// The hashmap key is a Minecraft version, and the value is the loader versions that work on -/// the specified Minecraft version -pub async fn fetch_maven_metadata( - url: Option<&str>, -) -> Result>, Error> { - Ok(serde_json::from_slice( - &download_file(url.unwrap_or(DEFAULT_MAVEN_METADATA_URL), None).await?, - )?) -} diff --git a/daedalus/src/lib.rs b/daedalus/src/lib.rs index 4f3147553..a4ac00599 100644 --- a/daedalus/src/lib.rs +++ b/daedalus/src/lib.rs @@ -4,12 +4,10 @@ #![warn(missing_docs, unused_import_braces, missing_debug_implementations)] -/// Models and methods for fetching metadata for the Fabric mod loader -pub mod fabric; -/// Models and methods for fetching metadata for the Forge mod loader -pub mod forge; /// Models and methods for fetching metadata for Minecraft pub mod minecraft; +/// Models and methods for fetching metadata for Minecraft mod loaders +pub mod modded; #[derive(thiserror::Error, Debug)] /// An error type representing possible errors when fetching metadata diff --git a/daedalus/src/modded.rs b/daedalus/src/modded.rs new file mode 100644 index 000000000..cff051fa6 --- /dev/null +++ b/daedalus/src/modded.rs @@ -0,0 +1,115 @@ +use crate::{download_file, Error}; + +use crate::minecraft::{Argument, ArgumentType, Library, VersionInfo, VersionType}; +use chrono::{DateTime, Utc}; +use serde::{Deserialize, Serialize}; +use std::collections::HashMap; + +/// The latest version of the format the fabric model structs deserialize to +pub const CURRENT_FABRIC_FORMAT_VERSION: usize = 0; +/// The latest version of the format the fabric model structs deserialize to +pub const CURRENT_FORGE_FORMAT_VERSION: usize = 0; + +#[derive(Serialize, Deserialize, Debug)] +#[serde(rename_all = "camelCase")] +/// A partial version returned by fabric meta +pub struct PartialVersionInfo { + /// The version ID of the version + pub id: String, + /// The version ID this partial version inherits from + pub inherits_from: String, + /// The time that the version was released + pub release_time: DateTime, + /// The latest time a file in this version was updated + pub time: DateTime, + /// The classpath to the main class to launch the game + pub main_class: Option, + /// Arguments passed to the game or JVM + pub arguments: Option>>, + /// Libraries that the version depends on + pub libraries: Vec, + #[serde(rename = "type")] + /// The type of version + pub type_: VersionType, +} + +/// Fetches the version manifest of a game version's URL +pub async fn fetch_partial_version(url: &str) -> Result { + Ok(serde_json::from_slice(&download_file(url, None).await?)?) +} + +/// Merges a partial version into a complete one +pub fn merge_partial_version(partial: PartialVersionInfo, merge: VersionInfo) -> VersionInfo { + VersionInfo { + arguments: if let Some(partial_args) = partial.arguments { + if let Some(merge_args) = merge.arguments { + Some(partial_args.into_iter().chain(merge_args).collect()) + } else { + Some(partial_args) + } + } else { + merge.arguments + }, + asset_index: merge.asset_index, + assets: merge.assets, + downloads: merge.downloads, + id: merge.id, + libraries: partial + .libraries + .into_iter() + .chain(merge.libraries) + .collect::>(), + main_class: if let Some(main_class) = partial.main_class { + main_class + } else { + merge.main_class + }, + minecraft_arguments: merge.minecraft_arguments, + minimum_launcher_version: merge.minimum_launcher_version, + release_time: partial.release_time, + time: partial.time, + type_: partial.type_, + } +} + +#[derive(Serialize, Deserialize, Debug)] +#[serde(rename_all = "camelCase")] +/// A manifest containing information about a mod loader's versions +pub struct Manifest { + /// The game versions the mod loader supports + pub game_versions: Vec, +} + +#[derive(Serialize, Deserialize, Debug, Eq, PartialEq, Hash, Clone)] +#[serde(rename_all = "camelCase")] +/// The version type of the loader +pub enum LoaderType { + /// The latest type is for experimental loader versions that may not be ready for normal use + Latest, + /// The stable type is for the most stable but recent loader version. For the forge mod loader, + /// this is never used + Stable, +} + +#[derive(Serialize, Deserialize, Debug)] +/// A game version of Minecraft +pub struct Version { + /// The minecraft version ID + pub id: String, + /// A map that contains loader versions for the game version + pub loaders: HashMap, +} + +#[derive(Serialize, Deserialize, Debug)] +/// A version of a Minecraft mod loader +pub struct LoaderVersion { + /// The version ID of the loader + pub id: String, + /// The URL of the version's manifest + pub url: String, +} + +/// Fetches the manifest of a mod loader +pub async fn fetch_manifest(url: &str) -> Result { + Ok(serde_json::from_slice(&download_file(url, None).await?)?) +} diff --git a/daedalus_client/src/fabric.rs b/daedalus_client/src/fabric.rs index 809ec832e..63f637f40 100644 --- a/daedalus_client/src/fabric.rs +++ b/daedalus_client/src/fabric.rs @@ -1,33 +1,47 @@ use crate::{format_url, upload_file_to_bucket, Error}; -use daedalus::fabric::PartialVersionInfo; +use daedalus::download_file; use daedalus::minecraft::Library; +use daedalus::modded::{LoaderType, LoaderVersion, Manifest, PartialVersionInfo, Version}; +use futures::lock::Mutex; +use serde::{Deserialize, Serialize}; use std::collections::HashMap; -use std::sync::{Arc}; +use std::sync::Arc; use std::time::{Duration, Instant}; -use futures::lock::Mutex; pub async fn retrieve_data() -> Result<(), Error> { - let mut list = daedalus::fabric::fetch_fabric_versions(None).await?; + let mut list = fetch_fabric_versions(None).await?; + let old_manifest = daedalus::modded::fetch_manifest(&*format!( + "fabric/v{}/manifest.json", + daedalus::modded::CURRENT_FABRIC_FORMAT_VERSION, + )) + .await + .ok(); + + let versions = Arc::new(Mutex::new(if let Some(old_manifest) = old_manifest { + old_manifest.game_versions + } else { + Vec::new() + })); if let Some(latest) = list.loader.get(0) { - let loaders_mutex = Arc::new(Mutex::new(Vec::new())); + let loaders_mutex = Arc::new(Mutex::new(HashMap::new())); let visited_artifacts_mutex = Arc::new(Mutex::new(Vec::new())); { let mut loaders = loaders_mutex.lock().await; - loaders.push(latest.version.clone()); + loaders.insert(LoaderType::Latest, latest.version.clone()); if !latest.stable { if let Some(stable) = list.loader.iter().find(|x| x.stable) { - loaders.push(stable.version.clone()); + loaders.insert(LoaderType::Stable, stable.version.clone()); } } list.loader = list .loader .into_iter() - .filter(|x| loaders.contains(&x.version)) + .filter(|x| loaders.values().any(|val| val == &x.version)) .collect(); } @@ -35,105 +49,127 @@ pub async fn retrieve_data() -> Result<(), Error> { for game_version in list.game.iter_mut() { let visited_artifacts_mutex = Arc::clone(&visited_artifacts_mutex); - let game_version_mutex = Mutex::new(HashMap::new()); let loaders_mutex = Arc::clone(&loaders_mutex); + + let versions_mutex = Arc::clone(&versions); version_futures.push(async move { - let versions = futures::future::try_join_all(loaders_mutex.lock().await.clone().into_iter().map( - |loader| async { - let version = daedalus::fabric::fetch_fabric_version( - &*game_version.version, - &*loader, - ) - .await - .expect(&*format!("{}, {}", game_version.version, loader)); - - Ok::<(String, PartialVersionInfo), Error>((loader, version)) - }, - )) - .await?; + let loader_version_mutex = Mutex::new(HashMap::new()); - futures::future::try_join_all(versions.into_iter().map( - |(loader, version)| async { - let libs = futures::future::try_join_all( - version.libraries.into_iter().map(|mut lib| async { + let versions = + futures::future::try_join_all( + loaders_mutex.lock().await.clone().into_iter().map( + |(type_, loader)| async { { - let mut visited_assets = visited_artifacts_mutex.lock().await; + if versions_mutex.lock().await.iter().any(|x| { + x.id == game_version.version + && x.loaders + .get(&type_) + .map(|x| x.id == loader) + .unwrap_or(false) + }) { + return Ok(None); + } + } - if visited_assets.contains(&lib.name) { - lib.url = Some(format_url("maven/")); + let version = + fetch_fabric_version(&*game_version.version, &*loader).await?; - return Ok(lib); - } else { - visited_assets.push(lib.name.clone()) - } + Ok::, Error>(Some( + (type_, loader, version), + )) + }, + ), + ) + .await? + .into_iter() + .flatten(); + + futures::future::try_join_all(versions.map(|(type_, loader, version)| async { + let libs = futures::future::try_join_all(version.libraries.into_iter().map( + |mut lib| async { + { + let mut visited_assets = visited_artifacts_mutex.lock().await; + + if visited_assets.contains(&lib.name) { + lib.url = Some(format_url("maven/")); + + return Ok(lib); + } else { + visited_assets.push(lib.name.clone()) } + } + + let artifact_path = daedalus::get_path_from_artifact(&*lib.name)?; + + let artifact = daedalus::download_file( + &*format!( + "{}{}", + lib.url.unwrap_or_else(|| { + "https://maven.fabricmc.net/".to_string() + }), + artifact_path + ), + None, + ) + .await?; + + lib.url = Some(format_url("maven/")); + + upload_file_to_bucket( + format!("{}/{}", "maven", artifact_path), + artifact.to_vec(), + Some("application/java-archive".to_string()), + ) + .await?; - let artifact_path = - daedalus::get_path_from_artifact(&*lib.name)?; - - let artifact = daedalus::download_file( - &*format!( - "{}{}", - lib.url.unwrap_or_else(|| { - "https://maven.fabricmc.net/".to_string() - }), - artifact_path - ), - None, - ) - .await?; - - lib.url = Some(format_url("maven/")); - - upload_file_to_bucket( - format!("{}/{}", "maven", artifact_path), - artifact.to_vec(), - Some("application/java-archive".to_string()), - ) - .await?; - - Ok::(lib) - }), - ) - .await?; - - let version_path = format!( - "fabric/v{}/versions/{}-{}.json", - daedalus::fabric::CURRENT_FORMAT_VERSION, - version.inherits_from, - loader + Ok::(lib) + }, + )) + .await?; + + let version_path = format!( + "fabric/v{}/versions/{}-{}.json", + daedalus::modded::CURRENT_FABRIC_FORMAT_VERSION, + version.inherits_from, + loader + ); + + upload_file_to_bucket( + version_path.clone(), + serde_json::to_vec(&PartialVersionInfo { + arguments: version.arguments, + id: version.id, + main_class: version.main_class, + release_time: version.release_time, + time: version.time, + type_: version.type_, + inherits_from: version.inherits_from, + libraries: libs, + })?, + Some("application/json".to_string()), + ) + .await?; + + { + let mut loader_version_map = loader_version_mutex.lock().await; + loader_version_map.insert( + type_, + LoaderVersion { + id: loader, + url: format_url(&*version_path), + }, ); + } - upload_file_to_bucket( - version_path.clone(), - serde_json::to_vec(&PartialVersionInfo { - arguments: version.arguments, - id: version.id, - main_class: version.main_class, - release_time: version.release_time, - time: version.time, - type_: version.type_, - inherits_from: version.inherits_from, - libraries: libs, - })?, - Some("application/json".to_string()), - ) - .await?; - - { - let mut game_version_map = game_version_mutex.lock().await; - game_version_map.insert(loader, format_url(&*version_path)); - } - - Ok::<(), Error>(()) - }, - )) + Ok::<(), Error>(()) + })) .await?; - game_version.urls = Some( - game_version_mutex.lock().await - .clone(), - ); + let mut versions = versions_mutex.lock().await; + versions.push(Version { + id: game_version.version.clone(), + loaders: loader_version_mutex.into_inner(), + }); Ok::<(), Error>(()) }); @@ -156,15 +192,80 @@ pub async fn retrieve_data() -> Result<(), Error> { } } - upload_file_to_bucket( - format!( - "fabric/v{}/manifest.json", - daedalus::fabric::CURRENT_FORMAT_VERSION, - ), - serde_json::to_vec(&list)?, - Some("application/json".to_string()), - ) - .await?; + if let Ok(versions) = Arc::try_unwrap(versions) { + upload_file_to_bucket( + format!( + "fabric/v{}/manifest.json", + daedalus::modded::CURRENT_FABRIC_FORMAT_VERSION, + ), + serde_json::to_vec(&Manifest { + game_versions: versions.into_inner(), + })?, + Some("application/json".to_string()), + ) + .await?; + } Ok(()) } + +const FABRIC_META_URL: &str = "https://meta.fabricmc.net/v2"; + +async fn fetch_fabric_version( + version_number: &str, + loader_version: &str, +) -> Result { + Ok(serde_json::from_slice( + &download_file( + &*format!( + "{}/versions/loader/{}/{}/profile/json", + FABRIC_META_URL, version_number, loader_version + ), + None, + ) + .await?, + )?) +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +/// Versions of fabric components +struct FabricVersions { + /// Versions of Minecraft that fabric supports + pub game: Vec, + /// Available versions of the fabric loader + pub loader: Vec, +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +/// A version of Minecraft that fabric supports +struct FabricGameVersion { + /// The version number of the game + pub version: String, + /// Whether the Minecraft version is stable or not + pub stable: bool, +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +/// A version of the fabric loader +struct FabricLoaderVersion { + /// The separator to get the build number + pub separator: String, + /// The build number + pub build: u32, + /// The maven artifact + pub maven: String, + /// The version number of the fabric loader + pub version: String, + /// Whether the loader is stable or not + pub stable: bool, +} +/// Fetches the list of fabric versions +async fn fetch_fabric_versions(url: Option<&str>) -> Result { + Ok(serde_json::from_slice( + &download_file( + url.unwrap_or(&*format!("{}/versions", FABRIC_META_URL)), + None, + ) + .await?, + )?) +} diff --git a/daedalus_client/src/forge.rs b/daedalus_client/src/forge.rs index 61dd1fc1b..f06d23b00 100644 --- a/daedalus_client/src/forge.rs +++ b/daedalus_client/src/forge.rs @@ -1,60 +1,39 @@ use crate::{format_url, upload_file_to_bucket, Error}; -use semver::{VersionReq, Version}; -use lazy_static::lazy_static; +use chrono::{DateTime, Utc}; use daedalus::download_file; +use daedalus::minecraft::{Argument, ArgumentType, Library, VersionType}; +use daedalus::modded::{LoaderType, LoaderVersion, Manifest, PartialVersionInfo}; +use lazy_static::lazy_static; +use semver::{Version, VersionReq}; +use serde::{Deserialize, Serialize}; +use std::collections::HashMap; use std::io::Read; -use tokio::sync::{Mutex}; -use std::sync::{Arc}; -use daedalus::minecraft::{Library, VersionType, ArgumentType, Argument}; -use chrono::{DateTime, Utc}; -use serde::{Serialize, Deserialize}; -use daedalus::fabric::PartialVersionInfo; -use std::time::{Instant, Duration}; - -#[derive(Serialize, Deserialize, Debug)] -#[serde(rename_all = "camelCase")] -struct ForgeInstallerProfileInstallDataV1 { - pub mirror_list: String, - pub target: String, - /// Path to the Forge universal library - pub file_path: String, - pub logo: String, - pub welcome: String, - pub version: String, - /// Maven coordinates of the Forge universal library - pub path: String, - pub profile_name: String, - pub minecraft: String, -} - -#[derive(Serialize, Deserialize, Debug)] -#[serde(rename_all = "camelCase")] -struct ForgeInstallerProfileManifestV1 { - pub id: String, - pub libraries: Vec, - pub main_class: Option, - pub minecraft_arguments: Option, - pub release_time: DateTime, - pub time: DateTime, - pub type_: VersionType, - pub assets: Option, - pub inherits_from: Option, - pub jar: Option, -} - -#[derive(Serialize, Deserialize, Debug)] -#[serde(rename_all = "camelCase")] -struct ForgeInstallerProfileV1 { - pub install: ForgeInstallerProfileInstallDataV1, - pub version_info: ForgeInstallerProfileManifestV1, -} +use std::sync::Arc; +use std::time::{Duration, Instant}; +use tokio::sync::Mutex; lazy_static! { - static ref FORGE_MANIFEST_V1_QUERY: VersionReq = VersionReq::parse(">=8.0.684, <23.5.2851").unwrap(); + static ref FORGE_MANIFEST_V1_QUERY: VersionReq = + VersionReq::parse(">=8.0.684, <23.5.2851").unwrap(); + static ref FORGE_MANIFEST_V2_QUERY: VersionReq = + VersionReq::parse(">=23.5.2851, <37.0.0").unwrap(); + static ref FORGE_MANIFEST_V3_QUERY: VersionReq = VersionReq::parse(">=37.0.0").unwrap(); } pub async fn retrieve_data() -> Result<(), Error> { - let maven_metadata = daedalus::forge::fetch_maven_metadata(None).await?; + let maven_metadata = fetch_maven_metadata(None).await?; + let old_manifest = daedalus::modded::fetch_manifest(&*format!( + "forge/v{}/manifest.json", + daedalus::modded::CURRENT_FORGE_FORMAT_VERSION, + )) + .await + .ok(); + + let versions = Arc::new(Mutex::new(if let Some(old_manifest) = old_manifest { + old_manifest.game_versions + } else { + Vec::new() + })); let visited_assets_mutex = Arc::new(Mutex::new(Vec::new())); @@ -69,8 +48,8 @@ pub async fn retrieve_data() -> Result<(), Error> { // Most of this is a hack anyways :( // Works for all forge versions! let split = loader_version_raw.split('.').collect::>(); - let loader_version =if split.len() >= 4 { - if split[0].parse::().unwrap() < 6 { + let loader_version = if split.len() >= 4 { + if split[0].parse::().unwrap_or(0) < 6 { format!("{}.{}.{}", split[0], split[1], split[3]) } else { format!("{}.{}.{}", split[1], split[2], split[3]) @@ -79,31 +58,46 @@ pub async fn retrieve_data() -> Result<(), Error> { loader_version_raw.to_string() }; - if FORGE_MANIFEST_V1_QUERY.matches(&Version::parse(&*loader_version).unwrap()) { - version_futures.push(async { - let visited_assets = Arc::clone(&visited_assets_mutex); - async move { - println!("installer start {}", loader_version_full.clone()); - let bytes = download_file(&*format!("https://maven.minecraftforge.net/net/minecraftforge/forge/{0}/forge-{0}-installer.jar", loader_version_full), None).await.unwrap(); + let version = Version::parse(&*loader_version)?; + + version_futures.push(async { + let versions_mutex = Arc::clone(&versions); + let visited_assets = Arc::clone(&visited_assets_mutex); + async move { + { + if versions_mutex.lock().await.iter().any(|x| { + x.id == minecraft_version + && x.loaders + .get(&LoaderType::Latest) + .map(|x| x.id == loader_version_full) + .unwrap_or(false) + }) { + return Ok(()); + } + } + + println!("installer start {}", loader_version_full.clone()); + let bytes = download_file(&*format!("https://maven.minecraftforge.net/net/minecraftforge/forge/{0}/forge-{0}-installer.jar", loader_version_full), None).await?; - let reader = std::io::Cursor::new(&*bytes); + let reader = std::io::Cursor::new(&*bytes); - if let Ok(mut archive) = zip::ZipArchive::new(reader) { + if let Ok(mut archive) = zip::ZipArchive::new(reader) { + if FORGE_MANIFEST_V1_QUERY.matches(&version) { let install_profile = { - let mut install_profile = archive.by_name("install_profile.json").unwrap(); + let mut install_profile = archive.by_name("install_profile.json")?; let mut contents = String::new(); - install_profile.read_to_string(&mut contents).unwrap(); + install_profile.read_to_string(&mut contents)?; contents }; - let profile = serde_json::from_str::(&*install_profile).unwrap(); + let profile = serde_json::from_str::(&*install_profile)?; let forge_universal_bytes = { - let mut forge_universal_file = archive.by_name(&*profile.install.file_path).unwrap(); + let mut forge_universal_file = archive.by_name(&*profile.install.file_path)?; let mut forge_universal = Vec::new(); - forge_universal_file.read_to_end(&mut forge_universal).unwrap(); + forge_universal_file.read_to_end(&mut forge_universal)?; bytes::Bytes::from(forge_universal) }; @@ -168,7 +162,7 @@ pub async fn retrieve_data() -> Result<(), Error> { let version_path = format!( "forge/v{}/versions/{}.json", - daedalus::forge::CURRENT_FORMAT_VERSION, + daedalus::modded::CURRENT_FORGE_FORMAT_VERSION, new_profile.id ); @@ -177,34 +171,127 @@ pub async fn retrieve_data() -> Result<(), Error> { serde_json::to_vec(&new_profile)?, Some("application/json".to_string()), ).await?; - } + let mut map = HashMap::new(); + map.insert(LoaderType::Latest, LoaderVersion { + id: loader_version_full, + url: format_url(&*version_path) + }); + versions_mutex.lock().await.push(daedalus::modded::Version { + id: minecraft_version, + loaders: map + }) + } else if FORGE_MANIFEST_V2_QUERY.matches(&version) { + let install_profile = { + let mut install_profile = archive.by_name("install_profile.json")?; + + let mut contents = String::new(); + install_profile.read_to_string(&mut contents)?; + + contents + }; + } else if FORGE_MANIFEST_V3_QUERY.matches(&version) { - Ok::<(), Error>(()) - }.await?; + } + } Ok::<(), Error>(()) - }); - } + }.await?; + + Ok::<(), Error>(()) + }); } } } - let mut versions = version_futures.into_iter().peekable(); - let mut chunk_index = 0; - while versions.peek().is_some() { - let now = Instant::now(); + { + let mut versions_peek = version_futures.into_iter().peekable(); + let mut chunk_index = 0; + while versions_peek.peek().is_some() { + let now = Instant::now(); - let chunk: Vec<_> = versions.by_ref().take(100).collect(); - futures::future::try_join_all(chunk).await?; + let chunk: Vec<_> = versions_peek.by_ref().take(100).collect(); + futures::future::try_join_all(chunk).await?; - std::thread::sleep(Duration::from_secs(1)); + std::thread::sleep(Duration::from_secs(1)); - chunk_index += 1; + chunk_index += 1; - let elapsed = now.elapsed(); - println!("Chunk {} Elapsed: {:.2?}", chunk_index, elapsed); + let elapsed = now.elapsed(); + println!("Chunk {} Elapsed: {:.2?}", chunk_index, elapsed); + } + } + + if let Ok(versions) = Arc::try_unwrap(versions) { + upload_file_to_bucket( + format!( + "forge/v{}/manifest.json", + daedalus::modded::CURRENT_FORGE_FORMAT_VERSION, + ), + serde_json::to_vec(&Manifest { + game_versions: versions.into_inner(), + })?, + Some("application/json".to_string()), + ) + .await?; } Ok(()) -} \ No newline at end of file +} + +const DEFAULT_MAVEN_METADATA_URL: &str = + "https://files.minecraftforge.net/net/minecraftforge/forge/maven-metadata.json"; + +/// Fetches the forge maven metadata from the specified URL. If no URL is specified, the default is used. +/// Returns a hashmap specifying the versions of the forge mod loader +/// The hashmap key is a Minecraft version, and the value is the loader versions that work on +/// the specified Minecraft version +pub async fn fetch_maven_metadata( + url: Option<&str>, +) -> Result>, Error> { + Ok(serde_json::from_slice( + &download_file(url.unwrap_or(DEFAULT_MAVEN_METADATA_URL), None).await?, + )?) +} + +#[derive(Serialize, Deserialize, Debug)] +#[serde(rename_all = "camelCase")] +struct ForgeInstallerProfileInstallDataV1 { + pub mirror_list: String, + pub target: String, + /// Path to the Forge universal library + pub file_path: String, + pub logo: String, + pub welcome: String, + pub version: String, + /// Maven coordinates of the Forge universal library + pub path: String, + pub profile_name: String, + pub minecraft: String, +} + +#[derive(Serialize, Deserialize, Debug)] +#[serde(rename_all = "camelCase")] +struct ForgeInstallerProfileManifestV1 { + pub id: String, + pub libraries: Vec, + pub main_class: Option, + pub minecraft_arguments: Option, + pub release_time: DateTime, + pub time: DateTime, + pub type_: VersionType, + pub assets: Option, + pub inherits_from: Option, + pub jar: Option, +} + +#[derive(Serialize, Deserialize, Debug)] +#[serde(rename_all = "camelCase")] +struct ForgeInstallerProfileV1 { + pub install: ForgeInstallerProfileInstallDataV1, + pub version_info: ForgeInstallerProfileManifestV1, +} + +#[derive(Serialize, Deserialize, Debug)] +#[serde(rename_all = "camelCase")] +struct ForgeInstallerProfileV2 {} diff --git a/daedalus_client/src/main.rs b/daedalus_client/src/main.rs index 321ef10b3..14f4fb133 100644 --- a/daedalus_client/src/main.rs +++ b/daedalus_client/src/main.rs @@ -6,8 +6,8 @@ use rusoto_s3::{PutObjectRequest, S3}; use std::time::Duration; mod fabric; -mod minecraft; mod forge; +mod minecraft; #[derive(thiserror::Error, Debug)] pub enum Error { @@ -24,6 +24,12 @@ pub enum Error { inner: RusotoError, file: String, }, + #[error("Error while parsing version as semver: {0}")] + SemVerError(#[from] semver::Error), + #[error("Error while reading zip file: {0}")] + ZipError(#[from] zip::result::ZipError), + #[error("Error while reading zip file: {0}")] + IoError(#[from] std::io::Error), } #[tokio::main] @@ -38,30 +44,20 @@ async fn main() { loop { timer.tick().await; - tokio::spawn( - async { - match fabric::retrieve_data().await { - Ok(..) => {} - Err(err) => error!("{:?}", err) - }; - } - ); - tokio::spawn( - async { - match minecraft::retrieve_data().await { - Ok(..) => {} - Err(err) => error!("{:?}", err) - }; - } - ); - tokio::spawn( - async { - match forge::retrieve_data().await { - Ok(..) => {} - Err(err) => error!("{:?}", err) - }; - } - ); + tokio::spawn(async { + match fabric::retrieve_data().await { + Ok(..) => {} + Err(err) => error!("{:?}", err), + }; + match minecraft::retrieve_data().await { + Ok(..) => {} + Err(err) => error!("{:?}", err), + }; + match forge::retrieve_data().await { + Ok(..) => {} + Err(err) => error!("{:?}", err), + }; + }); } } diff --git a/daedalus_client/src/minecraft.rs b/daedalus_client/src/minecraft.rs index 90b58806d..07907c4f3 100644 --- a/daedalus_client/src/minecraft.rs +++ b/daedalus_client/src/minecraft.rs @@ -1,8 +1,8 @@ use crate::{format_url, upload_file_to_bucket, Error}; use daedalus::download_file; -use std::sync::{Arc}; -use std::time::{Duration, Instant}; use futures::lock::Mutex; +use std::sync::Arc; +use std::time::{Duration, Instant}; pub async fn retrieve_data() -> Result<(), Error> { let old_manifest = @@ -22,10 +22,7 @@ pub async fn retrieve_data() -> Result<(), Error> { let mut version_futures = Vec::new(); - for version in manifest - .versions - .iter_mut() - { + for version in manifest.versions.iter_mut() { version_futures.push(async { let old_version = if let Some(old_manifest) = &old_manifest { old_manifest.versions.iter().find(|x| x.id == version.id) @@ -126,7 +123,7 @@ pub async fn retrieve_data() -> Result<(), Error> { Ok::<(), Error>(()) } - .await?; + .await?; Ok::<(), Error>(()) }) From d8332a27e5c8fb4ccad70695b03534c8312010a5 Mon Sep 17 00:00:00 2001 From: Jai A Date: Sun, 24 Oct 2021 14:25:24 -0700 Subject: [PATCH 06/85] Finish newer forge versions --- .env | 2 + daedalus/src/lib.rs | 2 +- daedalus/src/modded.rs | 35 +++- daedalus_client/Cargo.toml | 1 + daedalus_client/src/fabric.rs | 4 +- daedalus_client/src/forge.rs | 339 +++++++++++++++++++++---------- daedalus_client/src/main.rs | 2 + daedalus_client/src/minecraft.rs | 2 +- 8 files changed, 272 insertions(+), 115 deletions(-) diff --git a/.env b/.env index 4e9b803ba..56db8578a 100644 --- a/.env +++ b/.env @@ -1,3 +1,5 @@ +RUST_LOG=info,error + BASE_URL=https://modrinth-cdn-staging.nyc3.digitaloceanspaces.com BASE_FOLDER=gamedata diff --git a/daedalus/src/lib.rs b/daedalus/src/lib.rs index a4ac00599..5e0f5e422 100644 --- a/daedalus/src/lib.rs +++ b/daedalus/src/lib.rs @@ -90,7 +90,7 @@ pub async fn download_file_mirrors( pub async fn download_file(url: &str, sha1: Option<&str>) -> Result { let client = reqwest::Client::builder() .tcp_keepalive(Some(std::time::Duration::from_secs(10))) - .timeout(std::time::Duration::from_secs(30)) + .timeout(std::time::Duration::from_secs(15)) .build() .map_err(|err| Error::FetchError { inner: err, diff --git a/daedalus/src/modded.rs b/daedalus/src/modded.rs index cff051fa6..c631b6193 100644 --- a/daedalus/src/modded.rs +++ b/daedalus/src/modded.rs @@ -10,6 +10,15 @@ pub const CURRENT_FABRIC_FORMAT_VERSION: usize = 0; /// The latest version of the format the fabric model structs deserialize to pub const CURRENT_FORGE_FORMAT_VERSION: usize = 0; +/// A data variable entry that depends on the side of the installation +#[derive(Serialize, Deserialize, Debug)] +pub struct SidedDataEntry { + /// The value on the client + pub client: String, + /// The value on the server + pub server: String, +} + #[derive(Serialize, Deserialize, Debug)] #[serde(rename_all = "camelCase")] /// A partial version returned by fabric meta @@ -31,6 +40,26 @@ pub struct PartialVersionInfo { #[serde(rename = "type")] /// The type of version pub type_: VersionType, + /// (Forge-only) + pub data: Option>, + /// (Forge-only) The list of processors to run after downloading the files + pub processors: Option>, +} + +/// A processor to be ran after downloading the files +#[derive(Serialize, Deserialize, Debug)] +pub struct Processor { + /// Maven coordinates for the JAR library of this processor. + pub jar: String, + /// Maven coordinates for all the libraries that must be included in classpath when running this processor. + pub classpath: Vec, + /// Arguments for this processor. + pub args: Vec, + /// Represents a map of outputs. Keys and values can be data values + pub outputs: Option>, + /// Which sides this processor shall be ran on. + /// Valid values: client, server, extract + pub sides: Option>, } /// Fetches the version manifest of a game version's URL @@ -72,7 +101,7 @@ pub fn merge_partial_version(partial: PartialVersionInfo, merge: VersionInfo) -> } } -#[derive(Serialize, Deserialize, Debug)] +#[derive(Serialize, Deserialize, Debug, Clone)] #[serde(rename_all = "camelCase")] /// A manifest containing information about a mod loader's versions pub struct Manifest { @@ -91,7 +120,7 @@ pub enum LoaderType { Stable, } -#[derive(Serialize, Deserialize, Debug)] +#[derive(Serialize, Deserialize, Debug, Clone)] /// A game version of Minecraft pub struct Version { /// The minecraft version ID @@ -100,7 +129,7 @@ pub struct Version { pub loaders: HashMap, } -#[derive(Serialize, Deserialize, Debug)] +#[derive(Serialize, Deserialize, Debug, Clone)] /// A version of a Minecraft mod loader pub struct LoaderVersion { /// The version ID of the loader diff --git a/daedalus_client/Cargo.toml b/daedalus_client/Cargo.toml index 33f1c0638..f36f6f3b1 100644 --- a/daedalus_client/Cargo.toml +++ b/daedalus_client/Cargo.toml @@ -12,6 +12,7 @@ tokio = { version = "1", features = ["full"] } futures = "0.3.17" dotenv = "0.15.0" log = "0.4.8" +env_logger="0.9.0" serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" lazy_static = "1.4.0" diff --git a/daedalus_client/src/fabric.rs b/daedalus_client/src/fabric.rs index 63f637f40..5ac71f16f 100644 --- a/daedalus_client/src/fabric.rs +++ b/daedalus_client/src/fabric.rs @@ -145,6 +145,8 @@ pub async fn retrieve_data() -> Result<(), Error> { type_: version.type_, inherits_from: version.inherits_from, libraries: libs, + processors: None, + data: None })?, Some("application/json".to_string()), ) @@ -183,7 +185,7 @@ pub async fn retrieve_data() -> Result<(), Error> { let chunk: Vec<_> = versions.by_ref().take(10).collect(); futures::future::try_join_all(chunk).await?; - std::thread::sleep(Duration::from_secs(1)); + tokio::time::sleep(Duration::from_secs(1)).await; chunk_index += 1; diff --git a/daedalus_client/src/forge.rs b/daedalus_client/src/forge.rs index f06d23b00..af31f015f 100644 --- a/daedalus_client/src/forge.rs +++ b/daedalus_client/src/forge.rs @@ -2,7 +2,7 @@ use crate::{format_url, upload_file_to_bucket, Error}; use chrono::{DateTime, Utc}; use daedalus::download_file; use daedalus::minecraft::{Argument, ArgumentType, Library, VersionType}; -use daedalus::modded::{LoaderType, LoaderVersion, Manifest, PartialVersionInfo}; +use daedalus::modded::{LoaderType, LoaderVersion, Manifest, PartialVersionInfo, Processor, SidedDataEntry}; use lazy_static::lazy_static; use semver::{Version, VersionReq}; use serde::{Deserialize, Serialize}; @@ -15,17 +15,21 @@ use tokio::sync::Mutex; lazy_static! { static ref FORGE_MANIFEST_V1_QUERY: VersionReq = VersionReq::parse(">=8.0.684, <23.5.2851").unwrap(); - static ref FORGE_MANIFEST_V2_QUERY: VersionReq = - VersionReq::parse(">=23.5.2851, <37.0.0").unwrap(); + + static ref FORGE_MANIFEST_V2_QUERY_P1: VersionReq = + VersionReq::parse(">=23.5.2851, <31.2.52").unwrap(); + static ref FORGE_MANIFEST_V2_QUERY_P2: VersionReq = + VersionReq::parse(">=32.0.1, <37.0.0").unwrap(); + static ref FORGE_MANIFEST_V3_QUERY: VersionReq = VersionReq::parse(">=37.0.0").unwrap(); } pub async fn retrieve_data() -> Result<(), Error> { let maven_metadata = fetch_maven_metadata(None).await?; - let old_manifest = daedalus::modded::fetch_manifest(&*format!( + let old_manifest = daedalus::modded::fetch_manifest(&*format_url(&*format!( "forge/v{}/manifest.json", daedalus::modded::CURRENT_FORGE_FORMAT_VERSION, - )) + ))) .await .ok(); @@ -40,7 +44,9 @@ pub async fn retrieve_data() -> Result<(), Error> { let mut version_futures = Vec::new(); for (minecraft_version, loader_versions) in maven_metadata { - if let Some(loader_version_full) = loader_versions.into_iter().last() { + let mut loaders = Vec::new(); + + for loader_version_full in loader_versions { let loader_version = loader_version_full.split('-').into_iter().nth(1); if let Some(loader_version_raw) = loader_version { @@ -60,57 +66,173 @@ pub async fn retrieve_data() -> Result<(), Error> { let version = Version::parse(&*loader_version)?; - version_futures.push(async { - let versions_mutex = Arc::clone(&versions); - let visited_assets = Arc::clone(&visited_assets_mutex); - async move { - { - if versions_mutex.lock().await.iter().any(|x| { - x.id == minecraft_version - && x.loaders - .get(&LoaderType::Latest) - .map(|x| x.id == loader_version_full) - .unwrap_or(false) - }) { - return Ok(()); - } + if FORGE_MANIFEST_V1_QUERY.matches(&version) || FORGE_MANIFEST_V2_QUERY_P1.matches(&version) || FORGE_MANIFEST_V2_QUERY_P2.matches(&version) || FORGE_MANIFEST_V3_QUERY.matches(&version) { + loaders.push((loader_version_full, version)) + } + } + } + + if let Some((loader_version_full, version)) = loaders.into_iter().last() { + version_futures.push(async { + let versions_mutex = Arc::clone(&versions); + let visited_assets = Arc::clone(&visited_assets_mutex); + async move { + { + if versions_mutex.lock().await.iter().any(|x| { + x.id == minecraft_version + && x.loaders + .get(&LoaderType::Latest) + .map(|x| x.id == loader_version_full) + .unwrap_or(false) + }) { + return Ok(()); } + } + + println!("installer start {}", loader_version_full.clone()); + let bytes = download_file(&*format!("https://maven.minecraftforge.net/net/minecraftforge/forge/{0}/forge-{0}-installer.jar", loader_version_full), None).await?; + + let reader = std::io::Cursor::new(&*bytes); - println!("installer start {}", loader_version_full.clone()); - let bytes = download_file(&*format!("https://maven.minecraftforge.net/net/minecraftforge/forge/{0}/forge-{0}-installer.jar", loader_version_full), None).await?; + if let Ok(mut archive) = zip::ZipArchive::new(reader) { + if FORGE_MANIFEST_V1_QUERY.matches(&version) { + let profile = { + let mut install_profile = archive.by_name("install_profile.json")?; - let reader = std::io::Cursor::new(&*bytes); + let mut contents = String::new(); + install_profile.read_to_string(&mut contents)?; - if let Ok(mut archive) = zip::ZipArchive::new(reader) { - if FORGE_MANIFEST_V1_QUERY.matches(&version) { - let install_profile = { - let mut install_profile = archive.by_name("install_profile.json")?; + serde_json::from_str::(&*contents)? + }; - let mut contents = String::new(); - install_profile.read_to_string(&mut contents)?; + let forge_universal_bytes = { + let mut forge_universal_file = archive.by_name(&*profile.install.file_path)?; + let mut forge_universal = Vec::new(); + forge_universal_file.read_to_end(&mut forge_universal)?; - contents - }; + bytes::Bytes::from(forge_universal) + }; + let forge_universal_path = profile.install.file_path.clone(); - let profile = serde_json::from_str::(&*install_profile)?; + let now = Instant::now(); + let libs = futures::future::try_join_all(profile.version_info.libraries.into_iter().map(|mut lib| async { + if let Some(url) = lib.url { + { + let mut visited_assets = visited_assets.lock().await; - let forge_universal_bytes = { - let mut forge_universal_file = archive.by_name(&*profile.install.file_path)?; + if visited_assets.contains(&lib.name) { + lib.url = Some(format_url("maven/")); + + return Ok::(lib); + } else { + visited_assets.push(lib.name.clone()) + } + } + + let artifact_path = + daedalus::get_path_from_artifact(&*lib.name)?; + + let artifact = if lib.name == forge_universal_path { + forge_universal_bytes.clone() + } else { + let mirrors = vec![&*url, "https://maven.creeperhost.net/", "https://libraries.minecraft.net/"]; + + daedalus::download_file_mirrors( + &*artifact_path, + &mirrors, + None, + ) + .await? + }; + + lib.url = Some(format_url("maven/")); + + upload_file_to_bucket( + format!("{}/{}", "maven", artifact_path), + artifact.to_vec(), + Some("application/java-archive".to_string()), + ).await?; + } + + Ok::(lib) + })).await?; + + let elapsed = now.elapsed(); + println!("Elapsed lib DL: {:.2?}", elapsed); + + let new_profile = PartialVersionInfo { + id: profile.version_info.id, + inherits_from: profile.install.minecraft, + release_time: profile.version_info.release_time, + time: profile.version_info.time, + main_class: profile.version_info.main_class, + arguments: profile.version_info.minecraft_arguments.map(|x| [(ArgumentType::Game, x.split(' ').map(|x| Argument::Normal(x.to_string())).collect())].iter().cloned().collect()), + libraries: libs, + type_: profile.version_info.type_, + data: None, + processors: None + }; + + let version_path = format!( + "forge/v{}/versions/{}.json", + daedalus::modded::CURRENT_FORGE_FORMAT_VERSION, + new_profile.id + ); + + upload_file_to_bucket( + version_path.clone(), + serde_json::to_vec(&new_profile)?, + Some("application/json".to_string()), + ).await?; + + let mut map = HashMap::new(); + map.insert(LoaderType::Latest, LoaderVersion { + id: loader_version_full, + url: format_url(&*version_path) + }); + versions_mutex.lock().await.push(daedalus::modded::Version { + id: minecraft_version, + loaders: map + }) + } else if FORGE_MANIFEST_V2_QUERY_P1.matches(&version) || FORGE_MANIFEST_V2_QUERY_P2.matches(&version) || FORGE_MANIFEST_V3_QUERY.matches(&version) { + let profile = { + let mut install_profile = archive.by_name("install_profile.json")?; + + let mut contents = String::new(); + install_profile.read_to_string(&mut contents)?; + + serde_json::from_str::(&*contents)? + }; + + let version_info = { + let mut install_profile = archive.by_name("version.json")?; + + let mut contents = String::new(); + install_profile.read_to_string(&mut contents)?; + serde_json::from_str::(&*contents)? + }; + + let forge_universal_bytes = { + if let Some(path) = &profile.path { + let mut forge_universal_file = archive.by_name(&*format!("maven/{}", daedalus::get_path_from_artifact(&*path)?))?; let mut forge_universal = Vec::new(); forge_universal_file.read_to_end(&mut forge_universal)?; - bytes::Bytes::from(forge_universal) - }; - let forge_universal_path = profile.install.file_path.clone(); + Some(bytes::Bytes::from(forge_universal)) + } else { + None + } + }; - let now = Instant::now(); - let libs = futures::future::try_join_all(profile.version_info.libraries.into_iter().map(|mut lib| async { - if let Some(url) = lib.url { + let now = Instant::now(); + let libs = futures::future::try_join_all(profile.libraries.into_iter().chain(version_info.libraries).map(|mut lib| async { + if let Some(ref mut downloads) = lib.downloads { + if let Some(ref mut artifact) = downloads.artifact { { let mut visited_assets = visited_assets.lock().await; if visited_assets.contains(&lib.name) { - lib.url = Some(format_url("maven/")); + artifact.url = format_url(&*format!("maven/{}", artifact.path)); return Ok::(lib); } else { @@ -121,86 +243,74 @@ pub async fn retrieve_data() -> Result<(), Error> { let artifact_path = daedalus::get_path_from_artifact(&*lib.name)?; - let artifact = if lib.name == forge_universal_path { - forge_universal_bytes.clone() + let artifact_bytes = if &*artifact.url == "" { + forge_universal_bytes.clone().unwrap_or_default() } else { - let mirrors = vec![&*url, "https://maven.creeperhost.net/", "https://libraries.minecraft.net/"]; - - daedalus::download_file_mirrors( - &*artifact_path, - &mirrors, - None, + daedalus::download_file( + &*artifact.url, + Some(&*artifact.sha1), ) .await? }; - lib.url = Some(format_url("maven/")); + artifact.url = format_url(&*format!("maven/{}", artifact.path)); upload_file_to_bucket( format!("{}/{}", "maven", artifact_path), - artifact.to_vec(), + artifact_bytes.to_vec(), Some("application/java-archive".to_string()), ).await?; } - - Ok::(lib) - })).await?; - - let elapsed = now.elapsed(); - println!("Elapsed lib DL: {:.2?}", elapsed); - - let new_profile = PartialVersionInfo { - id: profile.version_info.id, - inherits_from: profile.install.minecraft, - release_time: profile.version_info.release_time, - time: profile.version_info.time, - main_class: profile.version_info.main_class, - arguments: profile.version_info.minecraft_arguments.map(|x| [(ArgumentType::Game, x.split(' ').map(|x| Argument::Normal(x.to_string())).collect())].iter().cloned().collect()), - libraries: libs, - type_: profile.version_info.type_, - }; - - let version_path = format!( - "forge/v{}/versions/{}.json", - daedalus::modded::CURRENT_FORGE_FORMAT_VERSION, - new_profile.id - ); - - upload_file_to_bucket( - version_path.clone(), - serde_json::to_vec(&new_profile)?, - Some("application/json".to_string()), - ).await?; - - let mut map = HashMap::new(); - map.insert(LoaderType::Latest, LoaderVersion { - id: loader_version_full, - url: format_url(&*version_path) - }); - versions_mutex.lock().await.push(daedalus::modded::Version { - id: minecraft_version, - loaders: map - }) - } else if FORGE_MANIFEST_V2_QUERY.matches(&version) { - let install_profile = { - let mut install_profile = archive.by_name("install_profile.json")?; - - let mut contents = String::new(); - install_profile.read_to_string(&mut contents)?; - - contents - }; - } else if FORGE_MANIFEST_V3_QUERY.matches(&version) { - - } + } + + Ok::(lib) + })).await?; + + let elapsed = now.elapsed(); + println!("Elapsed lib DL: {:.2?}", elapsed); + + let new_profile = PartialVersionInfo { + id: version_info.id, + inherits_from: version_info.inherits_from, + release_time: version_info.release_time, + time: version_info.time, + main_class: version_info.main_class, + arguments: version_info.arguments, + libraries: libs, + type_: version_info.type_, + data: Some(profile.data), + processors: Some(profile.processors), + }; + + let version_path = format!( + "forge/v{}/versions/{}.json", + daedalus::modded::CURRENT_FORGE_FORMAT_VERSION, + new_profile.id + ); + + upload_file_to_bucket( + version_path.clone(), + serde_json::to_vec(&new_profile)?, + Some("application/json".to_string()), + ).await?; + + let mut map = HashMap::new(); + map.insert(LoaderType::Latest, LoaderVersion { + id: loader_version_full, + url: format_url(&*version_path) + }); + versions_mutex.lock().await.push(daedalus::modded::Version { + id: minecraft_version, + loaders: map + }) } - - Ok::<(), Error>(()) - }.await?; + } Ok::<(), Error>(()) - }); - } + }.await?; + + Ok::<(), Error>(()) + }); } } @@ -208,12 +318,13 @@ pub async fn retrieve_data() -> Result<(), Error> { let mut versions_peek = version_futures.into_iter().peekable(); let mut chunk_index = 0; while versions_peek.peek().is_some() { + println!("Chunk {} Start", chunk_index); let now = Instant::now(); let chunk: Vec<_> = versions_peek.by_ref().take(100).collect(); futures::future::try_join_all(chunk).await?; - std::thread::sleep(Duration::from_secs(1)); + tokio::time::sleep(Duration::from_secs(1)).await; chunk_index += 1; @@ -294,4 +405,14 @@ struct ForgeInstallerProfileV1 { #[derive(Serialize, Deserialize, Debug)] #[serde(rename_all = "camelCase")] -struct ForgeInstallerProfileV2 {} +struct ForgeInstallerProfileV2 { + pub spec: i32, + pub profile: String, + pub version: String, + pub json: String, + pub path: Option, + pub minecraft: String, + pub data: HashMap, + pub libraries: Vec, + pub processors: Vec, +} \ No newline at end of file diff --git a/daedalus_client/src/main.rs b/daedalus_client/src/main.rs index 14f4fb133..7d0511422 100644 --- a/daedalus_client/src/main.rs +++ b/daedalus_client/src/main.rs @@ -34,6 +34,8 @@ pub enum Error { #[tokio::main] async fn main() { + env_logger::init(); + if check_env_vars() { error!("Some environment variables are missing!"); diff --git a/daedalus_client/src/minecraft.rs b/daedalus_client/src/minecraft.rs index 07907c4f3..c6062dbb2 100644 --- a/daedalus_client/src/minecraft.rs +++ b/daedalus_client/src/minecraft.rs @@ -137,7 +137,7 @@ pub async fn retrieve_data() -> Result<(), Error> { let chunk: Vec<_> = versions.by_ref().take(100).collect(); futures::future::try_join_all(chunk).await?; - std::thread::sleep(Duration::from_secs(1)); + tokio::time::sleep(Duration::from_secs(1)).await; chunk_index += 1; From 5218543c5864b22f6f002cbf84bed0eaba39d37b Mon Sep 17 00:00:00 2001 From: Jai A Date: Sun, 24 Oct 2021 16:10:38 -0700 Subject: [PATCH 07/85] Add GitHub Actions --- .github/workflows/clippy.yml | 0 .github/workflows/docker.yml | 0 .github/workflows/publish.yml | 0 .github/workflows/rust.yml | 0 Dockerfile | 0 5 files changed, 0 insertions(+), 0 deletions(-) create mode 100644 .github/workflows/clippy.yml create mode 100644 .github/workflows/docker.yml create mode 100644 .github/workflows/publish.yml create mode 100644 .github/workflows/rust.yml create mode 100644 Dockerfile diff --git a/.github/workflows/clippy.yml b/.github/workflows/clippy.yml new file mode 100644 index 000000000..e69de29bb diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml new file mode 100644 index 000000000..e69de29bb diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml new file mode 100644 index 000000000..e69de29bb diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml new file mode 100644 index 000000000..e69de29bb diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 000000000..e69de29bb From 4294081abbba4e5630d59d6b7cbd02a5fc649c24 Mon Sep 17 00:00:00 2001 From: Jai A Date: Sun, 24 Oct 2021 16:13:04 -0700 Subject: [PATCH 08/85] Add GitHub actions again --- .github/workflows/clippy.yml | 19 +++++++++++++++++++ .github/workflows/docker.yml | 33 +++++++++++++++++++++++++++++++++ .github/workflows/publish.yml | 30 ++++++++++++++++++++++++++++++ .github/workflows/rust.yml | 29 +++++++++++++++++++++++++++++ Dockerfile | 7 +++++++ 5 files changed, 118 insertions(+) diff --git a/.github/workflows/clippy.yml b/.github/workflows/clippy.yml index e69de29bb..36b9ad180 100644 --- a/.github/workflows/clippy.yml +++ b/.github/workflows/clippy.yml @@ -0,0 +1,19 @@ +name: Code quality + +on: + push: + tags: + - 'v*' +env: + CARGO_TERM_COLOR: always +jobs: + publish: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v1 + - run: cargo login ${CRATES_IO_TOKEN} + working-directory: ./daedalus + env: + CRATES_IO_TOKEN: ${{ secrets.CRATES_IO_TOKEN }} + - run: cargo publish + working-directory: ./daedalus diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index e69de29bb..0b7a22b14 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -0,0 +1,33 @@ +name: docker-build + +on: + push: + branches: + - '**' + tags: + - 'v*' + pull_request: + +jobs: + docker: + runs-on: ubuntu-latest + steps: + - + name: Set up QEMU + uses: docker/setup-qemu-action@v1 + - + name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + - + name: Login to GitHub Images + uses: docker/login-action@v1 + with: + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - + name: Build and push + id: docker_build + uses: docker/build-push-action@v2 + with: + push: ${{ github.event_name != 'pull_request' }} + tags: ghcr.io/modrinth/daedalus:latest diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index e69de29bb..d22210e6b 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -0,0 +1,30 @@ +name: Publish to crates.io + +on: + push: + branches: [ master ] + pull_request: +env: + CARGO_TERM_COLOR: always +jobs: + lint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Install toolchain + uses: actions-rs/toolchain@v1 + with: + toolchain: stable + override: true + components: rustfmt, clippy + - name: Cache build artifacts + id: cache-build + uses: actions/cache@v2 + with: + path: target/** + key: ${{ runner.os }}-build-cache + - name: Annotate commit with clippy warnings + uses: actions-rs/clippy-check@v1 + with: + token: ${{ secrets.GITHUB_TOKEN }} + args: --all-features diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index e69de29bb..a55aaf6e1 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -0,0 +1,29 @@ +name: Rust building + +on: + push: + branches: [ master ] + pull_request: +env: + CARGO_TERM_COLOR: always +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Get build cache + id: cache-build + uses: actions/cache@v2 + with: + path: target/** + key: ${{ runner.os }}-build-cache + - name: Install toolchain + uses: actions-rs/toolchain@v1 + with: + toolchain: stable + env: + SQLX_OFFLINE: true + - uses: actions-rs/cargo@v1 + name: Build program + with: + command: build diff --git a/Dockerfile b/Dockerfile index e69de29bb..c98c954f0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -0,0 +1,7 @@ +FROM rust:1.56 + +COPY ./ ./ + +RUN cargo build --release + +CMD ["./target/release/daedalus_client"] \ No newline at end of file From ecdfd65f50d534beb3eb377e590e28be35eb93a1 Mon Sep 17 00:00:00 2001 From: Jai A Date: Sun, 24 Oct 2021 16:16:23 -0700 Subject: [PATCH 09/85] Fix incorrect file names --- .github/workflows/clippy.yml | 19 ------------------- .github/workflows/lint.yml | 30 ++++++++++++++++++++++++++++++ .github/workflows/publish.yml | 33 +++++++++++---------------------- 3 files changed, 41 insertions(+), 41 deletions(-) delete mode 100644 .github/workflows/clippy.yml create mode 100644 .github/workflows/lint.yml diff --git a/.github/workflows/clippy.yml b/.github/workflows/clippy.yml deleted file mode 100644 index 36b9ad180..000000000 --- a/.github/workflows/clippy.yml +++ /dev/null @@ -1,19 +0,0 @@ -name: Code quality - -on: - push: - tags: - - 'v*' -env: - CARGO_TERM_COLOR: always -jobs: - publish: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v1 - - run: cargo login ${CRATES_IO_TOKEN} - working-directory: ./daedalus - env: - CRATES_IO_TOKEN: ${{ secrets.CRATES_IO_TOKEN }} - - run: cargo publish - working-directory: ./daedalus diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml new file mode 100644 index 000000000..d22210e6b --- /dev/null +++ b/.github/workflows/lint.yml @@ -0,0 +1,30 @@ +name: Publish to crates.io + +on: + push: + branches: [ master ] + pull_request: +env: + CARGO_TERM_COLOR: always +jobs: + lint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Install toolchain + uses: actions-rs/toolchain@v1 + with: + toolchain: stable + override: true + components: rustfmt, clippy + - name: Cache build artifacts + id: cache-build + uses: actions/cache@v2 + with: + path: target/** + key: ${{ runner.os }}-build-cache + - name: Annotate commit with clippy warnings + uses: actions-rs/clippy-check@v1 + with: + token: ${{ secrets.GITHUB_TOKEN }} + args: --all-features diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index d22210e6b..36b9ad180 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -1,30 +1,19 @@ -name: Publish to crates.io +name: Code quality on: push: - branches: [ master ] - pull_request: + tags: + - 'v*' env: CARGO_TERM_COLOR: always jobs: - lint: + publish: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 - - name: Install toolchain - uses: actions-rs/toolchain@v1 - with: - toolchain: stable - override: true - components: rustfmt, clippy - - name: Cache build artifacts - id: cache-build - uses: actions/cache@v2 - with: - path: target/** - key: ${{ runner.os }}-build-cache - - name: Annotate commit with clippy warnings - uses: actions-rs/clippy-check@v1 - with: - token: ${{ secrets.GITHUB_TOKEN }} - args: --all-features + - uses: actions/checkout@v1 + - run: cargo login ${CRATES_IO_TOKEN} + working-directory: ./daedalus + env: + CRATES_IO_TOKEN: ${{ secrets.CRATES_IO_TOKEN }} + - run: cargo publish + working-directory: ./daedalus From e36a191240187bab3c17574efb5ba20b8fb1d879 Mon Sep 17 00:00:00 2001 From: Jai A Date: Sun, 24 Oct 2021 16:16:49 -0700 Subject: [PATCH 10/85] Fix incorrect file names (again) --- .github/workflows/clippy.yml~ | 0 .github/workflows/docker.yml | 1 - .github/workflows/publish.yml | 2 +- 3 files changed, 1 insertion(+), 2 deletions(-) create mode 100644 .github/workflows/clippy.yml~ diff --git a/.github/workflows/clippy.yml~ b/.github/workflows/clippy.yml~ new file mode 100644 index 000000000..e69de29bb diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 0b7a22b14..e10d87942 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -7,7 +7,6 @@ on: tags: - 'v*' pull_request: - jobs: docker: runs-on: ubuntu-latest diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 36b9ad180..9e224e2ae 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -1,4 +1,4 @@ -name: Code quality +name: Publish to crates.io on: push: From 3c5edb6171d70b8ce47bb31920db7e7664bf12b3 Mon Sep 17 00:00:00 2001 From: Geometrically <18202329+Geometrically@users.noreply.github.com> Date: Sun, 24 Oct 2021 16:18:17 -0700 Subject: [PATCH 11/85] Delete clippy.yml~ --- .github/workflows/clippy.yml~ | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 .github/workflows/clippy.yml~ diff --git a/.github/workflows/clippy.yml~ b/.github/workflows/clippy.yml~ deleted file mode 100644 index e69de29bb..000000000 From e8057a5c8a414bffa814df683e63e12c32ebde22 Mon Sep 17 00:00:00 2001 From: Jai A Date: Sun, 24 Oct 2021 16:21:15 -0700 Subject: [PATCH 12/85] Fix incorrect docker registry --- .github/workflows/docker.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index e10d87942..4f4245c79 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -21,6 +21,7 @@ jobs: name: Login to GitHub Images uses: docker/login-action@v1 with: + registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - From fb16f25b070ebaede31941aaaf1845bd517482c9 Mon Sep 17 00:00:00 2001 From: Jai A Date: Tue, 2 Nov 2021 19:59:10 -0700 Subject: [PATCH 13/85] Fixes in forge universal lib + other things --- .env | 3 +- .github/workflows/docker.yml | 15 ++++---- daedalus/src/modded.rs | 2 +- daedalus_client/src/fabric.rs | 41 ++++++++++++++------- daedalus_client/src/forge.rs | 42 ++++++++++++++------- daedalus_client/src/main.rs | 63 ++++++++++++++++++++++++++++---- daedalus_client/src/minecraft.rs | 40 ++++++++++++-------- 7 files changed, 148 insertions(+), 58 deletions(-) diff --git a/.env b/.env index 56db8578a..dc87710fe 100644 --- a/.env +++ b/.env @@ -10,4 +10,5 @@ S3_REGION=none S3_BUCKET_NAME=none DO_INTEGRATION=false -DO_ACCESS_KEY=none \ No newline at end of file +DO_ACCESS_KEY=none +DO_ENDPOINT_ID=none \ No newline at end of file diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 4f4245c79..8465c522e 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -11,12 +11,12 @@ jobs: docker: runs-on: ubuntu-latest steps: - - - name: Set up QEMU - uses: docker/setup-qemu-action@v1 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 + - name: Checkout + uses: actions/checkout@v2 + - name: Fetch docker metadata + uses: docker/metadata-action@v3 + with: + images: ghcr.io/modrinth/daedalus - name: Login to GitHub Images uses: docker/login-action@v1 @@ -30,4 +30,5 @@ jobs: uses: docker/build-push-action@v2 with: push: ${{ github.event_name != 'pull_request' }} - tags: ghcr.io/modrinth/daedalus:latest + tags: ${{ steps.docker_meta.outputs.tags }} + labels: ${{ steps.docker_meta.outputs.labels }} diff --git a/daedalus/src/modded.rs b/daedalus/src/modded.rs index c631b6193..f9d9bc843 100644 --- a/daedalus/src/modded.rs +++ b/daedalus/src/modded.rs @@ -82,7 +82,7 @@ pub fn merge_partial_version(partial: PartialVersionInfo, merge: VersionInfo) -> asset_index: merge.asset_index, assets: merge.assets, downloads: merge.downloads, - id: merge.id, + id: partial.id, libraries: partial .libraries .into_iter() diff --git a/daedalus_client/src/fabric.rs b/daedalus_client/src/fabric.rs index 5ac71f16f..d9a2a1645 100644 --- a/daedalus_client/src/fabric.rs +++ b/daedalus_client/src/fabric.rs @@ -2,13 +2,13 @@ use crate::{format_url, upload_file_to_bucket, Error}; use daedalus::download_file; use daedalus::minecraft::Library; use daedalus::modded::{LoaderType, LoaderVersion, Manifest, PartialVersionInfo, Version}; -use futures::lock::Mutex; +use tokio::sync::Mutex; use serde::{Deserialize, Serialize}; use std::collections::HashMap; use std::sync::Arc; use std::time::{Duration, Instant}; -pub async fn retrieve_data() -> Result<(), Error> { +pub async fn retrieve_data(uploaded_files: &mut Vec) -> Result<(), Error> { let mut list = fetch_fabric_versions(None).await?; let old_manifest = daedalus::modded::fetch_manifest(&*format!( "fabric/v{}/manifest.json", @@ -23,6 +23,8 @@ pub async fn retrieve_data() -> Result<(), Error> { Vec::new() })); + let uploaded_files_mutex = Arc::new(Mutex::new(Vec::new())); + if let Some(latest) = list.loader.get(0) { let loaders_mutex = Arc::new(Mutex::new(HashMap::new())); let visited_artifacts_mutex = Arc::new(Mutex::new(Vec::new())); @@ -50,6 +52,7 @@ pub async fn retrieve_data() -> Result<(), Error> { for game_version in list.game.iter_mut() { let visited_artifacts_mutex = Arc::clone(&visited_artifacts_mutex); let loaders_mutex = Arc::clone(&loaders_mutex); + let uploaded_files_mutex = Arc::clone(&uploaded_files_mutex); let versions_mutex = Arc::clone(&versions); version_futures.push(async move { @@ -119,8 +122,9 @@ pub async fn retrieve_data() -> Result<(), Error> { format!("{}/{}", "maven", artifact_path), artifact.to_vec(), Some("application/java-archive".to_string()), + uploaded_files_mutex.as_ref(), ) - .await?; + .await?; Ok::(lib) }, @@ -131,9 +135,11 @@ pub async fn retrieve_data() -> Result<(), Error> { "fabric/v{}/versions/{}-{}.json", daedalus::modded::CURRENT_FABRIC_FORMAT_VERSION, version.inherits_from, - loader + &loader ); + let inherits_from = version.inherits_from.clone(); + upload_file_to_bucket( version_path.clone(), serde_json::to_vec(&PartialVersionInfo { @@ -146,21 +152,25 @@ pub async fn retrieve_data() -> Result<(), Error> { inherits_from: version.inherits_from, libraries: libs, processors: None, - data: None + data: None, })?, Some("application/json".to_string()), + uploaded_files_mutex.as_ref(), ) .await?; { let mut loader_version_map = loader_version_mutex.lock().await; - loader_version_map.insert( - type_, - LoaderVersion { - id: loader, - url: format_url(&*version_path), - }, - ); + async move { + loader_version_map.insert( + type_, + LoaderVersion { + id: format!("{}-{}", inherits_from, loader), + url: format_url(&*version_path), + }, + ); + } + .await; } Ok::<(), Error>(()) @@ -190,7 +200,7 @@ pub async fn retrieve_data() -> Result<(), Error> { chunk_index += 1; let elapsed = now.elapsed(); - println!("Chunk {} Elapsed: {:.2?}", chunk_index, elapsed); + info!("Chunk {} Elapsed: {:.2?}", chunk_index, elapsed); } } @@ -204,10 +214,15 @@ pub async fn retrieve_data() -> Result<(), Error> { game_versions: versions.into_inner(), })?, Some("application/json".to_string()), + uploaded_files_mutex.as_ref(), ) .await?; } + if let Ok(uploaded_files_mutex) = Arc::try_unwrap(uploaded_files_mutex) { + uploaded_files.extend(uploaded_files_mutex.into_inner()); + } + Ok(()) } diff --git a/daedalus_client/src/forge.rs b/daedalus_client/src/forge.rs index af31f015f..9ba8dfc94 100644 --- a/daedalus_client/src/forge.rs +++ b/daedalus_client/src/forge.rs @@ -2,7 +2,9 @@ use crate::{format_url, upload_file_to_bucket, Error}; use chrono::{DateTime, Utc}; use daedalus::download_file; use daedalus::minecraft::{Argument, ArgumentType, Library, VersionType}; -use daedalus::modded::{LoaderType, LoaderVersion, Manifest, PartialVersionInfo, Processor, SidedDataEntry}; +use daedalus::modded::{ + LoaderType, LoaderVersion, Manifest, PartialVersionInfo, Processor, SidedDataEntry, +}; use lazy_static::lazy_static; use semver::{Version, VersionReq}; use serde::{Deserialize, Serialize}; @@ -11,20 +13,19 @@ use std::io::Read; use std::sync::Arc; use std::time::{Duration, Instant}; use tokio::sync::Mutex; +use log::info; lazy_static! { static ref FORGE_MANIFEST_V1_QUERY: VersionReq = VersionReq::parse(">=8.0.684, <23.5.2851").unwrap(); - static ref FORGE_MANIFEST_V2_QUERY_P1: VersionReq = VersionReq::parse(">=23.5.2851, <31.2.52").unwrap(); static ref FORGE_MANIFEST_V2_QUERY_P2: VersionReq = VersionReq::parse(">=32.0.1, <37.0.0").unwrap(); - static ref FORGE_MANIFEST_V3_QUERY: VersionReq = VersionReq::parse(">=37.0.0").unwrap(); } -pub async fn retrieve_data() -> Result<(), Error> { +pub async fn retrieve_data(uploaded_files: &mut Vec) -> Result<(), Error> { let maven_metadata = fetch_maven_metadata(None).await?; let old_manifest = daedalus::modded::fetch_manifest(&*format_url(&*format!( "forge/v{}/manifest.json", @@ -40,6 +41,7 @@ pub async fn retrieve_data() -> Result<(), Error> { })); let visited_assets_mutex = Arc::new(Mutex::new(Vec::new())); + let uploaded_files_mutex = Arc::new(Mutex::new(Vec::new())); let mut version_futures = Vec::new(); @@ -66,7 +68,11 @@ pub async fn retrieve_data() -> Result<(), Error> { let version = Version::parse(&*loader_version)?; - if FORGE_MANIFEST_V1_QUERY.matches(&version) || FORGE_MANIFEST_V2_QUERY_P1.matches(&version) || FORGE_MANIFEST_V2_QUERY_P2.matches(&version) || FORGE_MANIFEST_V3_QUERY.matches(&version) { + if FORGE_MANIFEST_V1_QUERY.matches(&version) + || FORGE_MANIFEST_V2_QUERY_P1.matches(&version) + || FORGE_MANIFEST_V2_QUERY_P2.matches(&version) + || FORGE_MANIFEST_V3_QUERY.matches(&version) + { loaders.push((loader_version_full, version)) } } @@ -76,6 +82,7 @@ pub async fn retrieve_data() -> Result<(), Error> { version_futures.push(async { let versions_mutex = Arc::clone(&versions); let visited_assets = Arc::clone(&visited_assets_mutex); + let uploaded_files_mutex = Arc::clone(&uploaded_files_mutex); async move { { if versions_mutex.lock().await.iter().any(|x| { @@ -89,7 +96,7 @@ pub async fn retrieve_data() -> Result<(), Error> { } } - println!("installer start {}", loader_version_full.clone()); + info!("Forge - Installer Start {}", loader_version_full.clone()); let bytes = download_file(&*format!("https://maven.minecraftforge.net/net/minecraftforge/forge/{0}/forge-{0}-installer.jar", loader_version_full), None).await?; let reader = std::io::Cursor::new(&*bytes); @@ -112,7 +119,7 @@ pub async fn retrieve_data() -> Result<(), Error> { bytes::Bytes::from(forge_universal) }; - let forge_universal_path = profile.install.file_path.clone(); + let forge_universal_path = profile.install.path.clone(); let now = Instant::now(); let libs = futures::future::try_join_all(profile.version_info.libraries.into_iter().map(|mut lib| async { @@ -151,6 +158,7 @@ pub async fn retrieve_data() -> Result<(), Error> { format!("{}/{}", "maven", artifact_path), artifact.to_vec(), Some("application/java-archive".to_string()), + uploaded_files_mutex.as_ref(), ).await?; } @@ -158,7 +166,7 @@ pub async fn retrieve_data() -> Result<(), Error> { })).await?; let elapsed = now.elapsed(); - println!("Elapsed lib DL: {:.2?}", elapsed); + info!("Elapsed lib DL: {:.2?}", elapsed); let new_profile = PartialVersionInfo { id: profile.version_info.id, @@ -183,6 +191,7 @@ pub async fn retrieve_data() -> Result<(), Error> { version_path.clone(), serde_json::to_vec(&new_profile)?, Some("application/json".to_string()), + uploaded_files_mutex.as_ref() ).await?; let mut map = HashMap::new(); @@ -243,7 +252,7 @@ pub async fn retrieve_data() -> Result<(), Error> { let artifact_path = daedalus::get_path_from_artifact(&*lib.name)?; - let artifact_bytes = if &*artifact.url == "" { + let artifact_bytes = if artifact.url.is_empty() { forge_universal_bytes.clone().unwrap_or_default() } else { daedalus::download_file( @@ -259,6 +268,7 @@ pub async fn retrieve_data() -> Result<(), Error> { format!("{}/{}", "maven", artifact_path), artifact_bytes.to_vec(), Some("application/java-archive".to_string()), + uploaded_files_mutex.as_ref() ).await?; } } @@ -267,7 +277,7 @@ pub async fn retrieve_data() -> Result<(), Error> { })).await?; let elapsed = now.elapsed(); - println!("Elapsed lib DL: {:.2?}", elapsed); + info!("Elapsed lib DL: {:.2?}", elapsed); let new_profile = PartialVersionInfo { id: version_info.id, @@ -292,6 +302,7 @@ pub async fn retrieve_data() -> Result<(), Error> { version_path.clone(), serde_json::to_vec(&new_profile)?, Some("application/json".to_string()), + uploaded_files_mutex.as_ref() ).await?; let mut map = HashMap::new(); @@ -318,7 +329,7 @@ pub async fn retrieve_data() -> Result<(), Error> { let mut versions_peek = version_futures.into_iter().peekable(); let mut chunk_index = 0; while versions_peek.peek().is_some() { - println!("Chunk {} Start", chunk_index); + info!("Chunk {} Start", chunk_index); let now = Instant::now(); let chunk: Vec<_> = versions_peek.by_ref().take(100).collect(); @@ -329,7 +340,7 @@ pub async fn retrieve_data() -> Result<(), Error> { chunk_index += 1; let elapsed = now.elapsed(); - println!("Chunk {} Elapsed: {:.2?}", chunk_index, elapsed); + info!("Chunk {} Elapsed: {:.2?}", chunk_index, elapsed); } } @@ -343,10 +354,15 @@ pub async fn retrieve_data() -> Result<(), Error> { game_versions: versions.into_inner(), })?, Some("application/json".to_string()), + uploaded_files_mutex.as_ref() ) .await?; } + if let Ok(uploaded_files_mutex) = Arc::try_unwrap(uploaded_files_mutex) { + uploaded_files.extend(uploaded_files_mutex.into_inner()); + } + Ok(()) } @@ -415,4 +431,4 @@ struct ForgeInstallerProfileV2 { pub data: HashMap, pub libraries: Vec, pub processors: Vec, -} \ No newline at end of file +} diff --git a/daedalus_client/src/main.rs b/daedalus_client/src/main.rs index 7d0511422..b5419fe2d 100644 --- a/daedalus_client/src/main.rs +++ b/daedalus_client/src/main.rs @@ -47,15 +47,22 @@ async fn main() { loop { timer.tick().await; tokio::spawn(async { - match fabric::retrieve_data().await { + let mut uploaded_files = Vec::new(); + + match fabric::retrieve_data(&mut uploaded_files).await { + Ok(..) => {} + Err(err) => error!("{:?}", err), + }; + match minecraft::retrieve_data(&mut uploaded_files).await { Ok(..) => {} Err(err) => error!("{:?}", err), }; - match minecraft::retrieve_data().await { + match forge::retrieve_data(&mut uploaded_files).await { Ok(..) => {} Err(err) => error!("{:?}", err), }; - match forge::retrieve_data().await { + + match purge_digitalocean_cache(uploaded_files).await { Ok(..) => {} Err(err) => error!("{:?}", err), }; @@ -97,10 +104,12 @@ fn check_env_vars() -> bool { let do_integration = dotenv::var("DO_INTEGRATION") .ok() .map(|x| x.parse::().ok()) - .flatten(); + .flatten() + .unwrap_or(false); - if do_integration.unwrap_or(false) { - failed |= check_var::("DO_ACCESS_KEY"); + if do_integration { + failed |= check_var::("DO_ACCESS_KEY"); + failed |= check_var::("DO_ENDPOINT_ID"); } failed @@ -126,11 +135,14 @@ pub async fn upload_file_to_bucket( path: String, bytes: Vec, content_type: Option, + uploaded_files: &tokio::sync::Mutex>, ) -> Result<(), Error> { + let key = format!("{}/{}", &*dotenv::var("BASE_FOLDER").unwrap(), path); + CLIENT .put_object(PutObjectRequest { bucket: dotenv::var("S3_BUCKET_NAME").unwrap(), - key: format!("{}/{}", &*dotenv::var("BASE_FOLDER").unwrap(), path), + key: key.clone(), body: Some(bytes.into()), acl: Some("public-read".to_string()), content_type, @@ -142,6 +154,11 @@ pub async fn upload_file_to_bucket( file: format!("{}/{}", &*dotenv::var("BASE_FOLDER").unwrap(), path), })?; + { + let mut uploaded_files = uploaded_files.lock().await; + uploaded_files.push(key); + } + Ok(()) } @@ -153,3 +170,35 @@ pub fn format_url(path: &str) -> String { path ) } + +#[derive(serde::Serialize)] +struct PurgeCacheRequest { + pub files: Vec, +} + +pub async fn purge_digitalocean_cache(files: Vec) -> Result<(), Error> { + if !dotenv::var("DO_INTEGRATION") + .ok() + .map(|x| x.parse::().ok()) + .flatten() + .unwrap_or(false) { + + return Ok(()) + } + + let client = reqwest::Client::new(); + + client + .delete(&format!( + "https://api.digitalocean.com/v2/cdn/endpoints/{}/cache", + &*dotenv::var("DO_ENDPOINT_ID").unwrap() + )) + .header("Authorization", &*format!("Bearer {}", &*dotenv::var("DO_ACCESS_KEY").unwrap())) + .json(&PurgeCacheRequest { files }) + .send().await.map_err(|err| Error::FetchError { + inner: err, + item: "purging digital ocean cache".to_string() + })?; + + Ok(()) +} diff --git a/daedalus_client/src/minecraft.rs b/daedalus_client/src/minecraft.rs index c6062dbb2..3245238c7 100644 --- a/daedalus_client/src/minecraft.rs +++ b/daedalus_client/src/minecraft.rs @@ -1,10 +1,10 @@ use crate::{format_url, upload_file_to_bucket, Error}; use daedalus::download_file; -use futures::lock::Mutex; +use tokio::sync::Mutex; use std::sync::Arc; use std::time::{Duration, Instant}; -pub async fn retrieve_data() -> Result<(), Error> { +pub async fn retrieve_data(uploaded_files: &mut Vec) -> Result<(), Error> { let old_manifest = daedalus::minecraft::fetch_version_manifest(Some(&*crate::format_url(&*format!( "minecraft/v{}/manifest.json", @@ -17,6 +17,7 @@ pub async fn retrieve_data() -> Result<(), Error> { let cloned_manifest = Arc::new(Mutex::new(manifest.clone())); let visited_assets_mutex = Arc::new(Mutex::new(Vec::new())); + let uploaded_files_mutex = Arc::new(Mutex::new(Vec::new())); let now = Instant::now(); @@ -38,6 +39,7 @@ pub async fn retrieve_data() -> Result<(), Error> { let visited_assets_mutex = Arc::clone(&visited_assets_mutex); let cloned_manifest_mutex = Arc::clone(&cloned_manifest); + let uploaded_files_mutex = Arc::clone(&uploaded_files_mutex); let assets_hash = old_version.map(|x| x.assets_index_sha1.clone()).flatten(); @@ -104,6 +106,7 @@ pub async fn retrieve_data() -> Result<(), Error> { assets_path, assets_index.to_vec(), Some("application/json".to_string()), + uploaded_files_mutex.as_ref() )); } } @@ -113,13 +116,11 @@ pub async fn retrieve_data() -> Result<(), Error> { version_path, serde_json::to_vec(&version_info)?, Some("application/json".to_string()), + uploaded_files_mutex.as_ref() )); } - let now = Instant::now(); futures::future::try_join_all(upload_futures).await?; - let elapsed = now.elapsed(); - println!("Spaces Upload {} Elapsed: {:.2?}", version.id, elapsed); Ok::<(), Error>(()) } @@ -129,20 +130,22 @@ pub async fn retrieve_data() -> Result<(), Error> { }) } - let mut versions = version_futures.into_iter().peekable(); - let mut chunk_index = 0; - while versions.peek().is_some() { - let now = Instant::now(); + { + let mut versions = version_futures.into_iter().peekable(); + let mut chunk_index = 0; + while versions.peek().is_some() { + let now = Instant::now(); - let chunk: Vec<_> = versions.by_ref().take(100).collect(); - futures::future::try_join_all(chunk).await?; + let chunk: Vec<_> = versions.by_ref().take(100).collect(); + futures::future::try_join_all(chunk).await?; - tokio::time::sleep(Duration::from_secs(1)).await; + tokio::time::sleep(Duration::from_secs(1)).await; - chunk_index += 1; + chunk_index += 1; - let elapsed = now.elapsed(); - println!("Chunk {} Elapsed: {:.2?}", chunk_index, elapsed); + let elapsed = now.elapsed(); + info!("Chunk {} Elapsed: {:.2?}", chunk_index, elapsed); + } } upload_file_to_bucket( @@ -152,11 +155,16 @@ pub async fn retrieve_data() -> Result<(), Error> { ), serde_json::to_vec(&*cloned_manifest.lock().await)?, Some("application/json".to_string()), + uploaded_files_mutex.as_ref() ) .await?; + if let Ok(uploaded_files_mutex) = Arc::try_unwrap(uploaded_files_mutex) { + uploaded_files.extend(uploaded_files_mutex.into_inner()); + } + let elapsed = now.elapsed(); - println!("Elapsed: {:.2?}", elapsed); + info!("Elapsed: {:.2?}", elapsed); Ok(()) } From 8704eff632dd9cd8b5bed822c9b60c9a0943a5d2 Mon Sep 17 00:00:00 2001 From: Jai A Date: Tue, 2 Nov 2021 20:17:12 -0700 Subject: [PATCH 14/85] Fix docker action --- .github/workflows/docker.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 8465c522e..1aa837a1f 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -29,6 +29,8 @@ jobs: id: docker_build uses: docker/build-push-action@v2 with: + context: . + file: ./Dockerfile push: ${{ github.event_name != 'pull_request' }} tags: ${{ steps.docker_meta.outputs.tags }} labels: ${{ steps.docker_meta.outputs.labels }} From 061b88f5b5c9ddd4ba150d1b105949635c76e28e Mon Sep 17 00:00:00 2001 From: Jai A Date: Tue, 2 Nov 2021 20:19:43 -0700 Subject: [PATCH 15/85] Fix docker action again --- .github/workflows/docker.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 1aa837a1f..fb101422a 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -14,6 +14,7 @@ jobs: - name: Checkout uses: actions/checkout@v2 - name: Fetch docker metadata + id: docker_meta uses: docker/metadata-action@v3 with: images: ghcr.io/modrinth/daedalus From bec54b42837050252bb0dac192aaad86a1ce18f0 Mon Sep 17 00:00:00 2001 From: Jai A Date: Tue, 2 Nov 2021 20:20:55 -0700 Subject: [PATCH 16/85] Fix action title --- .github/workflows/lint.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index d22210e6b..8bf19521d 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -1,4 +1,4 @@ -name: Publish to crates.io +name: Rust lint on: push: From d596bdb454995f7e628e18dbc84b8973e8b50701 Mon Sep 17 00:00:00 2001 From: Jai A Date: Tue, 2 Nov 2021 20:24:51 -0700 Subject: [PATCH 17/85] Fix import errors --- daedalus_client/src/fabric.rs | 1 + daedalus_client/src/main.rs | 2 +- daedalus_client/src/minecraft.rs | 1 + 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/daedalus_client/src/fabric.rs b/daedalus_client/src/fabric.rs index d9a2a1645..5bf84c04c 100644 --- a/daedalus_client/src/fabric.rs +++ b/daedalus_client/src/fabric.rs @@ -7,6 +7,7 @@ use serde::{Deserialize, Serialize}; use std::collections::HashMap; use std::sync::Arc; use std::time::{Duration, Instant}; +use log::info; pub async fn retrieve_data(uploaded_files: &mut Vec) -> Result<(), Error> { let mut list = fetch_fabric_versions(None).await?; diff --git a/daedalus_client/src/main.rs b/daedalus_client/src/main.rs index b5419fe2d..f599b118b 100644 --- a/daedalus_client/src/main.rs +++ b/daedalus_client/src/main.rs @@ -1,4 +1,4 @@ -use log::{error, info, warn}; +use log::{error, warn}; use rusoto_core::credential::StaticProvider; use rusoto_core::{HttpClient, Region, RusotoError}; use rusoto_s3::{PutObjectError, S3Client}; diff --git a/daedalus_client/src/minecraft.rs b/daedalus_client/src/minecraft.rs index 3245238c7..69484d806 100644 --- a/daedalus_client/src/minecraft.rs +++ b/daedalus_client/src/minecraft.rs @@ -3,6 +3,7 @@ use daedalus::download_file; use tokio::sync::Mutex; use std::sync::Arc; use std::time::{Duration, Instant}; +use log::info; pub async fn retrieve_data(uploaded_files: &mut Vec) -> Result<(), Error> { let old_manifest = From c744dc8cc369aae5173a92189b09145031bee175 Mon Sep 17 00:00:00 2001 From: Jai A Date: Tue, 2 Nov 2021 20:25:55 -0700 Subject: [PATCH 18/85] Bump library version --- daedalus/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/daedalus/Cargo.toml b/daedalus/Cargo.toml index def9e0dcc..a248cf8f7 100644 --- a/daedalus/Cargo.toml +++ b/daedalus/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "daedalus" -version = "0.1.0" +version = "0.1.1" authors = ["Jai A "] edition = "2018" license = "MIT" From 240269eb25e3f320bf3aa7c2c0c10d6b872dc142 Mon Sep 17 00:00:00 2001 From: Jai A Date: Tue, 2 Nov 2021 21:44:31 -0700 Subject: [PATCH 19/85] Fix lib not parsing maven file extensions --- daedalus/src/lib.rs | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/daedalus/src/lib.rs b/daedalus/src/lib.rs index 5e0f5e422..b24d304a0 100644 --- a/daedalus/src/lib.rs +++ b/daedalus/src/lib.rs @@ -51,17 +51,22 @@ pub fn get_path_from_artifact(artifact: &str) -> Result { let name = name_items.get(1).ok_or_else(|| { Error::ParseError(format!("Unable to find name for library {}", &artifact)) })?; - let version = name_items.get(2).ok_or_else(|| { + let version_ext = name_items.get(2).ok_or_else(|| { + Error::ParseError(format!("Unable to find version for library {}", &artifact)) + })?.split('@').collect::>(); + let version = version_ext.get(0).ok_or_else(|| { Error::ParseError(format!("Unable to find version for library {}", &artifact)) })?; + let ext = version_ext.get(1); Ok(format!( - "{}/{}/{}/{}-{}.jar", + "{}/{}/{}/{}-{}.{}", package.replace(".", "/"), name, version, name, - version + version, + ext.unwrap_or(&"jar") )) } From 793e542312f6a71c177632fcf2d869f50e6b327f Mon Sep 17 00:00:00 2001 From: Jai A Date: Tue, 2 Nov 2021 21:44:59 -0700 Subject: [PATCH 20/85] Bump version --- daedalus/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/daedalus/Cargo.toml b/daedalus/Cargo.toml index a248cf8f7..336e30cbd 100644 --- a/daedalus/Cargo.toml +++ b/daedalus/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "daedalus" -version = "0.1.1" +version = "0.1.2" authors = ["Jai A "] edition = "2018" license = "MIT" From 2a7dbda133be3ad14eff311c857bea59fe4c4709 Mon Sep 17 00:00:00 2001 From: Jai A Date: Wed, 3 Nov 2021 17:49:58 -0700 Subject: [PATCH 21/85] Bump version + Fix partial version and full version argument joining --- daedalus/Cargo.toml | 2 +- daedalus/src/modded.rs | 23 +++++++++++++++++++++-- daedalus_client/src/fabric.rs | 1 + daedalus_client/src/forge.rs | 2 ++ 4 files changed, 25 insertions(+), 3 deletions(-) diff --git a/daedalus/Cargo.toml b/daedalus/Cargo.toml index 336e30cbd..d7252669b 100644 --- a/daedalus/Cargo.toml +++ b/daedalus/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "daedalus" -version = "0.1.2" +version = "0.1.3" authors = ["Jai A "] edition = "2018" license = "MIT" diff --git a/daedalus/src/modded.rs b/daedalus/src/modded.rs index f9d9bc843..471701c88 100644 --- a/daedalus/src/modded.rs +++ b/daedalus/src/modded.rs @@ -33,6 +33,8 @@ pub struct PartialVersionInfo { pub time: DateTime, /// The classpath to the main class to launch the game pub main_class: Option, + /// (Legacy) Arguments passed to the game + pub minecraft_arguments: Option, /// Arguments passed to the game or JVM pub arguments: Option>>, /// Libraries that the version depends on @@ -72,7 +74,24 @@ pub fn merge_partial_version(partial: PartialVersionInfo, merge: VersionInfo) -> VersionInfo { arguments: if let Some(partial_args) = partial.arguments { if let Some(merge_args) = merge.arguments { - Some(partial_args.into_iter().chain(merge_args).collect()) + let mut new_map = HashMap::new(); + + fn add_keys(new_map: &mut HashMap>, args: HashMap>) { + for (type_, arguments) in args { + for arg in arguments { + if let Some(vec) = new_map.get_mut(&type_) { + vec.push(arg); + } else { + new_map.insert(type_, vec![arg]); + } + } + } + } + + add_keys(&mut new_map, merge_args); + add_keys(&mut new_map, partial_args); + + Some(new_map) } else { Some(partial_args) } @@ -93,7 +112,7 @@ pub fn merge_partial_version(partial: PartialVersionInfo, merge: VersionInfo) -> } else { merge.main_class }, - minecraft_arguments: merge.minecraft_arguments, + minecraft_arguments: partial.minecraft_arguments, minimum_launcher_version: merge.minimum_launcher_version, release_time: partial.release_time, time: partial.time, diff --git a/daedalus_client/src/fabric.rs b/daedalus_client/src/fabric.rs index 5bf84c04c..2734bc26f 100644 --- a/daedalus_client/src/fabric.rs +++ b/daedalus_client/src/fabric.rs @@ -152,6 +152,7 @@ pub async fn retrieve_data(uploaded_files: &mut Vec) -> Result<(), Error type_: version.type_, inherits_from: version.inherits_from, libraries: libs, + minecraft_arguments: version.minecraft_arguments, processors: None, data: None, })?, diff --git a/daedalus_client/src/forge.rs b/daedalus_client/src/forge.rs index 9ba8dfc94..ff20be0cb 100644 --- a/daedalus_client/src/forge.rs +++ b/daedalus_client/src/forge.rs @@ -174,6 +174,7 @@ pub async fn retrieve_data(uploaded_files: &mut Vec) -> Result<(), Error release_time: profile.version_info.release_time, time: profile.version_info.time, main_class: profile.version_info.main_class, + minecraft_arguments: profile.version_info.minecraft_arguments.clone(), arguments: profile.version_info.minecraft_arguments.map(|x| [(ArgumentType::Game, x.split(' ').map(|x| Argument::Normal(x.to_string())).collect())].iter().cloned().collect()), libraries: libs, type_: profile.version_info.type_, @@ -285,6 +286,7 @@ pub async fn retrieve_data(uploaded_files: &mut Vec) -> Result<(), Error release_time: version_info.release_time, time: version_info.time, main_class: version_info.main_class, + minecraft_arguments: version_info.minecraft_arguments, arguments: version_info.arguments, libraries: libs, type_: version_info.type_, From e91f8f693b0e103ce4ebc317c61fa6f78551f386 Mon Sep 17 00:00:00 2001 From: Jai A Date: Sun, 7 Nov 2021 18:42:33 -0700 Subject: [PATCH 22/85] Add local libs to modrinth maven and other fixes --- daedalus/Cargo.toml | 2 +- daedalus/src/lib.rs | 68 +++++++--- daedalus/src/modded.rs | 5 +- daedalus_client/Cargo.toml | 2 +- daedalus_client/src/fabric.rs | 6 +- daedalus_client/src/forge.rs | 207 +++++++++++++++++++++++-------- daedalus_client/src/main.rs | 69 +++++++---- daedalus_client/src/minecraft.rs | 10 +- 8 files changed, 262 insertions(+), 107 deletions(-) diff --git a/daedalus/Cargo.toml b/daedalus/Cargo.toml index d7252669b..4f674a30a 100644 --- a/daedalus/Cargo.toml +++ b/daedalus/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "daedalus" -version = "0.1.3" +version = "0.1.4" authors = ["Jai A "] edition = "2018" license = "MIT" diff --git a/daedalus/src/lib.rs b/daedalus/src/lib.rs index b24d304a0..77657323d 100644 --- a/daedalus/src/lib.rs +++ b/daedalus/src/lib.rs @@ -51,23 +51,57 @@ pub fn get_path_from_artifact(artifact: &str) -> Result { let name = name_items.get(1).ok_or_else(|| { Error::ParseError(format!("Unable to find name for library {}", &artifact)) })?; - let version_ext = name_items.get(2).ok_or_else(|| { - Error::ParseError(format!("Unable to find version for library {}", &artifact)) - })?.split('@').collect::>(); - let version = version_ext.get(0).ok_or_else(|| { - Error::ParseError(format!("Unable to find version for library {}", &artifact)) - })?; - let ext = version_ext.get(1); - - Ok(format!( - "{}/{}/{}/{}-{}.{}", - package.replace(".", "/"), - name, - version, - name, - version, - ext.unwrap_or(&"jar") - )) + + if name_items.len() == 3 { + let version_ext = name_items + .get(2) + .ok_or_else(|| { + Error::ParseError(format!("Unable to find version for library {}", &artifact)) + })? + .split('@') + .collect::>(); + let version = version_ext.get(0).ok_or_else(|| { + Error::ParseError(format!("Unable to find version for library {}", &artifact)) + })?; + let ext = version_ext.get(1); + + Ok(format!( + "{}/{}/{}/{}-{}.{}", + package.replace(".", "/"), + name, + version, + name, + version, + ext.unwrap_or(&"jar") + )) + } else { + let version = name_items.get(2).ok_or_else(|| { + Error::ParseError(format!("Unable to find version for library {}", &artifact)) + })?; + + let data_ext = name_items + .get(3) + .ok_or_else(|| { + Error::ParseError(format!("Unable to find data for library {}", &artifact)) + })? + .split('@') + .collect::>(); + let data = data_ext.get(0).ok_or_else(|| { + Error::ParseError(format!("Unable to find data for library {}", &artifact)) + })?; + let ext = data_ext.get(1); + + Ok(format!( + "{}/{}/{}/{}-{}-{}.{}", + package.replace(".", "/"), + name, + version, + name, + version, + data, + ext.unwrap_or(&"jar") + )) + } } /// Downloads a file from specified mirrors diff --git a/daedalus/src/modded.rs b/daedalus/src/modded.rs index 471701c88..6cc42098b 100644 --- a/daedalus/src/modded.rs +++ b/daedalus/src/modded.rs @@ -76,7 +76,10 @@ pub fn merge_partial_version(partial: PartialVersionInfo, merge: VersionInfo) -> if let Some(merge_args) = merge.arguments { let mut new_map = HashMap::new(); - fn add_keys(new_map: &mut HashMap>, args: HashMap>) { + fn add_keys( + new_map: &mut HashMap>, + args: HashMap>, + ) { for (type_, arguments) in args { for arg in arguments { if let Some(vec) = new_map.get_mut(&type_) { diff --git a/daedalus_client/Cargo.toml b/daedalus_client/Cargo.toml index f36f6f3b1..f30de5b53 100644 --- a/daedalus_client/Cargo.toml +++ b/daedalus_client/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "daedalus_client" -version = "0.1.0" +version = "0.1.4" authors = ["Jai A "] edition = "2018" diff --git a/daedalus_client/src/fabric.rs b/daedalus_client/src/fabric.rs index 2734bc26f..0024a6ab2 100644 --- a/daedalus_client/src/fabric.rs +++ b/daedalus_client/src/fabric.rs @@ -2,12 +2,12 @@ use crate::{format_url, upload_file_to_bucket, Error}; use daedalus::download_file; use daedalus::minecraft::Library; use daedalus::modded::{LoaderType, LoaderVersion, Manifest, PartialVersionInfo, Version}; -use tokio::sync::Mutex; +use log::info; use serde::{Deserialize, Serialize}; use std::collections::HashMap; use std::sync::Arc; use std::time::{Duration, Instant}; -use log::info; +use tokio::sync::Mutex; pub async fn retrieve_data(uploaded_files: &mut Vec) -> Result<(), Error> { let mut list = fetch_fabric_versions(None).await?; @@ -125,7 +125,7 @@ pub async fn retrieve_data(uploaded_files: &mut Vec) -> Result<(), Error Some("application/java-archive".to_string()), uploaded_files_mutex.as_ref(), ) - .await?; + .await?; Ok::(lib) }, diff --git a/daedalus_client/src/forge.rs b/daedalus_client/src/forge.rs index ff20be0cb..48dae3abe 100644 --- a/daedalus_client/src/forge.rs +++ b/daedalus_client/src/forge.rs @@ -6,6 +6,7 @@ use daedalus::modded::{ LoaderType, LoaderVersion, Manifest, PartialVersionInfo, Processor, SidedDataEntry, }; use lazy_static::lazy_static; +use log::info; use semver::{Version, VersionReq}; use serde::{Deserialize, Serialize}; use std::collections::HashMap; @@ -13,7 +14,6 @@ use std::io::Read; use std::sync::Arc; use std::time::{Duration, Instant}; use tokio::sync::Mutex; -use log::info; lazy_static! { static ref FORGE_MANIFEST_V1_QUERY: VersionReq = @@ -99,26 +99,30 @@ pub async fn retrieve_data(uploaded_files: &mut Vec) -> Result<(), Error info!("Forge - Installer Start {}", loader_version_full.clone()); let bytes = download_file(&*format!("https://maven.minecraftforge.net/net/minecraftforge/forge/{0}/forge-{0}-installer.jar", loader_version_full), None).await?; - let reader = std::io::Cursor::new(&*bytes); + let reader = std::io::Cursor::new(bytes); - if let Ok(mut archive) = zip::ZipArchive::new(reader) { + if let Ok(archive) = zip::ZipArchive::new(reader) { if FORGE_MANIFEST_V1_QUERY.matches(&version) { - let profile = { - let mut install_profile = archive.by_name("install_profile.json")?; + let mut archive_clone = archive.clone(); + let profile = tokio::task::spawn_blocking(move || { + let mut install_profile = archive_clone.by_name("install_profile.json")?; let mut contents = String::new(); install_profile.read_to_string(&mut contents)?; - serde_json::from_str::(&*contents)? - }; + Ok::(serde_json::from_str::(&*contents)?) + }).await??; - let forge_universal_bytes = { - let mut forge_universal_file = archive.by_name(&*profile.install.file_path)?; + let mut archive_clone = archive.clone(); + let file_path = profile.install.file_path.clone(); + let forge_universal_bytes = tokio::task::spawn_blocking(move || { + let mut forge_universal_file = archive_clone.by_name(&*file_path)?; let mut forge_universal = Vec::new(); forge_universal_file.read_to_end(&mut forge_universal)?; - bytes::Bytes::from(forge_universal) - }; + + Ok::(bytes::Bytes::from(forge_universal)) + }).await??; let forge_universal_path = profile.install.path.clone(); let now = Instant::now(); @@ -205,73 +209,170 @@ pub async fn retrieve_data(uploaded_files: &mut Vec) -> Result<(), Error loaders: map }) } else if FORGE_MANIFEST_V2_QUERY_P1.matches(&version) || FORGE_MANIFEST_V2_QUERY_P2.matches(&version) || FORGE_MANIFEST_V3_QUERY.matches(&version) { - let profile = { - let mut install_profile = archive.by_name("install_profile.json")?; + let mut archive_clone = archive.clone(); + let mut profile = tokio::task::spawn_blocking(move || { + let mut install_profile = archive_clone.by_name("install_profile.json")?; let mut contents = String::new(); install_profile.read_to_string(&mut contents)?; - serde_json::from_str::(&*contents)? - }; + Ok::(serde_json::from_str::(&*contents)?) + }).await??; - let version_info = { - let mut install_profile = archive.by_name("version.json")?; + let mut archive_clone = archive.clone(); + let version_info = tokio::task::spawn_blocking(move || { + let mut install_profile = archive_clone.by_name("version.json")?; let mut contents = String::new(); install_profile.read_to_string(&mut contents)?; - serde_json::from_str::(&*contents)? - }; - let forge_universal_bytes = { - if let Some(path) = &profile.path { - let mut forge_universal_file = archive.by_name(&*format!("maven/{}", daedalus::get_path_from_artifact(&*path)?))?; - let mut forge_universal = Vec::new(); - forge_universal_file.read_to_end(&mut forge_universal)?; + Ok::(serde_json::from_str::(&*contents)?) + }).await??; + + + let mut libs : Vec = profile.libraries.into_iter().chain(version_info.libraries).collect(); + + let mut local_libs : HashMap = HashMap::new(); - Some(bytes::Bytes::from(forge_universal)) - } else { - None + for lib in &libs { + if lib.downloads.as_ref().map(|x| x.artifact.as_ref().map(|x| x.url.is_empty())).flatten().unwrap_or(false) { + let mut archive_clone = archive.clone(); + let lib_name_clone = lib.name.clone(); + + let lib_bytes = tokio::task::spawn_blocking(move || { + let mut lib_file = archive_clone.by_name(&*format!("maven/{}", daedalus::get_path_from_artifact(&*lib_name_clone)?))?; + let mut lib_bytes = Vec::new(); + lib_file.read_to_end(&mut lib_bytes)?; + + Ok::(bytes::Bytes::from(lib_bytes)) + }).await??; + + local_libs.insert(lib.name.clone(), lib_bytes); } - }; + } + + let path = profile.path.clone(); + let version = profile.version.clone(); + + for entry in profile.data.values_mut() { + if entry.client.starts_with('/') || entry.server.starts_with('/') { + macro_rules! read_data { + ($value:expr) => { + let mut archive_clone = archive.clone(); + let value_clone = $value.clone(); + let lib_bytes = tokio::task::spawn_blocking(move || { + let mut lib_file = archive_clone.by_name(&value_clone[1..value_clone.len()])?; + let mut lib_bytes = Vec::new(); + lib_file.read_to_end(&mut lib_bytes)?; + + Ok::(bytes::Bytes::from(lib_bytes)) + }).await??; + + let split = $value.split('/').last(); + + if let Some(last) = split { + let mut file = last.split('.'); + + if let Some(file_name) = file.next() { + if let Some(ext) = file.next() { + let path = format!("{}:{}@{}", path.as_deref().unwrap_or(&*format!("net.minecraftforge:forge:{}", version)), file_name, ext); + $value = format!("[{}]", &path); + local_libs.insert(path.clone(), bytes::Bytes::from(lib_bytes)); + + libs.push(Library { + downloads: None, + extract: None, + name: path, + url: Some("".to_string()), + natives: None, + rules: None, + checksums: None + }); + } + } + } + } + } + + if entry.client.starts_with('/') { + read_data!(entry.client); + } + + // Do we really need to support server installs? Keeping this here + // just in case + // + // if entry.server.starts_with('/') { + // read_data!(entry.server); + // } + } + } let now = Instant::now(); - let libs = futures::future::try_join_all(profile.libraries.into_iter().chain(version_info.libraries).map(|mut lib| async { - if let Some(ref mut downloads) = lib.downloads { - if let Some(ref mut artifact) = downloads.artifact { - { - let mut visited_assets = visited_assets.lock().await; + let libs = futures::future::try_join_all(libs.into_iter().map(|mut lib| async { + let artifact_path = + daedalus::get_path_from_artifact(&*lib.name)?; - if visited_assets.contains(&lib.name) { - artifact.url = format_url(&*format!("maven/{}", artifact.path)); + { + let mut visited_assets = visited_assets.lock().await; - return Ok::(lib); - } else { - visited_assets.push(lib.name.clone()) + if visited_assets.contains(&lib.name) { + if let Some(ref mut downloads) = lib.downloads { + if let Some(ref mut artifact) = downloads.artifact { + artifact.url = format_url(&*format!("maven/{}", artifact_path)); } + } else if lib.url.is_some() { + lib.url = Some(format_url(&*format!("maven/{}", artifact_path))); } - let artifact_path = - daedalus::get_path_from_artifact(&*lib.name)?; + return Ok::(lib); + } else { + visited_assets.push(lib.name.clone()) + } + } - let artifact_bytes = if artifact.url.is_empty() { - forge_universal_bytes.clone().unwrap_or_default() + let artifact_bytes = if let Some(ref mut downloads) = lib.downloads { + if let Some(ref mut artifact) = downloads.artifact { + let res = if artifact.url.is_empty() { + local_libs.get(&lib.name).cloned() } else { - daedalus::download_file( + Some(daedalus::download_file( &*artifact.url, Some(&*artifact.sha1), ) - .await? + .await?) }; - artifact.url = format_url(&*format!("maven/{}", artifact.path)); + if res.is_some() { + artifact.url = format_url(&*format!("maven/{}", artifact_path)); + } - upload_file_to_bucket( - format!("{}/{}", "maven", artifact_path), - artifact_bytes.to_vec(), - Some("application/java-archive".to_string()), - uploaded_files_mutex.as_ref() - ).await?; + res + } else { None } + } else if let Some(ref mut url) = lib.url { + let res = if url.is_empty() { + local_libs.get(&lib.name).cloned() + } else { + Some(daedalus::download_file( + url, + None, + ) + .await?) + }; + + if res.is_some() { + lib.url = Some(format_url(&*format!("maven/{}", artifact_path))); } + + res + } else { None }; + + if let Some(bytes) = artifact_bytes { + upload_file_to_bucket( + format!("{}/{}", "maven", artifact_path), + bytes.to_vec(), + Some("application/java-archive".to_string()), + uploaded_files_mutex.as_ref() + ).await?; } Ok::(lib) @@ -334,7 +435,7 @@ pub async fn retrieve_data(uploaded_files: &mut Vec) -> Result<(), Error info!("Chunk {} Start", chunk_index); let now = Instant::now(); - let chunk: Vec<_> = versions_peek.by_ref().take(100).collect(); + let chunk: Vec<_> = versions_peek.by_ref().take(10).collect(); futures::future::try_join_all(chunk).await?; tokio::time::sleep(Duration::from_secs(1)).await; @@ -356,7 +457,7 @@ pub async fn retrieve_data(uploaded_files: &mut Vec) -> Result<(), Error game_versions: versions.into_inner(), })?, Some("application/json".to_string()), - uploaded_files_mutex.as_ref() + uploaded_files_mutex.as_ref(), ) .await?; } diff --git a/daedalus_client/src/main.rs b/daedalus_client/src/main.rs index f599b118b..123e2b4f2 100644 --- a/daedalus_client/src/main.rs +++ b/daedalus_client/src/main.rs @@ -62,7 +62,7 @@ async fn main() { Err(err) => error!("{:?}", err), }; - match purge_digitalocean_cache(uploaded_files).await { + match purge_digitalocean_cache(uploaded_files).await { Ok(..) => {} Err(err) => error!("{:?}", err), }; @@ -139,27 +139,39 @@ pub async fn upload_file_to_bucket( ) -> Result<(), Error> { let key = format!("{}/{}", &*dotenv::var("BASE_FOLDER").unwrap(), path); - CLIENT - .put_object(PutObjectRequest { - bucket: dotenv::var("S3_BUCKET_NAME").unwrap(), - key: key.clone(), - body: Some(bytes.into()), - acl: Some("public-read".to_string()), - content_type, - ..Default::default() - }) - .await - .map_err(|err| Error::S3Error { - inner: err, - file: format!("{}/{}", &*dotenv::var("BASE_FOLDER").unwrap(), path), - })?; - - { - let mut uploaded_files = uploaded_files.lock().await; - uploaded_files.push(key); + for attempt in 1..=4 { + let result = CLIENT + .put_object(PutObjectRequest { + bucket: dotenv::var("S3_BUCKET_NAME").unwrap(), + key: key.clone(), + body: Some(bytes.clone().into()), + acl: Some("public-read".to_string()), + content_type: content_type.clone(), + ..Default::default() + }) + .await + .map_err(|err| Error::S3Error { + inner: err, + file: format!("{}/{}", &*dotenv::var("BASE_FOLDER").unwrap(), path), + }); + + match result { + Ok(_) => { + { + let mut uploaded_files = uploaded_files.lock().await; + uploaded_files.push(key); + } + + return Ok(()); + } + Err(_) if attempt <= 3 => continue, + Err(_) => { + result?; + } + } } - Ok(()) + unreachable!() } pub fn format_url(path: &str) -> String { @@ -181,9 +193,9 @@ pub async fn purge_digitalocean_cache(files: Vec) -> Result<(), Error> { .ok() .map(|x| x.parse::().ok()) .flatten() - .unwrap_or(false) { - - return Ok(()) + .unwrap_or(false) + { + return Ok(()); } let client = reqwest::Client::new(); @@ -193,11 +205,16 @@ pub async fn purge_digitalocean_cache(files: Vec) -> Result<(), Error> { "https://api.digitalocean.com/v2/cdn/endpoints/{}/cache", &*dotenv::var("DO_ENDPOINT_ID").unwrap() )) - .header("Authorization", &*format!("Bearer {}", &*dotenv::var("DO_ACCESS_KEY").unwrap())) + .header( + "Authorization", + &*format!("Bearer {}", &*dotenv::var("DO_ACCESS_KEY").unwrap()), + ) .json(&PurgeCacheRequest { files }) - .send().await.map_err(|err| Error::FetchError { + .send() + .await + .map_err(|err| Error::FetchError { inner: err, - item: "purging digital ocean cache".to_string() + item: "purging digital ocean cache".to_string(), })?; Ok(()) diff --git a/daedalus_client/src/minecraft.rs b/daedalus_client/src/minecraft.rs index 69484d806..babb6a189 100644 --- a/daedalus_client/src/minecraft.rs +++ b/daedalus_client/src/minecraft.rs @@ -1,9 +1,9 @@ use crate::{format_url, upload_file_to_bucket, Error}; use daedalus::download_file; -use tokio::sync::Mutex; +use log::info; use std::sync::Arc; use std::time::{Duration, Instant}; -use log::info; +use tokio::sync::Mutex; pub async fn retrieve_data(uploaded_files: &mut Vec) -> Result<(), Error> { let old_manifest = @@ -107,7 +107,7 @@ pub async fn retrieve_data(uploaded_files: &mut Vec) -> Result<(), Error assets_path, assets_index.to_vec(), Some("application/json".to_string()), - uploaded_files_mutex.as_ref() + uploaded_files_mutex.as_ref(), )); } } @@ -117,7 +117,7 @@ pub async fn retrieve_data(uploaded_files: &mut Vec) -> Result<(), Error version_path, serde_json::to_vec(&version_info)?, Some("application/json".to_string()), - uploaded_files_mutex.as_ref() + uploaded_files_mutex.as_ref(), )); } @@ -156,7 +156,7 @@ pub async fn retrieve_data(uploaded_files: &mut Vec) -> Result<(), Error ), serde_json::to_vec(&*cloned_manifest.lock().await)?, Some("application/json".to_string()), - uploaded_files_mutex.as_ref() + uploaded_files_mutex.as_ref(), ) .await?; From 0990ac4fc16677b86669f5bb03ed473891aee35f Mon Sep 17 00:00:00 2001 From: Jai A Date: Mon, 8 Nov 2021 20:17:20 -0700 Subject: [PATCH 23/85] Add forge data to main version info --- daedalus/Cargo.toml | 2 +- daedalus/src/minecraft.rs | 11 +++++++++++ daedalus/src/modded.rs | 9 +++++++++ 3 files changed, 21 insertions(+), 1 deletion(-) diff --git a/daedalus/Cargo.toml b/daedalus/Cargo.toml index 4f674a30a..ef5419919 100644 --- a/daedalus/Cargo.toml +++ b/daedalus/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "daedalus" -version = "0.1.4" +version = "0.1.5" authors = ["Jai A "] edition = "2018" license = "MIT" diff --git a/daedalus/src/minecraft.rs b/daedalus/src/minecraft.rs index d345f04c3..02d14424f 100644 --- a/daedalus/src/minecraft.rs +++ b/daedalus/src/minecraft.rs @@ -1,3 +1,4 @@ +use crate::modded::{Processor, SidedDataEntry}; use crate::{download_file, Error}; use chrono::{DateTime, Utc}; use serde::{Deserialize, Serialize}; @@ -51,9 +52,11 @@ pub struct Version { pub sha1: String, /// Whether the version supports the latest player safety features pub compliance_level: u32, + #[serde(skip_serializing_if = "Option::is_none")] /// (Modrinth Provided) The link to the assets index for this version /// This is only available when using the Modrinth mirror pub assets_index_url: Option, + #[serde(skip_serializing_if = "Option::is_none")] /// (Modrinth Provided) The SHA1 hash of the assets index for this version /// This is only available when using the Modrinth mirror pub assets_index_sha1: Option, @@ -147,8 +150,10 @@ pub struct LibraryDownload { #[derive(Serialize, Deserialize, Debug)] /// A list of files that should be downloaded for libraries pub struct LibraryDownloads { + #[serde(skip_serializing_if = "Option::is_none")] /// The primary library artifact pub artifact: Option, + #[serde(skip_serializing_if = "Option::is_none")] /// Conditional files that may be needed to be downloaded alongside the library /// The HashMap key specifies a classifier as additional information for downloading files pub classifiers: Option>, @@ -315,6 +320,12 @@ pub struct VersionInfo { #[serde(rename = "type")] /// The type of version pub type_: VersionType, + #[serde(skip_serializing_if = "Option::is_none")] + /// (Forge-only) + pub data: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + /// (Forge-only) The list of processors to run after downloading the files + pub processors: Option>, } /// Fetches detailed information about a version from the manifest diff --git a/daedalus/src/modded.rs b/daedalus/src/modded.rs index 6cc42098b..41289f343 100644 --- a/daedalus/src/modded.rs +++ b/daedalus/src/modded.rs @@ -31,10 +31,13 @@ pub struct PartialVersionInfo { pub release_time: DateTime, /// The latest time a file in this version was updated pub time: DateTime, + #[serde(skip_serializing_if = "Option::is_none")] /// The classpath to the main class to launch the game pub main_class: Option, + #[serde(skip_serializing_if = "Option::is_none")] /// (Legacy) Arguments passed to the game pub minecraft_arguments: Option, + #[serde(skip_serializing_if = "Option::is_none")] /// Arguments passed to the game or JVM pub arguments: Option>>, /// Libraries that the version depends on @@ -42,8 +45,10 @@ pub struct PartialVersionInfo { #[serde(rename = "type")] /// The type of version pub type_: VersionType, + #[serde(skip_serializing_if = "Option::is_none")] /// (Forge-only) pub data: Option>, + #[serde(skip_serializing_if = "Option::is_none")] /// (Forge-only) The list of processors to run after downloading the files pub processors: Option>, } @@ -57,8 +62,10 @@ pub struct Processor { pub classpath: Vec, /// Arguments for this processor. pub args: Vec, + #[serde(skip_serializing_if = "Option::is_none")] /// Represents a map of outputs. Keys and values can be data values pub outputs: Option>, + #[serde(skip_serializing_if = "Option::is_none")] /// Which sides this processor shall be ran on. /// Valid values: client, server, extract pub sides: Option>, @@ -120,6 +127,8 @@ pub fn merge_partial_version(partial: PartialVersionInfo, merge: VersionInfo) -> release_time: partial.release_time, time: partial.time, type_: partial.type_, + data: partial.data, + processors: partial.processors, } } From f6c611bbba1b93f96466babb2d2f811ededed069 Mon Sep 17 00:00:00 2001 From: Jai A Date: Wed, 10 Nov 2021 17:34:53 -0700 Subject: [PATCH 24/85] Add classpath variable for libraries --- daedalus/Cargo.toml | 2 +- daedalus/src/minecraft.rs | 7 +++++++ daedalus_client/src/forge.rs | 16 +++++++++++++--- 3 files changed, 21 insertions(+), 4 deletions(-) diff --git a/daedalus/Cargo.toml b/daedalus/Cargo.toml index ef5419919..dc5496aee 100644 --- a/daedalus/Cargo.toml +++ b/daedalus/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "daedalus" -version = "0.1.5" +version = "0.1.6" authors = ["Jai A "] edition = "2018" license = "MIT" diff --git a/daedalus/src/minecraft.rs b/daedalus/src/minecraft.rs index 02d14424f..0e9a863bf 100644 --- a/daedalus/src/minecraft.rs +++ b/daedalus/src/minecraft.rs @@ -252,6 +252,13 @@ pub struct Library { #[serde(skip_serializing_if = "Option::is_none")] /// SHA1 Checksums for validating the library's integrity. Only present for forge libraries pub checksums: Option>, + #[serde(default = "default_include_in_classpath")] + /// Whether the library should be included in the classpath at the game's launch + pub include_in_classpath: bool, +} + +fn default_include_in_classpath() -> bool { + true } #[derive(Serialize, Deserialize, Debug, Clone)] diff --git a/daedalus_client/src/forge.rs b/daedalus_client/src/forge.rs index 48dae3abe..f9854b84c 100644 --- a/daedalus_client/src/forge.rs +++ b/daedalus_client/src/forge.rs @@ -230,7 +230,16 @@ pub async fn retrieve_data(uploaded_files: &mut Vec) -> Result<(), Error }).await??; - let mut libs : Vec = profile.libraries.into_iter().chain(version_info.libraries).collect(); + let mut libs : Vec = version_info.libraries.into_iter().chain(profile.libraries.into_iter().map(|x| Library { + downloads: x.downloads, + extract: x.extract, + name: x.name, + url: x.url, + natives: x.natives, + rules: x.rules, + checksums: x.checksums, + include_in_classpath: false + })).collect(); let mut local_libs : HashMap = HashMap::new(); @@ -286,7 +295,8 @@ pub async fn retrieve_data(uploaded_files: &mut Vec) -> Result<(), Error url: Some("".to_string()), natives: None, rules: None, - checksums: None + checksums: None, + include_in_classpath: false, }); } } @@ -321,7 +331,7 @@ pub async fn retrieve_data(uploaded_files: &mut Vec) -> Result<(), Error artifact.url = format_url(&*format!("maven/{}", artifact_path)); } } else if lib.url.is_some() { - lib.url = Some(format_url(&*format!("maven/{}", artifact_path))); + lib.url = Some(format_url("maven/")); } return Ok::(lib); From d7e04687762b74ca542cf16b8fed20ae6eb1f203 Mon Sep 17 00:00:00 2001 From: Jai A Date: Wed, 10 Nov 2021 18:40:12 -0700 Subject: [PATCH 25/85] Fix library URL being set incorrectly --- daedalus/Cargo.toml | 2 +- daedalus_client/src/forge.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/daedalus/Cargo.toml b/daedalus/Cargo.toml index dc5496aee..f5e583bae 100644 --- a/daedalus/Cargo.toml +++ b/daedalus/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "daedalus" -version = "0.1.6" +version = "0.1.7" authors = ["Jai A "] edition = "2018" license = "MIT" diff --git a/daedalus_client/src/forge.rs b/daedalus_client/src/forge.rs index f9854b84c..baa816733 100644 --- a/daedalus_client/src/forge.rs +++ b/daedalus_client/src/forge.rs @@ -370,7 +370,7 @@ pub async fn retrieve_data(uploaded_files: &mut Vec) -> Result<(), Error }; if res.is_some() { - lib.url = Some(format_url(&*format!("maven/{}", artifact_path))); + lib.url = Some(format_url("maven/")); } res From 5a6c06c8a3dea11e5f10a491a40687b8ddcccba1 Mon Sep 17 00:00:00 2001 From: Jai A Date: Sat, 18 Dec 2021 22:55:03 -0700 Subject: [PATCH 26/85] Host all loaders for forge, fix stable markers, add java version to daedalus --- .idea/runConfigurations.xml | 10 --- daedalus/Cargo.toml | 2 +- daedalus/src/minecraft.rs | 12 ++++ daedalus/src/modded.rs | 16 ++--- daedalus_client/Cargo.toml | 2 +- daedalus_client/src/fabric.rs | 90 ++++++++++++++++-------- daedalus_client/src/forge.rs | 116 ++++++++++++++++++++----------- daedalus_client/src/main.rs | 31 ++++++--- daedalus_client/src/minecraft.rs | 7 +- 9 files changed, 180 insertions(+), 106 deletions(-) delete mode 100644 .idea/runConfigurations.xml diff --git a/.idea/runConfigurations.xml b/.idea/runConfigurations.xml deleted file mode 100644 index 797acea53..000000000 --- a/.idea/runConfigurations.xml +++ /dev/null @@ -1,10 +0,0 @@ - - - - - - \ No newline at end of file diff --git a/daedalus/Cargo.toml b/daedalus/Cargo.toml index f5e583bae..38aee9792 100644 --- a/daedalus/Cargo.toml +++ b/daedalus/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "daedalus" -version = "0.1.7" +version = "0.1.8" authors = ["Jai A "] edition = "2018" license = "MIT" diff --git a/daedalus/src/minecraft.rs b/daedalus/src/minecraft.rs index 0e9a863bf..250262cdc 100644 --- a/daedalus/src/minecraft.rs +++ b/daedalus/src/minecraft.rs @@ -229,6 +229,15 @@ pub struct LibraryExtract { pub exclude: Option>, } +#[derive(Serialize, Deserialize, Debug)] +/// Information about the java version the game needs +pub struct JavaVersion { + /// The component needed for the Java installation + component: String, + /// The major Java version number + major_version: u32, +} + #[derive(Serialize, Deserialize, Debug)] /// A library which the game relies on to run pub struct Library { @@ -311,6 +320,9 @@ pub struct VersionInfo { pub downloads: HashMap, /// The version ID of the version pub id: String, + + /// The Java version this version supports + pub java_version: JavaVersion, /// Libraries that the version depends on pub libraries: Vec, /// The classpath to the main class to launch the game diff --git a/daedalus/src/modded.rs b/daedalus/src/modded.rs index 41289f343..ea3e07c96 100644 --- a/daedalus/src/modded.rs +++ b/daedalus/src/modded.rs @@ -112,6 +112,7 @@ pub fn merge_partial_version(partial: PartialVersionInfo, merge: VersionInfo) -> assets: merge.assets, downloads: merge.downloads, id: partial.id, + java_version: merge.java_version, libraries: partial .libraries .into_iter() @@ -140,24 +141,13 @@ pub struct Manifest { pub game_versions: Vec, } -#[derive(Serialize, Deserialize, Debug, Eq, PartialEq, Hash, Clone)] -#[serde(rename_all = "camelCase")] -/// The version type of the loader -pub enum LoaderType { - /// The latest type is for experimental loader versions that may not be ready for normal use - Latest, - /// The stable type is for the most stable but recent loader version. For the forge mod loader, - /// this is never used - Stable, -} - #[derive(Serialize, Deserialize, Debug, Clone)] /// A game version of Minecraft pub struct Version { /// The minecraft version ID pub id: String, /// A map that contains loader versions for the game version - pub loaders: HashMap, + pub loaders: Vec, } #[derive(Serialize, Deserialize, Debug, Clone)] @@ -167,6 +157,8 @@ pub struct LoaderVersion { pub id: String, /// The URL of the version's manifest pub url: String, + /// Whether the loader is stable or not + pub stable: bool, } /// Fetches the manifest of a mod loader diff --git a/daedalus_client/Cargo.toml b/daedalus_client/Cargo.toml index f30de5b53..77ae6fd46 100644 --- a/daedalus_client/Cargo.toml +++ b/daedalus_client/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "daedalus_client" -version = "0.1.4" +version = "0.1.8" authors = ["Jai A "] edition = "2018" diff --git a/daedalus_client/src/fabric.rs b/daedalus_client/src/fabric.rs index 0024a6ab2..514252c96 100644 --- a/daedalus_client/src/fabric.rs +++ b/daedalus_client/src/fabric.rs @@ -1,15 +1,17 @@ use crate::{format_url, upload_file_to_bucket, Error}; use daedalus::download_file; -use daedalus::minecraft::Library; -use daedalus::modded::{LoaderType, LoaderVersion, Manifest, PartialVersionInfo, Version}; +use daedalus::minecraft::{Library, VersionManifest}; +use daedalus::modded::{LoaderVersion, Manifest, PartialVersionInfo, Version}; use log::info; use serde::{Deserialize, Serialize}; -use std::collections::HashMap; use std::sync::Arc; use std::time::{Duration, Instant}; -use tokio::sync::Mutex; +use tokio::sync::{Mutex, RwLock}; -pub async fn retrieve_data(uploaded_files: &mut Vec) -> Result<(), Error> { +pub async fn retrieve_data( + minecraft_versions: &VersionManifest, + uploaded_files: &mut Vec, +) -> Result<(), Error> { let mut list = fetch_fabric_versions(None).await?; let old_manifest = daedalus::modded::fetch_manifest(&*format!( "fabric/v{}/manifest.json", @@ -27,24 +29,28 @@ pub async fn retrieve_data(uploaded_files: &mut Vec) -> Result<(), Error let uploaded_files_mutex = Arc::new(Mutex::new(Vec::new())); if let Some(latest) = list.loader.get(0) { - let loaders_mutex = Arc::new(Mutex::new(HashMap::new())); + let loaders_mutex = Arc::new(RwLock::new(Vec::new())); let visited_artifacts_mutex = Arc::new(Mutex::new(Vec::new())); { - let mut loaders = loaders_mutex.lock().await; + let mut loaders = loaders_mutex.write().await; - loaders.insert(LoaderType::Latest, latest.version.clone()); + // for loader in &list.loader { + // loaders.push((Box::new(loader.stable), loader.version.clone())) + // } + + loaders.push((Box::new(latest.stable), latest.version.clone())); if !latest.stable { if let Some(stable) = list.loader.iter().find(|x| x.stable) { - loaders.insert(LoaderType::Stable, stable.version.clone()); + loaders.push((Box::new(stable.stable), stable.version.clone())); } } list.loader = list .loader .into_iter() - .filter(|x| loaders.values().any(|val| val == &x.version)) + .filter(|x| loaders.iter().any(|val| val.1 == x.version)) .collect(); } @@ -57,19 +63,16 @@ pub async fn retrieve_data(uploaded_files: &mut Vec) -> Result<(), Error let versions_mutex = Arc::clone(&versions); version_futures.push(async move { - let loader_version_mutex = Mutex::new(HashMap::new()); + let loader_version_mutex = Mutex::new(Vec::new()); let versions = futures::future::try_join_all( - loaders_mutex.lock().await.clone().into_iter().map( - |(type_, loader)| async { + loaders_mutex.read().await.clone().into_iter().map( + |(stable, loader)| async { { if versions_mutex.lock().await.iter().any(|x| { x.id == game_version.version - && x.loaders - .get(&type_) - .map(|x| x.id == loader) - .unwrap_or(false) + && x.loaders.iter().any(|x| x.id == loader) }) { return Ok(None); } @@ -78,8 +81,8 @@ pub async fn retrieve_data(uploaded_files: &mut Vec) -> Result<(), Error let version = fetch_fabric_version(&*game_version.version, &*loader).await?; - Ok::, Error>(Some( - (type_, loader, version), + Ok::, String, PartialVersionInfo)>, Error>(Some( + (stable, loader, version), )) }, ), @@ -88,7 +91,7 @@ pub async fn retrieve_data(uploaded_files: &mut Vec) -> Result<(), Error .into_iter() .flatten(); - futures::future::try_join_all(versions.map(|(type_, loader, version)| async { + futures::future::try_join_all(versions.map(|(stable, loader, version)| async { let libs = futures::future::try_join_all(version.libraries.into_iter().map( |mut lib| async { { @@ -164,13 +167,11 @@ pub async fn retrieve_data(uploaded_files: &mut Vec) -> Result<(), Error { let mut loader_version_map = loader_version_mutex.lock().await; async move { - loader_version_map.insert( - type_, - LoaderVersion { - id: format!("{}-{}", inherits_from, loader), - url: format_url(&*version_path), - }, - ); + loader_version_map.push(LoaderVersion { + id: format!("{}-{}", inherits_from, loader), + url: format_url(&*version_path), + stable: *stable, + }); } .await; } @@ -207,13 +208,46 @@ pub async fn retrieve_data(uploaded_files: &mut Vec) -> Result<(), Error } if let Ok(versions) = Arc::try_unwrap(versions) { + let mut versions = versions.into_inner(); + + versions.sort_by(|x, y| { + minecraft_versions + .versions + .iter() + .position(|z| x.id == z.id) + .unwrap_or_default() + .cmp( + &minecraft_versions + .versions + .iter() + .position(|z| y.id == z.id) + .unwrap_or_default(), + ) + }); + + for version in &mut versions { + version.loaders.sort_by(|x, y| { + list.loader + .iter() + .position(|z| x.id == z.version) + .unwrap_or_default() + .cmp( + &list + .loader + .iter() + .position(|z| y.id == z.version) + .unwrap_or_default(), + ) + }) + } + upload_file_to_bucket( format!( "fabric/v{}/manifest.json", daedalus::modded::CURRENT_FABRIC_FORMAT_VERSION, ), serde_json::to_vec(&Manifest { - game_versions: versions.into_inner(), + game_versions: versions, })?, Some("application/json".to_string()), uploaded_files_mutex.as_ref(), diff --git a/daedalus_client/src/forge.rs b/daedalus_client/src/forge.rs index baa816733..f8f8e188d 100644 --- a/daedalus_client/src/forge.rs +++ b/daedalus_client/src/forge.rs @@ -1,10 +1,8 @@ use crate::{format_url, upload_file_to_bucket, Error}; use chrono::{DateTime, Utc}; use daedalus::download_file; -use daedalus::minecraft::{Argument, ArgumentType, Library, VersionType}; -use daedalus::modded::{ - LoaderType, LoaderVersion, Manifest, PartialVersionInfo, Processor, SidedDataEntry, -}; +use daedalus::minecraft::{Argument, ArgumentType, Library, VersionManifest, VersionType}; +use daedalus::modded::{LoaderVersion, Manifest, PartialVersionInfo, Processor, SidedDataEntry}; use lazy_static::lazy_static; use log::info; use semver::{Version, VersionReq}; @@ -25,7 +23,10 @@ lazy_static! { static ref FORGE_MANIFEST_V3_QUERY: VersionReq = VersionReq::parse(">=37.0.0").unwrap(); } -pub async fn retrieve_data(uploaded_files: &mut Vec) -> Result<(), Error> { +pub async fn retrieve_data( + minecraft_versions: &VersionManifest, + uploaded_files: &mut Vec, +) -> Result<(), Error> { let maven_metadata = fetch_maven_metadata(None).await?; let old_manifest = daedalus::modded::fetch_manifest(&*format_url(&*format!( "forge/v{}/manifest.json", @@ -34,18 +35,20 @@ pub async fn retrieve_data(uploaded_files: &mut Vec) -> Result<(), Error .await .ok(); - let versions = Arc::new(Mutex::new(if let Some(old_manifest) = old_manifest { + let old_versions = Arc::new(Mutex::new(if let Some(old_manifest) = old_manifest { old_manifest.game_versions } else { Vec::new() })); + let versions = Arc::new(Mutex::new(Vec::new())); + let visited_assets_mutex = Arc::new(Mutex::new(Vec::new())); let uploaded_files_mutex = Arc::new(Mutex::new(Vec::new())); let mut version_futures = Vec::new(); - for (minecraft_version, loader_versions) in maven_metadata { + for (minecraft_version, loader_versions) in maven_metadata.clone() { let mut loaders = Vec::new(); for loader_version_full in loader_versions { @@ -77,22 +80,21 @@ pub async fn retrieve_data(uploaded_files: &mut Vec) -> Result<(), Error } } } - - if let Some((loader_version_full, version)) = loaders.into_iter().last() { - version_futures.push(async { - let versions_mutex = Arc::clone(&versions); + version_futures.push(async { + let loaders_versions = futures::future::try_join_all(loaders.into_iter().map(|(loader_version_full, version)| async { + let versions_mutex = Arc::clone(&old_versions); let visited_assets = Arc::clone(&visited_assets_mutex); let uploaded_files_mutex = Arc::clone(&uploaded_files_mutex); + let minecraft_version = minecraft_version.clone(); + async move { { - if versions_mutex.lock().await.iter().any(|x| { - x.id == minecraft_version - && x.loaders - .get(&LoaderType::Latest) - .map(|x| x.id == loader_version_full) - .unwrap_or(false) - }) { - return Ok(()); + let versions = versions_mutex.lock().await; + let version = versions.iter().find(|x| + x.id == minecraft_version).map(|x| x.loaders.iter().find(|x| x.id == loader_version_full)).flatten(); + + if let Some(version) = version { + return Ok::, Error>(Some(version.clone())); } } @@ -199,15 +201,11 @@ pub async fn retrieve_data(uploaded_files: &mut Vec) -> Result<(), Error uploaded_files_mutex.as_ref() ).await?; - let mut map = HashMap::new(); - map.insert(LoaderType::Latest, LoaderVersion { + return Ok(Some(LoaderVersion { id: loader_version_full, - url: format_url(&*version_path) - }); - versions_mutex.lock().await.push(daedalus::modded::Version { - id: minecraft_version, - loaders: map - }) + url: format_url(&*version_path), + stable: false + })); } else if FORGE_MANIFEST_V2_QUERY_P1.matches(&version) || FORGE_MANIFEST_V2_QUERY_P2.matches(&version) || FORGE_MANIFEST_V3_QUERY.matches(&version) { let mut archive_clone = archive.clone(); let mut profile = tokio::task::spawn_blocking(move || { @@ -342,7 +340,7 @@ pub async fn retrieve_data(uploaded_files: &mut Vec) -> Result<(), Error let artifact_bytes = if let Some(ref mut downloads) = lib.downloads { if let Some(ref mut artifact) = downloads.artifact { - let res = if artifact.url.is_empty() { + let res = if artifact.url.is_empty() { local_libs.get(&lib.name).cloned() } else { Some(daedalus::download_file( @@ -418,24 +416,25 @@ pub async fn retrieve_data(uploaded_files: &mut Vec) -> Result<(), Error uploaded_files_mutex.as_ref() ).await?; - let mut map = HashMap::new(); - map.insert(LoaderType::Latest, LoaderVersion { + return Ok(Some(LoaderVersion { id: loader_version_full, - url: format_url(&*version_path) - }); - versions_mutex.lock().await.push(daedalus::modded::Version { - id: minecraft_version, - loaders: map - }) + url: format_url(&*version_path), + stable: false + })); } } - Ok::<(), Error>(()) - }.await?; + Ok(None) + }.await + })).await?.into_iter().flatten().collect(); - Ok::<(), Error>(()) + versions.lock().await.push(daedalus::modded::Version { + id: minecraft_version, + loaders: loaders_versions }); - } + + Ok::<(), Error>(()) + }); } { @@ -458,13 +457,48 @@ pub async fn retrieve_data(uploaded_files: &mut Vec) -> Result<(), Error } if let Ok(versions) = Arc::try_unwrap(versions) { + let mut versions = versions.into_inner(); + + versions.sort_by(|x, y| { + minecraft_versions + .versions + .iter() + .position(|z| x.id == z.id) + .unwrap_or_default() + .cmp( + &minecraft_versions + .versions + .iter() + .position(|z| y.id == z.id) + .unwrap_or_default(), + ) + }); + + for version in &mut versions { + let loader_versions = maven_metadata.get(&version.id); + if let Some(loader_versions) = loader_versions { + version.loaders.sort_by(|x, y| { + loader_versions + .iter() + .position(|z| &y.id == z) + .unwrap_or_default() + .cmp( + &loader_versions + .iter() + .position(|z| &x.id == z) + .unwrap_or_default(), + ) + }) + } + } + upload_file_to_bucket( format!( "forge/v{}/manifest.json", daedalus::modded::CURRENT_FORGE_FORMAT_VERSION, ), serde_json::to_vec(&Manifest { - game_versions: versions.into_inner(), + game_versions: versions, })?, Some("application/json".to_string()), uploaded_files_mutex.as_ref(), diff --git a/daedalus_client/src/main.rs b/daedalus_client/src/main.rs index 123e2b4f2..5729cd157 100644 --- a/daedalus_client/src/main.rs +++ b/daedalus_client/src/main.rs @@ -30,6 +30,8 @@ pub enum Error { ZipError(#[from] zip::result::ZipError), #[error("Error while reading zip file: {0}")] IoError(#[from] std::io::Error), + #[error("Error while obtaining strong reference to Arc")] + ArcError, } #[tokio::main] @@ -49,19 +51,26 @@ async fn main() { tokio::spawn(async { let mut uploaded_files = Vec::new(); - match fabric::retrieve_data(&mut uploaded_files).await { - Ok(..) => {} - Err(err) => error!("{:?}", err), - }; - match minecraft::retrieve_data(&mut uploaded_files).await { - Ok(..) => {} - Err(err) => error!("{:?}", err), - }; - match forge::retrieve_data(&mut uploaded_files).await { - Ok(..) => {} - Err(err) => error!("{:?}", err), + let versions = match minecraft::retrieve_data(&mut uploaded_files).await { + Ok(res) => Some(res), + Err(err) => { + error!("{:?}", err); + + None + } }; + if let Some(manifest) = versions { + match fabric::retrieve_data(&manifest, &mut uploaded_files).await { + Ok(..) => {} + Err(err) => error!("{:?}", err), + }; + match forge::retrieve_data(&manifest, &mut uploaded_files).await { + Ok(..) => {} + Err(err) => error!("{:?}", err), + }; + } + match purge_digitalocean_cache(uploaded_files).await { Ok(..) => {} Err(err) => error!("{:?}", err), diff --git a/daedalus_client/src/minecraft.rs b/daedalus_client/src/minecraft.rs index babb6a189..d3f7c72fc 100644 --- a/daedalus_client/src/minecraft.rs +++ b/daedalus_client/src/minecraft.rs @@ -1,11 +1,12 @@ use crate::{format_url, upload_file_to_bucket, Error}; use daedalus::download_file; +use daedalus::minecraft::VersionManifest; use log::info; use std::sync::Arc; use std::time::{Duration, Instant}; use tokio::sync::Mutex; -pub async fn retrieve_data(uploaded_files: &mut Vec) -> Result<(), Error> { +pub async fn retrieve_data(uploaded_files: &mut Vec) -> Result { let old_manifest = daedalus::minecraft::fetch_version_manifest(Some(&*crate::format_url(&*format!( "minecraft/v{}/manifest.json", @@ -167,5 +168,7 @@ pub async fn retrieve_data(uploaded_files: &mut Vec) -> Result<(), Error let elapsed = now.elapsed(); info!("Elapsed: {:.2?}", elapsed); - Ok(()) + Ok(Arc::try_unwrap(cloned_manifest) + .map_err(|_| Error::ArcError)? + .into_inner()) } From 2a588d1e9a8d31079bd059a535dbd12172bae6ec Mon Sep 17 00:00:00 2001 From: Jai A Date: Sun, 19 Dec 2021 15:09:36 -0700 Subject: [PATCH 27/85] Make java stuff public, fix forge erroring out due to ratelimiting --- daedalus/Cargo.toml | 2 +- daedalus/src/minecraft.rs | 4 +- daedalus_client/Cargo.toml | 2 +- daedalus_client/src/forge.rs | 553 ++++++++++++++++++----------------- daedalus_client/src/main.rs | 41 ++- 5 files changed, 312 insertions(+), 290 deletions(-) diff --git a/daedalus/Cargo.toml b/daedalus/Cargo.toml index 38aee9792..c4f01bb45 100644 --- a/daedalus/Cargo.toml +++ b/daedalus/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "daedalus" -version = "0.1.8" +version = "0.1.9" authors = ["Jai A "] edition = "2018" license = "MIT" diff --git a/daedalus/src/minecraft.rs b/daedalus/src/minecraft.rs index 250262cdc..ce15217a1 100644 --- a/daedalus/src/minecraft.rs +++ b/daedalus/src/minecraft.rs @@ -233,9 +233,9 @@ pub struct LibraryExtract { /// Information about the java version the game needs pub struct JavaVersion { /// The component needed for the Java installation - component: String, + pub component: String, /// The major Java version number - major_version: u32, + pub major_version: u32, } #[derive(Serialize, Deserialize, Debug)] diff --git a/daedalus_client/Cargo.toml b/daedalus_client/Cargo.toml index 77ae6fd46..9d4cf1e43 100644 --- a/daedalus_client/Cargo.toml +++ b/daedalus_client/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "daedalus_client" -version = "0.1.8" +version = "0.1.9" authors = ["Jai A "] edition = "2018" diff --git a/daedalus_client/src/forge.rs b/daedalus_client/src/forge.rs index f8f8e188d..99429ac18 100644 --- a/daedalus_client/src/forge.rs +++ b/daedalus_client/src/forge.rs @@ -27,6 +27,7 @@ pub async fn retrieve_data( minecraft_versions: &VersionManifest, uploaded_files: &mut Vec, ) -> Result<(), Error> { + println!("forg"); let maven_metadata = fetch_maven_metadata(None).await?; let old_manifest = daedalus::modded::fetch_manifest(&*format_url(&*format!( "forge/v{}/manifest.json", @@ -81,189 +82,192 @@ pub async fn retrieve_data( } } version_futures.push(async { - let loaders_versions = futures::future::try_join_all(loaders.into_iter().map(|(loader_version_full, version)| async { - let versions_mutex = Arc::clone(&old_versions); - let visited_assets = Arc::clone(&visited_assets_mutex); - let uploaded_files_mutex = Arc::clone(&uploaded_files_mutex); - let minecraft_version = minecraft_version.clone(); - - async move { - { - let versions = versions_mutex.lock().await; - let version = versions.iter().find(|x| - x.id == minecraft_version).map(|x| x.loaders.iter().find(|x| x.id == loader_version_full)).flatten(); - - if let Some(version) = version { - return Ok::, Error>(Some(version.clone())); + let mut loaders_versions = Vec::new(); + + { + let mut loaders_futures = loaders.into_iter().map(|(loader_version_full, version)| async { + let versions_mutex = Arc::clone(&old_versions); + let visited_assets = Arc::clone(&visited_assets_mutex); + let uploaded_files_mutex = Arc::clone(&uploaded_files_mutex); + let minecraft_version = minecraft_version.clone(); + + async move { + { + let versions = versions_mutex.lock().await; + let version = versions.iter().find(|x| + x.id == minecraft_version).map(|x| x.loaders.iter().find(|x| x.id == loader_version_full)).flatten(); + + if let Some(version) = version { + return Ok::, Error>(Some(version.clone())); + } } - } - info!("Forge - Installer Start {}", loader_version_full.clone()); - let bytes = download_file(&*format!("https://maven.minecraftforge.net/net/minecraftforge/forge/{0}/forge-{0}-installer.jar", loader_version_full), None).await?; + info!("Forge - Installer Start {}", loader_version_full.clone()); + let bytes = download_file(&*format!("https://maven.minecraftforge.net/net/minecraftforge/forge/{0}/forge-{0}-installer.jar", loader_version_full), None).await?; - let reader = std::io::Cursor::new(bytes); + let reader = std::io::Cursor::new(bytes); - if let Ok(archive) = zip::ZipArchive::new(reader) { - if FORGE_MANIFEST_V1_QUERY.matches(&version) { - let mut archive_clone = archive.clone(); - let profile = tokio::task::spawn_blocking(move || { - let mut install_profile = archive_clone.by_name("install_profile.json")?; + if let Ok(archive) = zip::ZipArchive::new(reader) { + if FORGE_MANIFEST_V1_QUERY.matches(&version) { + let mut archive_clone = archive.clone(); + let profile = tokio::task::spawn_blocking(move || { + let mut install_profile = archive_clone.by_name("install_profile.json")?; - let mut contents = String::new(); - install_profile.read_to_string(&mut contents)?; + let mut contents = String::new(); + install_profile.read_to_string(&mut contents)?; - Ok::(serde_json::from_str::(&*contents)?) - }).await??; + Ok::(serde_json::from_str::(&*contents)?) + }).await??; - let mut archive_clone = archive.clone(); - let file_path = profile.install.file_path.clone(); - let forge_universal_bytes = tokio::task::spawn_blocking(move || { - let mut forge_universal_file = archive_clone.by_name(&*file_path)?; - let mut forge_universal = Vec::new(); - forge_universal_file.read_to_end(&mut forge_universal)?; + let mut archive_clone = archive.clone(); + let file_path = profile.install.file_path.clone(); + let forge_universal_bytes = tokio::task::spawn_blocking(move || { + let mut forge_universal_file = archive_clone.by_name(&*file_path)?; + let mut forge_universal = Vec::new(); + forge_universal_file.read_to_end(&mut forge_universal)?; - Ok::(bytes::Bytes::from(forge_universal)) - }).await??; - let forge_universal_path = profile.install.path.clone(); + Ok::(bytes::Bytes::from(forge_universal)) + }).await??; + let forge_universal_path = profile.install.path.clone(); - let now = Instant::now(); - let libs = futures::future::try_join_all(profile.version_info.libraries.into_iter().map(|mut lib| async { - if let Some(url) = lib.url { - { - let mut visited_assets = visited_assets.lock().await; + let now = Instant::now(); + let libs = futures::future::try_join_all(profile.version_info.libraries.into_iter().map(|mut lib| async { + if let Some(url) = lib.url { + { + let mut visited_assets = visited_assets.lock().await; - if visited_assets.contains(&lib.name) { - lib.url = Some(format_url("maven/")); + if visited_assets.contains(&lib.name) { + lib.url = Some(format_url("maven/")); - return Ok::(lib); - } else { - visited_assets.push(lib.name.clone()) + return Ok::(lib); + } else { + visited_assets.push(lib.name.clone()) + } } - } - let artifact_path = - daedalus::get_path_from_artifact(&*lib.name)?; + let artifact_path = + daedalus::get_path_from_artifact(&*lib.name)?; - let artifact = if lib.name == forge_universal_path { - forge_universal_bytes.clone() - } else { - let mirrors = vec![&*url, "https://maven.creeperhost.net/", "https://libraries.minecraft.net/"]; - - daedalus::download_file_mirrors( - &*artifact_path, - &mirrors, - None, - ) - .await? - }; - - lib.url = Some(format_url("maven/")); - - upload_file_to_bucket( - format!("{}/{}", "maven", artifact_path), - artifact.to_vec(), - Some("application/java-archive".to_string()), - uploaded_files_mutex.as_ref(), - ).await?; - } + let artifact = if lib.name == forge_universal_path { + forge_universal_bytes.clone() + } else { + let mirrors = vec![&*url, "https://maven.creeperhost.net/", "https://libraries.minecraft.net/"]; - Ok::(lib) - })).await?; - - let elapsed = now.elapsed(); - info!("Elapsed lib DL: {:.2?}", elapsed); - - let new_profile = PartialVersionInfo { - id: profile.version_info.id, - inherits_from: profile.install.minecraft, - release_time: profile.version_info.release_time, - time: profile.version_info.time, - main_class: profile.version_info.main_class, - minecraft_arguments: profile.version_info.minecraft_arguments.clone(), - arguments: profile.version_info.minecraft_arguments.map(|x| [(ArgumentType::Game, x.split(' ').map(|x| Argument::Normal(x.to_string())).collect())].iter().cloned().collect()), - libraries: libs, - type_: profile.version_info.type_, - data: None, - processors: None - }; - - let version_path = format!( - "forge/v{}/versions/{}.json", - daedalus::modded::CURRENT_FORGE_FORMAT_VERSION, - new_profile.id - ); - - upload_file_to_bucket( - version_path.clone(), - serde_json::to_vec(&new_profile)?, - Some("application/json".to_string()), - uploaded_files_mutex.as_ref() - ).await?; - - return Ok(Some(LoaderVersion { - id: loader_version_full, - url: format_url(&*version_path), - stable: false - })); - } else if FORGE_MANIFEST_V2_QUERY_P1.matches(&version) || FORGE_MANIFEST_V2_QUERY_P2.matches(&version) || FORGE_MANIFEST_V3_QUERY.matches(&version) { - let mut archive_clone = archive.clone(); - let mut profile = tokio::task::spawn_blocking(move || { - let mut install_profile = archive_clone.by_name("install_profile.json")?; - - let mut contents = String::new(); - install_profile.read_to_string(&mut contents)?; - - Ok::(serde_json::from_str::(&*contents)?) - }).await??; - - let mut archive_clone = archive.clone(); - let version_info = tokio::task::spawn_blocking(move || { - let mut install_profile = archive_clone.by_name("version.json")?; - - let mut contents = String::new(); - install_profile.read_to_string(&mut contents)?; - - Ok::(serde_json::from_str::(&*contents)?) - }).await??; - - - let mut libs : Vec = version_info.libraries.into_iter().chain(profile.libraries.into_iter().map(|x| Library { - downloads: x.downloads, - extract: x.extract, - name: x.name, - url: x.url, - natives: x.natives, - rules: x.rules, - checksums: x.checksums, - include_in_classpath: false - })).collect(); - - let mut local_libs : HashMap = HashMap::new(); - - for lib in &libs { - if lib.downloads.as_ref().map(|x| x.artifact.as_ref().map(|x| x.url.is_empty())).flatten().unwrap_or(false) { - let mut archive_clone = archive.clone(); - let lib_name_clone = lib.name.clone(); - - let lib_bytes = tokio::task::spawn_blocking(move || { - let mut lib_file = archive_clone.by_name(&*format!("maven/{}", daedalus::get_path_from_artifact(&*lib_name_clone)?))?; - let mut lib_bytes = Vec::new(); - lib_file.read_to_end(&mut lib_bytes)?; - - Ok::(bytes::Bytes::from(lib_bytes)) - }).await??; - - local_libs.insert(lib.name.clone(), lib_bytes); + daedalus::download_file_mirrors( + &*artifact_path, + &mirrors, + None, + ) + .await? + }; + + lib.url = Some(format_url("maven/")); + + upload_file_to_bucket( + format!("{}/{}", "maven", artifact_path), + artifact.to_vec(), + Some("application/java-archive".to_string()), + uploaded_files_mutex.as_ref(), + ).await?; + } + + Ok::(lib) + })).await?; + + let elapsed = now.elapsed(); + info!("Elapsed lib DL: {:.2?}", elapsed); + + let new_profile = PartialVersionInfo { + id: profile.version_info.id, + inherits_from: profile.install.minecraft, + release_time: profile.version_info.release_time, + time: profile.version_info.time, + main_class: profile.version_info.main_class, + minecraft_arguments: profile.version_info.minecraft_arguments.clone(), + arguments: profile.version_info.minecraft_arguments.map(|x| [(ArgumentType::Game, x.split(' ').map(|x| Argument::Normal(x.to_string())).collect())].iter().cloned().collect()), + libraries: libs, + type_: profile.version_info.type_, + data: None, + processors: None + }; + + let version_path = format!( + "forge/v{}/versions/{}.json", + daedalus::modded::CURRENT_FORGE_FORMAT_VERSION, + new_profile.id + ); + + upload_file_to_bucket( + version_path.clone(), + serde_json::to_vec(&new_profile)?, + Some("application/json".to_string()), + uploaded_files_mutex.as_ref() + ).await?; + + return Ok(Some(LoaderVersion { + id: loader_version_full, + url: format_url(&*version_path), + stable: false + })); + } else if FORGE_MANIFEST_V2_QUERY_P1.matches(&version) || FORGE_MANIFEST_V2_QUERY_P2.matches(&version) || FORGE_MANIFEST_V3_QUERY.matches(&version) { + let mut archive_clone = archive.clone(); + let mut profile = tokio::task::spawn_blocking(move || { + let mut install_profile = archive_clone.by_name("install_profile.json")?; + + let mut contents = String::new(); + install_profile.read_to_string(&mut contents)?; + + Ok::(serde_json::from_str::(&*contents)?) + }).await??; + + let mut archive_clone = archive.clone(); + let version_info = tokio::task::spawn_blocking(move || { + let mut install_profile = archive_clone.by_name("version.json")?; + + let mut contents = String::new(); + install_profile.read_to_string(&mut contents)?; + + Ok::(serde_json::from_str::(&*contents)?) + }).await??; + + + let mut libs : Vec = version_info.libraries.into_iter().chain(profile.libraries.into_iter().map(|x| Library { + downloads: x.downloads, + extract: x.extract, + name: x.name, + url: x.url, + natives: x.natives, + rules: x.rules, + checksums: x.checksums, + include_in_classpath: false + })).collect(); + + let mut local_libs : HashMap = HashMap::new(); + + for lib in &libs { + if lib.downloads.as_ref().map(|x| x.artifact.as_ref().map(|x| x.url.is_empty())).flatten().unwrap_or(false) { + let mut archive_clone = archive.clone(); + let lib_name_clone = lib.name.clone(); + + let lib_bytes = tokio::task::spawn_blocking(move || { + let mut lib_file = archive_clone.by_name(&*format!("maven/{}", daedalus::get_path_from_artifact(&*lib_name_clone)?))?; + let mut lib_bytes = Vec::new(); + lib_file.read_to_end(&mut lib_bytes)?; + + Ok::(bytes::Bytes::from(lib_bytes)) + }).await??; + + local_libs.insert(lib.name.clone(), lib_bytes); + } } - } - let path = profile.path.clone(); - let version = profile.version.clone(); + let path = profile.path.clone(); + let version = profile.version.clone(); - for entry in profile.data.values_mut() { - if entry.client.starts_with('/') || entry.server.starts_with('/') { - macro_rules! read_data { + for entry in profile.data.values_mut() { + if entry.client.starts_with('/') || entry.server.starts_with('/') { + macro_rules! read_data { ($value:expr) => { let mut archive_clone = archive.clone(); let value_clone = $value.clone(); @@ -302,131 +306,150 @@ pub async fn retrieve_data( } } - if entry.client.starts_with('/') { - read_data!(entry.client); - } + if entry.client.starts_with('/') { + read_data!(entry.client); + } - // Do we really need to support server installs? Keeping this here - // just in case - // - // if entry.server.starts_with('/') { - // read_data!(entry.server); - // } + // Do we really need to support server installs? Keeping this here + // just in case + // + // if entry.server.starts_with('/') { + // read_data!(entry.server); + // } + } } - } - let now = Instant::now(); - let libs = futures::future::try_join_all(libs.into_iter().map(|mut lib| async { - let artifact_path = - daedalus::get_path_from_artifact(&*lib.name)?; + let now = Instant::now(); + let libs = futures::future::try_join_all(libs.into_iter().map(|mut lib| async { + let artifact_path = + daedalus::get_path_from_artifact(&*lib.name)?; - { - let mut visited_assets = visited_assets.lock().await; + { + let mut visited_assets = visited_assets.lock().await; - if visited_assets.contains(&lib.name) { - if let Some(ref mut downloads) = lib.downloads { - if let Some(ref mut artifact) = downloads.artifact { - artifact.url = format_url(&*format!("maven/{}", artifact_path)); + if visited_assets.contains(&lib.name) { + if let Some(ref mut downloads) = lib.downloads { + if let Some(ref mut artifact) = downloads.artifact { + artifact.url = format_url(&*format!("maven/{}", artifact_path)); + } + } else if lib.url.is_some() { + lib.url = Some(format_url("maven/")); } - } else if lib.url.is_some() { - lib.url = Some(format_url("maven/")); - } - return Ok::(lib); - } else { - visited_assets.push(lib.name.clone()) + return Ok::(lib); + } else { + visited_assets.push(lib.name.clone()) + } } - } - let artifact_bytes = if let Some(ref mut downloads) = lib.downloads { - if let Some(ref mut artifact) = downloads.artifact { - let res = if artifact.url.is_empty() { + let artifact_bytes = if let Some(ref mut downloads) = lib.downloads { + if let Some(ref mut artifact) = downloads.artifact { + let res = if artifact.url.is_empty() { + local_libs.get(&lib.name).cloned() + } else { + Some(daedalus::download_file( + &*artifact.url, + Some(&*artifact.sha1), + ) + .await?) + }; + + if res.is_some() { + artifact.url = format_url(&*format!("maven/{}", artifact_path)); + } + + res + } else { None } + } else if let Some(ref mut url) = lib.url { + let res = if url.is_empty() { local_libs.get(&lib.name).cloned() } else { Some(daedalus::download_file( - &*artifact.url, - Some(&*artifact.sha1), + url, + None, ) .await?) }; if res.is_some() { - artifact.url = format_url(&*format!("maven/{}", artifact_path)); + lib.url = Some(format_url("maven/")); } res - } else { None } - } else if let Some(ref mut url) = lib.url { - let res = if url.is_empty() { - local_libs.get(&lib.name).cloned() - } else { - Some(daedalus::download_file( - url, - None, - ) - .await?) - }; - - if res.is_some() { - lib.url = Some(format_url("maven/")); + } else { None }; + + if let Some(bytes) = artifact_bytes { + upload_file_to_bucket( + format!("{}/{}", "maven", artifact_path), + bytes.to_vec(), + Some("application/java-archive".to_string()), + uploaded_files_mutex.as_ref() + ).await?; } - res - } else { None }; + Ok::(lib) + })).await?; + + let elapsed = now.elapsed(); + info!("Elapsed lib DL: {:.2?}", elapsed); + + let new_profile = PartialVersionInfo { + id: version_info.id, + inherits_from: version_info.inherits_from, + release_time: version_info.release_time, + time: version_info.time, + main_class: version_info.main_class, + minecraft_arguments: version_info.minecraft_arguments, + arguments: version_info.arguments, + libraries: libs, + type_: version_info.type_, + data: Some(profile.data), + processors: Some(profile.processors), + }; + + let version_path = format!( + "forge/v{}/versions/{}.json", + daedalus::modded::CURRENT_FORGE_FORMAT_VERSION, + new_profile.id + ); + + upload_file_to_bucket( + version_path.clone(), + serde_json::to_vec(&new_profile)?, + Some("application/json".to_string()), + uploaded_files_mutex.as_ref() + ).await?; + + return Ok(Some(LoaderVersion { + id: loader_version_full, + url: format_url(&*version_path), + stable: false + })); + } + } - if let Some(bytes) = artifact_bytes { - upload_file_to_bucket( - format!("{}/{}", "maven", artifact_path), - bytes.to_vec(), - Some("application/java-archive".to_string()), - uploaded_files_mutex.as_ref() - ).await?; - } + Ok(None) + }.await + }).into_iter().peekable()/*.into_iter().flatten().collect()*/; - Ok::(lib) - })).await?; - - let elapsed = now.elapsed(); - info!("Elapsed lib DL: {:.2?}", elapsed); - - let new_profile = PartialVersionInfo { - id: version_info.id, - inherits_from: version_info.inherits_from, - release_time: version_info.release_time, - time: version_info.time, - main_class: version_info.main_class, - minecraft_arguments: version_info.minecraft_arguments, - arguments: version_info.arguments, - libraries: libs, - type_: version_info.type_, - data: Some(profile.data), - processors: Some(profile.processors), - }; - - let version_path = format!( - "forge/v{}/versions/{}.json", - daedalus::modded::CURRENT_FORGE_FORMAT_VERSION, - new_profile.id - ); - - upload_file_to_bucket( - version_path.clone(), - serde_json::to_vec(&new_profile)?, - Some("application/json".to_string()), - uploaded_files_mutex.as_ref() - ).await?; - - return Ok(Some(LoaderVersion { - id: loader_version_full, - url: format_url(&*version_path), - stable: false - })); - } - } + let mut chunk_index = 0; + while loaders_futures.peek().is_some() { + info!("Loader Chunk {} Start", chunk_index); + let now = Instant::now(); + + let chunk: Vec<_> = loaders_futures.by_ref().take(10).collect(); + + let res = futures::future::try_join_all(chunk).await?; + loaders_versions.extend(res.into_iter().flatten()); + + tokio::time::sleep(Duration::from_secs(1)).await; - Ok(None) - }.await - })).await?.into_iter().flatten().collect(); + chunk_index += 1; + + let elapsed = now.elapsed(); + info!("Loader Chunk {} Elapsed: {:.2?}", chunk_index, elapsed); + } + } versions.lock().await.push(daedalus::modded::Version { id: minecraft_version, @@ -444,7 +467,7 @@ pub async fn retrieve_data( info!("Chunk {} Start", chunk_index); let now = Instant::now(); - let chunk: Vec<_> = versions_peek.by_ref().take(10).collect(); + let chunk: Vec<_> = versions_peek.by_ref().take(1).collect(); futures::future::try_join_all(chunk).await?; tokio::time::sleep(Duration::from_secs(1)).await; diff --git a/daedalus_client/src/main.rs b/daedalus_client/src/main.rs index 5729cd157..ee0586e0a 100644 --- a/daedalus_client/src/main.rs +++ b/daedalus_client/src/main.rs @@ -48,34 +48,33 @@ async fn main() { loop { timer.tick().await; - tokio::spawn(async { - let mut uploaded_files = Vec::new(); - let versions = match minecraft::retrieve_data(&mut uploaded_files).await { - Ok(res) => Some(res), - Err(err) => { - error!("{:?}", err); + let mut uploaded_files = Vec::new(); - None - } - }; + let versions = match minecraft::retrieve_data(&mut uploaded_files).await { + Ok(res) => Some(res), + Err(err) => { + error!("{:?}", err); - if let Some(manifest) = versions { - match fabric::retrieve_data(&manifest, &mut uploaded_files).await { - Ok(..) => {} - Err(err) => error!("{:?}", err), - }; - match forge::retrieve_data(&manifest, &mut uploaded_files).await { - Ok(..) => {} - Err(err) => error!("{:?}", err), - }; + None } - - match purge_digitalocean_cache(uploaded_files).await { + }; + + if let Some(manifest) = versions { + // match fabric::retrieve_data(&manifest, &mut uploaded_files).await { + // Ok(..) => {} + // Err(err) => error!("{:?}", err), + // }; + match forge::retrieve_data(&manifest, &mut uploaded_files).await { Ok(..) => {} Err(err) => error!("{:?}", err), }; - }); + } + + match purge_digitalocean_cache(uploaded_files).await { + Ok(..) => {} + Err(err) => error!("{:?}", err), + }; } } From c9598b674c803fbac031f88138f0eb34ecaedeaf Mon Sep 17 00:00:00 2001 From: Jai A Date: Sun, 19 Dec 2021 18:52:00 -0700 Subject: [PATCH 28/85] Fix forge erroring out due to wonky version --- daedalus/Cargo.toml | 2 +- daedalus_client/Cargo.toml | 2 +- daedalus_client/src/forge.rs | 10 ++++++++++ daedalus_client/src/main.rs | 8 ++++---- 4 files changed, 16 insertions(+), 6 deletions(-) diff --git a/daedalus/Cargo.toml b/daedalus/Cargo.toml index c4f01bb45..b6b09a1aa 100644 --- a/daedalus/Cargo.toml +++ b/daedalus/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "daedalus" -version = "0.1.9" +version = "0.1.10" authors = ["Jai A "] edition = "2018" license = "MIT" diff --git a/daedalus_client/Cargo.toml b/daedalus_client/Cargo.toml index 9d4cf1e43..e196417de 100644 --- a/daedalus_client/Cargo.toml +++ b/daedalus_client/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "daedalus_client" -version = "0.1.9" +version = "0.1.10" authors = ["Jai A "] edition = "2018" diff --git a/daedalus_client/src/forge.rs b/daedalus_client/src/forge.rs index 99429ac18..a2fba717f 100644 --- a/daedalus_client/src/forge.rs +++ b/daedalus_client/src/forge.rs @@ -92,6 +92,16 @@ pub async fn retrieve_data( let minecraft_version = minecraft_version.clone(); async move { + /// These forge versions are not worth supporting! + const WHITELIST : [&str; 1] = [ + // Not supported due to `data` field being `[]` even though the type is a map + "1.12.2-14.23.5.2851" + ]; + + if !WHITELIST.contains(&&*loader_version_full) { + return Ok(None); + } + { let versions = versions_mutex.lock().await; let version = versions.iter().find(|x| diff --git a/daedalus_client/src/main.rs b/daedalus_client/src/main.rs index ee0586e0a..4a4f67667 100644 --- a/daedalus_client/src/main.rs +++ b/daedalus_client/src/main.rs @@ -61,10 +61,10 @@ async fn main() { }; if let Some(manifest) = versions { - // match fabric::retrieve_data(&manifest, &mut uploaded_files).await { - // Ok(..) => {} - // Err(err) => error!("{:?}", err), - // }; + match fabric::retrieve_data(&manifest, &mut uploaded_files).await { + Ok(..) => {} + Err(err) => error!("{:?}", err), + }; match forge::retrieve_data(&manifest, &mut uploaded_files).await { Ok(..) => {} Err(err) => error!("{:?}", err), From 80173634a02a86f1c4ecb2240e598e3541b51e5d Mon Sep 17 00:00:00 2001 From: Jai A Date: Sun, 19 Dec 2021 19:04:42 -0700 Subject: [PATCH 29/85] Fix java version specifier being in snake case --- daedalus/Cargo.toml | 2 +- daedalus/src/minecraft.rs | 1 + daedalus_client/Cargo.toml | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/daedalus/Cargo.toml b/daedalus/Cargo.toml index b6b09a1aa..fedac092b 100644 --- a/daedalus/Cargo.toml +++ b/daedalus/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "daedalus" -version = "0.1.10" +version = "0.1.11" authors = ["Jai A "] edition = "2018" license = "MIT" diff --git a/daedalus/src/minecraft.rs b/daedalus/src/minecraft.rs index ce15217a1..73ee8097d 100644 --- a/daedalus/src/minecraft.rs +++ b/daedalus/src/minecraft.rs @@ -230,6 +230,7 @@ pub struct LibraryExtract { } #[derive(Serialize, Deserialize, Debug)] +#[serde(rename_all = "camelCase")] /// Information about the java version the game needs pub struct JavaVersion { /// The component needed for the Java installation diff --git a/daedalus_client/Cargo.toml b/daedalus_client/Cargo.toml index e196417de..60943cb63 100644 --- a/daedalus_client/Cargo.toml +++ b/daedalus_client/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "daedalus_client" -version = "0.1.10" +version = "0.1.11" authors = ["Jai A "] edition = "2018" From 1d86aac33800d84e2f227f3d17f8d6c046d452e3 Mon Sep 17 00:00:00 2001 From: Jai A Date: Sun, 19 Dec 2021 19:56:56 -0700 Subject: [PATCH 30/85] Fix java version not being optional --- daedalus/Cargo.toml | 2 +- daedalus/src/minecraft.rs | 2 +- daedalus_client/Cargo.toml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/daedalus/Cargo.toml b/daedalus/Cargo.toml index fedac092b..3190f9e09 100644 --- a/daedalus/Cargo.toml +++ b/daedalus/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "daedalus" -version = "0.1.11" +version = "0.1.12" authors = ["Jai A "] edition = "2018" license = "MIT" diff --git a/daedalus/src/minecraft.rs b/daedalus/src/minecraft.rs index 73ee8097d..339cbfef4 100644 --- a/daedalus/src/minecraft.rs +++ b/daedalus/src/minecraft.rs @@ -323,7 +323,7 @@ pub struct VersionInfo { pub id: String, /// The Java version this version supports - pub java_version: JavaVersion, + pub java_version: Option, /// Libraries that the version depends on pub libraries: Vec, /// The classpath to the main class to launch the game diff --git a/daedalus_client/Cargo.toml b/daedalus_client/Cargo.toml index 60943cb63..c7f1d2b65 100644 --- a/daedalus_client/Cargo.toml +++ b/daedalus_client/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "daedalus_client" -version = "0.1.11" +version = "0.1.12" authors = ["Jai A "] edition = "2018" From 09aef18999b1904c6e0639435b188c7d130a92f4 Mon Sep 17 00:00:00 2001 From: Jai A Date: Sun, 19 Dec 2021 20:51:34 -0700 Subject: [PATCH 31/85] Fix incorrect condition for forge and incorrect fabric loader version ordering --- daedalus/Cargo.toml | 2 +- daedalus_client/Cargo.toml | 2 +- daedalus_client/src/fabric.rs | 14 ++++++++++++-- daedalus_client/src/forge.rs | 3 +-- 4 files changed, 15 insertions(+), 6 deletions(-) diff --git a/daedalus/Cargo.toml b/daedalus/Cargo.toml index 3190f9e09..3daee5cfc 100644 --- a/daedalus/Cargo.toml +++ b/daedalus/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "daedalus" -version = "0.1.12" +version = "0.1.13" authors = ["Jai A "] edition = "2018" license = "MIT" diff --git a/daedalus_client/Cargo.toml b/daedalus_client/Cargo.toml index c7f1d2b65..cf925cfcb 100644 --- a/daedalus_client/Cargo.toml +++ b/daedalus_client/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "daedalus_client" -version = "0.1.12" +version = "0.1.13" authors = ["Jai A "] edition = "2018" diff --git a/daedalus_client/src/fabric.rs b/daedalus_client/src/fabric.rs index 514252c96..fdc3cf46c 100644 --- a/daedalus_client/src/fabric.rs +++ b/daedalus_client/src/fabric.rs @@ -229,13 +229,23 @@ pub async fn retrieve_data( version.loaders.sort_by(|x, y| { list.loader .iter() - .position(|z| x.id == z.version) + .position(|z| { + x.id.split('-') + .next() + .unwrap_or_default() + == &*z.version + }) .unwrap_or_default() .cmp( &list .loader .iter() - .position(|z| y.id == z.version) + .position(|z| { + y.id.split('-') + .next() + .unwrap_or_default() + == z.version + }) .unwrap_or_default(), ) }) diff --git a/daedalus_client/src/forge.rs b/daedalus_client/src/forge.rs index a2fba717f..c0e841c5e 100644 --- a/daedalus_client/src/forge.rs +++ b/daedalus_client/src/forge.rs @@ -27,7 +27,6 @@ pub async fn retrieve_data( minecraft_versions: &VersionManifest, uploaded_files: &mut Vec, ) -> Result<(), Error> { - println!("forg"); let maven_metadata = fetch_maven_metadata(None).await?; let old_manifest = daedalus::modded::fetch_manifest(&*format_url(&*format!( "forge/v{}/manifest.json", @@ -98,7 +97,7 @@ pub async fn retrieve_data( "1.12.2-14.23.5.2851" ]; - if !WHITELIST.contains(&&*loader_version_full) { + if WHITELIST.contains(&&*loader_version_full) { return Ok(None); } From 00adf3631a5e3111798349f0dea5d416c56f122d Mon Sep 17 00:00:00 2001 From: Jai A Date: Mon, 20 Dec 2021 09:11:40 -0700 Subject: [PATCH 32/85] Remove replacing assets index URL in manifest to preserve file integrity --- daedalus/Cargo.toml | 2 +- daedalus_client/Cargo.toml | 2 +- daedalus_client/src/minecraft.rs | 1 - 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/daedalus/Cargo.toml b/daedalus/Cargo.toml index 3daee5cfc..448ead6e6 100644 --- a/daedalus/Cargo.toml +++ b/daedalus/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "daedalus" -version = "0.1.13" +version = "0.1.14" authors = ["Jai A "] edition = "2018" license = "MIT" diff --git a/daedalus_client/Cargo.toml b/daedalus_client/Cargo.toml index cf925cfcb..bf4473328 100644 --- a/daedalus_client/Cargo.toml +++ b/daedalus_client/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "daedalus_client" -version = "0.1.13" +version = "0.1.14" authors = ["Jai A "] edition = "2018" diff --git a/daedalus_client/src/minecraft.rs b/daedalus_client/src/minecraft.rs index d3f7c72fc..58fae665b 100644 --- a/daedalus_client/src/minecraft.rs +++ b/daedalus_client/src/minecraft.rs @@ -75,7 +75,6 @@ pub async fn retrieve_data(uploaded_files: &mut Vec) -> Result Date: Sun, 26 Jun 2022 16:23:41 -0700 Subject: [PATCH 33/85] Add Bincode feature for efficient binary storage --- daedalus/Cargo.toml | 1 + daedalus/src/minecraft.rs | 48 +++++++++++++++++++++++++++++++++++---- daedalus/src/modded.rs | 24 +++++++++++++++++--- rustfmt.toml | 2 ++ 4 files changed, 68 insertions(+), 7 deletions(-) create mode 100644 rustfmt.toml diff --git a/daedalus/Cargo.toml b/daedalus/Cargo.toml index 448ead6e6..abd0f5af2 100644 --- a/daedalus/Cargo.toml +++ b/daedalus/Cargo.toml @@ -22,3 +22,4 @@ bytes = "1" thiserror = "1.0" tokio = { version = "1", features = ["full"] } sha1 = { version = "0.6.0", features = ["std"]} +bincode = {version = "2.0.0-rc", features = ["serde"], optional = true} diff --git a/daedalus/src/minecraft.rs b/daedalus/src/minecraft.rs index 339cbfef4..c86c39a25 100644 --- a/daedalus/src/minecraft.rs +++ b/daedalus/src/minecraft.rs @@ -4,9 +4,13 @@ use chrono::{DateTime, Utc}; use serde::{Deserialize, Serialize}; use std::collections::HashMap; +#[cfg(feature = "bincode")] +use bincode::{Decode, Encode}; + /// The latest version of the format the model structs deserialize to pub const CURRENT_FORMAT_VERSION: usize = 0; +#[cfg_attr(feature = "bincode", derive(Encode, Decode))] #[derive(Serialize, Deserialize, Debug, Clone)] #[serde(rename_all = "snake_case")] /// The version type @@ -33,6 +37,7 @@ impl VersionType { } } +#[cfg_attr(feature = "bincode", derive(Encode, Decode))] #[derive(Serialize, Deserialize, Debug, Clone)] #[serde(rename_all = "camelCase")] /// A game version of Minecraft @@ -45,8 +50,10 @@ pub struct Version { /// A link to additional information about the version pub url: String, /// The latest time a file in this version was updated + #[bincode(with_serde)] pub time: DateTime, /// The time this version was released + #[bincode(with_serde)] pub release_time: DateTime, /// The SHA1 hash of the additional information about the version pub sha1: String, @@ -62,6 +69,7 @@ pub struct Version { pub assets_index_sha1: Option, } +#[cfg_attr(feature = "bincode", derive(Encode, Decode))] #[derive(Serialize, Deserialize, Debug, Clone)] /// The latest snapshot and release of the game pub struct LatestVersion { @@ -71,6 +79,7 @@ pub struct LatestVersion { pub snapshot: String, } +#[cfg_attr(feature = "bincode", derive(Encode, Decode))] #[derive(Serialize, Deserialize, Debug, Clone)] /// Data of all game versions of Minecraft pub struct VersionManifest { @@ -85,12 +94,15 @@ pub const VERSION_MANIFEST_URL: &str = "https://launchermeta.mojang.com/mc/game/version_manifest_v2.json"; /// Fetches a version manifest from the specified URL. If no URL is specified, the default is used. -pub async fn fetch_version_manifest(url: Option<&str>) -> Result { +pub async fn fetch_version_manifest( + url: Option<&str>, +) -> Result { Ok(serde_json::from_slice( &download_file(url.unwrap_or(VERSION_MANIFEST_URL), None).await?, )?) } +#[cfg_attr(feature = "bincode", derive(Encode, Decode))] #[derive(Serialize, Deserialize, Debug)] #[serde(rename_all = "camelCase")] /// Information about the assets of the game @@ -107,6 +119,7 @@ pub struct AssetIndex { pub url: String, } +#[cfg_attr(feature = "bincode", derive(Encode, Decode))] #[derive(Serialize, Deserialize, Debug, Eq, PartialEq, Hash)] #[serde(rename_all = "snake_case")] /// The type of download @@ -123,6 +136,7 @@ pub enum DownloadType { WindowsServer, } +#[cfg_attr(feature = "bincode", derive(Encode, Decode))] #[derive(Serialize, Deserialize, Debug)] /// Download information of a file pub struct Download { @@ -134,6 +148,7 @@ pub struct Download { pub url: String, } +#[cfg_attr(feature = "bincode", derive(Encode, Decode))] #[derive(Serialize, Deserialize, Debug)] /// Download information of a library pub struct LibraryDownload { @@ -147,6 +162,7 @@ pub struct LibraryDownload { pub url: String, } +#[cfg_attr(feature = "bincode", derive(Encode, Decode))] #[derive(Serialize, Deserialize, Debug)] /// A list of files that should be downloaded for libraries pub struct LibraryDownloads { @@ -159,6 +175,7 @@ pub struct LibraryDownloads { pub classifiers: Option>, } +#[cfg_attr(feature = "bincode", derive(Encode, Decode))] #[derive(Serialize, Deserialize, Debug, Clone)] #[serde(rename_all = "snake_case")] /// The action a rule can follow @@ -169,6 +186,7 @@ pub enum RuleAction { Disallow, } +#[cfg_attr(feature = "bincode", derive(Encode, Decode))] #[derive(Serialize, Deserialize, Debug, Eq, PartialEq, Hash, Clone)] #[serde(rename_all = "snake_case")] /// An enum representing the different types of operating systems @@ -183,6 +201,7 @@ pub enum Os { Unknown, } +#[cfg_attr(feature = "bincode", derive(Encode, Decode))] #[derive(Serialize, Deserialize, Debug, Clone)] /// A rule which depends on what OS the user is on pub struct OsRule { @@ -197,6 +216,7 @@ pub struct OsRule { pub arch: Option, } +#[cfg_attr(feature = "bincode", derive(Encode, Decode))] #[derive(Serialize, Deserialize, Debug, Clone)] /// A rule which depends on the toggled features of the launcher pub struct FeatureRule { @@ -208,6 +228,7 @@ pub struct FeatureRule { pub has_demo_resolution: Option, } +#[cfg_attr(feature = "bincode", derive(Encode, Decode))] #[derive(Serialize, Deserialize, Debug, Clone)] /// A rule deciding whether a file is downloaded, an argument is used, etc. pub struct Rule { @@ -221,6 +242,7 @@ pub struct Rule { pub features: Option, } +#[cfg_attr(feature = "bincode", derive(Encode, Decode))] #[derive(Serialize, Deserialize, Debug)] /// Information delegating the extraction of the library pub struct LibraryExtract { @@ -229,6 +251,7 @@ pub struct LibraryExtract { pub exclude: Option>, } +#[cfg_attr(feature = "bincode", derive(Encode, Decode))] #[derive(Serialize, Deserialize, Debug)] #[serde(rename_all = "camelCase")] /// Information about the java version the game needs @@ -239,6 +262,7 @@ pub struct JavaVersion { pub major_version: u32, } +#[cfg_attr(feature = "bincode", derive(Encode, Decode))] #[derive(Serialize, Deserialize, Debug)] /// A library which the game relies on to run pub struct Library { @@ -271,6 +295,7 @@ fn default_include_in_classpath() -> bool { true } +#[cfg_attr(feature = "bincode", derive(Encode, Decode))] #[derive(Serialize, Deserialize, Debug, Clone)] #[serde(untagged)] /// A container for an argument or multiple arguments @@ -281,6 +306,7 @@ pub enum ArgumentValue { Many(Vec), } +#[cfg_attr(feature = "bincode", derive(Encode, Decode))] #[derive(Serialize, Deserialize, Debug, Clone)] #[serde(untagged)] /// A command line argument passed to a program @@ -296,6 +322,7 @@ pub enum Argument { }, } +#[cfg_attr(feature = "bincode", derive(Encode, Decode))] #[derive(Serialize, Deserialize, Debug, Eq, PartialEq, Hash, Clone, Copy)] #[serde(rename_all = "snake_case")] /// The type of argument @@ -306,6 +333,7 @@ pub enum ArgumentType { Jvm, } +#[cfg_attr(feature = "bincode", derive(Encode, Decode))] #[derive(Serialize, Deserialize, Debug)] #[serde(rename_all = "camelCase")] /// Information about a version @@ -334,8 +362,10 @@ pub struct VersionInfo { /// The minimum version of the Minecraft Launcher that can run this version of the game pub minimum_launcher_version: u32, /// The time that the version was released + #[bincode(with_serde)] pub release_time: DateTime, /// The latest time a file in this version was updated + #[bincode(with_serde)] pub time: DateTime, #[serde(rename = "type")] /// The type of version @@ -349,12 +379,15 @@ pub struct VersionInfo { } /// Fetches detailed information about a version from the manifest -pub async fn fetch_version_info(version: &Version) -> Result { +pub async fn fetch_version_info( + version: &Version, +) -> Result { Ok(serde_json::from_slice( &download_file(&version.url, Some(&version.sha1)).await?, )?) } +#[cfg_attr(feature = "bincode", derive(Encode, Decode))] #[derive(Serialize, Deserialize, Debug)] /// An asset of the game pub struct Asset { @@ -364,6 +397,7 @@ pub struct Asset { pub size: u32, } +#[cfg_attr(feature = "bincode", derive(Encode, Decode))] #[derive(Serialize, Deserialize, Debug)] /// An index containing all assets the game needs pub struct AssetsIndex { @@ -372,8 +406,14 @@ pub struct AssetsIndex { } /// Fetches the assets index from the version info -pub async fn fetch_assets_index(version: &VersionInfo) -> Result { +pub async fn fetch_assets_index( + version: &VersionInfo, +) -> Result { Ok(serde_json::from_slice( - &download_file(&version.asset_index.url, Some(&version.asset_index.sha1)).await?, + &download_file( + &version.asset_index.url, + Some(&version.asset_index.sha1), + ) + .await?, )?) } diff --git a/daedalus/src/modded.rs b/daedalus/src/modded.rs index ea3e07c96..4e51604ed 100644 --- a/daedalus/src/modded.rs +++ b/daedalus/src/modded.rs @@ -1,16 +1,22 @@ use crate::{download_file, Error}; -use crate::minecraft::{Argument, ArgumentType, Library, VersionInfo, VersionType}; +use crate::minecraft::{ + Argument, ArgumentType, Library, VersionInfo, VersionType, +}; use chrono::{DateTime, Utc}; use serde::{Deserialize, Serialize}; use std::collections::HashMap; +#[cfg(feature = "bincode")] +use bincode::{Decode, Encode}; + /// The latest version of the format the fabric model structs deserialize to pub const CURRENT_FABRIC_FORMAT_VERSION: usize = 0; /// The latest version of the format the fabric model structs deserialize to pub const CURRENT_FORGE_FORMAT_VERSION: usize = 0; /// A data variable entry that depends on the side of the installation +#[cfg_attr(feature = "bincode", derive(Encode, Decode))] #[derive(Serialize, Deserialize, Debug)] pub struct SidedDataEntry { /// The value on the client @@ -19,6 +25,7 @@ pub struct SidedDataEntry { pub server: String, } +#[cfg_attr(feature = "bincode", derive(Encode, Decode))] #[derive(Serialize, Deserialize, Debug)] #[serde(rename_all = "camelCase")] /// A partial version returned by fabric meta @@ -28,8 +35,10 @@ pub struct PartialVersionInfo { /// The version ID this partial version inherits from pub inherits_from: String, /// The time that the version was released + #[bincode(with_serde)] pub release_time: DateTime, /// The latest time a file in this version was updated + #[bincode(with_serde)] pub time: DateTime, #[serde(skip_serializing_if = "Option::is_none")] /// The classpath to the main class to launch the game @@ -54,6 +63,7 @@ pub struct PartialVersionInfo { } /// A processor to be ran after downloading the files +#[cfg_attr(feature = "bincode", derive(Encode, Decode))] #[derive(Serialize, Deserialize, Debug)] pub struct Processor { /// Maven coordinates for the JAR library of this processor. @@ -72,12 +82,17 @@ pub struct Processor { } /// Fetches the version manifest of a game version's URL -pub async fn fetch_partial_version(url: &str) -> Result { +pub async fn fetch_partial_version( + url: &str, +) -> Result { Ok(serde_json::from_slice(&download_file(url, None).await?)?) } /// Merges a partial version into a complete one -pub fn merge_partial_version(partial: PartialVersionInfo, merge: VersionInfo) -> VersionInfo { +pub fn merge_partial_version( + partial: PartialVersionInfo, + merge: VersionInfo, +) -> VersionInfo { VersionInfo { arguments: if let Some(partial_args) = partial.arguments { if let Some(merge_args) = merge.arguments { @@ -133,6 +148,7 @@ pub fn merge_partial_version(partial: PartialVersionInfo, merge: VersionInfo) -> } } +#[cfg_attr(feature = "bincode", derive(Encode, Decode))] #[derive(Serialize, Deserialize, Debug, Clone)] #[serde(rename_all = "camelCase")] /// A manifest containing information about a mod loader's versions @@ -141,6 +157,7 @@ pub struct Manifest { pub game_versions: Vec, } +#[cfg_attr(feature = "bincode", derive(Encode, Decode))] #[derive(Serialize, Deserialize, Debug, Clone)] /// A game version of Minecraft pub struct Version { @@ -150,6 +167,7 @@ pub struct Version { pub loaders: Vec, } +#[cfg_attr(feature = "bincode", derive(Encode, Decode))] #[derive(Serialize, Deserialize, Debug, Clone)] /// A version of a Minecraft mod loader pub struct LoaderVersion { diff --git a/rustfmt.toml b/rustfmt.toml new file mode 100644 index 000000000..f5a8b8674 --- /dev/null +++ b/rustfmt.toml @@ -0,0 +1,2 @@ +edition = "2018" +max_width = 80 \ No newline at end of file From 18153e0fcc4d0dd0460b098f503717075b5dd538 Mon Sep 17 00:00:00 2001 From: Danielle Hutzley Date: Sun, 26 Jun 2022 16:23:57 -0700 Subject: [PATCH 34/85] Bump version --- daedalus/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/daedalus/Cargo.toml b/daedalus/Cargo.toml index abd0f5af2..694da1265 100644 --- a/daedalus/Cargo.toml +++ b/daedalus/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "daedalus" -version = "0.1.14" +version = "0.1.15" authors = ["Jai A "] edition = "2018" license = "MIT" From 93817ba92f90ac5bb126869aba4a3dfbfd390656 Mon Sep 17 00:00:00 2001 From: Danielle Hutzley Date: Sun, 26 Jun 2022 16:41:22 -0700 Subject: [PATCH 35/85] Actually compile without Bincode --- daedalus/src/minecraft.rs | 8 ++++---- daedalus/src/modded.rs | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/daedalus/src/minecraft.rs b/daedalus/src/minecraft.rs index c86c39a25..478f69046 100644 --- a/daedalus/src/minecraft.rs +++ b/daedalus/src/minecraft.rs @@ -50,10 +50,10 @@ pub struct Version { /// A link to additional information about the version pub url: String, /// The latest time a file in this version was updated - #[bincode(with_serde)] + #[cfg_attr(feature = "bincode", bincode(with_serde))] pub time: DateTime, /// The time this version was released - #[bincode(with_serde)] + #[cfg_attr(feature = "bincode", bincode(with_serde))] pub release_time: DateTime, /// The SHA1 hash of the additional information about the version pub sha1: String, @@ -362,10 +362,10 @@ pub struct VersionInfo { /// The minimum version of the Minecraft Launcher that can run this version of the game pub minimum_launcher_version: u32, /// The time that the version was released - #[bincode(with_serde)] + #[cfg_attr(feature = "bincode", bincode(with_serde))] pub release_time: DateTime, /// The latest time a file in this version was updated - #[bincode(with_serde)] + #[cfg_attr(feature = "bincode", bincode(with_serde))] pub time: DateTime, #[serde(rename = "type")] /// The type of version diff --git a/daedalus/src/modded.rs b/daedalus/src/modded.rs index 4e51604ed..2e62c981e 100644 --- a/daedalus/src/modded.rs +++ b/daedalus/src/modded.rs @@ -35,10 +35,10 @@ pub struct PartialVersionInfo { /// The version ID this partial version inherits from pub inherits_from: String, /// The time that the version was released - #[bincode(with_serde)] + #[cfg_attr(feature = "bincode", bincode(with_serde))] pub release_time: DateTime, /// The latest time a file in this version was updated - #[bincode(with_serde)] + #[cfg_attr(feature = "bincode", bincode(with_serde))] pub time: DateTime, #[serde(skip_serializing_if = "Option::is_none")] /// The classpath to the main class to launch the game From c2a1ed926e62c9713aeda3a148a247e0d43c49d8 Mon Sep 17 00:00:00 2001 From: Danielle Hutzley Date: Mon, 27 Jun 2022 13:54:09 -0700 Subject: [PATCH 36/85] Migrate to piston-meta --- daedalus/.#Cargo.toml | 1 + daedalus/src/minecraft.rs | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) create mode 120000 daedalus/.#Cargo.toml diff --git a/daedalus/.#Cargo.toml b/daedalus/.#Cargo.toml new file mode 120000 index 000000000..655ff6ece --- /dev/null +++ b/daedalus/.#Cargo.toml @@ -0,0 +1 @@ +enderger@DannyPC.2785515:1655912895 \ No newline at end of file diff --git a/daedalus/src/minecraft.rs b/daedalus/src/minecraft.rs index 478f69046..247274a54 100644 --- a/daedalus/src/minecraft.rs +++ b/daedalus/src/minecraft.rs @@ -91,7 +91,7 @@ pub struct VersionManifest { /// The URL to the version manifest pub const VERSION_MANIFEST_URL: &str = - "https://launchermeta.mojang.com/mc/game/version_manifest_v2.json"; + "https://piston-meta.mojang.com/mc/game/version_manifest_v2.json"; /// Fetches a version manifest from the specified URL. If no URL is specified, the default is used. pub async fn fetch_version_manifest( From 827c4e31ee7375b5c2e70ebf67dc45d7927eb0ae Mon Sep 17 00:00:00 2001 From: Danielle Hutzley Date: Mon, 27 Jun 2022 13:59:06 -0700 Subject: [PATCH 37/85] fixup! Migrate to piston-meta --- daedalus/.#Cargo.toml | 1 - daedalus/Cargo.toml | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) delete mode 120000 daedalus/.#Cargo.toml diff --git a/daedalus/.#Cargo.toml b/daedalus/.#Cargo.toml deleted file mode 120000 index 655ff6ece..000000000 --- a/daedalus/.#Cargo.toml +++ /dev/null @@ -1 +0,0 @@ -enderger@DannyPC.2785515:1655912895 \ No newline at end of file diff --git a/daedalus/Cargo.toml b/daedalus/Cargo.toml index 694da1265..9b5929c52 100644 --- a/daedalus/Cargo.toml +++ b/daedalus/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "daedalus" -version = "0.1.15" +version = "0.1.16" authors = ["Jai A "] edition = "2018" license = "MIT" From 45dbf5393f62fb019ff6ff82b3495c18e856d0d7 Mon Sep 17 00:00:00 2001 From: Jai A Date: Thu, 22 Dec 2022 19:01:41 -0700 Subject: [PATCH 38/85] Bump versions + switch AWS/S3 library --- .env | 4 - daedalus/Cargo.toml | 6 +- daedalus/src/lib.rs | 12 +-- daedalus_client/Cargo.toml | 20 +++-- daedalus_client/src/fabric.rs | 18 ++--- daedalus_client/src/forge.rs | 36 ++++----- daedalus_client/src/main.rs | 135 ++++++++++--------------------- daedalus_client/src/minecraft.rs | 6 +- 8 files changed, 90 insertions(+), 147 deletions(-) diff --git a/.env b/.env index dc87710fe..34bf98cd1 100644 --- a/.env +++ b/.env @@ -8,7 +8,3 @@ S3_SECRET=none S3_URL=none S3_REGION=none S3_BUCKET_NAME=none - -DO_INTEGRATION=false -DO_ACCESS_KEY=none -DO_ENDPOINT_ID=none \ No newline at end of file diff --git a/daedalus/Cargo.toml b/daedalus/Cargo.toml index 9b5929c52..3909d8403 100644 --- a/daedalus/Cargo.toml +++ b/daedalus/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "daedalus" -version = "0.1.16" +version = "0.1.17" authors = ["Jai A "] edition = "2018" license = "MIT" @@ -21,5 +21,5 @@ chrono = { version = "0.4", features = ["serde"] } bytes = "1" thiserror = "1.0" tokio = { version = "1", features = ["full"] } -sha1 = { version = "0.6.0", features = ["std"]} -bincode = {version = "2.0.0-rc", features = ["serde"], optional = true} +sha1 = { version = "0.6.1", features = ["std"]} +bincode = {version = "2.0.0-rc.2", features = ["serde"], optional = true} diff --git a/daedalus/src/lib.rs b/daedalus/src/lib.rs index 77657323d..678bf1c4a 100644 --- a/daedalus/src/lib.rs +++ b/daedalus/src/lib.rs @@ -45,7 +45,7 @@ pub enum Error { pub fn get_path_from_artifact(artifact: &str) -> Result { let name_items = artifact.split(':').collect::>(); - let package = name_items.get(0).ok_or_else(|| { + let package = name_items.first().ok_or_else(|| { Error::ParseError(format!("Unable to find package for library {}", &artifact)) })?; let name = name_items.get(1).ok_or_else(|| { @@ -60,14 +60,14 @@ pub fn get_path_from_artifact(artifact: &str) -> Result { })? .split('@') .collect::>(); - let version = version_ext.get(0).ok_or_else(|| { + let version = version_ext.first().ok_or_else(|| { Error::ParseError(format!("Unable to find version for library {}", &artifact)) })?; let ext = version_ext.get(1); Ok(format!( "{}/{}/{}/{}-{}.{}", - package.replace(".", "/"), + package.replace('.', "/"), name, version, name, @@ -86,14 +86,14 @@ pub fn get_path_from_artifact(artifact: &str) -> Result { })? .split('@') .collect::>(); - let data = data_ext.get(0).ok_or_else(|| { + let data = data_ext.first().ok_or_else(|| { Error::ParseError(format!("Unable to find data for library {}", &artifact)) })?; let ext = data_ext.get(1); Ok(format!( "{}/{}/{}/{}-{}-{}.{}", - package.replace(".", "/"), + package.replace('.', "/"), name, version, name, @@ -115,7 +115,7 @@ pub async fn download_file_mirrors( } for (index, mirror) in mirrors.iter().enumerate() { - let result = download_file(&*format!("{}{}", mirror, base), sha1).await; + let result = download_file(&format!("{}{}", mirror, base), sha1).await; if result.is_ok() || (result.is_err() && index == (mirrors.len() - 1)) { return result; diff --git a/daedalus_client/Cargo.toml b/daedalus_client/Cargo.toml index bf4473328..035588722 100644 --- a/daedalus_client/Cargo.toml +++ b/daedalus_client/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "daedalus_client" -version = "0.1.14" +version = "0.1.17" authors = ["Jai A "] edition = "2018" @@ -9,19 +9,17 @@ edition = "2018" [dependencies] daedalus = { path = "../daedalus" } tokio = { version = "1", features = ["full"] } -futures = "0.3.17" -dotenv = "0.15.0" -log = "0.4.8" -env_logger="0.9.0" +futures = "0.3.25" +dotenvy = "0.15.6" +log = "0.4.17" +env_logger= "0.10.0" serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" lazy_static = "1.4.0" thiserror = "1.0" -reqwest = "0.11.4" -zip = "0.5.13" +reqwest = "0.11.13" +zip = "0.6.3" semver = "1.0" chrono = { version = "0.4", features = ["serde"] } -bytes = "1.1.0" - -rusoto_core = "0.47.0" -rusoto_s3 = "0.47.0" +bytes = "1.3.0" +rust-s3 = "0.32.3" diff --git a/daedalus_client/src/fabric.rs b/daedalus_client/src/fabric.rs index fdc3cf46c..3aef4fdc4 100644 --- a/daedalus_client/src/fabric.rs +++ b/daedalus_client/src/fabric.rs @@ -13,7 +13,7 @@ pub async fn retrieve_data( uploaded_files: &mut Vec, ) -> Result<(), Error> { let mut list = fetch_fabric_versions(None).await?; - let old_manifest = daedalus::modded::fetch_manifest(&*format!( + let old_manifest = daedalus::modded::fetch_manifest(&format!( "fabric/v{}/manifest.json", daedalus::modded::CURRENT_FABRIC_FORMAT_VERSION, )) @@ -47,11 +47,7 @@ pub async fn retrieve_data( } } - list.loader = list - .loader - .into_iter() - .filter(|x| loaders.iter().any(|val| val.1 == x.version)) - .collect(); + list.loader.retain(|x| loaders.iter().any(|val| val.1 == x.version)) } let mut version_futures = Vec::new(); @@ -79,7 +75,7 @@ pub async fn retrieve_data( } let version = - fetch_fabric_version(&*game_version.version, &*loader).await?; + fetch_fabric_version(&game_version.version, &loader).await?; Ok::, String, PartialVersionInfo)>, Error>(Some( (stable, loader, version), @@ -106,10 +102,10 @@ pub async fn retrieve_data( } } - let artifact_path = daedalus::get_path_from_artifact(&*lib.name)?; + let artifact_path = daedalus::get_path_from_artifact(&lib.name)?; let artifact = daedalus::download_file( - &*format!( + &format!( "{}{}", lib.url.unwrap_or_else(|| { "https://maven.fabricmc.net/".to_string() @@ -169,7 +165,7 @@ pub async fn retrieve_data( async move { loader_version_map.push(LoaderVersion { id: format!("{}-{}", inherits_from, loader), - url: format_url(&*version_path), + url: format_url(&version_path), stable: *stable, }); } @@ -280,7 +276,7 @@ async fn fetch_fabric_version( ) -> Result { Ok(serde_json::from_slice( &download_file( - &*format!( + &format!( "{}/versions/loader/{}/{}/profile/json", FABRIC_META_URL, version_number, loader_version ), diff --git a/daedalus_client/src/forge.rs b/daedalus_client/src/forge.rs index c0e841c5e..25da023af 100644 --- a/daedalus_client/src/forge.rs +++ b/daedalus_client/src/forge.rs @@ -28,7 +28,7 @@ pub async fn retrieve_data( uploaded_files: &mut Vec, ) -> Result<(), Error> { let maven_metadata = fetch_maven_metadata(None).await?; - let old_manifest = daedalus::modded::fetch_manifest(&*format_url(&*format!( + let old_manifest = daedalus::modded::fetch_manifest(&format_url(&format!( "forge/v{}/manifest.json", daedalus::modded::CURRENT_FORGE_FORMAT_VERSION, ))) @@ -69,7 +69,7 @@ pub async fn retrieve_data( loader_version_raw.to_string() }; - let version = Version::parse(&*loader_version)?; + let version = Version::parse(&loader_version)?; if FORGE_MANIFEST_V1_QUERY.matches(&version) || FORGE_MANIFEST_V2_QUERY_P1.matches(&version) @@ -104,7 +104,7 @@ pub async fn retrieve_data( { let versions = versions_mutex.lock().await; let version = versions.iter().find(|x| - x.id == minecraft_version).map(|x| x.loaders.iter().find(|x| x.id == loader_version_full)).flatten(); + x.id == minecraft_version).and_then(|x| x.loaders.iter().find(|x| x.id == loader_version_full)); if let Some(version) = version { return Ok::, Error>(Some(version.clone())); @@ -112,7 +112,7 @@ pub async fn retrieve_data( } info!("Forge - Installer Start {}", loader_version_full.clone()); - let bytes = download_file(&*format!("https://maven.minecraftforge.net/net/minecraftforge/forge/{0}/forge-{0}-installer.jar", loader_version_full), None).await?; + let bytes = download_file(&format!("https://maven.minecraftforge.net/net/minecraftforge/forge/{0}/forge-{0}-installer.jar", loader_version_full), None).await?; let reader = std::io::Cursor::new(bytes); @@ -125,13 +125,13 @@ pub async fn retrieve_data( let mut contents = String::new(); install_profile.read_to_string(&mut contents)?; - Ok::(serde_json::from_str::(&*contents)?) + Ok::(serde_json::from_str::(&contents)?) }).await??; let mut archive_clone = archive.clone(); let file_path = profile.install.file_path.clone(); let forge_universal_bytes = tokio::task::spawn_blocking(move || { - let mut forge_universal_file = archive_clone.by_name(&*file_path)?; + let mut forge_universal_file = archive_clone.by_name(&file_path)?; let mut forge_universal = Vec::new(); forge_universal_file.read_to_end(&mut forge_universal)?; @@ -156,7 +156,7 @@ pub async fn retrieve_data( } let artifact_path = - daedalus::get_path_from_artifact(&*lib.name)?; + daedalus::get_path_from_artifact(&lib.name)?; let artifact = if lib.name == forge_universal_path { forge_universal_bytes.clone() @@ -164,7 +164,7 @@ pub async fn retrieve_data( let mirrors = vec![&*url, "https://maven.creeperhost.net/", "https://libraries.minecraft.net/"]; daedalus::download_file_mirrors( - &*artifact_path, + &artifact_path, &mirrors, None, ) @@ -216,7 +216,7 @@ pub async fn retrieve_data( return Ok(Some(LoaderVersion { id: loader_version_full, - url: format_url(&*version_path), + url: format_url(&version_path), stable: false })); } else if FORGE_MANIFEST_V2_QUERY_P1.matches(&version) || FORGE_MANIFEST_V2_QUERY_P2.matches(&version) || FORGE_MANIFEST_V3_QUERY.matches(&version) { @@ -227,7 +227,7 @@ pub async fn retrieve_data( let mut contents = String::new(); install_profile.read_to_string(&mut contents)?; - Ok::(serde_json::from_str::(&*contents)?) + Ok::(serde_json::from_str::(&contents)?) }).await??; let mut archive_clone = archive.clone(); @@ -237,7 +237,7 @@ pub async fn retrieve_data( let mut contents = String::new(); install_profile.read_to_string(&mut contents)?; - Ok::(serde_json::from_str::(&*contents)?) + Ok::(serde_json::from_str::(&contents)?) }).await??; @@ -255,12 +255,12 @@ pub async fn retrieve_data( let mut local_libs : HashMap = HashMap::new(); for lib in &libs { - if lib.downloads.as_ref().map(|x| x.artifact.as_ref().map(|x| x.url.is_empty())).flatten().unwrap_or(false) { + if lib.downloads.as_ref().and_then(|x| x.artifact.as_ref().map(|x| x.url.is_empty())).unwrap_or(false) { let mut archive_clone = archive.clone(); let lib_name_clone = lib.name.clone(); let lib_bytes = tokio::task::spawn_blocking(move || { - let mut lib_file = archive_clone.by_name(&*format!("maven/{}", daedalus::get_path_from_artifact(&*lib_name_clone)?))?; + let mut lib_file = archive_clone.by_name(&format!("maven/{}", daedalus::get_path_from_artifact(&lib_name_clone)?))?; let mut lib_bytes = Vec::new(); lib_file.read_to_end(&mut lib_bytes)?; @@ -331,7 +331,7 @@ pub async fn retrieve_data( let now = Instant::now(); let libs = futures::future::try_join_all(libs.into_iter().map(|mut lib| async { let artifact_path = - daedalus::get_path_from_artifact(&*lib.name)?; + daedalus::get_path_from_artifact(&lib.name)?; { let mut visited_assets = visited_assets.lock().await; @@ -339,7 +339,7 @@ pub async fn retrieve_data( if visited_assets.contains(&lib.name) { if let Some(ref mut downloads) = lib.downloads { if let Some(ref mut artifact) = downloads.artifact { - artifact.url = format_url(&*format!("maven/{}", artifact_path)); + artifact.url = format_url(&format!("maven/{}", artifact_path)); } } else if lib.url.is_some() { lib.url = Some(format_url("maven/")); @@ -357,14 +357,14 @@ pub async fn retrieve_data( local_libs.get(&lib.name).cloned() } else { Some(daedalus::download_file( - &*artifact.url, + &artifact.url, Some(&*artifact.sha1), ) .await?) }; if res.is_some() { - artifact.url = format_url(&*format!("maven/{}", artifact_path)); + artifact.url = format_url(&format!("maven/{}", artifact_path)); } res @@ -431,7 +431,7 @@ pub async fn retrieve_data( return Ok(Some(LoaderVersion { id: loader_version_full, - url: format_url(&*version_path), + url: format_url(&version_path), stable: false })); } diff --git a/daedalus_client/src/main.rs b/daedalus_client/src/main.rs index 4a4f67667..70a4fa6df 100644 --- a/daedalus_client/src/main.rs +++ b/daedalus_client/src/main.rs @@ -1,9 +1,8 @@ use log::{error, warn}; -use rusoto_core::credential::StaticProvider; -use rusoto_core::{HttpClient, Region, RusotoError}; -use rusoto_s3::{PutObjectError, S3Client}; -use rusoto_s3::{PutObjectRequest, S3}; use std::time::Duration; +use s3::{Bucket, Region}; +use s3::creds::Credentials; +use s3::error::S3Error; mod fabric; mod forge; @@ -21,7 +20,7 @@ pub enum Error { TaskError(#[from] tokio::task::JoinError), #[error("Error while uploading file to S3")] S3Error { - inner: RusotoError, + inner: S3Error, file: String, }, #[error("Error while parsing version as semver: {0}")] @@ -70,11 +69,6 @@ async fn main() { Err(err) => error!("{:?}", err), }; } - - match purge_digitalocean_cache(uploaded_files).await { - Ok(..) => {} - Err(err) => error!("{:?}", err), - }; } } @@ -82,13 +76,13 @@ fn check_env_vars() -> bool { let mut failed = false; fn check_var(var: &str) -> bool { - if dotenv::var(var) + if dotenvy::var(var) .ok() .and_then(|s| s.parse::().ok()) .is_none() { warn!( - "Variable `{}` missing in dotenv or not of type `{}`", + "Variable `{}` missing in dotenvy or not of type `{}`", var, std::any::type_name::() ); @@ -107,36 +101,31 @@ fn check_env_vars() -> bool { failed |= check_var::("S3_REGION"); failed |= check_var::("S3_BUCKET_NAME"); - failed |= check_var::("DO_INTEGRATION"); - - let do_integration = dotenv::var("DO_INTEGRATION") - .ok() - .map(|x| x.parse::().ok()) - .flatten() - .unwrap_or(false); - - if do_integration { - failed |= check_var::("DO_ACCESS_KEY"); - failed |= check_var::("DO_ENDPOINT_ID"); - } - failed } + lazy_static::lazy_static! { - static ref CLIENT : S3Client = S3Client::new_with( - HttpClient::new().unwrap(), - StaticProvider::new( - dotenv::var("S3_ACCESS_TOKEN").unwrap(), - dotenv::var("S3_SECRET").unwrap(), - None, - None, - ), + static ref CLIENT : Bucket = Bucket::new( + &dotenvy::var("S3_BUCKET_NAME").unwrap(), + if &*dotenvy::var("S3_REGION").unwrap() == "r2" { + Region::R2 { + account_id: dotenvy::var("S3_URL").unwrap(), + } + } else { Region::Custom { - name: dotenv::var("S3_REGION").unwrap(), - endpoint: dotenv::var("S3_URL").unwrap(), - }, - ); + region: dotenvy::var("S3_REGION").unwrap(), + endpoint: dotenvy::var("S3_URL").unwrap(), + } + }, + Credentials::new( + Some(&*dotenvy::var("S3_ACCESS_TOKEN").unwrap()), + Some(&*dotenvy::var("S3_SECRET").unwrap()), + None, + None, + None, + ).unwrap(), + ).unwrap(); } pub async fn upload_file_to_bucket( @@ -145,23 +134,24 @@ pub async fn upload_file_to_bucket( content_type: Option, uploaded_files: &tokio::sync::Mutex>, ) -> Result<(), Error> { - let key = format!("{}/{}", &*dotenv::var("BASE_FOLDER").unwrap(), path); + let key = format!("{}/{}", &*dotenvy::var("BASE_FOLDER").unwrap(), path); for attempt in 1..=4 { - let result = CLIENT - .put_object(PutObjectRequest { - bucket: dotenv::var("S3_BUCKET_NAME").unwrap(), - key: key.clone(), - body: Some(bytes.clone().into()), - acl: Some("public-read".to_string()), - content_type: content_type.clone(), - ..Default::default() - }) - .await - .map_err(|err| Error::S3Error { - inner: err, - file: format!("{}/{}", &*dotenv::var("BASE_FOLDER").unwrap(), path), - }); + let result = if let Some(ref content_type) = content_type { + CLIENT.put_object_with_content_type( + key.clone(), + &bytes, + content_type, + ).await + } else { + CLIENT.put_object( + key.clone(), + &bytes, + ).await + }.map_err(|err| Error::S3Error { + inner: err, + file: format!("{}/{}", &*dotenvy::var("BASE_FOLDER").unwrap(), path), + }); match result { Ok(_) => { @@ -185,45 +175,8 @@ pub async fn upload_file_to_bucket( pub fn format_url(path: &str) -> String { format!( "{}/{}/{}", - &*dotenv::var("BASE_URL").unwrap(), - &*dotenv::var("BASE_FOLDER").unwrap(), + &*dotenvy::var("BASE_URL").unwrap(), + &*dotenvy::var("BASE_FOLDER").unwrap(), path ) } - -#[derive(serde::Serialize)] -struct PurgeCacheRequest { - pub files: Vec, -} - -pub async fn purge_digitalocean_cache(files: Vec) -> Result<(), Error> { - if !dotenv::var("DO_INTEGRATION") - .ok() - .map(|x| x.parse::().ok()) - .flatten() - .unwrap_or(false) - { - return Ok(()); - } - - let client = reqwest::Client::new(); - - client - .delete(&format!( - "https://api.digitalocean.com/v2/cdn/endpoints/{}/cache", - &*dotenv::var("DO_ENDPOINT_ID").unwrap() - )) - .header( - "Authorization", - &*format!("Bearer {}", &*dotenv::var("DO_ACCESS_KEY").unwrap()), - ) - .json(&PurgeCacheRequest { files }) - .send() - .await - .map_err(|err| Error::FetchError { - inner: err, - item: "purging digital ocean cache".to_string(), - })?; - - Ok(()) -} diff --git a/daedalus_client/src/minecraft.rs b/daedalus_client/src/minecraft.rs index 58fae665b..56007b9f9 100644 --- a/daedalus_client/src/minecraft.rs +++ b/daedalus_client/src/minecraft.rs @@ -8,7 +8,7 @@ use tokio::sync::Mutex; pub async fn retrieve_data(uploaded_files: &mut Vec) -> Result { let old_manifest = - daedalus::minecraft::fetch_version_manifest(Some(&*crate::format_url(&*format!( + daedalus::minecraft::fetch_version_manifest(Some(&*crate::format_url(&format!( "minecraft/v{}/manifest.json", daedalus::minecraft::CURRENT_FORMAT_VERSION )))) @@ -43,12 +43,12 @@ pub async fn retrieve_data(uploaded_files: &mut Vec) -> Result Date: Thu, 22 Dec 2022 20:11:22 -0700 Subject: [PATCH 39/85] bump rust version in container --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index c98c954f0..6aed673d5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM rust:1.56 +FROM rust:1.65-slim COPY ./ ./ From fb5f7a336d24c5183feb0b2631d005e30b1c1700 Mon Sep 17 00:00:00 2001 From: Jai A Date: Thu, 22 Dec 2022 20:12:05 -0700 Subject: [PATCH 40/85] bump rust version in container (2) --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 6aed673d5..a63ddaec0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM rust:1.65-slim +FROM rust:1.65 COPY ./ ./ From b7e2d7fb8e3deb686ee96198e4764cda27e89430 Mon Sep 17 00:00:00 2001 From: Jai A Date: Thu, 22 Dec 2022 20:14:26 -0700 Subject: [PATCH 41/85] Update copyright --- LICENSE | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LICENSE b/LICENSE index 6d21f198e..1bc38c7a1 100644 --- a/LICENSE +++ b/LICENSE @@ -1,4 +1,4 @@ -Copyright © 2021 Guavy LLC +Copyright © 2022 Rinth, Inc. Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: From 79ceb56c60a83a8aaff661e7f1fdd5fa6e763636 Mon Sep 17 00:00:00 2001 From: Jai A Date: Tue, 4 Apr 2023 20:25:17 -0700 Subject: [PATCH 42/85] Fix issues --- Dockerfile | 2 +- daedalus/Cargo.toml | 2 +- daedalus/src/lib.rs | 52 ++- daedalus/src/modded.rs | 19 +- daedalus_client/Cargo.toml | 2 +- daedalus_client/src/fabric.rs | 456 +++++++++++---------- daedalus_client/src/forge.rs | 675 +++++++++++++++---------------- daedalus_client/src/main.rs | 85 +++- daedalus_client/src/minecraft.rs | 73 ++-- 9 files changed, 741 insertions(+), 625 deletions(-) diff --git a/Dockerfile b/Dockerfile index a63ddaec0..eae1ea34b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM rust:1.65 +FROM rust:1.68.2 COPY ./ ./ diff --git a/daedalus/Cargo.toml b/daedalus/Cargo.toml index 3909d8403..ada94ed8c 100644 --- a/daedalus/Cargo.toml +++ b/daedalus/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "daedalus" -version = "0.1.17" +version = "0.1.18" authors = ["Jai A "] edition = "2018" license = "MIT" diff --git a/daedalus/src/lib.rs b/daedalus/src/lib.rs index 678bf1c4a..51221eede 100644 --- a/daedalus/src/lib.rs +++ b/daedalus/src/lib.rs @@ -46,22 +46,34 @@ pub fn get_path_from_artifact(artifact: &str) -> Result { let name_items = artifact.split(':').collect::>(); let package = name_items.first().ok_or_else(|| { - Error::ParseError(format!("Unable to find package for library {}", &artifact)) + Error::ParseError(format!( + "Unable to find package for library {}", + &artifact + )) })?; let name = name_items.get(1).ok_or_else(|| { - Error::ParseError(format!("Unable to find name for library {}", &artifact)) + Error::ParseError(format!( + "Unable to find name for library {}", + &artifact + )) })?; if name_items.len() == 3 { let version_ext = name_items .get(2) .ok_or_else(|| { - Error::ParseError(format!("Unable to find version for library {}", &artifact)) + Error::ParseError(format!( + "Unable to find version for library {}", + &artifact + )) })? .split('@') .collect::>(); let version = version_ext.first().ok_or_else(|| { - Error::ParseError(format!("Unable to find version for library {}", &artifact)) + Error::ParseError(format!( + "Unable to find version for library {}", + &artifact + )) })?; let ext = version_ext.get(1); @@ -76,18 +88,27 @@ pub fn get_path_from_artifact(artifact: &str) -> Result { )) } else { let version = name_items.get(2).ok_or_else(|| { - Error::ParseError(format!("Unable to find version for library {}", &artifact)) + Error::ParseError(format!( + "Unable to find version for library {}", + &artifact + )) })?; let data_ext = name_items .get(3) .ok_or_else(|| { - Error::ParseError(format!("Unable to find data for library {}", &artifact)) + Error::ParseError(format!( + "Unable to find data for library {}", + &artifact + )) })? .split('@') .collect::>(); let data = data_ext.first().ok_or_else(|| { - Error::ParseError(format!("Unable to find data for library {}", &artifact)) + Error::ParseError(format!( + "Unable to find data for library {}", + &artifact + )) })?; let ext = data_ext.get(1); @@ -126,10 +147,21 @@ pub async fn download_file_mirrors( } /// Downloads a file with retry and checksum functionality -pub async fn download_file(url: &str, sha1: Option<&str>) -> Result { +pub async fn download_file( + url: &str, + sha1: Option<&str>, +) -> Result { + let mut headers = reqwest::header::HeaderMap::new(); + if let Ok(header) = reqwest::header::HeaderValue::from_str(&format!( + "modrinth/daedalus/{} (support@modrinth.com)", + env!("CARGO_PKG_VERSION") + )) { + headers.insert(reqwest::header::USER_AGENT, header); + } let client = reqwest::Client::builder() .tcp_keepalive(Some(std::time::Duration::from_secs(10))) .timeout(std::time::Duration::from_secs(15)) + .default_headers(headers) .build() .map_err(|err| Error::FetchError { inner: err, @@ -183,7 +215,9 @@ pub async fn download_file(url: &str, sha1: Option<&str>) -> Result Result { - let hash = tokio::task::spawn_blocking(|| sha1::Sha1::from(bytes).hexdigest()).await?; + let hash = + tokio::task::spawn_blocking(|| sha1::Sha1::from(bytes).hexdigest()) + .await?; Ok(hash) } diff --git a/daedalus/src/modded.rs b/daedalus/src/modded.rs index 2e62c981e..2494ded4d 100644 --- a/daedalus/src/modded.rs +++ b/daedalus/src/modded.rs @@ -15,6 +15,9 @@ pub const CURRENT_FABRIC_FORMAT_VERSION: usize = 0; /// The latest version of the format the fabric model structs deserialize to pub const CURRENT_FORGE_FORMAT_VERSION: usize = 0; +/// The dummy replace string library names, inheritsFrom, and version names should be replaced with +pub const DUMMY_REPLACE_STRING: &str = "${modrinth.gameVersion}"; + /// A data variable entry that depends on the side of the installation #[cfg_attr(feature = "bincode", derive(Encode, Decode))] #[derive(Serialize, Deserialize, Debug)] @@ -93,6 +96,8 @@ pub fn merge_partial_version( partial: PartialVersionInfo, merge: VersionInfo, ) -> VersionInfo { + let merge_id = merge.id.clone(); + VersionInfo { arguments: if let Some(partial_args) = partial.arguments { if let Some(merge_args) = merge.arguments { @@ -126,12 +131,22 @@ pub fn merge_partial_version( asset_index: merge.asset_index, assets: merge.assets, downloads: merge.downloads, - id: partial.id, + id: merge.id, java_version: merge.java_version, libraries: partial .libraries .into_iter() .chain(merge.libraries) + .map(|x| Library { + downloads: x.downloads, + extract: x.extract, + name: x.name.replace(DUMMY_REPLACE_STRING, &merge_id), + url: x.url, + natives: x.natives, + rules: x.rules, + checksums: x.checksums, + include_in_classpath: x.include_in_classpath, + }) .collect::>(), main_class: if let Some(main_class) = partial.main_class { main_class @@ -163,6 +178,8 @@ pub struct Manifest { pub struct Version { /// The minecraft version ID pub id: String, + /// Whether the release is stable or not + pub stable: bool, /// A map that contains loader versions for the game version pub loaders: Vec, } diff --git a/daedalus_client/Cargo.toml b/daedalus_client/Cargo.toml index 035588722..389c45a64 100644 --- a/daedalus_client/Cargo.toml +++ b/daedalus_client/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "daedalus_client" -version = "0.1.17" +version = "0.1.18" authors = ["Jai A "] edition = "2018" diff --git a/daedalus_client/src/fabric.rs b/daedalus_client/src/fabric.rs index 3aef4fdc4..b40c31ee0 100644 --- a/daedalus_client/src/fabric.rs +++ b/daedalus_client/src/fabric.rs @@ -1,266 +1,284 @@ -use crate::{format_url, upload_file_to_bucket, Error}; -use daedalus::download_file; +use crate::{download_file, format_url, upload_file_to_bucket, Error}; use daedalus::minecraft::{Library, VersionManifest}; -use daedalus::modded::{LoaderVersion, Manifest, PartialVersionInfo, Version}; -use log::info; +use daedalus::modded::{ + LoaderVersion, Manifest, PartialVersionInfo, Version, DUMMY_REPLACE_STRING, +}; use serde::{Deserialize, Serialize}; use std::sync::Arc; -use std::time::{Duration, Instant}; -use tokio::sync::{Mutex, RwLock}; +use tokio::sync::{Mutex, RwLock, Semaphore}; pub async fn retrieve_data( minecraft_versions: &VersionManifest, uploaded_files: &mut Vec, + semaphore: Arc, ) -> Result<(), Error> { - let mut list = fetch_fabric_versions(None).await?; - let old_manifest = daedalus::modded::fetch_manifest(&format!( + let mut list = fetch_fabric_versions(None, semaphore.clone()).await?; + let old_manifest = daedalus::modded::fetch_manifest(&format_url(&format!( "fabric/v{}/manifest.json", daedalus::modded::CURRENT_FABRIC_FORMAT_VERSION, - )) + ))) .await .ok(); - let versions = Arc::new(Mutex::new(if let Some(old_manifest) = old_manifest { + let mut versions = if let Some(old_manifest) = old_manifest { old_manifest.game_versions } else { Vec::new() - })); + }; - let uploaded_files_mutex = Arc::new(Mutex::new(Vec::new())); + let loaders_mutex = RwLock::new(Vec::new()); - if let Some(latest) = list.loader.get(0) { - let loaders_mutex = Arc::new(RwLock::new(Vec::new())); - let visited_artifacts_mutex = Arc::new(Mutex::new(Vec::new())); + { + let mut loaders = loaders_mutex.write().await; - { - let mut loaders = loaders_mutex.write().await; + for loader in &list.loader { + loaders.push((Box::new(loader.stable), loader.version.clone())) + } - // for loader in &list.loader { - // loaders.push((Box::new(loader.stable), loader.version.clone())) - // } + list.loader + .retain(|x| loaders.iter().any(|val| val.1 == x.version)) + } - loaders.push((Box::new(latest.stable), latest.version.clone())); + const DUMMY_GAME_VERSION: &str = "1.19.4-rc2"; - if !latest.stable { - if let Some(stable) = list.loader.iter().find(|x| x.stable) { - loaders.push((Box::new(stable.stable), stable.version.clone())); + let loader_version_mutex = Mutex::new(Vec::new()); + let uploaded_files_mutex = Arc::new(Mutex::new(Vec::new())); + + let loader_versions = futures::future::try_join_all( + loaders_mutex.read().await.clone().into_iter().map( + |(stable, loader)| async { + { + if versions.iter().any(|x| { + x.id == DUMMY_REPLACE_STRING + && x.loaders.iter().any(|x| x.id == loader) + }) { + return Ok(None); + } } - } - list.loader.retain(|x| loaders.iter().any(|val| val.1 == x.version)) - } + let version = fetch_fabric_version( + DUMMY_GAME_VERSION, + &loader, + semaphore.clone(), + ) + .await?; - let mut version_futures = Vec::new(); - - for game_version in list.game.iter_mut() { - let visited_artifacts_mutex = Arc::clone(&visited_artifacts_mutex); - let loaders_mutex = Arc::clone(&loaders_mutex); - let uploaded_files_mutex = Arc::clone(&uploaded_files_mutex); - - let versions_mutex = Arc::clone(&versions); - version_futures.push(async move { - let loader_version_mutex = Mutex::new(Vec::new()); - - let versions = - futures::future::try_join_all( - loaders_mutex.read().await.clone().into_iter().map( - |(stable, loader)| async { - { - if versions_mutex.lock().await.iter().any(|x| { - x.id == game_version.version - && x.loaders.iter().any(|x| x.id == loader) - }) { - return Ok(None); - } - } - - let version = - fetch_fabric_version(&game_version.version, &loader).await?; - - Ok::, String, PartialVersionInfo)>, Error>(Some( - (stable, loader, version), - )) - }, - ), - ) - .await? - .into_iter() - .flatten(); - - futures::future::try_join_all(versions.map(|(stable, loader, version)| async { - let libs = futures::future::try_join_all(version.libraries.into_iter().map( - |mut lib| async { - { - let mut visited_assets = visited_artifacts_mutex.lock().await; - - if visited_assets.contains(&lib.name) { - lib.url = Some(format_url("maven/")); - - return Ok(lib); - } else { - visited_assets.push(lib.name.clone()) - } - } - - let artifact_path = daedalus::get_path_from_artifact(&lib.name)?; - - let artifact = daedalus::download_file( - &format!( - "{}{}", - lib.url.unwrap_or_else(|| { - "https://maven.fabricmc.net/".to_string() - }), - artifact_path - ), - None, - ) - .await?; + //println!("{}", loader); + Ok::, String, PartialVersionInfo)>, Error>( + Some((stable, loader, version)), + ) + }, + ), + ) + .await?; + + let visited_artifacts_mutex = Arc::new(Mutex::new(Vec::new())); + futures::future::try_join_all(loader_versions.into_iter() + .flatten().map( + |(stable, loader, version)| async { + let libs = futures::future::try_join_all( + version.libraries.into_iter().map(|mut lib| async { + { + let mut visited_assets = + visited_artifacts_mutex.lock().await; + if visited_assets.contains(&lib.name) { lib.url = Some(format_url("maven/")); - upload_file_to_bucket( - format!("{}/{}", "maven", artifact_path), - artifact.to_vec(), - Some("application/java-archive".to_string()), - uploaded_files_mutex.as_ref(), - ) - .await?; - - Ok::(lib) - }, - )) - .await?; - - let version_path = format!( - "fabric/v{}/versions/{}-{}.json", - daedalus::modded::CURRENT_FABRIC_FORMAT_VERSION, - version.inherits_from, - &loader - ); - - let inherits_from = version.inherits_from.clone(); - - upload_file_to_bucket( - version_path.clone(), - serde_json::to_vec(&PartialVersionInfo { - arguments: version.arguments, - id: version.id, - main_class: version.main_class, - release_time: version.release_time, - time: version.time, - type_: version.type_, - inherits_from: version.inherits_from, - libraries: libs, - minecraft_arguments: version.minecraft_arguments, - processors: None, - data: None, - })?, - Some("application/json".to_string()), - uploaded_files_mutex.as_ref(), - ) - .await?; - - { - let mut loader_version_map = loader_version_mutex.lock().await; - async move { - loader_version_map.push(LoaderVersion { - id: format!("{}-{}", inherits_from, loader), - url: format_url(&version_path), - stable: *stable, - }); + return Ok(lib); + } else { + visited_assets.push(lib.name.clone()) } - .await; } - Ok::<(), Error>(()) - })) - .await?; + if lib.name.contains(DUMMY_GAME_VERSION) { + lib.name = lib.name.replace(DUMMY_GAME_VERSION, DUMMY_REPLACE_STRING); + lib.url = Some(format_url("maven/")); + futures::future::try_join_all(list.game.clone().into_iter().map(|game_version| async { + let semaphore = semaphore.clone(); + let uploaded_files_mutex = uploaded_files_mutex.clone(); + let lib_name = lib.name.clone(); + let lib_url = lib.url.clone(); + + async move { + let artifact_path = + daedalus::get_path_from_artifact(&lib_name.replace(DUMMY_REPLACE_STRING, &game_version.version))?; + + let artifact = download_file( + &format!( + "{}{}", + lib_url.unwrap_or_else(|| { + "https://maven.fabricmc.net/".to_string() + }), + artifact_path + ), + None, + semaphore.clone(), + ) + .await?; + + upload_file_to_bucket( + format!("{}/{}", "maven", artifact_path), + artifact.to_vec(), + Some("application/java-archive".to_string()), + &uploaded_files_mutex, + semaphore.clone(), + ) + .await?; + + Ok::<(), Error>(()) + }.await?; + + Ok::<(), Error>(()) + })).await?; + + return Ok(lib); + } - let mut versions = versions_mutex.lock().await; - versions.push(Version { - id: game_version.version.clone(), - loaders: loader_version_mutex.into_inner(), - }); + let artifact_path = + daedalus::get_path_from_artifact(&lib.name)?; - Ok::<(), Error>(()) - }); - } + let artifact = download_file( + &format!( + "{}{}", + lib.url.unwrap_or_else(|| { + "https://maven.fabricmc.net/".to_string() + }), + artifact_path + ), + None, + semaphore.clone(), + ) + .await?; - let mut versions = version_futures.into_iter().peekable(); - let mut chunk_index = 0; - while versions.peek().is_some() { - let now = Instant::now(); + lib.url = Some(format_url("maven/")); - let chunk: Vec<_> = versions.by_ref().take(10).collect(); - futures::future::try_join_all(chunk).await?; + upload_file_to_bucket( + format!("{}/{}", "maven", artifact_path), + artifact.to_vec(), + Some("application/java-archive".to_string()), + &uploaded_files_mutex, + semaphore.clone(), + ) + .await?; - tokio::time::sleep(Duration::from_secs(1)).await; + Ok::(lib) + }), + ) + .await?; - chunk_index += 1; + let version_path = format!( + "fabric/v{}/versions/{}.json", + daedalus::modded::CURRENT_FABRIC_FORMAT_VERSION, + &loader + ); + + upload_file_to_bucket( + version_path.clone(), + serde_json::to_vec(&PartialVersionInfo { + arguments: version.arguments, + id: version + .id + .replace(DUMMY_GAME_VERSION, DUMMY_REPLACE_STRING), + main_class: version.main_class, + release_time: version.release_time, + time: version.time, + type_: version.type_, + inherits_from: version + .inherits_from + .replace(DUMMY_GAME_VERSION, DUMMY_REPLACE_STRING), + libraries: libs, + minecraft_arguments: version.minecraft_arguments, + processors: None, + data: None, + })?, + Some("application/json".to_string()), + &uploaded_files_mutex, + semaphore.clone(), + ) + .await?; + + { + let mut loader_version_map = loader_version_mutex.lock().await; + async move { + loader_version_map.push(LoaderVersion { + id: loader.to_string(), + url: format_url(&version_path), + stable: *stable, + }); + } + .await; + } - let elapsed = now.elapsed(); - info!("Chunk {} Elapsed: {:.2?}", chunk_index, elapsed); - } + Ok::<(), Error>(()) + }, + )) + .await?; + + versions.push(Version { + id: DUMMY_REPLACE_STRING.to_string(), + stable: true, + loaders: loader_version_mutex.into_inner(), + }); + + for version in &list.game { + versions.push(Version { + id: version.version.clone(), + stable: version.stable, + loaders: vec![], + }); } - if let Ok(versions) = Arc::try_unwrap(versions) { - let mut versions = versions.into_inner(); - - versions.sort_by(|x, y| { - minecraft_versions - .versions + versions.sort_by(|x, y| { + minecraft_versions + .versions + .iter() + .position(|z| x.id == z.id) + .unwrap_or_default() + .cmp( + &minecraft_versions + .versions + .iter() + .position(|z| y.id == z.id) + .unwrap_or_default(), + ) + }); + + for version in &mut versions { + version.loaders.sort_by(|x, y| { + list.loader .iter() - .position(|z| x.id == z.id) + .position(|z| { + x.id.split('-').next().unwrap_or_default() == &*z.version + }) .unwrap_or_default() .cmp( - &minecraft_versions - .versions + &list + .loader .iter() - .position(|z| y.id == z.id) + .position(|z| { + y.id.split('-').next().unwrap_or_default() + == z.version + }) .unwrap_or_default(), ) - }); - - for version in &mut versions { - version.loaders.sort_by(|x, y| { - list.loader - .iter() - .position(|z| { - x.id.split('-') - .next() - .unwrap_or_default() - == &*z.version - }) - .unwrap_or_default() - .cmp( - &list - .loader - .iter() - .position(|z| { - y.id.split('-') - .next() - .unwrap_or_default() - == z.version - }) - .unwrap_or_default(), - ) - }) - } - - upload_file_to_bucket( - format!( - "fabric/v{}/manifest.json", - daedalus::modded::CURRENT_FABRIC_FORMAT_VERSION, - ), - serde_json::to_vec(&Manifest { - game_versions: versions, - })?, - Some("application/json".to_string()), - uploaded_files_mutex.as_ref(), - ) - .await?; + }) } + upload_file_to_bucket( + format!( + "fabric/v{}/manifest.json", + daedalus::modded::CURRENT_FABRIC_FORMAT_VERSION, + ), + serde_json::to_vec(&Manifest { + game_versions: versions, + })?, + Some("application/json".to_string()), + &uploaded_files_mutex, + semaphore, + ) + .await?; + if let Ok(uploaded_files_mutex) = Arc::try_unwrap(uploaded_files_mutex) { uploaded_files.extend(uploaded_files_mutex.into_inner()); } @@ -273,6 +291,7 @@ const FABRIC_META_URL: &str = "https://meta.fabricmc.net/v2"; async fn fetch_fabric_version( version_number: &str, loader_version: &str, + semaphore: Arc, ) -> Result { Ok(serde_json::from_slice( &download_file( @@ -281,6 +300,7 @@ async fn fetch_fabric_version( FABRIC_META_URL, version_number, loader_version ), None, + semaphore, ) .await?, )?) @@ -319,11 +339,15 @@ struct FabricLoaderVersion { pub stable: bool, } /// Fetches the list of fabric versions -async fn fetch_fabric_versions(url: Option<&str>) -> Result { +async fn fetch_fabric_versions( + url: Option<&str>, + semaphore: Arc, +) -> Result { Ok(serde_json::from_slice( &download_file( url.unwrap_or(&*format!("{}/versions", FABRIC_META_URL)), None, + semaphore, ) .await?, )?) diff --git a/daedalus_client/src/forge.rs b/daedalus_client/src/forge.rs index 25da023af..cb903f15d 100644 --- a/daedalus_client/src/forge.rs +++ b/daedalus_client/src/forge.rs @@ -1,8 +1,14 @@ -use crate::{format_url, upload_file_to_bucket, Error}; +use crate::{ + download_file, download_file_mirrors, format_url, upload_file_to_bucket, + Error, +}; use chrono::{DateTime, Utc}; -use daedalus::download_file; -use daedalus::minecraft::{Argument, ArgumentType, Library, VersionManifest, VersionType}; -use daedalus::modded::{LoaderVersion, Manifest, PartialVersionInfo, Processor, SidedDataEntry}; +use daedalus::minecraft::{ + Argument, ArgumentType, Library, VersionManifest, VersionType, +}; +use daedalus::modded::{ + LoaderVersion, Manifest, PartialVersionInfo, Processor, SidedDataEntry, +}; use lazy_static::lazy_static; use log::info; use semver::{Version, VersionReq}; @@ -10,8 +16,8 @@ use serde::{Deserialize, Serialize}; use std::collections::HashMap; use std::io::Read; use std::sync::Arc; -use std::time::{Duration, Instant}; -use tokio::sync::Mutex; +use std::time::Instant; +use tokio::sync::{Mutex, Semaphore}; lazy_static! { static ref FORGE_MANIFEST_V1_QUERY: VersionReq = @@ -20,14 +26,16 @@ lazy_static! { VersionReq::parse(">=23.5.2851, <31.2.52").unwrap(); static ref FORGE_MANIFEST_V2_QUERY_P2: VersionReq = VersionReq::parse(">=32.0.1, <37.0.0").unwrap(); - static ref FORGE_MANIFEST_V3_QUERY: VersionReq = VersionReq::parse(">=37.0.0").unwrap(); + static ref FORGE_MANIFEST_V3_QUERY: VersionReq = + VersionReq::parse(">=37.0.0").unwrap(); } pub async fn retrieve_data( minecraft_versions: &VersionManifest, uploaded_files: &mut Vec, + semaphore: Arc, ) -> Result<(), Error> { - let maven_metadata = fetch_maven_metadata(None).await?; + let maven_metadata = fetch_maven_metadata(None, semaphore.clone()).await?; let old_manifest = daedalus::modded::fetch_manifest(&format_url(&format!( "forge/v{}/manifest.json", daedalus::modded::CURRENT_FORGE_FORMAT_VERSION, @@ -35,11 +43,12 @@ pub async fn retrieve_data( .await .ok(); - let old_versions = Arc::new(Mutex::new(if let Some(old_manifest) = old_manifest { - old_manifest.game_versions - } else { - Vec::new() - })); + let old_versions = + Arc::new(Mutex::new(if let Some(old_manifest) = old_manifest { + old_manifest.game_versions + } else { + Vec::new() + })); let versions = Arc::new(Mutex::new(Vec::new())); @@ -52,13 +61,14 @@ pub async fn retrieve_data( let mut loaders = Vec::new(); for loader_version_full in loader_versions { - let loader_version = loader_version_full.split('-').into_iter().nth(1); + let loader_version = loader_version_full.split('-').nth(1); if let Some(loader_version_raw) = loader_version { // This is a dirty hack to get around Forge not complying with SemVer, but whatever // Most of this is a hack anyways :( // Works for all forge versions! - let split = loader_version_raw.split('.').collect::>(); + let split = + loader_version_raw.split('.').collect::>(); let loader_version = if split.len() >= 4 { if split[0].parse::().unwrap_or(0) < 6 { format!("{}.{}.{}", split[0], split[1], split[3]) @@ -80,203 +90,209 @@ pub async fn retrieve_data( } } } - version_futures.push(async { - let mut loaders_versions = Vec::new(); - - { - let mut loaders_futures = loaders.into_iter().map(|(loader_version_full, version)| async { - let versions_mutex = Arc::clone(&old_versions); - let visited_assets = Arc::clone(&visited_assets_mutex); - let uploaded_files_mutex = Arc::clone(&uploaded_files_mutex); - let minecraft_version = minecraft_version.clone(); - - async move { - /// These forge versions are not worth supporting! - const WHITELIST : [&str; 1] = [ - // Not supported due to `data` field being `[]` even though the type is a map - "1.12.2-14.23.5.2851" - ]; - - if WHITELIST.contains(&&*loader_version_full) { - return Ok(None); - } - - { - let versions = versions_mutex.lock().await; - let version = versions.iter().find(|x| - x.id == minecraft_version).and_then(|x| x.loaders.iter().find(|x| x.id == loader_version_full)); - - if let Some(version) = version { - return Ok::, Error>(Some(version.clone())); + + if !loaders.is_empty() { + version_futures.push(async { + let loaders_versions = Vec::new(); + + { + let loaders_futures = loaders.into_iter().map(|(loader_version_full, version)| async { + let versions_mutex = Arc::clone(&old_versions); + let visited_assets = Arc::clone(&visited_assets_mutex); + let uploaded_files_mutex = Arc::clone(&uploaded_files_mutex); + let semaphore = Arc::clone(&semaphore); + let minecraft_version = minecraft_version.clone(); + + async move { + /// These forge versions are not worth supporting! + const WHITELIST : [&str; 1] = [ + // Not supported due to `data` field being `[]` even though the type is a map + "1.12.2-14.23.5.2851", + ]; + + if WHITELIST.contains(&&*loader_version_full) { + return Ok(None); } - } - info!("Forge - Installer Start {}", loader_version_full.clone()); - let bytes = download_file(&format!("https://maven.minecraftforge.net/net/minecraftforge/forge/{0}/forge-{0}-installer.jar", loader_version_full), None).await?; + { + let versions = versions_mutex.lock().await; + let version = versions.iter().find(|x| + x.id == minecraft_version).and_then(|x| x.loaders.iter().find(|x| x.id == loader_version_full)); - let reader = std::io::Cursor::new(bytes); + if let Some(version) = version { + return Ok::, Error>(Some(version.clone())); + } + } - if let Ok(archive) = zip::ZipArchive::new(reader) { - if FORGE_MANIFEST_V1_QUERY.matches(&version) { - let mut archive_clone = archive.clone(); - let profile = tokio::task::spawn_blocking(move || { - let mut install_profile = archive_clone.by_name("install_profile.json")?; + info!("Forge - Installer Start {}", loader_version_full.clone()); + let bytes = download_file(&format!("https://maven.minecraftforge.net/net/minecraftforge/forge/{0}/forge-{0}-installer.jar", loader_version_full), None, semaphore.clone()).await?; - let mut contents = String::new(); - install_profile.read_to_string(&mut contents)?; + let reader = std::io::Cursor::new(bytes); - Ok::(serde_json::from_str::(&contents)?) - }).await??; + if let Ok(archive) = zip::ZipArchive::new(reader) { + if FORGE_MANIFEST_V1_QUERY.matches(&version) { + let mut archive_clone = archive.clone(); + let profile = tokio::task::spawn_blocking(move || { + let mut install_profile = archive_clone.by_name("install_profile.json")?; - let mut archive_clone = archive.clone(); - let file_path = profile.install.file_path.clone(); - let forge_universal_bytes = tokio::task::spawn_blocking(move || { - let mut forge_universal_file = archive_clone.by_name(&file_path)?; - let mut forge_universal = Vec::new(); - forge_universal_file.read_to_end(&mut forge_universal)?; + let mut contents = String::new(); + install_profile.read_to_string(&mut contents)?; + Ok::(serde_json::from_str::(&contents)?) + }).await??; - Ok::(bytes::Bytes::from(forge_universal)) - }).await??; - let forge_universal_path = profile.install.path.clone(); + let mut archive_clone = archive.clone(); + let file_path = profile.install.file_path.clone(); + let forge_universal_bytes = tokio::task::spawn_blocking(move || { + let mut forge_universal_file = archive_clone.by_name(&file_path)?; + let mut forge_universal = Vec::new(); + forge_universal_file.read_to_end(&mut forge_universal)?; - let now = Instant::now(); - let libs = futures::future::try_join_all(profile.version_info.libraries.into_iter().map(|mut lib| async { - if let Some(url) = lib.url { - { - let mut visited_assets = visited_assets.lock().await; - if visited_assets.contains(&lib.name) { - lib.url = Some(format_url("maven/")); + Ok::(bytes::Bytes::from(forge_universal)) + }).await??; + let forge_universal_path = profile.install.path.clone(); - return Ok::(lib); - } else { - visited_assets.push(lib.name.clone()) + let now = Instant::now(); + let libs = futures::future::try_join_all(profile.version_info.libraries.into_iter().map(|mut lib| async { + if let Some(url) = lib.url { + { + let mut visited_assets = visited_assets.lock().await; + + if visited_assets.contains(&lib.name) { + lib.url = Some(format_url("maven/")); + + return Ok::(lib); + } else { + visited_assets.push(lib.name.clone()) + } } + + let artifact_path = + daedalus::get_path_from_artifact(&lib.name)?; + + let artifact = if lib.name == forge_universal_path { + forge_universal_bytes.clone() + } else { + let mirrors = vec![&*url, "https://maven.creeperhost.net/", "https://libraries.minecraft.net/"]; + + download_file_mirrors( + &artifact_path, + &mirrors, + None, + semaphore.clone(), + ) + .await? + }; + + lib.url = Some(format_url("maven/")); + + upload_file_to_bucket( + format!("{}/{}", "maven", artifact_path), + artifact.to_vec(), + Some("application/java-archive".to_string()), + uploaded_files_mutex.as_ref(), + semaphore.clone(), + ).await?; } - let artifact_path = - daedalus::get_path_from_artifact(&lib.name)?; + Ok::(lib) + })).await?; + + let elapsed = now.elapsed(); + info!("Elapsed lib DL: {:.2?}", elapsed); + + let new_profile = PartialVersionInfo { + id: profile.version_info.id, + inherits_from: profile.install.minecraft, + release_time: profile.version_info.release_time, + time: profile.version_info.time, + main_class: profile.version_info.main_class, + minecraft_arguments: profile.version_info.minecraft_arguments.clone(), + arguments: profile.version_info.minecraft_arguments.map(|x| [(ArgumentType::Game, x.split(' ').map(|x| Argument::Normal(x.to_string())).collect())].iter().cloned().collect()), + libraries: libs, + type_: profile.version_info.type_, + data: None, + processors: None + }; + + let version_path = format!( + "forge/v{}/versions/{}.json", + daedalus::modded::CURRENT_FORGE_FORMAT_VERSION, + new_profile.id + ); + + upload_file_to_bucket( + version_path.clone(), + serde_json::to_vec(&new_profile)?, + Some("application/json".to_string()), + uploaded_files_mutex.as_ref(), + semaphore.clone(), + ).await?; + + return Ok(Some(LoaderVersion { + id: loader_version_full, + url: format_url(&version_path), + stable: false + })); + } else if FORGE_MANIFEST_V2_QUERY_P1.matches(&version) || FORGE_MANIFEST_V2_QUERY_P2.matches(&version) || FORGE_MANIFEST_V3_QUERY.matches(&version) { + let mut archive_clone = archive.clone(); + let mut profile = tokio::task::spawn_blocking(move || { + let mut install_profile = archive_clone.by_name("install_profile.json")?; + + let mut contents = String::new(); + install_profile.read_to_string(&mut contents)?; + + Ok::(serde_json::from_str::(&contents)?) + }).await??; + + let mut archive_clone = archive.clone(); + let version_info = tokio::task::spawn_blocking(move || { + let mut install_profile = archive_clone.by_name("version.json")?; + + let mut contents = String::new(); + install_profile.read_to_string(&mut contents)?; + + Ok::(serde_json::from_str::(&contents)?) + }).await??; + + + let mut libs : Vec = version_info.libraries.into_iter().chain(profile.libraries.into_iter().map(|x| Library { + downloads: x.downloads, + extract: x.extract, + name: x.name, + url: x.url, + natives: x.natives, + rules: x.rules, + checksums: x.checksums, + include_in_classpath: false + })).collect(); + + let mut local_libs : HashMap = HashMap::new(); + + for lib in &libs { + if lib.downloads.as_ref().and_then(|x| x.artifact.as_ref().map(|x| x.url.is_empty())).unwrap_or(false) { + let mut archive_clone = archive.clone(); + let lib_name_clone = lib.name.clone(); - let artifact = if lib.name == forge_universal_path { - forge_universal_bytes.clone() - } else { - let mirrors = vec![&*url, "https://maven.creeperhost.net/", "https://libraries.minecraft.net/"]; - - daedalus::download_file_mirrors( - &artifact_path, - &mirrors, - None, - ) - .await? - }; - - lib.url = Some(format_url("maven/")); - - upload_file_to_bucket( - format!("{}/{}", "maven", artifact_path), - artifact.to_vec(), - Some("application/java-archive".to_string()), - uploaded_files_mutex.as_ref(), - ).await?; - } + let lib_bytes = tokio::task::spawn_blocking(move || { + let mut lib_file = archive_clone.by_name(&format!("maven/{}", daedalus::get_path_from_artifact(&lib_name_clone)?))?; + let mut lib_bytes = Vec::new(); + lib_file.read_to_end(&mut lib_bytes)?; - Ok::(lib) - })).await?; - - let elapsed = now.elapsed(); - info!("Elapsed lib DL: {:.2?}", elapsed); - - let new_profile = PartialVersionInfo { - id: profile.version_info.id, - inherits_from: profile.install.minecraft, - release_time: profile.version_info.release_time, - time: profile.version_info.time, - main_class: profile.version_info.main_class, - minecraft_arguments: profile.version_info.minecraft_arguments.clone(), - arguments: profile.version_info.minecraft_arguments.map(|x| [(ArgumentType::Game, x.split(' ').map(|x| Argument::Normal(x.to_string())).collect())].iter().cloned().collect()), - libraries: libs, - type_: profile.version_info.type_, - data: None, - processors: None - }; - - let version_path = format!( - "forge/v{}/versions/{}.json", - daedalus::modded::CURRENT_FORGE_FORMAT_VERSION, - new_profile.id - ); - - upload_file_to_bucket( - version_path.clone(), - serde_json::to_vec(&new_profile)?, - Some("application/json".to_string()), - uploaded_files_mutex.as_ref() - ).await?; - - return Ok(Some(LoaderVersion { - id: loader_version_full, - url: format_url(&version_path), - stable: false - })); - } else if FORGE_MANIFEST_V2_QUERY_P1.matches(&version) || FORGE_MANIFEST_V2_QUERY_P2.matches(&version) || FORGE_MANIFEST_V3_QUERY.matches(&version) { - let mut archive_clone = archive.clone(); - let mut profile = tokio::task::spawn_blocking(move || { - let mut install_profile = archive_clone.by_name("install_profile.json")?; - - let mut contents = String::new(); - install_profile.read_to_string(&mut contents)?; - - Ok::(serde_json::from_str::(&contents)?) - }).await??; - - let mut archive_clone = archive.clone(); - let version_info = tokio::task::spawn_blocking(move || { - let mut install_profile = archive_clone.by_name("version.json")?; - - let mut contents = String::new(); - install_profile.read_to_string(&mut contents)?; - - Ok::(serde_json::from_str::(&contents)?) - }).await??; - - - let mut libs : Vec = version_info.libraries.into_iter().chain(profile.libraries.into_iter().map(|x| Library { - downloads: x.downloads, - extract: x.extract, - name: x.name, - url: x.url, - natives: x.natives, - rules: x.rules, - checksums: x.checksums, - include_in_classpath: false - })).collect(); - - let mut local_libs : HashMap = HashMap::new(); - - for lib in &libs { - if lib.downloads.as_ref().and_then(|x| x.artifact.as_ref().map(|x| x.url.is_empty())).unwrap_or(false) { - let mut archive_clone = archive.clone(); - let lib_name_clone = lib.name.clone(); - - let lib_bytes = tokio::task::spawn_blocking(move || { - let mut lib_file = archive_clone.by_name(&format!("maven/{}", daedalus::get_path_from_artifact(&lib_name_clone)?))?; - let mut lib_bytes = Vec::new(); - lib_file.read_to_end(&mut lib_bytes)?; - - Ok::(bytes::Bytes::from(lib_bytes)) - }).await??; - - local_libs.insert(lib.name.clone(), lib_bytes); + Ok::(bytes::Bytes::from(lib_bytes)) + }).await??; + + local_libs.insert(lib.name.clone(), lib_bytes); + } } - } - let path = profile.path.clone(); - let version = profile.version.clone(); + let path = profile.path.clone(); + let version = profile.version.clone(); - for entry in profile.data.values_mut() { - if entry.client.starts_with('/') || entry.server.starts_with('/') { - macro_rules! read_data { + for entry in profile.data.values_mut() { + if entry.client.starts_with('/') || entry.server.starts_with('/') { + macro_rules! read_data { ($value:expr) => { let mut archive_clone = archive.clone(); let value_clone = $value.clone(); @@ -315,179 +331,149 @@ pub async fn retrieve_data( } } - if entry.client.starts_with('/') { - read_data!(entry.client); - } + if entry.client.starts_with('/') { + read_data!(entry.client); + } - // Do we really need to support server installs? Keeping this here - // just in case - // - // if entry.server.starts_with('/') { - // read_data!(entry.server); - // } + if entry.server.starts_with('/') { + read_data!(entry.server); + } + } } - } - let now = Instant::now(); - let libs = futures::future::try_join_all(libs.into_iter().map(|mut lib| async { - let artifact_path = - daedalus::get_path_from_artifact(&lib.name)?; + let now = Instant::now(); + let libs = futures::future::try_join_all(libs.into_iter().map(|mut lib| async { + let artifact_path = + daedalus::get_path_from_artifact(&lib.name)?; - { - let mut visited_assets = visited_assets.lock().await; + { + let mut visited_assets = visited_assets.lock().await; - if visited_assets.contains(&lib.name) { - if let Some(ref mut downloads) = lib.downloads { - if let Some(ref mut artifact) = downloads.artifact { - artifact.url = format_url(&format!("maven/{}", artifact_path)); + if visited_assets.contains(&lib.name) { + if let Some(ref mut downloads) = lib.downloads { + if let Some(ref mut artifact) = downloads.artifact { + artifact.url = format_url(&format!("maven/{}", artifact_path)); + } + } else if lib.url.is_some() { + lib.url = Some(format_url("maven/")); } - } else if lib.url.is_some() { - lib.url = Some(format_url("maven/")); - } - return Ok::(lib); - } else { - visited_assets.push(lib.name.clone()) + return Ok::(lib); + } else { + visited_assets.push(lib.name.clone()) + } } - } - let artifact_bytes = if let Some(ref mut downloads) = lib.downloads { - if let Some(ref mut artifact) = downloads.artifact { - let res = if artifact.url.is_empty() { + let artifact_bytes = if let Some(ref mut downloads) = lib.downloads { + if let Some(ref mut artifact) = downloads.artifact { + let res = if artifact.url.is_empty() { + local_libs.get(&lib.name).cloned() + } else { + Some(download_file( + &artifact.url, + Some(&*artifact.sha1), + semaphore.clone(), + ) + .await?) + }; + + if res.is_some() { + artifact.url = format_url(&format!("maven/{}", artifact_path)); + } + + res + } else { None } + } else if let Some(ref mut url) = lib.url { + let res = if url.is_empty() { local_libs.get(&lib.name).cloned() } else { - Some(daedalus::download_file( - &artifact.url, - Some(&*artifact.sha1), + Some(download_file( + url, + None, + semaphore.clone(), ) .await?) }; if res.is_some() { - artifact.url = format_url(&format!("maven/{}", artifact_path)); + lib.url = Some(format_url("maven/")); } res - } else { None } - } else if let Some(ref mut url) = lib.url { - let res = if url.is_empty() { - local_libs.get(&lib.name).cloned() - } else { - Some(daedalus::download_file( - url, - None, - ) - .await?) - }; - - if res.is_some() { - lib.url = Some(format_url("maven/")); + } else { None }; + + if let Some(bytes) = artifact_bytes { + upload_file_to_bucket( + format!("{}/{}", "maven", artifact_path), + bytes.to_vec(), + Some("application/java-archive".to_string()), + uploaded_files_mutex.as_ref(), + semaphore.clone(), + ).await?; } - res - } else { None }; - - if let Some(bytes) = artifact_bytes { - upload_file_to_bucket( - format!("{}/{}", "maven", artifact_path), - bytes.to_vec(), - Some("application/java-archive".to_string()), - uploaded_files_mutex.as_ref() - ).await?; - } - - Ok::(lib) - })).await?; - - let elapsed = now.elapsed(); - info!("Elapsed lib DL: {:.2?}", elapsed); - - let new_profile = PartialVersionInfo { - id: version_info.id, - inherits_from: version_info.inherits_from, - release_time: version_info.release_time, - time: version_info.time, - main_class: version_info.main_class, - minecraft_arguments: version_info.minecraft_arguments, - arguments: version_info.arguments, - libraries: libs, - type_: version_info.type_, - data: Some(profile.data), - processors: Some(profile.processors), - }; - - let version_path = format!( - "forge/v{}/versions/{}.json", - daedalus::modded::CURRENT_FORGE_FORMAT_VERSION, - new_profile.id - ); - - upload_file_to_bucket( - version_path.clone(), - serde_json::to_vec(&new_profile)?, - Some("application/json".to_string()), - uploaded_files_mutex.as_ref() - ).await?; - - return Ok(Some(LoaderVersion { - id: loader_version_full, - url: format_url(&version_path), - stable: false - })); + Ok::(lib) + })).await?; + + let elapsed = now.elapsed(); + info!("Elapsed lib DL: {:.2?}", elapsed); + + let new_profile = PartialVersionInfo { + id: version_info.id, + inherits_from: version_info.inherits_from, + release_time: version_info.release_time, + time: version_info.time, + main_class: version_info.main_class, + minecraft_arguments: version_info.minecraft_arguments, + arguments: version_info.arguments, + libraries: libs, + type_: version_info.type_, + data: Some(profile.data), + processors: Some(profile.processors), + }; + + let version_path = format!( + "forge/v{}/versions/{}.json", + daedalus::modded::CURRENT_FORGE_FORMAT_VERSION, + new_profile.id + ); + + upload_file_to_bucket( + version_path.clone(), + serde_json::to_vec(&new_profile)?, + Some("application/json".to_string()), + uploaded_files_mutex.as_ref(), + semaphore.clone(), + ).await?; + + return Ok(Some(LoaderVersion { + id: loader_version_full, + url: format_url(&version_path), + stable: false + })); + } } - } - - Ok(None) - }.await - }).into_iter().peekable()/*.into_iter().flatten().collect()*/; - - let mut chunk_index = 0; - while loaders_futures.peek().is_some() { - info!("Loader Chunk {} Start", chunk_index); - let now = Instant::now(); - - let chunk: Vec<_> = loaders_futures.by_ref().take(10).collect(); - - let res = futures::future::try_join_all(chunk).await?; - loaders_versions.extend(res.into_iter().flatten()); - tokio::time::sleep(Duration::from_secs(1)).await; + Ok(None) + }.await + }); - chunk_index += 1; - - let elapsed = now.elapsed(); - info!("Loader Chunk {} Elapsed: {:.2?}", chunk_index, elapsed); + futures::future::try_join_all(loaders_futures).await?; } - } - - versions.lock().await.push(daedalus::modded::Version { - id: minecraft_version, - loaders: loaders_versions - }); - - Ok::<(), Error>(()) - }); - } - - { - let mut versions_peek = version_futures.into_iter().peekable(); - let mut chunk_index = 0; - while versions_peek.peek().is_some() { - info!("Chunk {} Start", chunk_index); - let now = Instant::now(); - - let chunk: Vec<_> = versions_peek.by_ref().take(1).collect(); - futures::future::try_join_all(chunk).await?; - - tokio::time::sleep(Duration::from_secs(1)).await; - chunk_index += 1; + versions.lock().await.push(daedalus::modded::Version { + id: minecraft_version, + stable: true, + loaders: loaders_versions + }); - let elapsed = now.elapsed(); - info!("Chunk {} Elapsed: {:.2?}", chunk_index, elapsed); + Ok::<(), Error>(()) + }); } } + futures::future::try_join_all(version_futures).await?; + if let Ok(versions) = Arc::try_unwrap(versions) { let mut versions = versions.into_inner(); @@ -495,13 +481,17 @@ pub async fn retrieve_data( minecraft_versions .versions .iter() - .position(|z| x.id == z.id) + .position(|z| { + x.id.replace("1.7.10_pre4", "1.7.10-pre4") == z.id + }) .unwrap_or_default() .cmp( &minecraft_versions .versions .iter() - .position(|z| y.id == z.id) + .position(|z| { + y.id.replace("1.7.10_pre4", "1.7.10-pre4") == z.id + }) .unwrap_or_default(), ) }); @@ -524,6 +514,8 @@ pub async fn retrieve_data( } } + println!("{:?}", versions); + upload_file_to_bucket( format!( "forge/v{}/manifest.json", @@ -534,6 +526,7 @@ pub async fn retrieve_data( })?, Some("application/json".to_string()), uploaded_files_mutex.as_ref(), + semaphore, ) .await?; } @@ -554,9 +547,15 @@ const DEFAULT_MAVEN_METADATA_URL: &str = /// the specified Minecraft version pub async fn fetch_maven_metadata( url: Option<&str>, + semaphore: Arc, ) -> Result>, Error> { Ok(serde_json::from_slice( - &download_file(url.unwrap_or(DEFAULT_MAVEN_METADATA_URL), None).await?, + &download_file( + url.unwrap_or(DEFAULT_MAVEN_METADATA_URL), + None, + semaphore, + ) + .await?, )?) } diff --git a/daedalus_client/src/main.rs b/daedalus_client/src/main.rs index 70a4fa6df..cefa4e179 100644 --- a/daedalus_client/src/main.rs +++ b/daedalus_client/src/main.rs @@ -1,8 +1,10 @@ use log::{error, warn}; -use std::time::Duration; -use s3::{Bucket, Region}; use s3::creds::Credentials; use s3::error::S3Error; +use s3::{Bucket, Region}; +use std::sync::Arc; +use std::time::Duration; +use tokio::sync::Semaphore; mod fabric; mod forge; @@ -19,10 +21,7 @@ pub enum Error { #[error("Error while managing asynchronous tasks")] TaskError(#[from] tokio::task::JoinError), #[error("Error while uploading file to S3")] - S3Error { - inner: S3Error, - file: String, - }, + S3Error { inner: S3Error, file: String }, #[error("Error while parsing version as semver: {0}")] SemVerError(#[from] semver::Error), #[error("Error while reading zip file: {0}")] @@ -31,6 +30,8 @@ pub enum Error { IoError(#[from] std::io::Error), #[error("Error while obtaining strong reference to Arc")] ArcError, + #[error("Error acquiring semaphore: {0}")] + AcquireError(#[from] tokio::sync::AcquireError), } #[tokio::main] @@ -43,14 +44,20 @@ async fn main() { return; } - let mut timer = tokio::time::interval(Duration::from_secs(10 * 60)); + let mut timer = tokio::time::interval(Duration::from_secs(30 * 60)); + let semaphore = Arc::new(Semaphore::new(50)); loop { timer.tick().await; let mut uploaded_files = Vec::new(); - let versions = match minecraft::retrieve_data(&mut uploaded_files).await { + let versions = match minecraft::retrieve_data( + &mut uploaded_files, + semaphore.clone(), + ) + .await + { Ok(res) => Some(res), Err(err) => { error!("{:?}", err); @@ -60,15 +67,22 @@ async fn main() { }; if let Some(manifest) = versions { - match fabric::retrieve_data(&manifest, &mut uploaded_files).await { + match fabric::retrieve_data( + &manifest, + &mut uploaded_files, + semaphore.clone(), + ) + .await + { Ok(..) => {} Err(err) => error!("{:?}", err), }; - match forge::retrieve_data(&manifest, &mut uploaded_files).await { + match forge::retrieve_data(&manifest, &mut uploaded_files, semaphore.clone()).await { Ok(..) => {} Err(err) => error!("{:?}", err), }; } + break; } } @@ -104,7 +118,6 @@ fn check_env_vars() -> bool { failed } - lazy_static::lazy_static! { static ref CLIENT : Bucket = Bucket::new( &dotenvy::var("S3_BUCKET_NAME").unwrap(), @@ -133,24 +146,26 @@ pub async fn upload_file_to_bucket( bytes: Vec, content_type: Option, uploaded_files: &tokio::sync::Mutex>, + semaphore: Arc, ) -> Result<(), Error> { + let _permit = semaphore.acquire().await?; let key = format!("{}/{}", &*dotenvy::var("BASE_FOLDER").unwrap(), path); for attempt in 1..=4 { let result = if let Some(ref content_type) = content_type { - CLIENT.put_object_with_content_type( - key.clone(), - &bytes, - content_type, - ).await + CLIENT + .put_object_with_content_type(key.clone(), &bytes, content_type) + .await } else { - CLIENT.put_object( - key.clone(), - &bytes, - ).await - }.map_err(|err| Error::S3Error { + CLIENT.put_object(key.clone(), &bytes).await + } + .map_err(|err| Error::S3Error { inner: err, - file: format!("{}/{}", &*dotenvy::var("BASE_FOLDER").unwrap(), path), + file: format!( + "{}/{}", + &*dotenvy::var("BASE_FOLDER").unwrap(), + path + ), }); match result { @@ -180,3 +195,29 @@ pub fn format_url(path: &str) -> String { path ) } + +pub async fn download_file( + url: &str, + sha1: Option<&str>, + semaphore: Arc, +) -> Result { + let _permit = semaphore.acquire().await?; + println!("{} url start", url); + + let val = daedalus::download_file(url, sha1).await?; + println!("{} url end", url); + Ok(val) +} + +pub async fn download_file_mirrors( + base: &str, + mirrors: &[&str], + sha1: Option<&str>, + semaphore: Arc, +) -> Result { + let _permit = semaphore.acquire().await?; + + let val = daedalus::download_file_mirrors(base, mirrors, sha1).await?; + + Ok(val) +} diff --git a/daedalus_client/src/minecraft.rs b/daedalus_client/src/minecraft.rs index 56007b9f9..256eef713 100644 --- a/daedalus_client/src/minecraft.rs +++ b/daedalus_client/src/minecraft.rs @@ -1,21 +1,26 @@ +use crate::download_file; use crate::{format_url, upload_file_to_bucket, Error}; -use daedalus::download_file; use daedalus::minecraft::VersionManifest; use log::info; use std::sync::Arc; -use std::time::{Duration, Instant}; -use tokio::sync::Mutex; - -pub async fn retrieve_data(uploaded_files: &mut Vec) -> Result { - let old_manifest = - daedalus::minecraft::fetch_version_manifest(Some(&*crate::format_url(&format!( +use std::time::Instant; +use tokio::sync::{Mutex, Semaphore}; + +pub async fn retrieve_data( + uploaded_files: &mut Vec, + semaphore: Arc, +) -> Result { + let old_manifest = daedalus::minecraft::fetch_version_manifest(Some( + &*format_url(&format!( "minecraft/v{}/manifest.json", daedalus::minecraft::CURRENT_FORMAT_VERSION - )))) - .await - .ok(); + )), + )) + .await + .ok(); - let mut manifest = daedalus::minecraft::fetch_version_manifest(None).await?; + let mut manifest = + daedalus::minecraft::fetch_version_manifest(None).await?; let cloned_manifest = Arc::new(Mutex::new(manifest.clone())); let visited_assets_mutex = Arc::new(Mutex::new(Vec::new())); @@ -42,13 +47,16 @@ pub async fn retrieve_data(uploaded_files: &mut Vec) -> Result) -> Result) -> Result) -> Result) -> Result) -> Result = versions.by_ref().take(100).collect(); - futures::future::try_join_all(chunk).await?; - - tokio::time::sleep(Duration::from_secs(1)).await; - - chunk_index += 1; - - let elapsed = now.elapsed(); - info!("Chunk {} Elapsed: {:.2?}", chunk_index, elapsed); - } - } + futures::future::try_join_all(version_futures).await?; upload_file_to_bucket( format!( @@ -157,6 +157,7 @@ pub async fn retrieve_data(uploaded_files: &mut Vec) -> Result Date: Tue, 4 Apr 2023 20:26:10 -0700 Subject: [PATCH 43/85] fix debug stuff --- daedalus_client/src/fabric.rs | 1 - daedalus_client/src/forge.rs | 2 -- daedalus_client/src/main.rs | 12 +++++++----- 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/daedalus_client/src/fabric.rs b/daedalus_client/src/fabric.rs index b40c31ee0..e3625dabc 100644 --- a/daedalus_client/src/fabric.rs +++ b/daedalus_client/src/fabric.rs @@ -63,7 +63,6 @@ pub async fn retrieve_data( ) .await?; - //println!("{}", loader); Ok::, String, PartialVersionInfo)>, Error>( Some((stable, loader, version)), ) diff --git a/daedalus_client/src/forge.rs b/daedalus_client/src/forge.rs index cb903f15d..0de8cf1e1 100644 --- a/daedalus_client/src/forge.rs +++ b/daedalus_client/src/forge.rs @@ -514,8 +514,6 @@ pub async fn retrieve_data( } } - println!("{:?}", versions); - upload_file_to_bucket( format!( "forge/v{}/manifest.json", diff --git a/daedalus_client/src/main.rs b/daedalus_client/src/main.rs index cefa4e179..de97bc071 100644 --- a/daedalus_client/src/main.rs +++ b/daedalus_client/src/main.rs @@ -77,12 +77,17 @@ async fn main() { Ok(..) => {} Err(err) => error!("{:?}", err), }; - match forge::retrieve_data(&manifest, &mut uploaded_files, semaphore.clone()).await { + match forge::retrieve_data( + &manifest, + &mut uploaded_files, + semaphore.clone(), + ) + .await + { Ok(..) => {} Err(err) => error!("{:?}", err), }; } - break; } } @@ -202,10 +207,7 @@ pub async fn download_file( semaphore: Arc, ) -> Result { let _permit = semaphore.acquire().await?; - println!("{} url start", url); - let val = daedalus::download_file(url, sha1).await?; - println!("{} url end", url); Ok(val) } From 9754f2d1c51dee70c3833192926809fbd2608619 Mon Sep 17 00:00:00 2001 From: Jai A Date: Tue, 4 Apr 2023 21:17:19 -0700 Subject: [PATCH 44/85] Add limiter for forge downloading --- .env | 1 - daedalus_client/src/forge.rs | 36 +++++++++++++++++++++++++++++--- daedalus_client/src/main.rs | 22 +++++++++---------- daedalus_client/src/minecraft.rs | 17 ++++++++++++++- 4 files changed, 59 insertions(+), 17 deletions(-) diff --git a/.env b/.env index 34bf98cd1..c7298bc74 100644 --- a/.env +++ b/.env @@ -1,7 +1,6 @@ RUST_LOG=info,error BASE_URL=https://modrinth-cdn-staging.nyc3.digitaloceanspaces.com -BASE_FOLDER=gamedata S3_ACCESS_TOKEN=none S3_SECRET=none diff --git a/daedalus_client/src/forge.rs b/daedalus_client/src/forge.rs index 0de8cf1e1..7cbe84790 100644 --- a/daedalus_client/src/forge.rs +++ b/daedalus_client/src/forge.rs @@ -16,7 +16,7 @@ use serde::{Deserialize, Serialize}; use std::collections::HashMap; use std::io::Read; use std::sync::Arc; -use std::time::Instant; +use std::time::{Duration, Instant}; use tokio::sync::{Mutex, Semaphore}; lazy_static! { @@ -458,7 +458,22 @@ pub async fn retrieve_data( }.await }); - futures::future::try_join_all(loaders_futures).await?; + { + let mut versions = loaders_futures.into_iter().peekable(); + let mut chunk_index = 0; + while versions.peek().is_some() { + let now = Instant::now(); + + let chunk: Vec<_> = versions.by_ref().take(1).collect(); + futures::future::try_join_all(chunk).await?; + + chunk_index += 1; + + let elapsed = now.elapsed(); + info!("Chunk {} Elapsed: {:.2?}", chunk_index, elapsed); + } + } + //futures::future::try_join_all(loaders_futures).await?; } versions.lock().await.push(daedalus::modded::Version { @@ -472,7 +487,22 @@ pub async fn retrieve_data( } } - futures::future::try_join_all(version_futures).await?; + { + let mut versions = version_futures.into_iter().peekable(); + let mut chunk_index = 0; + while versions.peek().is_some() { + let now = Instant::now(); + + let chunk: Vec<_> = versions.by_ref().take(10).collect(); + futures::future::try_join_all(chunk).await?; + + chunk_index += 1; + + let elapsed = now.elapsed(); + info!("Chunk {} Elapsed: {:.2?}", chunk_index, elapsed); + } + } + //futures::future::try_join_all(version_futures).await?; if let Ok(versions) = Arc::try_unwrap(versions) { let mut versions = versions.into_inner(); diff --git a/daedalus_client/src/main.rs b/daedalus_client/src/main.rs index de97bc071..56b524bfb 100644 --- a/daedalus_client/src/main.rs +++ b/daedalus_client/src/main.rs @@ -1,4 +1,4 @@ -use log::{error, warn}; +use log::{error, info, warn}; use s3::creds::Credentials; use s3::error::S3Error; use s3::{Bucket, Region}; @@ -112,7 +112,6 @@ fn check_env_vars() -> bool { } failed |= check_var::("BASE_URL"); - failed |= check_var::("BASE_FOLDER"); failed |= check_var::("S3_ACCESS_TOKEN"); failed |= check_var::("S3_SECRET"); @@ -154,7 +153,8 @@ pub async fn upload_file_to_bucket( semaphore: Arc, ) -> Result<(), Error> { let _permit = semaphore.acquire().await?; - let key = format!("{}/{}", &*dotenvy::var("BASE_FOLDER").unwrap(), path); + info!("{} started uploading", path); + let key = path.clone(); for attempt in 1..=4 { let result = if let Some(ref content_type) = content_type { @@ -166,16 +166,13 @@ pub async fn upload_file_to_bucket( } .map_err(|err| Error::S3Error { inner: err, - file: format!( - "{}/{}", - &*dotenvy::var("BASE_FOLDER").unwrap(), - path - ), + file: path.clone(), }); match result { Ok(_) => { { + info!("{} done uploading", path); let mut uploaded_files = uploaded_files.lock().await; uploaded_files.push(key); } @@ -188,15 +185,13 @@ pub async fn upload_file_to_bucket( } } } - unreachable!() } pub fn format_url(path: &str) -> String { format!( - "{}/{}/{}", + "{}/{}", &*dotenvy::var("BASE_URL").unwrap(), - &*dotenvy::var("BASE_FOLDER").unwrap(), path ) } @@ -207,7 +202,9 @@ pub async fn download_file( semaphore: Arc, ) -> Result { let _permit = semaphore.acquire().await?; + info!("{} started downloading", url); let val = daedalus::download_file(url, sha1).await?; + info!("{} finished downloading", url); Ok(val) } @@ -218,8 +215,9 @@ pub async fn download_file_mirrors( semaphore: Arc, ) -> Result { let _permit = semaphore.acquire().await?; - + info!("{} started downloading", base); let val = daedalus::download_file_mirrors(base, mirrors, sha1).await?; + info!("{} finished downloading", base); Ok(val) } diff --git a/daedalus_client/src/minecraft.rs b/daedalus_client/src/minecraft.rs index 256eef713..024ce3d05 100644 --- a/daedalus_client/src/minecraft.rs +++ b/daedalus_client/src/minecraft.rs @@ -147,7 +147,22 @@ pub async fn retrieve_data( }) } - futures::future::try_join_all(version_futures).await?; + { + let mut versions = version_futures.into_iter().peekable(); + let mut chunk_index = 0; + while versions.peek().is_some() { + let now = Instant::now(); + + let chunk: Vec<_> = versions.by_ref().take(100).collect(); + futures::future::try_join_all(chunk).await?; + + chunk_index += 1; + + let elapsed = now.elapsed(); + info!("Chunk {} Elapsed: {:.2?}", chunk_index, elapsed); + } + } + //futures::future::try_join_all(version_futures).await?; upload_file_to_bucket( format!( From aa84f21fde4954775f2f7b442a5a45eb442315e7 Mon Sep 17 00:00:00 2001 From: Jai A Date: Wed, 5 Apr 2023 12:02:00 -0700 Subject: [PATCH 45/85] Fix forge syncing not working --- .env | 2 +- Dockerfile | 22 ++++++++++++++++++---- daedalus/Cargo.toml | 2 +- daedalus_client/Cargo.toml | 2 +- daedalus_client/src/fabric.rs | 3 ++- daedalus_client/src/forge.rs | 11 +++++++---- daedalus_client/src/main.rs | 2 +- 7 files changed, 31 insertions(+), 13 deletions(-) diff --git a/.env b/.env index c7298bc74..bbcfb5cbf 100644 --- a/.env +++ b/.env @@ -1,4 +1,4 @@ -RUST_LOG=info,error +RUST_LOG=info BASE_URL=https://modrinth-cdn-staging.nyc3.digitaloceanspaces.com diff --git a/Dockerfile b/Dockerfile index eae1ea34b..a2a341da0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,21 @@ -FROM rust:1.68.2 - -COPY ./ ./ +FROM rust:1.68.2 as build +ENV PKG_CONFIG_ALLOW_CROSS=1 +WORKDIR /usr/src/daedalus +COPY . . RUN cargo build --release -CMD ["./target/release/daedalus_client"] \ No newline at end of file + +FROM debian:bullseye-slim + +RUN apt-get update \ + && apt-get install -y --no-install-recommends ca-certificates \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + +RUN update-ca-certificates + +COPY --from=build /usr/src/daedalus/target/release/daedalus_client /daedalus/daedalus_client +WORKDIR /daedalus_client + +CMD /daedalus/daedalus_client diff --git a/daedalus/Cargo.toml b/daedalus/Cargo.toml index ada94ed8c..09c54758c 100644 --- a/daedalus/Cargo.toml +++ b/daedalus/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "daedalus" -version = "0.1.18" +version = "0.1.19" authors = ["Jai A "] edition = "2018" license = "MIT" diff --git a/daedalus_client/Cargo.toml b/daedalus_client/Cargo.toml index 389c45a64..8b531712c 100644 --- a/daedalus_client/Cargo.toml +++ b/daedalus_client/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "daedalus_client" -version = "0.1.18" +version = "0.1.19" authors = ["Jai A "] edition = "2018" diff --git a/daedalus_client/src/fabric.rs b/daedalus_client/src/fabric.rs index e3625dabc..6aa8f91fe 100644 --- a/daedalus_client/src/fabric.rs +++ b/daedalus_client/src/fabric.rs @@ -82,6 +82,7 @@ pub async fn retrieve_data( visited_artifacts_mutex.lock().await; if visited_assets.contains(&lib.name) { + lib.name = lib.name.replace(DUMMY_GAME_VERSION, DUMMY_REPLACE_STRING); lib.url = Some(format_url("maven/")); return Ok(lib); @@ -92,7 +93,6 @@ pub async fn retrieve_data( if lib.name.contains(DUMMY_GAME_VERSION) { lib.name = lib.name.replace(DUMMY_GAME_VERSION, DUMMY_REPLACE_STRING); - lib.url = Some(format_url("maven/")); futures::future::try_join_all(list.game.clone().into_iter().map(|game_version| async { let semaphore = semaphore.clone(); let uploaded_files_mutex = uploaded_files_mutex.clone(); @@ -130,6 +130,7 @@ pub async fn retrieve_data( Ok::<(), Error>(()) })).await?; + lib.url = Some(format_url("maven/")); return Ok(lib); } diff --git a/daedalus_client/src/forge.rs b/daedalus_client/src/forge.rs index 7cbe84790..5335f20db 100644 --- a/daedalus_client/src/forge.rs +++ b/daedalus_client/src/forge.rs @@ -93,7 +93,7 @@ pub async fn retrieve_data( if !loaders.is_empty() { version_futures.push(async { - let loaders_versions = Vec::new(); + let mut loaders_versions = Vec::new(); { let loaders_futures = loaders.into_iter().map(|(loader_version_full, version)| async { @@ -464,8 +464,9 @@ pub async fn retrieve_data( while versions.peek().is_some() { let now = Instant::now(); - let chunk: Vec<_> = versions.by_ref().take(1).collect(); - futures::future::try_join_all(chunk).await?; + let chunk: Vec<_> = versions.by_ref().take(10).collect(); + let res = futures::future::try_join_all(chunk).await?; + loaders_versions.extend(res.into_iter().flatten()); chunk_index += 1; @@ -493,7 +494,7 @@ pub async fn retrieve_data( while versions.peek().is_some() { let now = Instant::now(); - let chunk: Vec<_> = versions.by_ref().take(10).collect(); + let chunk: Vec<_> = versions.by_ref().take(1).collect(); futures::future::try_join_all(chunk).await?; chunk_index += 1; @@ -544,6 +545,8 @@ pub async fn retrieve_data( } } + println!("{}", serde_json::to_string(&versions).unwrap()); + upload_file_to_bucket( format!( "forge/v{}/manifest.json", diff --git a/daedalus_client/src/main.rs b/daedalus_client/src/main.rs index 56b524bfb..000f003a0 100644 --- a/daedalus_client/src/main.rs +++ b/daedalus_client/src/main.rs @@ -44,7 +44,7 @@ async fn main() { return; } - let mut timer = tokio::time::interval(Duration::from_secs(30 * 60)); + let mut timer = tokio::time::interval(Duration::from_secs(60 * 60)); let semaphore = Arc::new(Semaphore::new(50)); loop { From bf5a25a96f0da836b20e16f1998cc9eb7bebc3f9 Mon Sep 17 00:00:00 2001 From: Jai A Date: Wed, 5 Apr 2023 12:02:47 -0700 Subject: [PATCH 46/85] fmt + clippy --- daedalus_client/src/forge.rs | 4 +--- daedalus_client/src/main.rs | 6 +----- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/daedalus_client/src/forge.rs b/daedalus_client/src/forge.rs index 5335f20db..7415cc651 100644 --- a/daedalus_client/src/forge.rs +++ b/daedalus_client/src/forge.rs @@ -16,7 +16,7 @@ use serde::{Deserialize, Serialize}; use std::collections::HashMap; use std::io::Read; use std::sync::Arc; -use std::time::{Duration, Instant}; +use std::time::Instant; use tokio::sync::{Mutex, Semaphore}; lazy_static! { @@ -545,8 +545,6 @@ pub async fn retrieve_data( } } - println!("{}", serde_json::to_string(&versions).unwrap()); - upload_file_to_bucket( format!( "forge/v{}/manifest.json", diff --git a/daedalus_client/src/main.rs b/daedalus_client/src/main.rs index 000f003a0..92fee4cd7 100644 --- a/daedalus_client/src/main.rs +++ b/daedalus_client/src/main.rs @@ -189,11 +189,7 @@ pub async fn upload_file_to_bucket( } pub fn format_url(path: &str) -> String { - format!( - "{}/{}", - &*dotenvy::var("BASE_URL").unwrap(), - path - ) + format!("{}/{}", &*dotenvy::var("BASE_URL").unwrap(), path) } pub async fn download_file( From 0c2e9137a2cb45ac9cb422d822e8c9eb8d676486 Mon Sep 17 00:00:00 2001 From: Jai A Date: Wed, 5 Apr 2023 12:39:16 -0700 Subject: [PATCH 47/85] Update id merging --- daedalus/src/modded.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/daedalus/src/modded.rs b/daedalus/src/modded.rs index 2494ded4d..2519719c8 100644 --- a/daedalus/src/modded.rs +++ b/daedalus/src/modded.rs @@ -131,7 +131,7 @@ pub fn merge_partial_version( asset_index: merge.asset_index, assets: merge.assets, downloads: merge.downloads, - id: merge.id, + id: partial.id.replace(DUMMY_REPLACE_STRING, &merge_id), java_version: merge.java_version, libraries: partial .libraries From 89e56ae279392f41abfc8e35658099449071a3e6 Mon Sep 17 00:00:00 2001 From: Jai A Date: Tue, 25 Apr 2023 19:36:21 -0700 Subject: [PATCH 48/85] Support for ARM + Quilt --- LICENSE | 2 +- daedalus/Cargo.toml | 2 +- daedalus/src/minecraft.rs | 82 ++++++- daedalus/src/modded.rs | 2 + daedalus_client/Cargo.toml | 2 +- daedalus_client/src/fabric.rs | 29 ++- daedalus_client/src/forge.rs | 17 +- daedalus_client/src/main.rs | 13 +- daedalus_client/src/minecraft.rs | 61 +++++- daedalus_client/src/quilt.rs | 362 +++++++++++++++++++++++++++++++ 10 files changed, 542 insertions(+), 30 deletions(-) create mode 100644 daedalus_client/src/quilt.rs diff --git a/LICENSE b/LICENSE index 1bc38c7a1..75e4f7799 100644 --- a/LICENSE +++ b/LICENSE @@ -1,4 +1,4 @@ -Copyright © 2022 Rinth, Inc. +Copyright © 2023 Rinth, Inc. Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: diff --git a/daedalus/Cargo.toml b/daedalus/Cargo.toml index 09c54758c..94cbde710 100644 --- a/daedalus/Cargo.toml +++ b/daedalus/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "daedalus" -version = "0.1.19" +version = "0.1.20" authors = ["Jai A "] edition = "2018" license = "MIT" diff --git a/daedalus/src/minecraft.rs b/daedalus/src/minecraft.rs index 247274a54..49c87fe4c 100644 --- a/daedalus/src/minecraft.rs +++ b/daedalus/src/minecraft.rs @@ -149,11 +149,12 @@ pub struct Download { } #[cfg_attr(feature = "bincode", derive(Encode, Decode))] -#[derive(Serialize, Deserialize, Debug)] +#[derive(Serialize, Deserialize, Debug, Clone)] /// Download information of a library pub struct LibraryDownload { + #[serde(skip_serializing_if = "Option::is_none")] /// The path that the library should be saved to - pub path: String, + pub path: Option, /// The SHA1 hash of the library pub sha1: String, /// The size of the library @@ -163,7 +164,7 @@ pub struct LibraryDownload { } #[cfg_attr(feature = "bincode", derive(Encode, Decode))] -#[derive(Serialize, Deserialize, Debug)] +#[derive(Serialize, Deserialize, Debug, Clone)] /// A list of files that should be downloaded for libraries pub struct LibraryDownloads { #[serde(skip_serializing_if = "Option::is_none")] @@ -188,17 +189,25 @@ pub enum RuleAction { #[cfg_attr(feature = "bincode", derive(Encode, Decode))] #[derive(Serialize, Deserialize, Debug, Eq, PartialEq, Hash, Clone)] -#[serde(rename_all = "snake_case")] +#[serde(rename_all = "kebab-case")] /// An enum representing the different types of operating systems pub enum Os { - /// MacOS + /// MacOS (x86) Osx, - /// Windows + /// M1-Based Macs + OsxArm64, + /// Windows (x86) Windows, - /// Linux and its derivatives + /// Windows ARM + WindowsArm64, + /// Linux (x86) and its derivatives Linux, + /// Linux ARM 64 + LinuxArm64, + /// Linux ARM 32 + LinuxArm32, /// The OS is unknown - Unknown, + Unknown } #[cfg_attr(feature = "bincode", derive(Encode, Decode))] @@ -243,7 +252,7 @@ pub struct Rule { } #[cfg_attr(feature = "bincode", derive(Encode, Decode))] -#[derive(Serialize, Deserialize, Debug)] +#[derive(Serialize, Deserialize, Debug, Clone)] /// Information delegating the extraction of the library pub struct LibraryExtract { #[serde(skip_serializing_if = "Option::is_none")] @@ -263,7 +272,7 @@ pub struct JavaVersion { } #[cfg_attr(feature = "bincode", derive(Encode, Decode))] -#[derive(Serialize, Deserialize, Debug)] +#[derive(Serialize, Deserialize, Debug, Clone)] /// A library which the game relies on to run pub struct Library { #[serde(skip_serializing_if = "Option::is_none")] @@ -291,6 +300,59 @@ pub struct Library { pub include_in_classpath: bool, } +#[derive(Deserialize, Debug, Clone)] +/// A partial library which should be merged with a full library +pub struct PartialLibrary { + /// The files the library has + pub downloads: Option, + /// Rules of the extraction of the file + pub extract: Option, + /// The maven name of the library. The format is `groupId:artifactId:version` + pub name: Option, + /// The URL to the repository where the library can be downloaded + pub url: Option, + /// Native files that the library relies on + pub natives: Option>, + /// Rules deciding whether the library should be downloaded or not + pub rules: Option>, + /// SHA1 Checksums for validating the library's integrity. Only present for forge libraries + pub checksums: Option>, + /// Whether the library should be included in the classpath at the game's launch + pub include_in_classpath: Option, +} + +pub fn merge_partial_library( + partial: PartialLibrary, + mut merge: Library, +) -> Library { + if let Some(downloads) = partial.downloads { + merge.downloads = Some(downloads) + } + if let Some(extract) = partial.extract { + merge.extract = Some(extract) + } + if let Some(name) = partial.name { + merge.name = name + } + if let Some(url) = partial.url { + merge.url = Some(url) + } + if let Some(natives) = partial.natives { + merge.natives = Some(natives) + } + if let Some(rules) = partial.rules { + merge.rules = Some(rules) + } + if let Some(checksums) = partial.checksums { + merge.checksums = Some(checksums) + } + if let Some(include_in_classpath) = partial.include_in_classpath { + merge.include_in_classpath = include_in_classpath + } + + merge +} + fn default_include_in_classpath() -> bool { true } diff --git a/daedalus/src/modded.rs b/daedalus/src/modded.rs index 2519719c8..8f91798fc 100644 --- a/daedalus/src/modded.rs +++ b/daedalus/src/modded.rs @@ -14,6 +14,8 @@ use bincode::{Decode, Encode}; pub const CURRENT_FABRIC_FORMAT_VERSION: usize = 0; /// The latest version of the format the fabric model structs deserialize to pub const CURRENT_FORGE_FORMAT_VERSION: usize = 0; +/// The latest version of the format the quilt model structs deserialize to +pub const CURRENT_QUILT_FORMAT_VERSION: usize = 0; /// The dummy replace string library names, inheritsFrom, and version names should be replaced with pub const DUMMY_REPLACE_STRING: &str = "${modrinth.gameVersion}"; diff --git a/daedalus_client/Cargo.toml b/daedalus_client/Cargo.toml index 8b531712c..42b93a80b 100644 --- a/daedalus_client/Cargo.toml +++ b/daedalus_client/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "daedalus_client" -version = "0.1.19" +version = "0.1.20" authors = ["Jai A "] edition = "2018" diff --git a/daedalus_client/src/fabric.rs b/daedalus_client/src/fabric.rs index 6aa8f91fe..d57cfd851 100644 --- a/daedalus_client/src/fabric.rs +++ b/daedalus_client/src/fabric.rs @@ -215,18 +215,27 @@ pub async fn retrieve_data( )) .await?; - versions.push(Version { - id: DUMMY_REPLACE_STRING.to_string(), - stable: true, - loaders: loader_version_mutex.into_inner(), - }); + let mut loader_version_mutex = loader_version_mutex.into_inner(); + if !loader_version_mutex.is_empty() { + if let Some(version) = versions.iter_mut().find(|x| x.id == DUMMY_REPLACE_STRING) { + version.loaders.append(&mut loader_version_mutex); + } else { + versions.push(Version { + id: DUMMY_REPLACE_STRING.to_string(), + stable: true, + loaders: loader_version_mutex, + }); + } + } for version in &list.game { - versions.push(Version { - id: version.version.clone(), - stable: version.stable, - loaders: vec![], - }); + if !versions.iter().any(|x| x.id == version.version) { + versions.push(Version { + id: version.version.clone(), + stable: version.stable, + loaders: vec![], + }); + } } versions.sort_by(|x, y| { diff --git a/daedalus_client/src/forge.rs b/daedalus_client/src/forge.rs index 7415cc651..fc3bf2f76 100644 --- a/daedalus_client/src/forge.rs +++ b/daedalus_client/src/forge.rs @@ -105,9 +105,16 @@ pub async fn retrieve_data( async move { /// These forge versions are not worth supporting! - const WHITELIST : [&str; 1] = [ + const WHITELIST : &[&str] = &[ // Not supported due to `data` field being `[]` even though the type is a map "1.12.2-14.23.5.2851", + // Malformed Archives + "1.6.1-8.9.0.749", + "1.6.1-8.9.0.751", + "1.6.4-9.11.1.960", + "1.6.4-9.11.1.961", + "1.6.4-9.11.1.963", + "1.6.4-9.11.1.964", ]; if WHITELIST.contains(&&*loader_version_full) { @@ -459,19 +466,20 @@ pub async fn retrieve_data( }); { + let len = loaders_futures.len(); let mut versions = loaders_futures.into_iter().peekable(); let mut chunk_index = 0; while versions.peek().is_some() { let now = Instant::now(); - let chunk: Vec<_> = versions.by_ref().take(10).collect(); + let chunk: Vec<_> = versions.by_ref().take(1).collect(); let res = futures::future::try_join_all(chunk).await?; loaders_versions.extend(res.into_iter().flatten()); chunk_index += 1; let elapsed = now.elapsed(); - info!("Chunk {} Elapsed: {:.2?}", chunk_index, elapsed); + info!("Loader Chunk {}/{len} Elapsed: {:.2?}", chunk_index, elapsed); } } //futures::future::try_join_all(loaders_futures).await?; @@ -489,6 +497,7 @@ pub async fn retrieve_data( } { + let len = version_futures.len(); let mut versions = version_futures.into_iter().peekable(); let mut chunk_index = 0; while versions.peek().is_some() { @@ -500,7 +509,7 @@ pub async fn retrieve_data( chunk_index += 1; let elapsed = now.elapsed(); - info!("Chunk {} Elapsed: {:.2?}", chunk_index, elapsed); + info!("Chunk {}/{len} Elapsed: {:.2?}", chunk_index, elapsed); } } //futures::future::try_join_all(version_futures).await?; diff --git a/daedalus_client/src/main.rs b/daedalus_client/src/main.rs index 92fee4cd7..c4bb24719 100644 --- a/daedalus_client/src/main.rs +++ b/daedalus_client/src/main.rs @@ -9,6 +9,7 @@ use tokio::sync::Semaphore; mod fabric; mod forge; mod minecraft; +mod quilt; #[derive(thiserror::Error, Debug)] pub enum Error { @@ -45,7 +46,7 @@ async fn main() { } let mut timer = tokio::time::interval(Duration::from_secs(60 * 60)); - let semaphore = Arc::new(Semaphore::new(50)); + let semaphore = Arc::new(Semaphore::new(10)); loop { timer.tick().await; @@ -87,6 +88,16 @@ async fn main() { Ok(..) => {} Err(err) => error!("{:?}", err), }; + match quilt::retrieve_data( + &manifest, + &mut uploaded_files, + semaphore.clone(), + ) + .await + { + Ok(..) => {} + Err(err) => error!("{:?}", err), + }; } } } diff --git a/daedalus_client/src/minecraft.rs b/daedalus_client/src/minecraft.rs index 024ce3d05..282fd9346 100644 --- a/daedalus_client/src/minecraft.rs +++ b/daedalus_client/src/minecraft.rs @@ -1,10 +1,12 @@ use crate::download_file; use crate::{format_url, upload_file_to_bucket, Error}; -use daedalus::minecraft::VersionManifest; +use daedalus::minecraft::{Library, merge_partial_library, PartialLibrary, VersionManifest}; use log::info; use std::sync::Arc; use std::time::Instant; use tokio::sync::{Mutex, Semaphore}; +use serde::Deserialize; +use daedalus::get_hash; pub async fn retrieve_data( uploaded_files: &mut Vec, @@ -23,6 +25,9 @@ pub async fn retrieve_data( daedalus::minecraft::fetch_version_manifest(None).await?; let cloned_manifest = Arc::new(Mutex::new(manifest.clone())); + let patches = fetch_library_patches(None, semaphore.clone()).await?; + let cloned_patches = Arc::new(&patches); + let visited_assets_mutex = Arc::new(Mutex::new(Vec::new())); let uploaded_files_mutex = Arc::new(Mutex::new(Vec::new())); @@ -48,6 +53,7 @@ pub async fn retrieve_data( let cloned_manifest_mutex = Arc::clone(&cloned_manifest); let uploaded_files_mutex = Arc::clone(&uploaded_files_mutex); let semaphore = Arc::clone(&semaphore); + let patches = Arc::clone(&cloned_patches); let assets_hash = old_version.and_then(|x| x.assets_index_sha1.clone()); @@ -55,9 +61,30 @@ pub async fn retrieve_data( async move { let mut upload_futures = Vec::new(); - let version_info = + let mut version_info = daedalus::minecraft::fetch_version_info(version).await?; + let mut new_libraries = Vec::new(); + for library in version_info.libraries { + if let Some(patch) = patches.iter().find(|x| x.match_.contains(&library.name)) { + if let Some(additional_libraries) = &patch.additional_libraries { + new_libraries.push(library); + for additional_library in additional_libraries { + new_libraries.push(additional_library.clone()); + } + } else if let Some(override_) = &patch.override_ { + new_libraries.push(merge_partial_library(override_.clone(), library)); + } else { + new_libraries.push(library); + } + } else { + new_libraries.push(library); + } + } + version_info.libraries = new_libraries; + + let version_info_hash = get_hash(bytes::Bytes::from(serde_json::to_vec(&version_info)?)).await?; + let version_path = format!( "minecraft/v{}/versions/{}.json", daedalus::minecraft::CURRENT_FORMAT_VERSION, @@ -85,6 +112,7 @@ pub async fn retrieve_data( Some(version_info.asset_index.sha1.clone()); cloned_manifest.versions[position].assets_index_url = Some(format_url(&assets_path)); + cloned_manifest.versions[position].sha1 = version_info_hash; } let mut download_assets = false; @@ -187,3 +215,32 @@ pub async fn retrieve_data( .map_err(|_| Error::ArcError)? .into_inner()) } + +#[derive(Deserialize, Debug)] +#[serde(rename_all = "camelCase")] +/// A version of the fabric loader +struct LibraryPatch { + #[serde(rename = "_comment")] + pub _comment: String, + #[serde(rename = "match")] + pub match_: Vec, + pub additional_libraries: Option>, + #[serde(rename = "override")] + pub override_: Option, + pub patch_additional_libraries: Option, +} + +/// Fetches the list of fabric versions +async fn fetch_library_patches( + url: Option<&str>, + semaphore: Arc, +) -> Result, Error> { + Ok(serde_json::from_slice( + &download_file( + url.unwrap_or(&format_url("library-patches.json")), + None, + semaphore, + ) + .await?, + )?) +} diff --git a/daedalus_client/src/quilt.rs b/daedalus_client/src/quilt.rs new file mode 100644 index 000000000..9b122967b --- /dev/null +++ b/daedalus_client/src/quilt.rs @@ -0,0 +1,362 @@ +use crate::{download_file, format_url, upload_file_to_bucket, Error}; +use daedalus::minecraft::{Library, VersionManifest}; +use daedalus::modded::{ + LoaderVersion, Manifest, PartialVersionInfo, Version, DUMMY_REPLACE_STRING, +}; +use serde::{Deserialize, Serialize}; +use std::sync::Arc; +use tokio::sync::{Mutex, RwLock, Semaphore}; + +pub async fn retrieve_data( + minecraft_versions: &VersionManifest, + uploaded_files: &mut Vec, + semaphore: Arc, +) -> Result<(), Error> { + let mut list = fetch_quilt_versions(None, semaphore.clone()).await?; + let old_manifest = daedalus::modded::fetch_manifest(&format_url(&format!( + "quilt/v{}/manifest.json", + daedalus::modded::CURRENT_QUILT_FORMAT_VERSION, + ))) + .await + .ok(); + + let mut versions = if let Some(old_manifest) = old_manifest { + old_manifest.game_versions + } else { + Vec::new() + }; + + let loaders_mutex = RwLock::new(Vec::new()); + + { + let mut loaders = loaders_mutex.write().await; + + for loader in &list.loader { + loaders.push((Box::new(false), loader.version.clone())) + } + + list.loader + .retain(|x| loaders.iter().any(|val| val.1 == x.version)) + } + + const DUMMY_GAME_VERSION: &str = "1.19.4-rc2"; + + let loader_version_mutex = Mutex::new(Vec::new()); + let uploaded_files_mutex = Arc::new(Mutex::new(Vec::new())); + + let loader_versions = futures::future::try_join_all( + loaders_mutex.read().await.clone().into_iter().map( + |(stable, loader)| async { + { + if versions.iter().any(|x| { + x.id == DUMMY_REPLACE_STRING + && x.loaders.iter().any(|x| x.id == loader) + }) { + return Ok(None); + } + } + + let version = fetch_quilt_version( + DUMMY_GAME_VERSION, + &loader, + semaphore.clone(), + ) + .await?; + + Ok::, String, PartialVersionInfo)>, Error>( + Some((stable, loader, version)), + ) + }, + ), + ) + .await?; + + let visited_artifacts_mutex = Arc::new(Mutex::new(Vec::new())); + futures::future::try_join_all(loader_versions.into_iter() + .flatten().map( + |(stable, loader, version)| async { + let libs = futures::future::try_join_all( + version.libraries.into_iter().map(|mut lib| async { + { + let mut visited_assets = + visited_artifacts_mutex.lock().await; + + if visited_assets.contains(&lib.name) { + lib.name = lib.name.replace(DUMMY_GAME_VERSION, DUMMY_REPLACE_STRING); + lib.url = Some(format_url("maven/")); + + return Ok(lib); + } else { + visited_assets.push(lib.name.clone()) + } + } + + if lib.name.contains(DUMMY_GAME_VERSION) { + lib.name = lib.name.replace(DUMMY_GAME_VERSION, DUMMY_REPLACE_STRING); + futures::future::try_join_all(list.game.clone().into_iter().map(|game_version| async { + let semaphore = semaphore.clone(); + let uploaded_files_mutex = uploaded_files_mutex.clone(); + let lib_name = lib.name.clone(); + let lib_url = lib.url.clone(); + + async move { + let artifact_path = + daedalus::get_path_from_artifact(&lib_name.replace(DUMMY_REPLACE_STRING, &game_version.version))?; + + let artifact = download_file( + &format!( + "{}{}", + lib_url.unwrap_or_else(|| { + "https://maven.quiltmc.org/".to_string() + }), + artifact_path + ), + None, + semaphore.clone(), + ) + .await?; + + upload_file_to_bucket( + format!("{}/{}", "maven", artifact_path), + artifact.to_vec(), + Some("application/java-archive".to_string()), + &uploaded_files_mutex, + semaphore.clone(), + ) + .await?; + + Ok::<(), Error>(()) + }.await?; + + Ok::<(), Error>(()) + })).await?; + lib.url = Some(format_url("maven/")); + + return Ok(lib); + } + + let artifact_path = + daedalus::get_path_from_artifact(&lib.name)?; + + let artifact = download_file( + &format!( + "{}{}", + lib.url.unwrap_or_else(|| { + "https://maven.quiltmc.org/".to_string() + }), + artifact_path + ), + None, + semaphore.clone(), + ) + .await?; + + lib.url = Some(format_url("maven/")); + + upload_file_to_bucket( + format!("{}/{}", "maven", artifact_path), + artifact.to_vec(), + Some("application/java-archive".to_string()), + &uploaded_files_mutex, + semaphore.clone(), + ) + .await?; + + Ok::(lib) + }), + ) + .await?; + + let version_path = format!( + "quilt/v{}/versions/{}.json", + daedalus::modded::CURRENT_QUILT_FORMAT_VERSION, + &loader + ); + + upload_file_to_bucket( + version_path.clone(), + serde_json::to_vec(&PartialVersionInfo { + arguments: version.arguments, + id: version + .id + .replace(DUMMY_GAME_VERSION, DUMMY_REPLACE_STRING), + main_class: version.main_class, + release_time: version.release_time, + time: version.time, + type_: version.type_, + inherits_from: version + .inherits_from + .replace(DUMMY_GAME_VERSION, DUMMY_REPLACE_STRING), + libraries: libs, + minecraft_arguments: version.minecraft_arguments, + processors: None, + data: None, + })?, + Some("application/json".to_string()), + &uploaded_files_mutex, + semaphore.clone(), + ) + .await?; + + { + let mut loader_version_map = loader_version_mutex.lock().await; + async move { + loader_version_map.push(LoaderVersion { + id: loader.to_string(), + url: format_url(&version_path), + stable: *stable, + }); + } + .await; + } + + Ok::<(), Error>(()) + }, + )) + .await?; + + let mut loader_version_mutex = loader_version_mutex.into_inner(); + if !loader_version_mutex.is_empty() { + if let Some(version) = versions.iter_mut().find(|x| x.id == DUMMY_REPLACE_STRING) { + version.loaders.append(&mut loader_version_mutex); + } else { + versions.push(Version { + id: DUMMY_REPLACE_STRING.to_string(), + stable: true, + loaders: loader_version_mutex, + }); + } + } + + for version in &list.game { + if !versions.iter().any(|x| x.id == version.version) { + versions.push(Version { + id: version.version.clone(), + stable: version.stable, + loaders: vec![], + }); + } + } + + versions.sort_by(|x, y| { + minecraft_versions + .versions + .iter() + .position(|z| x.id == z.id) + .unwrap_or_default() + .cmp( + &minecraft_versions + .versions + .iter() + .position(|z| y.id == z.id) + .unwrap_or_default(), + ) + }); + + for version in &mut versions { + version.loaders.sort_by(|x, y| { + list.loader + .iter() + .position(|z| { + x.id.split('-').next().unwrap_or_default() == &*z.version + }) + .unwrap_or_default() + .cmp( + &list + .loader + .iter() + .position(|z| { + y.id.split('-').next().unwrap_or_default() + == z.version + }) + .unwrap_or_default(), + ) + }) + } + + upload_file_to_bucket( + format!( + "quilt/v{}/manifest.json", + daedalus::modded::CURRENT_QUILT_FORMAT_VERSION, + ), + serde_json::to_vec(&Manifest { + game_versions: versions, + })?, + Some("application/json".to_string()), + &uploaded_files_mutex, + semaphore, + ) + .await?; + + if let Ok(uploaded_files_mutex) = Arc::try_unwrap(uploaded_files_mutex) { + uploaded_files.extend(uploaded_files_mutex.into_inner()); + } + + Ok(()) +} + +const QUILT_META_URL: &str = "https://meta.quiltmc.org/v3"; + +async fn fetch_quilt_version( + version_number: &str, + loader_version: &str, + semaphore: Arc, +) -> Result { + Ok(serde_json::from_slice( + &download_file( + &format!( + "{}/versions/loader/{}/{}/profile/json", + QUILT_META_URL, version_number, loader_version + ), + None, + semaphore, + ) + .await?, + )?) +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +/// Versions of quilt components +struct QuiltVersions { + /// Versions of Minecraft that quilt supports + pub game: Vec, + /// Available versions of the quilt loader + pub loader: Vec, +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +/// A version of Minecraft that quilt supports +struct QuiltGameVersion { + /// The version number of the game + pub version: String, + /// Whether the Minecraft version is stable or not + pub stable: bool, +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +/// A version of the quilt loader +struct QuiltLoaderVersion { + /// The separator to get the build number + pub separator: String, + /// The build number + pub build: u32, + /// The maven artifact + pub maven: String, + /// The version number of the quilt loader + pub version: String, +} + +/// Fetches the list of quilt versions +async fn fetch_quilt_versions( + url: Option<&str>, + semaphore: Arc, +) -> Result { + Ok(serde_json::from_slice( + &download_file( + url.unwrap_or(&*format!("{}/versions", QUILT_META_URL)), + None, + semaphore, + ) + .await?, + )?) +} From 568e5a9bb89b7b2bd18f419d752eb2fc319224f7 Mon Sep 17 00:00:00 2001 From: Jai A Date: Tue, 25 Apr 2023 19:37:56 -0700 Subject: [PATCH 49/85] run fmt --- daedalus/src/minecraft.rs | 3 ++- daedalus_client/src/fabric.rs | 4 +++- daedalus_client/src/main.rs | 2 +- daedalus_client/src/minecraft.rs | 31 ++++++++++++++++++++++--------- daedalus_client/src/quilt.rs | 4 +++- 5 files changed, 31 insertions(+), 13 deletions(-) diff --git a/daedalus/src/minecraft.rs b/daedalus/src/minecraft.rs index 49c87fe4c..efa89ed98 100644 --- a/daedalus/src/minecraft.rs +++ b/daedalus/src/minecraft.rs @@ -207,7 +207,7 @@ pub enum Os { /// Linux ARM 32 LinuxArm32, /// The OS is unknown - Unknown + Unknown, } #[cfg_attr(feature = "bincode", derive(Encode, Decode))] @@ -321,6 +321,7 @@ pub struct PartialLibrary { pub include_in_classpath: Option, } +/// Merges a partial library to make a complete library pub fn merge_partial_library( partial: PartialLibrary, mut merge: Library, diff --git a/daedalus_client/src/fabric.rs b/daedalus_client/src/fabric.rs index d57cfd851..39c4d413f 100644 --- a/daedalus_client/src/fabric.rs +++ b/daedalus_client/src/fabric.rs @@ -217,7 +217,9 @@ pub async fn retrieve_data( let mut loader_version_mutex = loader_version_mutex.into_inner(); if !loader_version_mutex.is_empty() { - if let Some(version) = versions.iter_mut().find(|x| x.id == DUMMY_REPLACE_STRING) { + if let Some(version) = + versions.iter_mut().find(|x| x.id == DUMMY_REPLACE_STRING) + { version.loaders.append(&mut loader_version_mutex); } else { versions.push(Version { diff --git a/daedalus_client/src/main.rs b/daedalus_client/src/main.rs index c4bb24719..db3639de1 100644 --- a/daedalus_client/src/main.rs +++ b/daedalus_client/src/main.rs @@ -93,7 +93,7 @@ async fn main() { &mut uploaded_files, semaphore.clone(), ) - .await + .await { Ok(..) => {} Err(err) => error!("{:?}", err), diff --git a/daedalus_client/src/minecraft.rs b/daedalus_client/src/minecraft.rs index 282fd9346..30cf7f099 100644 --- a/daedalus_client/src/minecraft.rs +++ b/daedalus_client/src/minecraft.rs @@ -1,12 +1,14 @@ use crate::download_file; use crate::{format_url, upload_file_to_bucket, Error}; -use daedalus::minecraft::{Library, merge_partial_library, PartialLibrary, VersionManifest}; +use daedalus::get_hash; +use daedalus::minecraft::{ + merge_partial_library, Library, PartialLibrary, VersionManifest, +}; use log::info; +use serde::Deserialize; use std::sync::Arc; use std::time::Instant; use tokio::sync::{Mutex, Semaphore}; -use serde::Deserialize; -use daedalus::get_hash; pub async fn retrieve_data( uploaded_files: &mut Vec, @@ -66,14 +68,22 @@ pub async fn retrieve_data( let mut new_libraries = Vec::new(); for library in version_info.libraries { - if let Some(patch) = patches.iter().find(|x| x.match_.contains(&library.name)) { - if let Some(additional_libraries) = &patch.additional_libraries { + if let Some(patch) = patches + .iter() + .find(|x| x.match_.contains(&library.name)) + { + if let Some(additional_libraries) = + &patch.additional_libraries + { new_libraries.push(library); for additional_library in additional_libraries { new_libraries.push(additional_library.clone()); } } else if let Some(override_) = &patch.override_ { - new_libraries.push(merge_partial_library(override_.clone(), library)); + new_libraries.push(merge_partial_library( + override_.clone(), + library, + )); } else { new_libraries.push(library); } @@ -83,7 +93,10 @@ pub async fn retrieve_data( } version_info.libraries = new_libraries; - let version_info_hash = get_hash(bytes::Bytes::from(serde_json::to_vec(&version_info)?)).await?; + let version_info_hash = get_hash(bytes::Bytes::from( + serde_json::to_vec(&version_info)?, + )) + .await?; let version_path = format!( "minecraft/v{}/versions/{}.json", @@ -227,7 +240,7 @@ struct LibraryPatch { pub additional_libraries: Option>, #[serde(rename = "override")] pub override_: Option, - pub patch_additional_libraries: Option, + // pub patch_additional_libraries: Option, } /// Fetches the list of fabric versions @@ -241,6 +254,6 @@ async fn fetch_library_patches( None, semaphore, ) - .await?, + .await?, )?) } diff --git a/daedalus_client/src/quilt.rs b/daedalus_client/src/quilt.rs index 9b122967b..19ebeb45d 100644 --- a/daedalus_client/src/quilt.rs +++ b/daedalus_client/src/quilt.rs @@ -217,7 +217,9 @@ pub async fn retrieve_data( let mut loader_version_mutex = loader_version_mutex.into_inner(); if !loader_version_mutex.is_empty() { - if let Some(version) = versions.iter_mut().find(|x| x.id == DUMMY_REPLACE_STRING) { + if let Some(version) = + versions.iter_mut().find(|x| x.id == DUMMY_REPLACE_STRING) + { version.loaders.append(&mut loader_version_mutex); } else { versions.push(Version { From 3afda713499294d6e1f9744019df7b5d72ccfda6 Mon Sep 17 00:00:00 2001 From: Jai A Date: Wed, 26 Apr 2023 10:41:43 -0700 Subject: [PATCH 50/85] fix some data --- daedalus_client/src/minecraft.rs | 56 +++++++++++++++++++++----------- 1 file changed, 37 insertions(+), 19 deletions(-) diff --git a/daedalus_client/src/minecraft.rs b/daedalus_client/src/minecraft.rs index 30cf7f099..113456a04 100644 --- a/daedalus_client/src/minecraft.rs +++ b/daedalus_client/src/minecraft.rs @@ -66,30 +66,48 @@ pub async fn retrieve_data( let mut version_info = daedalus::minecraft::fetch_version_info(version).await?; - let mut new_libraries = Vec::new(); - for library in version_info.libraries { - if let Some(patch) = patches + fn patch_library(patches: &Vec, mut library: Library) -> Vec { + let mut val = Vec::new(); + + let actual_patches = patches .iter() - .find(|x| x.match_.contains(&library.name)) + .filter(|x| x.match_.contains(&library.name)) + .collect::>(); + + if !actual_patches.is_empty() { - if let Some(additional_libraries) = - &patch.additional_libraries - { - new_libraries.push(library); - for additional_library in additional_libraries { - new_libraries.push(additional_library.clone()); + for patch in actual_patches { + if let Some(additional_libraries) = + &patch.additional_libraries + { + for additional_library in additional_libraries { + if patch.patch_additional_libraries.unwrap_or(false) { + let mut libs = patch_library(patches, additional_library.clone()); + val.append(&mut libs) + } else { + val.push(additional_library.clone()); + } + } + } else if let Some(override_) = &patch.override_ { + library = merge_partial_library( + override_.clone(), + library, + ); } - } else if let Some(override_) = &patch.override_ { - new_libraries.push(merge_partial_library( - override_.clone(), - library, - )); - } else { - new_libraries.push(library); } + + val.push(library); } else { - new_libraries.push(library); + val.push(library); } + + val + } + + let mut new_libraries = Vec::new(); + for library in version_info.libraries { + let mut libs = patch_library(&patches, library); + new_libraries.append(&mut libs) } version_info.libraries = new_libraries; @@ -240,7 +258,7 @@ struct LibraryPatch { pub additional_libraries: Option>, #[serde(rename = "override")] pub override_: Option, - // pub patch_additional_libraries: Option, + pub patch_additional_libraries: Option, } /// Fetches the list of fabric versions From 3db00534c24edfab910f0b02dfca75b1adb9448c Mon Sep 17 00:00:00 2001 From: Geometrically <18202329+Geometrically@users.noreply.github.com> Date: Fri, 12 May 2023 20:42:18 -0700 Subject: [PATCH 51/85] Fix launcher overrides (#7) * Fix launcher overrides * Remove none set * Patch to work with legacy versions * remove mc --- daedalus/Cargo.toml | 2 +- daedalus/src/minecraft.rs | 33 +- daedalus_client/Cargo.toml | 2 +- daedalus_client/library-patches.json | 2020 ++++++++++++++++++++++++++ daedalus_client/src/minecraft.rs | 36 +- 5 files changed, 2068 insertions(+), 25 deletions(-) create mode 100644 daedalus_client/library-patches.json diff --git a/daedalus/Cargo.toml b/daedalus/Cargo.toml index 94cbde710..fbb6c0e4d 100644 --- a/daedalus/Cargo.toml +++ b/daedalus/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "daedalus" -version = "0.1.20" +version = "0.1.21" authors = ["Jai A "] edition = "2018" license = "MIT" diff --git a/daedalus/src/minecraft.rs b/daedalus/src/minecraft.rs index efa89ed98..87506720c 100644 --- a/daedalus/src/minecraft.rs +++ b/daedalus/src/minecraft.rs @@ -327,7 +327,22 @@ pub fn merge_partial_library( mut merge: Library, ) -> Library { if let Some(downloads) = partial.downloads { - merge.downloads = Some(downloads) + if let Some(merge_downloads) = &mut merge.downloads { + if let Some(artifact) = downloads.artifact { + merge_downloads.artifact = Some(artifact); + } + if let Some(classifiers) = downloads.classifiers { + if let Some(merge_classifiers) = &mut merge_downloads.classifiers { + for classifier in classifiers { + merge_classifiers.insert(classifier.0, classifier.1); + } + } else { + merge_downloads.classifiers = Some(classifiers); + } + } + } else { + merge.downloads = Some(downloads) + } } if let Some(extract) = partial.extract { merge.extract = Some(extract) @@ -339,10 +354,22 @@ pub fn merge_partial_library( merge.url = Some(url) } if let Some(natives) = partial.natives { - merge.natives = Some(natives) + if let Some(merge_natives) = &mut merge.natives { + for native in natives { + merge_natives.insert(native.0, native.1); + } + } else { + merge.natives = Some(natives); + } } if let Some(rules) = partial.rules { - merge.rules = Some(rules) + if let Some(merge_rules) = &mut merge.rules { + for rule in rules { + merge_rules.push(rule); + } + } else { + merge.rules = Some(rules) + } } if let Some(checksums) = partial.checksums { merge.checksums = Some(checksums) diff --git a/daedalus_client/Cargo.toml b/daedalus_client/Cargo.toml index 42b93a80b..48a599755 100644 --- a/daedalus_client/Cargo.toml +++ b/daedalus_client/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "daedalus_client" -version = "0.1.20" +version = "0.1.21" authors = ["Jai A "] edition = "2018" diff --git a/daedalus_client/library-patches.json b/daedalus_client/library-patches.json new file mode 100644 index 000000000..31a6a8cb8 --- /dev/null +++ b/daedalus_client/library-patches.json @@ -0,0 +1,2020 @@ +[ + { + "_comment": "Add missing tinyfd to the broken LWJGL 3.2.2 variant", + "match": [ + "org.lwjgl:lwjgl:3.2.2" + ], + "additionalLibraries": [ + { + "downloads": { + "artifact": { + "sha1": "fcbe606c8f8da6f8f9a05e2c540eb1ee8632b0e9", + "size": 7092, + "url": "https://libraries.minecraft.net/org/lwjgl/lwjgl-tinyfd/3.2.2/lwjgl-tinyfd-3.2.2.jar" + } + }, + "name": "org.lwjgl:lwjgl-tinyfd:3.2.2" + }, + { + "downloads": { + "artifact": { + "sha1": "fcbe606c8f8da6f8f9a05e2c540eb1ee8632b0e9", + "size": 7092, + "url": "https://libraries.minecraft.net/org/lwjgl/lwjgl-tinyfd/3.2.2/lwjgl-tinyfd-3.2.2.jar" + }, + "classifiers": { + "natives-linux": { + "sha1": "39e35b161c130635d9c8918ce04e887a30c5b687", + "size": 38804, + "url": "https://libraries.minecraft.net/org/lwjgl/lwjgl-tinyfd/3.2.2/lwjgl-tinyfd-3.2.2-natives-linux.jar" + }, + "natives-macos": { + "sha1": "46d0798228b8a28e857a2a0f02310fd6ba2a4eab", + "size": 42136, + "url": "https://libraries.minecraft.net/org/lwjgl/lwjgl-tinyfd/3.2.2/lwjgl-tinyfd-3.2.2-natives-macos.jar" + }, + "natives-windows": { + "sha1": "e9115958773644e863332a6a06488d26f9e1fc9f", + "size": 208314, + "url": + "https://libraries.minecraft.net/org/lwjgl/lwjgl-tinyfd/3.2.2/lwjgl-tinyfd-3.2.2-natives-windows.jar" + } + } + }, + "name": "org.lwjgl:lwjgl-tinyfd:3.2.2", + "natives": { + "linux": "natives-linux", + "osx": "natives-macos", + "windows": "natives-windows" + } + } + ], + "patchAdditionalLibraries": true + }, + { + "_comment": "Add additional library just for osx-arm64. No override needed", + "match": [ + "ca.weblite:java-objc-bridge:1.0.0" + ], + "additionalLibraries": [ + { + "downloads": { + "artifact": { + "sha1": "369a83621e3c65496348491e533cb97fe5f2f37d", + "size": 91947, + "url": + "https://github.com/MinecraftMachina/Java-Objective-C-Bridge/releases/download/1.1.0-mmachina.1/java-objc-bridge-1.1.jar" + } + }, + "name": "ca.weblite:java-objc-bridge:1.1.0-mmachina.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "osx-arm64" + } + } + ] + } + ] + }, + { + "_comment": "Add additional classifiers for jinput-platform", + "match": [ + "net.java.jinput:jinput-platform:2.0.5" + ], + "override": { + "downloads": { + "classifiers": { + "natives-osx-arm64": { + "sha1": "5189eb40db3087fb11ca063b68fa4f4c20b199dd", + "size": 10031, + "url": "https://github.com/r58Playz/jinput-m1/raw/main/plugins/OSX/bin/jinput-platform-2.0.5.jar" + }, + "natives-linux-arm64": { + "sha1": "42b388ccb7c63cec4e9f24f4dddef33325f8b212", + "size": 10932, + "url": + "https://github.com/theofficialgman/lwjgl3-binaries-arm64/raw/lwjgl-2.9.4/jinput-platform-2.0.5-natives-linux.jar" + }, + "natives-linux-arm32": { + "sha1": "f3c455b71c5146acb5f8a9513247fc06db182fd5", + "size": 4521, + "url": + "https://github.com/theofficialgman/lwjgl3-binaries-arm32/raw/lwjgl-2.9.4/jinput-platform-2.0.5-natives-linux.jar" + } + } + }, + "natives": { + "linux-arm64": "natives-linux-arm64", + "linux-arm32": "natives-linux-arm32", + "osx-arm64": "natives-osx-arm64" + } + } + }, + { + "_comment": "Use a newer version on osx-arm64", + "match": [ + "com.mojang:text2speech:1.0.10", + "com.mojang:text2speech:1.5", + "com.mojang:text2speech:1.6", + "com.mojang:text2speech:1.7", + "com.mojang:text2speech:1.10.1", + "com.mojang:text2speech:1.10.3", + "com.mojang:text2speech:1.11.2" + ], + "override": { + "rules": [ + { + "action": "allow" + }, + { + "action": "disallow", + "os": { + "name": "osx-arm64" + } + } + ] + }, + "additionalLibraries": [ + { + "downloads": { + "artifact": { + "sha1": "f378f889797edd7df8d32272c06ca80a1b6b0f58", + "size": 13164, + "url": "https://libraries.minecraft.net/com/mojang/text2speech/1.11.3/text2speech-1.11.3.jar" + } + }, + "name": "com.mojang:text2speech:1.11.3", + "rules": [ + { + "action": "allow", + "os": { + "name": "osx-arm64" + } + } + ] + } + ] + }, + { + "_comment": "Use a newer version on osx-arm64, linux-arm64, and linux-arm32", + "match": [ + "org.lwjgl.lwjgl:lwjgl:2.9.3", + "org.lwjgl.lwjgl:lwjgl:2.9.1-nightly-20131120", + "org.lwjgl.lwjgl:lwjgl:2.9.1-nightly-20131017", + "org.lwjgl.lwjgl:lwjgl:2.9.1-nightly-20130708-debug3", + "org.lwjgl.lwjgl:lwjgl:2.9.1", + "org.lwjgl.lwjgl:lwjgl:2.9.0" + ], + "override": { + "rules": [ + { + "action": "allow" + }, + { + "action": "disallow", + "os": { + "name": "osx-arm64" + } + }, + { + "action": "disallow", + "os": { + "name": "linux-arm64" + } + }, + { + "action": "disallow", + "os": { + "name": "linux-arm32" + } + } + ] + }, + "additionalLibraries": [ + { + "downloads": { + "artifact": { + "sha1": "697517568c68e78ae0b4544145af031c81082dfe", + "size": 1047168, + "url": + "https://libraries.minecraft.net/org/lwjgl/lwjgl/lwjgl/2.9.4-nightly-20150209/lwjgl-2.9.4-nightly-20150209.jar" + } + }, + "name": "org.lwjgl.lwjgl:lwjgl:2.9.4-nightly-20150209", + "rules": [ + { + "action": "allow", + "os": { + "name": "osx-arm64" + } + }, + { + "action": "allow", + "os": { + "name": "linux-arm64" + } + }, + { + "action": "allow", + "os": { + "name": "linux-arm32" + } + } + ] + } + ] + }, + { + "_comment": "Use a newer version on osx-arm64, linux-arm64, and linux-arm32", + "match": [ + "org.lwjgl.lwjgl:lwjgl_util:2.9.3", + "org.lwjgl.lwjgl:lwjgl_util:2.9.1-nightly-20131120", + "org.lwjgl.lwjgl:lwjgl_util:2.9.1-nightly-20131017", + "org.lwjgl.lwjgl:lwjgl_util:2.9.1-nightly-20130708-debug3", + "org.lwjgl.lwjgl:lwjgl_util:2.9.1", + "org.lwjgl.lwjgl:lwjgl_util:2.9.0" + ], + "override": { + "rules": [ + { + "action": "allow" + }, + { + "action": "disallow", + "os": { + "name": "osx-arm64" + } + }, + { + "action": "disallow", + "os": { + "name": "linux-arm64" + } + }, + { + "action": "disallow", + "os": { + "name": "linux-arm32" + } + } + ] + }, + "additionalLibraries": [ + { + "downloads": { + "artifact": { + "sha1": "d51a7c040a721d13efdfbd34f8b257b2df882ad0", + "size": 173887, + "url": + "https://libraries.minecraft.net/org/lwjgl/lwjgl/lwjgl_util/2.9.4-nightly-20150209/lwjgl_util-2.9.4-nightly-20150209.jar" + } + }, + "name": "org.lwjgl.lwjgl:lwjgl_util:2.9.4-nightly-20150209", + "rules": [ + { + "action": "allow", + "os": { + "name": "osx-arm64" + } + }, + { + "action": "allow", + "os": { + "name": "linux-arm64" + } + }, + { + "action": "allow", + "os": { + "name": "linux-arm32" + } + } + ] + } + ] + }, + { + "_comment": "Use a newer patched version on osx-arm64, linux-arm64, and linux-arm32", + "match": [ + "org.lwjgl.lwjgl:lwjgl-platform:2.9.4-nightly-20150209", + "org.lwjgl.lwjgl:lwjgl-platform:2.9.3", + "org.lwjgl.lwjgl:lwjgl-platform:2.9.1-nightly-20131120", + "org.lwjgl.lwjgl:lwjgl-platform:2.9.1-nightly-20131017", + "org.lwjgl.lwjgl:lwjgl-platform:2.9.1-nightly-20130708-debug3", + "org.lwjgl.lwjgl:lwjgl-platform:2.9.1", + "org.lwjgl.lwjgl:lwjgl-platform:2.9.0" + ], + "override": { + "downloads": { + "classifiers": { + "natives-osx-arm64": { + "sha1": "eff546c0b319d6ffc7a835652124c18089c67f36", + "size": 488316, + "url": + "https://github.com/MinecraftMachina/lwjgl/releases/download/2.9.4-20150209-mmachina.2/lwjgl-platform-2.9.4-nightly-20150209-natives-osx.jar" + }, + "natives-linux-arm64": { + "sha1": "63ac7da0f4a4785c7eadc0f8edc1e9dcc4dd08cb", + "size": 579979, + "url": + "https://github.com/theofficialgman/lwjgl3-binaries-arm64/raw/lwjgl-2.9.4/lwjgl-platform-2.9.4-nightly-20150209-natives-linux.jar" + }, + "natives-linux-arm32": { + "sha1": "fa483e540a9a753a5ffbb23dcf7879a5bf752611", + "size": 475177, + "url": + "https://github.com/theofficialgman/lwjgl3-binaries-arm32/raw/lwjgl-2.9.4/lwjgl-platform-2.9.4-nightly-20150209-natives-linux.jar" + } + } + }, + "natives": { + "linux-arm64": "natives-linux-arm64", + "linux-arm32": "natives-linux-arm32", + "osx-arm64": "natives-osx-arm64" + } + } + }, + { + "_comment": "Use a newer patched version on osx-arm64, linux-arm64, and linux-arm32", + "match": [ + "org.lwjgl:lwjgl-glfw:3.2.2", + "org.lwjgl:lwjgl-glfw:3.2.1", + "org.lwjgl:lwjgl-glfw:3.1.6", + "org.lwjgl:lwjgl-glfw:3.1.2" + ], + "override": { + "rules": [ + { + "action": "allow" + }, + { + "action": "disallow", + "os": { + "name": "linux-arm64" + } + }, + { + "action": "disallow", + "os": { + "name": "linux-arm32" + } + }, + { + "action": "disallow", + "os": { + "name": "osx-arm64" + } + } + ] + }, + "additionalLibraries": [ + { + "downloads": { + "artifact": { + "sha1": "155d175037efc76630940c197ca6dea2b17d7e18", + "size": 108691, + "url": "https://github.com/theofficialgman/lwjgl3-binaries-arm64/raw/lwjgl-3.2.2/lwjgl-glfw.jar" + } + }, + "name": "org.lwjgl:lwjgl-glfw:3.2.2-gman64.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm64" + } + } + ] + }, + { + "downloads": { + "artifact": { + "sha1": "155d175037efc76630940c197ca6dea2b17d7e18", + "size": 108691, + "url": "https://github.com/theofficialgman/lwjgl3-binaries-arm64/raw/lwjgl-3.2.2/lwjgl-glfw.jar" + }, + "classifiers": { + "natives-linux-arm64": { + "sha1": "074ad243761147df0d060fbefc814614d2ff75cc", + "size": 85072, + "url": + "https://github.com/theofficialgman/lwjgl3-binaries-arm64/raw/lwjgl-3.2.2/lwjgl-glfw-natives-linux-arm64.jar" + } + } + }, + "name": "org.lwjgl:lwjgl-glfw:3.2.2-gman64.1", + "natives": { + "linux-arm64": "natives-linux-arm64" + }, + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm64" + } + } + ] + }, + { + "downloads": { + "artifact": { + "sha1": "99e9a39fa8ed4167e3ff9e04d47eb32c9e69804d", + "size": 108691, + "url": "https://github.com/theofficialgman/lwjgl3-binaries-arm32/raw/lwjgl-3.2.2/lwjgl-glfw.jar" + } + }, + "name": "org.lwjgl:lwjgl-glfw:3.2.2-gman32.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm32" + } + } + ] + }, + { + "downloads": { + "artifact": { + "sha1": "99e9a39fa8ed4167e3ff9e04d47eb32c9e69804d", + "size": 108691, + "url": "https://github.com/theofficialgman/lwjgl3-binaries-arm32/raw/lwjgl-3.2.2/lwjgl-glfw.jar" + }, + "classifiers": { + "natives-linux-arm32": { + "sha1": "4265f2fbe3b9d642591165165a17cf406cf7b98e", + "size": 80186, + "url": + "https://github.com/theofficialgman/lwjgl3-binaries-arm32/raw/lwjgl-3.2.2/lwjgl-glfw-natives-linux-arm32.jar" + } + } + }, + "name": "org.lwjgl:lwjgl-glfw:3.2.2-gman32.1", + "natives": { + "linux-arm32": "natives-linux-arm32" + }, + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm32" + } + } + ] + }, + { + "downloads": { + "artifact": { + "sha1": "e9a101bca4fa30d26b21b526ff28e7c2d8927f1b", + "size": 130128, + "url": "https://github.com/MinecraftMachina/lwjgl3/releases/download/3.3.1-mmachina.1/lwjgl-glfw.jar" + } + }, + "name": "org.lwjgl:lwjgl-glfw:3.3.1-mmachina.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "osx-arm64" + } + } + ] + }, + { + "downloads": { + "artifact": { + "sha1": "e9a101bca4fa30d26b21b526ff28e7c2d8927f1b", + "size": 130128, + "url": "https://github.com/MinecraftMachina/lwjgl3/releases/download/3.3.1-mmachina.1/lwjgl-glfw.jar" + }, + "classifiers": { + "natives-osx-arm64": { + "sha1": "71d793d0a5a42e3dfe78eb882abc2523a2c6b496", + "size": 129076, + "url": + "https://github.com/MinecraftMachina/lwjgl3/releases/download/3.3.1-mmachina.1/lwjgl-glfw-natives-macos-arm64.jar" + } + } + }, + "name": "org.lwjgl:lwjgl-glfw:3.3.1-mmachina.1", + "natives": { + "osx-arm64": "natives-osx-arm64" + }, + "rules": [ + { + "action": "allow", + "os": { + "name": "osx-arm64" + } + } + ] + } + ] + }, + { + "_comment": "Use a newer patched version on osx-arm64, linux-arm64, and linux-arm32", + "match": [ + "org.lwjgl:lwjgl-jemalloc:3.2.2", + "org.lwjgl:lwjgl-jemalloc:3.2.1", + "org.lwjgl:lwjgl-jemalloc:3.1.6", + "org.lwjgl:lwjgl-jemalloc:3.1.2" + ], + "override": { + "rules": [ + { + "action": "allow" + }, + { + "action": "disallow", + "os": { + "name": "linux-arm64" + } + }, + { + "action": "disallow", + "os": { + "name": "linux-arm32" + } + }, + { + "action": "disallow", + "os": { + "name": "osx-arm64" + } + } + ] + }, + "additionalLibraries": [ + { + "downloads": { + "artifact": { + "sha1": "cc04eec29b2fa8c298791af9800a3766d9617954", + "size": 33790, + "url": "https://github.com/theofficialgman/lwjgl3-binaries-arm64/raw/lwjgl-3.2.2/lwjgl-jemalloc.jar" + } + }, + "name": "org.lwjgl:lwjgl-jemalloc:3.2.2-gman64.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm64" + } + } + ] + }, + { + "downloads": { + "artifact": { + "sha1": "cc04eec29b2fa8c298791af9800a3766d9617954", + "size": 33790, + "url": "https://github.com/theofficialgman/lwjgl3-binaries-arm64/raw/lwjgl-3.2.2/lwjgl-jemalloc.jar" + }, + "classifiers": { + "natives-linux-arm64": { + "sha1": "762d7d80c9cdf3a3f3fc80c8a5f86612255edfe0", + "size": 156343, + "url": + "https://github.com/theofficialgman/lwjgl3-binaries-arm64/raw/lwjgl-3.2.2/lwjgl-jemalloc-patched-natives-linux-arm64.jar" + } + } + }, + "name": "org.lwjgl:lwjgl-jemalloc:3.2.2-gman64.2", + "natives": { + "linux-arm64": "natives-linux-arm64" + }, + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm64" + } + } + ] + }, + { + "downloads": { + "artifact": { + "sha1": "8224ae2e8fc6d8e1a0fc7d84dc917aa3c440620c", + "size": 33790, + "url": "https://github.com/theofficialgman/lwjgl3-binaries-arm32/raw/lwjgl-3.2.2/lwjgl-jemalloc.jar" + } + }, + "name": "org.lwjgl:lwjgl-jemalloc:3.2.2-gman32.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm32" + } + } + ] + }, + { + "downloads": { + "artifact": { + "sha1": "8224ae2e8fc6d8e1a0fc7d84dc917aa3c440620c", + "size": 33790, + "url": "https://github.com/theofficialgman/lwjgl3-binaries-arm32/raw/lwjgl-3.2.2/lwjgl-jemalloc.jar" + }, + "classifiers": { + "natives-linux-arm32": { + "sha1": "9163a2a5559ef87bc13ead8fea84417ea3928748", + "size": 134237, + "url": + "https://github.com/theofficialgman/lwjgl3-binaries-arm32/raw/lwjgl-3.2.2/lwjgl-jemalloc-natives-linux-arm32.jar" + } + } + }, + "name": "org.lwjgl:lwjgl-jemalloc:3.2.2-gman32.1", + "natives": { + "linux-arm32": "natives-linux-arm32" + }, + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm32" + } + } + ] + }, + { + "downloads": { + "artifact": { + "sha1": "4fb94224378d3588d52d2beb172f2eeafea2d546", + "size": 36976, + "url": "https://github.com/MinecraftMachina/lwjgl3/releases/download/3.3.1-mmachina.1/lwjgl-jemalloc.jar" + } + }, + "name": "org.lwjgl:lwjgl-jemalloc:3.3.1-mmachina.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "osx-arm64" + } + } + ] + }, + { + "downloads": { + "artifact": { + "sha1": "4fb94224378d3588d52d2beb172f2eeafea2d546", + "size": 36976, + "url": "https://github.com/MinecraftMachina/lwjgl3/releases/download/3.3.1-mmachina.1/lwjgl-jemalloc.jar" + }, + "classifiers": { + "natives-osx-arm64": { + "sha1": "b0be721188d2e7195798780b1c5fe7eafe8091c1", + "size": 103478, + "url": + "https://github.com/MinecraftMachina/lwjgl3/releases/download/3.3.1-mmachina.1/lwjgl-jemalloc-natives-macos-arm64.jar" + } + } + }, + "name": "org.lwjgl:lwjgl-jemalloc:3.3.1-mmachina.1", + "natives": { + "osx-arm64": "natives-osx-arm64" + }, + "rules": [ + { + "action": "allow", + "os": { + "name": "osx-arm64" + } + } + ] + } + ] + }, + { + "_comment": "Use a newer patched version on osx-arm64, linux-arm64, and linux-arm32", + "match": [ + "org.lwjgl:lwjgl-openal:3.2.2", + "org.lwjgl:lwjgl-openal:3.2.1", + "org.lwjgl:lwjgl-openal:3.1.6", + "org.lwjgl:lwjgl-openal:3.1.2" + ], + "override": { + "rules": [ + { + "action": "allow" + }, + { + "action": "disallow", + "os": { + "name": "linux-arm64" + } + }, + { + "action": "disallow", + "os": { + "name": "linux-arm32" + } + }, + { + "action": "disallow", + "os": { + "name": "osx-arm64" + } + } + ] + }, + "additionalLibraries": [ + { + "downloads": { + "artifact": { + "sha1": "6dfce9dc6a9629c75b2ae01a8df7e7be80ba0261", + "size": 79582, + "url": "https://github.com/theofficialgman/lwjgl3-binaries-arm64/raw/lwjgl-3.2.2/lwjgl-openal.jar" + } + }, + "name": "org.lwjgl:lwjgl-openal:3.2.2-gman64.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm64" + } + } + ] + }, + { + "downloads": { + "artifact": { + "sha1": "6dfce9dc6a9629c75b2ae01a8df7e7be80ba0261", + "size": 79582, + "url": "https://github.com/theofficialgman/lwjgl3-binaries-arm64/raw/lwjgl-3.2.2/lwjgl-openal.jar" + }, + "classifiers": { + "natives-linux-arm64": { + "sha1": "948e415b5b2a2c650c25b377a4a9f443b21ce92e", + "size": 469432, + "url": + "https://github.com/theofficialgman/lwjgl3-binaries-arm64/raw/lwjgl-3.2.2/lwjgl-openal-natives-linux-arm64.jar" + } + } + }, + "name": "org.lwjgl:lwjgl-openal:3.2.2-gman64.1", + "natives": { + "linux-arm64": "natives-linux-arm64" + }, + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm64" + } + } + ] + }, + { + "downloads": { + "artifact": { + "sha1": "304f0571fd5971621ee6da86a4c1e90f6f52e2ee", + "size": 79582, + "url": "https://github.com/theofficialgman/lwjgl3-binaries-arm32/raw/lwjgl-3.2.2/lwjgl-openal.jar" + } + }, + "name": "org.lwjgl:lwjgl-openal:3.2.2-gman32.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm32" + } + } + ] + }, + { + "downloads": { + "artifact": { + "sha1": "304f0571fd5971621ee6da86a4c1e90f6f52e2ee", + "size": 79582, + "url": "https://github.com/theofficialgman/lwjgl3-binaries-arm32/raw/lwjgl-3.2.2/lwjgl-openal.jar" + }, + "classifiers": { + "natives-linux-arm32": { + "sha1": "ecbc981fdd996492a1f6334f003ed62e5a8c0cd5", + "size": 398418, + "url": + "https://github.com/theofficialgman/lwjgl3-binaries-arm32/raw/lwjgl-3.2.2/lwjgl-openal-natives-linux-arm32.jar" + } + } + }, + "name": "org.lwjgl:lwjgl-openal:3.2.2-gman32.1", + "natives": { + "linux-arm32": "natives-linux-arm32" + }, + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm32" + } + } + ] + }, + { + "downloads": { + "artifact": { + "sha1": "d48e753d85916fc8a200ccddc709b36e3865cc4e", + "size": 88880, + "url": "https://github.com/MinecraftMachina/lwjgl3/releases/download/3.3.1-mmachina.1/lwjgl-openal.jar" + } + }, + "name": "org.lwjgl:lwjgl-openal:3.3.1-mmachina.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "osx-arm64" + } + } + ] + }, + { + "downloads": { + "artifact": { + "sha1": "d48e753d85916fc8a200ccddc709b36e3865cc4e", + "size": 88880, + "url": "https://github.com/MinecraftMachina/lwjgl3/releases/download/3.3.1-mmachina.1/lwjgl-openal.jar" + }, + "classifiers": { + "natives-osx-arm64": { + "sha1": "6b80fc0b982a0723b141e88859c42d6f71bd723f", + "size": 346131, + "url": + "https://github.com/MinecraftMachina/lwjgl3/releases/download/3.3.1-mmachina.1/lwjgl-openal-natives-macos-arm64.jar" + } + } + }, + "name": "org.lwjgl:lwjgl-openal:3.3.1-mmachina.1", + "natives": { + "osx-arm64": "natives-osx-arm64" + }, + "rules": [ + { + "action": "allow", + "os": { + "name": "osx-arm64" + } + } + ] + } + ] + }, + { + "_comment": "Use a newer patched version on osx-arm64, linux-arm64, and linux-arm32", + "match": [ + "org.lwjgl:lwjgl-opengl:3.2.2", + "org.lwjgl:lwjgl-opengl:3.2.1", + "org.lwjgl:lwjgl-opengl:3.1.6", + "org.lwjgl:lwjgl-opengl:3.1.2" + ], + "override": { + "rules": [ + { + "action": "allow" + }, + { + "action": "disallow", + "os": { + "name": "linux-arm64" + } + }, + { + "action": "disallow", + "os": { + "name": "linux-arm32" + } + }, + { + "action": "disallow", + "os": { + "name": "osx-arm64" + } + } + ] + }, + "additionalLibraries": [ + { + "downloads": { + "artifact": { + "sha1": "198bc2f72e0b2eb401eb6f5999aea52909b31ac4", + "size": 937609, + "url": "https://github.com/theofficialgman/lwjgl3-binaries-arm64/raw/lwjgl-3.2.2/lwjgl-opengl.jar" + } + }, + "name": "org.lwjgl:lwjgl-opengl:3.2.2-gman64.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm64" + } + } + ] + }, + { + "downloads": { + "artifact": { + "sha1": "198bc2f72e0b2eb401eb6f5999aea52909b31ac4", + "size": 937609, + "url": "https://github.com/theofficialgman/lwjgl3-binaries-arm64/raw/lwjgl-3.2.2/lwjgl-opengl.jar" + }, + "classifiers": { + "natives-linux-arm64": { + "sha1": "bd40897077bf7d12f562da898b18ac2c68e1f9d7", + "size": 56109, + "url": + "https://github.com/theofficialgman/lwjgl3-binaries-arm64/raw/lwjgl-3.2.2/lwjgl-opengl-natives-linux-arm64.jar" + } + } + }, + "name": "org.lwjgl:lwjgl-opengl:3.2.2-gman64.1", + "natives": { + "linux-arm64": "natives-linux-arm64" + }, + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm64" + } + } + ] + }, + { + "downloads": { + "artifact": { + "sha1": "9762ae928d02147e716cd82e929b74a97ea9600a", + "size": 937609, + "url": "https://github.com/theofficialgman/lwjgl3-binaries-arm32/raw/lwjgl-3.2.2/lwjgl-opengl.jar" + } + }, + "name": "org.lwjgl:lwjgl-opengl:3.2.2-gman32.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm32" + } + } + ] + }, + { + "downloads": { + "artifact": { + "sha1": "9762ae928d02147e716cd82e929b74a97ea9600a", + "size": 937609, + "url": "https://github.com/theofficialgman/lwjgl3-binaries-arm32/raw/lwjgl-3.2.2/lwjgl-opengl.jar" + }, + "classifiers": { + "natives-linux-arm32": { + "sha1": "3af5599c74dd76dd8dbb567b3f9b4963a6abeed5", + "size": 56388, + "url": + "https://github.com/theofficialgman/lwjgl3-binaries-arm32/raw/lwjgl-3.2.2/lwjgl-opengl-natives-linux-arm32.jar" + } + } + }, + "name": "org.lwjgl:lwjgl-opengl:3.2.2-gman32.1", + "natives": { + "linux-arm32": "natives-linux-arm32" + }, + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm32" + } + } + ] + }, + { + "downloads": { + "artifact": { + "sha1": "962c2a8d2a8cdd3b89de3d78d766ab5e2133c2f4", + "size": 929233, + "url": "https://github.com/MinecraftMachina/lwjgl3/releases/download/3.3.1-mmachina.1/lwjgl-opengl.jar" + } + }, + "name": "org.lwjgl:lwjgl-opengl:3.3.1-mmachina.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "osx-arm64" + } + } + ] + }, + { + "downloads": { + "artifact": { + "sha1": "962c2a8d2a8cdd3b89de3d78d766ab5e2133c2f4", + "size": 929233, + "url": "https://github.com/MinecraftMachina/lwjgl3/releases/download/3.3.1-mmachina.1/lwjgl-opengl.jar" + }, + "classifiers": { + "natives-osx-arm64": { + "sha1": "bb575058e0372f515587b5d2d04ff7db185f3ffe", + "size": 41667, + "url": + "https://github.com/MinecraftMachina/lwjgl3/releases/download/3.3.1-mmachina.1/lwjgl-opengl-natives-macos-arm64.jar" + } + } + }, + "name": "org.lwjgl:lwjgl-opengl:3.3.1-mmachina.1", + "natives": { + "osx-arm64": "natives-osx-arm64" + }, + "rules": [ + { + "action": "allow", + "os": { + "name": "osx-arm64" + } + } + ] + } + ] + }, + { + "_comment": "Use a newer patched version on osx-arm64, linux-arm64, and linux-arm32", + "match": [ + "org.lwjgl:lwjgl-stb:3.2.2", + "org.lwjgl:lwjgl-stb:3.2.1", + "org.lwjgl:lwjgl-stb:3.1.6", + "org.lwjgl:lwjgl-stb:3.1.2" + ], + "override": { + "rules": [ + { + "action": "allow" + }, + { + "action": "disallow", + "os": { + "name": "linux-arm64" + } + }, + { + "action": "disallow", + "os": { + "name": "linux-arm32" + } + }, + { + "action": "disallow", + "os": { + "name": "osx-arm64" + } + } + ] + }, + "additionalLibraries": [ + { + "downloads": { + "artifact": { + "sha1": "46a5735f3eb9d17eb5dcbdd5afa194066d2a6555", + "size": 104075, + "url": "https://github.com/theofficialgman/lwjgl3-binaries-arm64/raw/lwjgl-3.2.2/lwjgl-stb.jar" + } + }, + "name": "org.lwjgl:lwjgl-stb:3.2.2-gman64.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm64" + } + } + ] + }, + { + "downloads": { + "artifact": { + "sha1": "46a5735f3eb9d17eb5dcbdd5afa194066d2a6555", + "size": 104075, + "url": "https://github.com/theofficialgman/lwjgl3-binaries-arm64/raw/lwjgl-3.2.2/lwjgl-stb.jar" + }, + "classifiers": { + "natives-linux-arm64": { + "sha1": "077efa7d7ea41b32df5c6078e912e724cccd06db", + "size": 202038, + "url": + "https://github.com/theofficialgman/lwjgl3-binaries-arm64/raw/lwjgl-3.2.2/lwjgl-stb-natives-linux-arm64.jar" + } + } + }, + "name": "org.lwjgl:lwjgl-stb:3.2.2-gman64.1", + "natives": { + "linux-arm64": "natives-linux-arm64" + }, + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm64" + } + } + ] + }, + { + "downloads": { + "artifact": { + "sha1": "ea979b0af45b8e689f5f47c989aa8550c148d8a2", + "size": 104075, + "url": "https://github.com/theofficialgman/lwjgl3-binaries-arm32/raw/lwjgl-3.2.2/lwjgl-stb.jar" + } + }, + "name": "org.lwjgl:lwjgl-stb:3.2.2-gman32.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm32" + } + } + ] + }, + { + "downloads": { + "artifact": { + "sha1": "ea979b0af45b8e689f5f47c989aa8550c148d8a2", + "size": 104075, + "url": "https://github.com/theofficialgman/lwjgl3-binaries-arm32/raw/lwjgl-3.2.2/lwjgl-stb.jar" + }, + "classifiers": { + "natives-linux-arm32": { + "sha1": "ec9d70aaebd0ff76dfeecf8f00b56118bf3706b1", + "size": 149387, + "url": + "https://github.com/theofficialgman/lwjgl3-binaries-arm32/raw/lwjgl-3.2.2/lwjgl-stb-natives-linux-arm32.jar" + } + } + }, + "name": "org.lwjgl:lwjgl-stb:3.2.2-gman32.1", + "natives": { + "linux-arm32": "natives-linux-arm32" + }, + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm32" + } + } + ] + }, + { + "downloads": { + "artifact": { + "sha1": "703e4b533e2542560e9f94d6d8bd148be1c1d572", + "size": 113273, + "url": "https://github.com/MinecraftMachina/lwjgl3/releases/download/3.3.1-mmachina.1/lwjgl-stb.jar" + } + }, + "name": "org.lwjgl:lwjgl-stb:3.3.1-mmachina.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "osx-arm64" + } + } + ] + }, + { + "downloads": { + "artifact": { + "sha1": "703e4b533e2542560e9f94d6d8bd148be1c1d572", + "size": 113273, + "url": "https://github.com/MinecraftMachina/lwjgl3/releases/download/3.3.1-mmachina.1/lwjgl-stb.jar" + }, + "classifiers": { + "natives-osx-arm64": { + "sha1": "98f0ad956c754723ef354d50057cc30417ef376a", + "size": 178409, + "url": + "https://github.com/MinecraftMachina/lwjgl3/releases/download/3.3.1-mmachina.1/lwjgl-stb-natives-macos-arm64.jar" + } + } + }, + "name": "org.lwjgl:lwjgl-stb:3.3.1-mmachina.1", + "natives": { + "osx-arm64": "natives-osx-arm64" + }, + "rules": [ + { + "action": "allow", + "os": { + "name": "osx-arm64" + } + } + ] + } + ] + }, + { + "_comment": "Use a newer patched version on osx-arm64, linux-arm64, and linux-arm32", + "match": [ + "org.lwjgl:lwjgl-tinyfd:3.2.2", + "org.lwjgl:lwjgl-tinyfd:3.2.1", + "org.lwjgl:lwjgl-tinyfd:3.1.6", + "org.lwjgl:lwjgl-tinyfd:3.1.2" + ], + "override": { + "rules": [ + { + "action": "allow" + }, + { + "action": "disallow", + "os": { + "name": "linux-arm64" + } + }, + { + "action": "disallow", + "os": { + "name": "linux-arm32" + } + }, + { + "action": "disallow", + "os": { + "name": "osx-arm64" + } + } + ] + }, + "additionalLibraries": [ + { + "downloads": { + "artifact": { + "sha1": "3a75b9811607633bf33c978f53964df1534a4bc1", + "size": 5571, + "url": "https://github.com/theofficialgman/lwjgl3-binaries-arm64/raw/lwjgl-3.2.2/lwjgl-tinyfd.jar" + } + }, + "name": "org.lwjgl:lwjgl-tinyfd:3.2.2-gman64.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm64" + } + } + ] + }, + { + "downloads": { + "artifact": { + "sha1": "3a75b9811607633bf33c978f53964df1534a4bc1", + "size": 5571, + "url": "https://github.com/theofficialgman/lwjgl3-binaries-arm64/raw/lwjgl-3.2.2/lwjgl-tinyfd.jar" + }, + "classifiers": { + "natives-linux-arm64": { + "sha1": "37c744ca289b5d7ae155d79e39029488b3254e5b", + "size": 37893, + "url": + "https://github.com/theofficialgman/lwjgl3-binaries-arm64/raw/lwjgl-3.2.2/lwjgl-tinyfd-natives-linux-arm64.jar" + } + } + }, + "name": "org.lwjgl:lwjgl-tinyfd:3.2.2-gman64.1", + "natives": { + "linux-arm64": "natives-linux-arm64" + }, + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm64" + } + } + ] + }, + { + "downloads": { + "artifact": { + "sha1": "a8c09f5b7fa24bd53ec329c231b566497a163d5b", + "size": 5571, + "url": "https://github.com/theofficialgman/lwjgl3-binaries-arm32/raw/lwjgl-3.2.2/lwjgl-tinyfd.jar" + } + }, + "name": "org.lwjgl:lwjgl-tinyfd:3.2.2-gman32.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm32" + } + } + ] + }, + { + "downloads": { + "artifact": { + "sha1": "a8c09f5b7fa24bd53ec329c231b566497a163d5b", + "size": 5571, + "url": "https://github.com/theofficialgman/lwjgl3-binaries-arm32/raw/lwjgl-3.2.2/lwjgl-tinyfd.jar" + }, + "classifiers": { + "natives-linux-arm32": { + "sha1": "82d16054ada6633297a3108fb6d8bae98800c76f", + "size": 41663, + "url": + "https://github.com/theofficialgman/lwjgl3-binaries-arm32/raw/lwjgl-3.2.2/lwjgl-tinyfd-natives-linux-arm32.jar" + } + } + }, + "name": "org.lwjgl:lwjgl-tinyfd:3.2.2-gman32.1", + "natives": { + "linux-arm32": "natives-linux-arm32" + }, + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm32" + } + } + ] + }, + { + "downloads": { + "artifact": { + "sha1": "1203660b3131cbb8681b17ce6437412545be95e0", + "size": 6802, + "url": "https://github.com/MinecraftMachina/lwjgl3/releases/download/3.3.1-mmachina.1/lwjgl-tinyfd.jar" + } + }, + "name": "org.lwjgl:lwjgl-tinyfd:3.3.1-mmachina.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "osx-arm64" + } + } + ] + }, + { + "downloads": { + "artifact": { + "sha1": "1203660b3131cbb8681b17ce6437412545be95e0", + "size": 6802, + "url": "https://github.com/MinecraftMachina/lwjgl3/releases/download/3.3.1-mmachina.1/lwjgl-tinyfd.jar" + }, + "classifiers": { + "natives-osx-arm64": { + "sha1": "015b931a2daba8f0c317d84c9d14e8e98ae56e0c", + "size": 41384, + "url": + "https://github.com/MinecraftMachina/lwjgl3/releases/download/3.3.1-mmachina.1/lwjgl-tinyfd-natives-macos-arm64.jar" + } + } + }, + "name": "org.lwjgl:lwjgl-tinyfd:3.3.1-mmachina.1", + "natives": { + "osx-arm64": "natives-osx-arm64" + }, + "rules": [ + { + "action": "allow", + "os": { + "name": "osx-arm64" + } + } + ] + } + ] + }, + { + "_comment": "Use a newer patched version on osx-arm64, linux-arm64, and linux-arm32", + "match": [ + "org.lwjgl:lwjgl:3.2.2", + "org.lwjgl:lwjgl:3.2.1", + "org.lwjgl:lwjgl:3.1.6", + "org.lwjgl:lwjgl:3.1.2" + ], + "override": { + "rules": [ + { + "action": "allow" + }, + { + "action": "disallow", + "os": { + "name": "linux-arm64" + } + }, + { + "action": "disallow", + "os": { + "name": "linux-arm32" + } + }, + { + "action": "disallow", + "os": { + "name": "osx-arm64" + } + } + ] + }, + "additionalLibraries": [ + { + "downloads": { + "artifact": { + "sha1": "360899386df83d6a8407844a94478607af937f97", + "size": 318833, + "url": "https://github.com/theofficialgman/lwjgl3-binaries-arm64/raw/lwjgl-3.2.2/lwjgl-core.jar" + } + }, + "name": "org.lwjgl:lwjgl:3.2.2-gman64.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm64" + } + } + ] + }, + { + "downloads": { + "artifact": { + "sha1": "360899386df83d6a8407844a94478607af937f97", + "size": 318833, + "url": "https://github.com/theofficialgman/lwjgl3-binaries-arm64/raw/lwjgl-3.2.2/lwjgl-core.jar" + }, + "classifiers": { + "natives-linux-arm64": { + "sha1": "612efd57d12b2e48e554858eb35e7e2eb46ebb4c", + "size": 87121, + "url": + "https://github.com/theofficialgman/lwjgl3-binaries-arm64/raw/lwjgl-3.2.2/lwjgl-natives-linux-arm64.jar" + } + } + }, + "name": "org.lwjgl:lwjgl:3.2.2-gman64.1", + "natives": { + "linux-arm64": "natives-linux-arm64" + }, + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm64" + } + } + ] + }, + { + "downloads": { + "artifact": { + "sha1": "16ea3934fca417368250d1ddac01a30c1809d317", + "size": 318413, + "url": "https://github.com/theofficialgman/lwjgl3-binaries-arm32/raw/lwjgl-3.2.2/lwjgl-core.jar" + } + }, + "name": "org.lwjgl:lwjgl:3.2.2-gman32.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm32" + } + } + ] + }, + { + "downloads": { + "artifact": { + "sha1": "16ea3934fca417368250d1ddac01a30c1809d317", + "size": 318413, + "url": "https://github.com/theofficialgman/lwjgl3-binaries-arm32/raw/lwjgl-3.2.2/lwjgl-core.jar" + }, + "classifiers": { + "natives-linux-arm32": { + "sha1": "6bd0b37fef777a309936a72dc7f63126e8c79ea5", + "size": 90296, + "url": + "https://github.com/theofficialgman/lwjgl3-binaries-arm32/raw/lwjgl-3.2.2/lwjgl-natives-linux-arm32.jar" + } + } + }, + "name": "org.lwjgl:lwjgl:3.2.2-gman32.1", + "natives": { + "linux-arm32": "natives-linux-arm32" + }, + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm32" + } + } + ] + }, + { + "downloads": { + "artifact": { + "sha1": "8e664dd69ad7bbcf2053da23efc7848e39e498db", + "size": 719038, + "url": "https://github.com/MinecraftMachina/lwjgl3/releases/download/3.3.1-mmachina.1/lwjgl.jar" + } + }, + "name": "org.lwjgl:lwjgl:3.3.1-mmachina.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "osx-arm64" + } + } + ] + }, + { + "downloads": { + "artifact": { + "sha1": "8e664dd69ad7bbcf2053da23efc7848e39e498db", + "size": 719038, + "url": "https://github.com/MinecraftMachina/lwjgl3/releases/download/3.3.1-mmachina.1/lwjgl.jar" + }, + "classifiers": { + "natives-osx-arm64": { + "sha1": "984df31fadaab86838877b112e5b4e4f68a00ccf", + "size": 42693, + "url": + "https://github.com/MinecraftMachina/lwjgl3/releases/download/3.3.1-mmachina.1/lwjgl-natives-macos-arm64.jar" + } + } + }, + "name": "org.lwjgl:lwjgl:3.3.1-mmachina.1", + "natives": { + "osx-arm64": "natives-osx-arm64" + }, + "rules": [ + { + "action": "allow", + "os": { + "name": "osx-arm64" + } + } + ] + } + ] + }, + { + "_comment": "Only allow osx-arm64 for existing LWJGL 3.3.1", + "match": [ + "org.lwjgl:lwjgl-glfw-natives-macos-arm64:3.3.1", + "org.lwjgl:lwjgl-jemalloc-natives-macos-arm64:3.3.1", + "org.lwjgl:lwjgl-openal-natives-macos-arm64:3.3.1", + "org.lwjgl:lwjgl-opengl-natives-macos-arm64:3.3.1", + "org.lwjgl:lwjgl-stb-natives-macos-arm64:3.3.1", + "org.lwjgl:lwjgl-tinyfd-natives-macos-arm64:3.3.1", + "org.lwjgl:lwjgl-natives-macos-arm64:3.3.1" + ], + "override": { + "rules": [ + { + "action": "allow", + "os": { + "name": "osx-arm64" + } + } + ] + } + }, + { + "_comment": "Only allow osx-arm64 for existing java-objc-bridge:1.1", + "match": [ + "ca.weblite:java-objc-bridge:1.1" + ], + "override": { + "rules": [ + { + "action": "allow", + "os": { + "name": "osx-arm64" + } + } + ] + } + }, + { + "_comment": "Only allow windows-arm64 for existing LWJGL 3.3.1", + "match": [ + "org.lwjgl:lwjgl-glfw-natives-windows-arm64:3.3.1", + "org.lwjgl:lwjgl-jemalloc-natives-windows-arm64:3.3.1", + "org.lwjgl:lwjgl-openal-natives-windows-arm64:3.3.1", + "org.lwjgl:lwjgl-opengl-natives-windows-arm64:3.3.1", + "org.lwjgl:lwjgl-stb-natives-windows-arm64:3.3.1", + "org.lwjgl:lwjgl-tinyfd-natives-windows-arm64:3.3.1", + "org.lwjgl:lwjgl-natives-windows-arm64:3.3.1" + ], + "override": { + "rules": [ + { + "action": "allow", + "os": { + "name": "windows-arm64" + } + } + ] + } + }, + { + "_comment": "Add linux-arm64 support for LWJGL 3.3.1", + "match": [ + "org.lwjgl:lwjgl-glfw:3.3.1" + ], + "additionalLibraries": [ + { + "downloads": { + "artifact": { + "sha1": "513eb39b866d0fe131a18d5c517087805433b029", + "size": 112350, + "url": "https://build.lwjgl.org/release/3.3.1/bin/lwjgl-glfw/lwjgl-glfw-natives-linux-arm64.jar" + } + }, + "name": "org.lwjgl:lwjgl-glfw-natives-linux-arm64:3.3.1-lwjgl.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm64" + } + } + ] + } + ] + }, + { + "_comment": "Add linux-arm64 support for LWJGL 3.3.1", + "match": [ + "org.lwjgl:lwjgl-jemalloc:3.3.1" + ], + "additionalLibraries": [ + { + "downloads": { + "artifact": { + "sha1": "749be48a9b86ee2c3a2da5fd77511208adcfb33b", + "size": 159993, + "url": + "https://github.com/theofficialgman/lwjgl3-binaries-arm64/raw/lwjgl-3.3.1/lwjgl-jemalloc-patched-natives-linux-arm64.jar" + } + }, + "name": "org.lwjgl:lwjgl-jemalloc-natives-linux-arm64:3.3.1-gman64.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm64" + } + } + ] + } + ] + }, + { + "_comment": "Add linux-arm64 support for LWJGL 3.3.1", + "match": [ + "org.lwjgl:lwjgl-openal:3.3.1" + ], + "additionalLibraries": [ + { + "downloads": { + "artifact": { + "sha1": "cf4e303257e82981b8b2e31bba3d7f8f7b8f42b2", + "size": 470743, + "url": "https://build.lwjgl.org/release/3.3.1/bin/lwjgl-openal/lwjgl-openal-natives-linux-arm64.jar" + } + }, + "name": "org.lwjgl:lwjgl-openal-natives-linux-arm64:3.3.1-lwjgl.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm64" + } + } + ] + } + ] + }, + { + "_comment": "Add linux-arm64 support for LWJGL 3.3.1", + "match": [ + "org.lwjgl:lwjgl-opengl:3.3.1" + ], + "additionalLibraries": [ + { + "downloads": { + "artifact": { + "sha1": "1c528fb258a6e63e8fceb4482d8db0f3af10a634", + "size": 57908, + "url": "https://build.lwjgl.org/release/3.3.1/bin/lwjgl-opengl/lwjgl-opengl-natives-linux-arm64.jar" + } + }, + "name": "org.lwjgl:lwjgl-opengl-natives-linux-arm64:3.3.1-lwjgl.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm64" + } + } + ] + } + ] + }, + { + "_comment": "Add linux-arm64 support for LWJGL 3.3.1", + "match": [ + "org.lwjgl:lwjgl-stb:3.3.1" + ], + "additionalLibraries": [ + { + "downloads": { + "artifact": { + "sha1": "8e8348a1813aad7f30aaf75ea197151ebb7beba9", + "size": 205491, + "url": "https://build.lwjgl.org/release/3.3.1/bin/lwjgl-stb/lwjgl-stb-natives-linux-arm64.jar" + } + }, + "name": "org.lwjgl:lwjgl-stb-natives-linux-arm64:3.3.1-lwjgl.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm64" + } + } + ] + } + ] + }, + { + "_comment": "Add linux-arm64 support for LWJGL 3.3.1", + "match": [ + "org.lwjgl:lwjgl-tinyfd:3.3.1" + ], + "additionalLibraries": [ + { + "downloads": { + "artifact": { + "sha1": "964f628b7a82fd909def086c0dd9a4b84bb259ae", + "size": 42654, + "url": "https://build.lwjgl.org/release/3.3.1/bin/lwjgl-tinyfd/lwjgl-tinyfd-natives-linux-arm64.jar" + } + }, + "name": "org.lwjgl:lwjgl-tinyfd-natives-linux-arm64:3.3.1-lwjgl.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm64" + } + } + ] + } + ] + }, + { + "_comment": "Add linux-arm64 support for LWJGL 3.3.1", + "match": [ + "org.lwjgl:lwjgl:3.3.1" + ], + "additionalLibraries": [ + { + "downloads": { + "artifact": { + "sha1": "b597401014acb7196c76d97e15a6288f54f1f692", + "size": 86308, + "url": "https://build.lwjgl.org/release/3.3.1/bin/lwjgl/lwjgl-natives-linux-arm64.jar" + } + }, + "name": "org.lwjgl:lwjgl-natives-linux-arm64:3.3.1-lwjgl.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm64" + } + } + ] + } + ] + }, + { + "_comment": "Add linux-arm32 support for LWJGL 3.3.1", + "match": [ + "org.lwjgl:lwjgl-glfw:3.3.1" + ], + "additionalLibraries": [ + { + "downloads": { + "artifact": { + "sha1": "816d935933f2dd743074c4e717cc25b55720f294", + "size": 104027, + "url": "https://build.lwjgl.org/release/3.3.1/bin/lwjgl-glfw/lwjgl-glfw-natives-linux-arm32.jar" + } + }, + "name": "org.lwjgl:lwjgl-glfw-natives-linux-arm32:3.3.1-lwjgl.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm32" + } + } + ] + } + ] + }, + { + "_comment": "Add linux-arm32 support for LWJGL 3.3.1", + "match": [ + "org.lwjgl:lwjgl-jemalloc:3.3.1" + ], + "additionalLibraries": [ + { + "downloads": { + "artifact": { + "sha1": "a96a6d6cb3876d7813fcee53c3c24f246aeba3b3", + "size": 136157, + "url": "https://build.lwjgl.org/release/3.3.1/bin/lwjgl-jemalloc/lwjgl-jemalloc-natives-linux-arm32.jar" + } + }, + "name": "org.lwjgl:lwjgl-jemalloc-natives-linux-arm32:3.3.1-lwjgl.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm32" + } + } + ] + } + ] + }, + { + "_comment": "Add linux-arm32 support for LWJGL 3.3.1", + "match": [ + "org.lwjgl:lwjgl-openal:3.3.1" + ], + "additionalLibraries": [ + { + "downloads": { + "artifact": { + "sha1": "ffbe35d7fa5ec9b7eca136a7c71f24d4025a510b", + "size": 400129, + "url": "https://build.lwjgl.org/release/3.3.1/bin/lwjgl-openal/lwjgl-openal-natives-linux-arm32.jar" + } + }, + "name": "org.lwjgl:lwjgl-openal-natives-linux-arm32:3.3.1-lwjgl.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm32" + } + } + ] + } + ] + }, + { + "_comment": "Add linux-arm32 support for LWJGL 3.3.1", + "match": [ + "org.lwjgl:lwjgl-opengl:3.3.1" + ], + "additionalLibraries": [ + { + "downloads": { + "artifact": { + "sha1": "e3550fa91097fd56e361b4370fa822220fef3595", + "size": 58474, + "url": "https://build.lwjgl.org/release/3.3.1/bin/lwjgl-opengl/lwjgl-opengl-natives-linux-arm32.jar" + } + }, + "name": "org.lwjgl:lwjgl-opengl-natives-linux-arm32:3.3.1-lwjgl.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm32" + } + } + ] + } + ] + }, + { + "_comment": "Add linux-arm32 support for LWJGL 3.3.1", + "match": [ + "org.lwjgl:lwjgl-stb:3.3.1" + ], + "additionalLibraries": [ + { + "downloads": { + "artifact": { + "sha1": "b08226bab162c06ae69337d8a1b0ee0a3fdf0b90", + "size": 153889, + "url": "https://build.lwjgl.org/release/3.3.1/bin/lwjgl-stb/lwjgl-stb-natives-linux-arm32.jar" + } + }, + "name": "org.lwjgl:lwjgl-stb-natives-linux-arm32:3.3.1-lwjgl.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm32" + } + } + ] + } + ] + }, + { + "_comment": "Add linux-arm32 support for LWJGL 3.3.1", + "match": [ + "org.lwjgl:lwjgl-tinyfd:3.3.1" + ], + "additionalLibraries": [ + { + "downloads": { + "artifact": { + "sha1": "d53d331e859217a61298fcbcf8d79137f3df345c", + "size": 48061, + "url": "https://build.lwjgl.org/release/3.3.1/bin/lwjgl-tinyfd/lwjgl-tinyfd-natives-linux-arm32.jar" + } + }, + "name": "org.lwjgl:lwjgl-tinyfd-natives-linux-arm32:3.3.1-lwjgl.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm32" + } + } + ] + } + ] + }, + { + "_comment": "Add linux-arm32 support for LWJGL 3.3.1", + "match": [ + "org.lwjgl:lwjgl:3.3.1" + ], + "additionalLibraries": [ + { + "downloads": { + "artifact": { + "sha1": "41a3c1dd15d6b964eb8196dde69720a3e3e5e969", + "size": 82374, + "url": "https://build.lwjgl.org/release/3.3.1/bin/lwjgl/lwjgl-natives-linux-arm32.jar" + } + }, + "name": "org.lwjgl:lwjgl-natives-linux-arm32:3.3.1-lwjgl.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm32" + } + } + ] + } + ] + }, + { + "_comment": "Replace glfw from 3.3.1 with version from 3.3.2 to prevent stack smashing", + "match": [ + "org.lwjgl:lwjgl-glfw-natives-linux:3.3.1" + ], + "override": { + "downloads": { + "artifact": { + "sha1": "0766bb0e8e829598b1c8052fd8173c62af741c52", + "size": 115553, + "url": "https://build.lwjgl.org/release/3.3.2/bin/lwjgl-glfw/lwjgl-glfw-natives-linux.jar" + } + }, + "name": "org.lwjgl:lwjgl-glfw-natives-linux:3.3.2-lwjgl.1" + } + } +] + diff --git a/daedalus_client/src/minecraft.rs b/daedalus_client/src/minecraft.rs index 113456a04..352bd6fca 100644 --- a/daedalus_client/src/minecraft.rs +++ b/daedalus_client/src/minecraft.rs @@ -1,9 +1,7 @@ use crate::download_file; use crate::{format_url, upload_file_to_bucket, Error}; use daedalus::get_hash; -use daedalus::minecraft::{ - merge_partial_library, Library, PartialLibrary, VersionManifest, -}; +use daedalus::minecraft::{merge_partial_library, Library, PartialLibrary, VersionManifest, VersionInfo}; use log::info; use serde::Deserialize; use std::sync::Arc; @@ -66,7 +64,7 @@ pub async fn retrieve_data( let mut version_info = daedalus::minecraft::fetch_version_info(version).await?; - fn patch_library(patches: &Vec, mut library: Library) -> Vec { + fn patch_library(patches: &Vec, mut library: Library, version_info: &VersionInfo) -> Vec { let mut val = Vec::new(); let actual_patches = patches @@ -77,22 +75,26 @@ pub async fn retrieve_data( if !actual_patches.is_empty() { for patch in actual_patches { + println!("{} {}", version_info.id, patch._comment); + + if let Some(override_) = &patch.override_ { + library = merge_partial_library( + override_.clone(), + library, + ); + } + if let Some(additional_libraries) = &patch.additional_libraries { for additional_library in additional_libraries { if patch.patch_additional_libraries.unwrap_or(false) { - let mut libs = patch_library(patches, additional_library.clone()); + let mut libs = patch_library(patches, additional_library.clone(), &version_info); val.append(&mut libs) } else { val.push(additional_library.clone()); } } - } else if let Some(override_) = &patch.override_ { - library = merge_partial_library( - override_.clone(), - library, - ); } } @@ -105,8 +107,8 @@ pub async fn retrieve_data( } let mut new_libraries = Vec::new(); - for library in version_info.libraries { - let mut libs = patch_library(&patches, library); + for library in version_info.libraries.clone() { + let mut libs = patch_library(&patches, library, &version_info); new_libraries.append(&mut libs) } version_info.libraries = new_libraries; @@ -266,12 +268,6 @@ async fn fetch_library_patches( url: Option<&str>, semaphore: Arc, ) -> Result, Error> { - Ok(serde_json::from_slice( - &download_file( - url.unwrap_or(&format_url("library-patches.json")), - None, - semaphore, - ) - .await?, - )?) + let patches = include_bytes!("../library-patches.json"); + Ok(serde_json::from_slice(patches)?) } From 6c628afe5d9067b19ccfb03291b9bbb29e94548b Mon Sep 17 00:00:00 2001 From: Geometrically <18202329+Geometrically@users.noreply.github.com> Date: Fri, 2 Jun 2023 16:13:00 -0700 Subject: [PATCH 52/85] Auto doing fixes (#8) --- daedalus/Cargo.toml | 2 +- daedalus/src/minecraft.rs | 4 +++- daedalus_client/Cargo.toml | 4 ++-- daedalus_client/src/minecraft.rs | 41 ++++++++++++++++++-------------- 4 files changed, 29 insertions(+), 22 deletions(-) diff --git a/daedalus/Cargo.toml b/daedalus/Cargo.toml index fbb6c0e4d..2ecbc1924 100644 --- a/daedalus/Cargo.toml +++ b/daedalus/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "daedalus" -version = "0.1.21" +version = "0.1.22" authors = ["Jai A "] edition = "2018" license = "MIT" diff --git a/daedalus/src/minecraft.rs b/daedalus/src/minecraft.rs index 87506720c..18ff41fa8 100644 --- a/daedalus/src/minecraft.rs +++ b/daedalus/src/minecraft.rs @@ -332,7 +332,9 @@ pub fn merge_partial_library( merge_downloads.artifact = Some(artifact); } if let Some(classifiers) = downloads.classifiers { - if let Some(merge_classifiers) = &mut merge_downloads.classifiers { + if let Some(merge_classifiers) = + &mut merge_downloads.classifiers + { for classifier in classifiers { merge_classifiers.insert(classifier.0, classifier.1); } diff --git a/daedalus_client/Cargo.toml b/daedalus_client/Cargo.toml index 48a599755..fc7f2f415 100644 --- a/daedalus_client/Cargo.toml +++ b/daedalus_client/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "daedalus_client" -version = "0.1.21" +version = "0.1.22" authors = ["Jai A "] edition = "2018" @@ -22,4 +22,4 @@ zip = "0.6.3" semver = "1.0" chrono = { version = "0.4", features = ["serde"] } bytes = "1.3.0" -rust-s3 = "0.32.3" +rust-s3 = "0.33.0" diff --git a/daedalus_client/src/minecraft.rs b/daedalus_client/src/minecraft.rs index 352bd6fca..231a03430 100644 --- a/daedalus_client/src/minecraft.rs +++ b/daedalus_client/src/minecraft.rs @@ -1,7 +1,10 @@ use crate::download_file; use crate::{format_url, upload_file_to_bucket, Error}; use daedalus::get_hash; -use daedalus::minecraft::{merge_partial_library, Library, PartialLibrary, VersionManifest, VersionInfo}; +use daedalus::minecraft::{ + merge_partial_library, Library, PartialLibrary, + VersionManifest, +}; use log::info; use serde::Deserialize; use std::sync::Arc; @@ -25,7 +28,7 @@ pub async fn retrieve_data( daedalus::minecraft::fetch_version_manifest(None).await?; let cloned_manifest = Arc::new(Mutex::new(manifest.clone())); - let patches = fetch_library_patches(None, semaphore.clone()).await?; + let patches = fetch_library_patches()?; let cloned_patches = Arc::new(&patches); let visited_assets_mutex = Arc::new(Mutex::new(Vec::new())); @@ -43,10 +46,8 @@ pub async fn retrieve_data( None }; - if let Some(old_version) = old_version { - if old_version.sha1 == version.sha1 { - return Ok(()); - } + if old_version.is_some() { + return Ok(()); } let visited_assets_mutex = Arc::clone(&visited_assets_mutex); @@ -64,7 +65,10 @@ pub async fn retrieve_data( let mut version_info = daedalus::minecraft::fetch_version_info(version).await?; - fn patch_library(patches: &Vec, mut library: Library, version_info: &VersionInfo) -> Vec { + fn patch_library( + patches: &Vec, + mut library: Library, + ) -> Vec { let mut val = Vec::new(); let actual_patches = patches @@ -72,11 +76,8 @@ pub async fn retrieve_data( .filter(|x| x.match_.contains(&library.name)) .collect::>(); - if !actual_patches.is_empty() - { + if !actual_patches.is_empty() { for patch in actual_patches { - println!("{} {}", version_info.id, patch._comment); - if let Some(override_) = &patch.override_ { library = merge_partial_library( override_.clone(), @@ -88,8 +89,14 @@ pub async fn retrieve_data( &patch.additional_libraries { for additional_library in additional_libraries { - if patch.patch_additional_libraries.unwrap_or(false) { - let mut libs = patch_library(patches, additional_library.clone(), &version_info); + if patch + .patch_additional_libraries + .unwrap_or(false) + { + let mut libs = patch_library( + patches, + additional_library.clone(), + ); val.append(&mut libs) } else { val.push(additional_library.clone()); @@ -108,7 +115,8 @@ pub async fn retrieve_data( let mut new_libraries = Vec::new(); for library in version_info.libraries.clone() { - let mut libs = patch_library(&patches, library, &version_info); + let mut libs = + patch_library(&patches, library); new_libraries.append(&mut libs) } version_info.libraries = new_libraries; @@ -264,10 +272,7 @@ struct LibraryPatch { } /// Fetches the list of fabric versions -async fn fetch_library_patches( - url: Option<&str>, - semaphore: Arc, -) -> Result, Error> { +fn fetch_library_patches() -> Result, Error> { let patches = include_bytes!("../library-patches.json"); Ok(serde_json::from_slice(patches)?) } From 0d56127758944ff0d8fb23bf2fa12daa9d2d05fc Mon Sep 17 00:00:00 2001 From: Geometrically <18202329+Geometrically@users.noreply.github.com> Date: Sun, 11 Jun 2023 13:40:31 -0700 Subject: [PATCH 53/85] Add new mojang args (#9) --- daedalus/Cargo.toml | 2 +- daedalus/src/minecraft.rs | 15 +++++++++++++-- daedalus_client/Cargo.toml | 2 +- 3 files changed, 15 insertions(+), 4 deletions(-) diff --git a/daedalus/Cargo.toml b/daedalus/Cargo.toml index 2ecbc1924..f00c0ae85 100644 --- a/daedalus/Cargo.toml +++ b/daedalus/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "daedalus" -version = "0.1.22" +version = "0.1.23" authors = ["Jai A "] edition = "2018" license = "MIT" diff --git a/daedalus/src/minecraft.rs b/daedalus/src/minecraft.rs index 18ff41fa8..747fa8a06 100644 --- a/daedalus/src/minecraft.rs +++ b/daedalus/src/minecraft.rs @@ -233,8 +233,19 @@ pub struct FeatureRule { /// Whether the user is in demo mode pub is_demo_user: Option, #[serde(skip_serializing_if = "Option::is_none")] - /// Whether the user is using the demo resolution - pub has_demo_resolution: Option, + /// Whether the user is using a custom resolution + pub has_custom_resolution: Option, + #[serde(skip_serializing_if = "Option::is_none")] + /// Whether the launcher has quick plays support + pub has_quick_plays_support: Option, + #[serde(skip_serializing_if = "Option::is_none")] + /// Whether the instance is being launched to a single-player world + pub is_quick_play_singleplayer: Option, + #[serde(skip_serializing_if = "Option::is_none")] + /// Whether the instance is being launched to a multi-player world + pub is_quick_play_multiplayer: Option, + /// Whether the instance is being launched to a realms world + pub is_quick_play_realms: Option, } #[cfg_attr(feature = "bincode", derive(Encode, Decode))] diff --git a/daedalus_client/Cargo.toml b/daedalus_client/Cargo.toml index fc7f2f415..a0d749118 100644 --- a/daedalus_client/Cargo.toml +++ b/daedalus_client/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "daedalus_client" -version = "0.1.22" +version = "0.1.23" authors = ["Jai A "] edition = "2018" From 10e7b66f38217f8ab735a80b1d67de956bd6aed6 Mon Sep 17 00:00:00 2001 From: Geometrically <18202329+Geometrically@users.noreply.github.com> Date: Fri, 28 Jul 2023 16:22:38 -0700 Subject: [PATCH 54/85] Fix loader ordering (#10) --- daedalus_client/src/fabric.rs | 9 ++------- daedalus_client/src/minecraft.rs | 6 ++---- daedalus_client/src/quilt.rs | 9 ++------- 3 files changed, 6 insertions(+), 18 deletions(-) diff --git a/daedalus_client/src/fabric.rs b/daedalus_client/src/fabric.rs index 39c4d413f..94d2488f4 100644 --- a/daedalus_client/src/fabric.rs +++ b/daedalus_client/src/fabric.rs @@ -259,18 +259,13 @@ pub async fn retrieve_data( version.loaders.sort_by(|x, y| { list.loader .iter() - .position(|z| { - x.id.split('-').next().unwrap_or_default() == &*z.version - }) + .position(|z| x.id == *z.version) .unwrap_or_default() .cmp( &list .loader .iter() - .position(|z| { - y.id.split('-').next().unwrap_or_default() - == z.version - }) + .position(|z| y.id == z.version) .unwrap_or_default(), ) }) diff --git a/daedalus_client/src/minecraft.rs b/daedalus_client/src/minecraft.rs index 231a03430..6badc9c4a 100644 --- a/daedalus_client/src/minecraft.rs +++ b/daedalus_client/src/minecraft.rs @@ -2,8 +2,7 @@ use crate::download_file; use crate::{format_url, upload_file_to_bucket, Error}; use daedalus::get_hash; use daedalus::minecraft::{ - merge_partial_library, Library, PartialLibrary, - VersionManifest, + merge_partial_library, Library, PartialLibrary, VersionManifest, }; use log::info; use serde::Deserialize; @@ -115,8 +114,7 @@ pub async fn retrieve_data( let mut new_libraries = Vec::new(); for library in version_info.libraries.clone() { - let mut libs = - patch_library(&patches, library); + let mut libs = patch_library(&patches, library); new_libraries.append(&mut libs) } version_info.libraries = new_libraries; diff --git a/daedalus_client/src/quilt.rs b/daedalus_client/src/quilt.rs index 19ebeb45d..827ad00e6 100644 --- a/daedalus_client/src/quilt.rs +++ b/daedalus_client/src/quilt.rs @@ -259,18 +259,13 @@ pub async fn retrieve_data( version.loaders.sort_by(|x, y| { list.loader .iter() - .position(|z| { - x.id.split('-').next().unwrap_or_default() == &*z.version - }) + .position(|z| x.id == *z.version) .unwrap_or_default() .cmp( &list .loader .iter() - .position(|z| { - y.id.split('-').next().unwrap_or_default() - == z.version - }) + .position(|z| y.id == z.version) .unwrap_or_default(), ) }) From 2fa8371bae0e91c79f2c64b237fabe50e42188f0 Mon Sep 17 00:00:00 2001 From: Geometrically <18202329+Geometrically@users.noreply.github.com> Date: Mon, 21 Aug 2023 14:34:22 -0400 Subject: [PATCH 55/85] Neoforge support (#11) --- daedalus/src/modded.rs | 2 + daedalus_client/Cargo.toml | 1 + daedalus_client/src/main.rs | 13 + daedalus_client/src/neo.rs | 470 ++++++++++++++++++++++++++++++++++++ 4 files changed, 486 insertions(+) create mode 100644 daedalus_client/src/neo.rs diff --git a/daedalus/src/modded.rs b/daedalus/src/modded.rs index 8f91798fc..83c579ab5 100644 --- a/daedalus/src/modded.rs +++ b/daedalus/src/modded.rs @@ -16,6 +16,8 @@ pub const CURRENT_FABRIC_FORMAT_VERSION: usize = 0; pub const CURRENT_FORGE_FORMAT_VERSION: usize = 0; /// The latest version of the format the quilt model structs deserialize to pub const CURRENT_QUILT_FORMAT_VERSION: usize = 0; +/// The latest version of the format the neoforge model structs deserialize to +pub const CURRENT_NEOFORGE_FORMAT_VERSION: usize = 0; /// The dummy replace string library names, inheritsFrom, and version names should be replaced with pub const DUMMY_REPLACE_STRING: &str = "${modrinth.gameVersion}"; diff --git a/daedalus_client/Cargo.toml b/daedalus_client/Cargo.toml index a0d749118..ca19f0147 100644 --- a/daedalus_client/Cargo.toml +++ b/daedalus_client/Cargo.toml @@ -15,6 +15,7 @@ log = "0.4.17" env_logger= "0.10.0" serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" +serde-xml-rs = "0.6.0" lazy_static = "1.4.0" thiserror = "1.0" reqwest = "0.11.13" diff --git a/daedalus_client/src/main.rs b/daedalus_client/src/main.rs index db3639de1..e706fa078 100644 --- a/daedalus_client/src/main.rs +++ b/daedalus_client/src/main.rs @@ -9,6 +9,7 @@ use tokio::sync::Semaphore; mod fabric; mod forge; mod minecraft; +mod neo; mod quilt; #[derive(thiserror::Error, Debug)] @@ -17,6 +18,8 @@ pub enum Error { DaedalusError(#[from] daedalus::Error), #[error("Error while deserializing JSON")] SerdeError(#[from] serde_json::Error), + #[error("Error while deserializing XML")] + XMLError(#[from] serde_xml_rs::Error), #[error("Unable to fetch {item}")] FetchError { inner: reqwest::Error, item: String }, #[error("Error while managing asynchronous tasks")] @@ -98,6 +101,16 @@ async fn main() { Ok(..) => {} Err(err) => error!("{:?}", err), }; + match neo::retrieve_data( + &manifest, + &mut uploaded_files, + semaphore.clone(), + ) + .await + { + Ok(..) => {} + Err(err) => error!("{:?}", err), + }; } } } diff --git a/daedalus_client/src/neo.rs b/daedalus_client/src/neo.rs new file mode 100644 index 000000000..172a56676 --- /dev/null +++ b/daedalus_client/src/neo.rs @@ -0,0 +1,470 @@ +use crate::{download_file, format_url, upload_file_to_bucket, Error}; +use daedalus::minecraft::{Library, VersionManifest}; +use daedalus::modded::{ + LoaderVersion, Manifest, PartialVersionInfo, Processor, SidedDataEntry, +}; +use log::info; +use semver::Version; +use serde::{Deserialize, Serialize}; +use std::collections::HashMap; +use std::io::Read; +use std::sync::Arc; +use std::time::Instant; +use tokio::sync::{Mutex, Semaphore}; + +pub async fn retrieve_data( + minecraft_versions: &VersionManifest, + uploaded_files: &mut Vec, + semaphore: Arc, +) -> Result<(), Error> { + let maven_metadata = fetch_maven_metadata(None, semaphore.clone()).await?; + let old_manifest = daedalus::modded::fetch_manifest(&format_url(&format!( + "neo/v{}/manifest.json", + daedalus::modded::CURRENT_NEOFORGE_FORMAT_VERSION, + ))) + .await + .ok(); + + let old_versions = + Arc::new(Mutex::new(if let Some(old_manifest) = old_manifest { + old_manifest.game_versions + } else { + Vec::new() + })); + + let versions = Arc::new(Mutex::new(Vec::new())); + + let visited_assets_mutex = Arc::new(Mutex::new(Vec::new())); + let uploaded_files_mutex = Arc::new(Mutex::new(Vec::new())); + + let mut version_futures = Vec::new(); + + for (minecraft_version, loader_versions) in maven_metadata.clone() { + let mut loaders = Vec::new(); + + for (full, loader_version) in loader_versions { + let version = Version::parse(&loader_version)?; + + loaders.push((full, version)) + } + + if !loaders.is_empty() { + version_futures.push(async { + let mut loaders_versions = Vec::new(); + + { + let loaders_futures = loaders.into_iter().map(|(loader_version_full, _)| async { + let versions_mutex = Arc::clone(&old_versions); + let visited_assets = Arc::clone(&visited_assets_mutex); + let uploaded_files_mutex = Arc::clone(&uploaded_files_mutex); + let semaphore = Arc::clone(&semaphore); + let minecraft_version = minecraft_version.clone(); + + async move { + { + let versions = versions_mutex.lock().await; + let version = versions.iter().find(|x| + x.id == minecraft_version).and_then(|x| x.loaders.iter().find(|x| x.id == loader_version_full)); + + if let Some(version) = version { + return Ok::, Error>(Some(version.clone())); + } + } + + info!("Forge - Installer Start {}", loader_version_full.clone()); + let bytes = download_file(&format!("https://maven.neoforged.net/net/neoforged/forge/{0}/forge-{0}-installer.jar", loader_version_full), None, semaphore.clone()).await?; + + let reader = std::io::Cursor::new(bytes); + + if let Ok(archive) = zip::ZipArchive::new(reader) { + let mut archive_clone = archive.clone(); + let mut profile = tokio::task::spawn_blocking(move || { + let mut install_profile = archive_clone.by_name("install_profile.json")?; + + let mut contents = String::new(); + install_profile.read_to_string(&mut contents)?; + + Ok::(serde_json::from_str::(&contents)?) + }).await??; + + let mut archive_clone = archive.clone(); + let version_info = tokio::task::spawn_blocking(move || { + let mut install_profile = archive_clone.by_name("version.json")?; + + let mut contents = String::new(); + install_profile.read_to_string(&mut contents)?; + + Ok::(serde_json::from_str::(&contents)?) + }).await??; + + + let mut libs : Vec = version_info.libraries.into_iter().chain(profile.libraries.into_iter().map(|x| Library { + downloads: x.downloads, + extract: x.extract, + name: x.name, + url: x.url, + natives: x.natives, + rules: x.rules, + checksums: x.checksums, + include_in_classpath: false + })).collect(); + + let mut local_libs : HashMap = HashMap::new(); + + for lib in &libs { + if lib.downloads.as_ref().and_then(|x| x.artifact.as_ref().map(|x| x.url.is_empty())).unwrap_or(false) { + let mut archive_clone = archive.clone(); + let lib_name_clone = lib.name.clone(); + + let lib_bytes = tokio::task::spawn_blocking(move || { + let mut lib_file = archive_clone.by_name(&format!("maven/{}", daedalus::get_path_from_artifact(&lib_name_clone)?))?; + let mut lib_bytes = Vec::new(); + lib_file.read_to_end(&mut lib_bytes)?; + + Ok::(bytes::Bytes::from(lib_bytes)) + }).await??; + + local_libs.insert(lib.name.clone(), lib_bytes); + } + } + + let path = profile.path.clone(); + let version = profile.version.clone(); + + for entry in profile.data.values_mut() { + if entry.client.starts_with('/') || entry.server.starts_with('/') { + macro_rules! read_data { + ($value:expr) => { + let mut archive_clone = archive.clone(); + let value_clone = $value.clone(); + let lib_bytes = tokio::task::spawn_blocking(move || { + let mut lib_file = archive_clone.by_name(&value_clone[1..value_clone.len()])?; + let mut lib_bytes = Vec::new(); + lib_file.read_to_end(&mut lib_bytes)?; + + Ok::(bytes::Bytes::from(lib_bytes)) + }).await??; + + let split = $value.split('/').last(); + + if let Some(last) = split { + let mut file = last.split('.'); + + if let Some(file_name) = file.next() { + if let Some(ext) = file.next() { + let path = format!("{}:{}@{}", path.as_deref().unwrap_or(&*format!("net.minecraftforge:forge:{}", version)), file_name, ext); + $value = format!("[{}]", &path); + local_libs.insert(path.clone(), bytes::Bytes::from(lib_bytes)); + + libs.push(Library { + downloads: None, + extract: None, + name: path, + url: Some("".to_string()), + natives: None, + rules: None, + checksums: None, + include_in_classpath: false, + }); + } + } + } + } + } + + if entry.client.starts_with('/') { + read_data!(entry.client); + } + + if entry.server.starts_with('/') { + read_data!(entry.server); + } + } + } + + let now = Instant::now(); + let libs = futures::future::try_join_all(libs.into_iter().map(|mut lib| async { + let artifact_path = + daedalus::get_path_from_artifact(&lib.name)?; + + { + let mut visited_assets = visited_assets.lock().await; + + if visited_assets.contains(&lib.name) { + if let Some(ref mut downloads) = lib.downloads { + if let Some(ref mut artifact) = downloads.artifact { + artifact.url = format_url(&format!("maven/{}", artifact_path)); + } + } else if lib.url.is_some() { + lib.url = Some(format_url("maven/")); + } + + return Ok::(lib); + } else { + visited_assets.push(lib.name.clone()) + } + } + + let artifact_bytes = if let Some(ref mut downloads) = lib.downloads { + if let Some(ref mut artifact) = downloads.artifact { + let res = if artifact.url.is_empty() { + local_libs.get(&lib.name).cloned() + } else { + Some(download_file( + &artifact.url, + Some(&*artifact.sha1), + semaphore.clone(), + ) + .await?) + }; + + if res.is_some() { + artifact.url = format_url(&format!("maven/{}", artifact_path)); + } + + res + } else { None } + } else if let Some(ref mut url) = lib.url { + let res = if url.is_empty() { + local_libs.get(&lib.name).cloned() + } else { + Some(download_file( + url, + None, + semaphore.clone(), + ) + .await?) + }; + + if res.is_some() { + lib.url = Some(format_url("maven/")); + } + + res + } else { None }; + + if let Some(bytes) = artifact_bytes { + upload_file_to_bucket( + format!("{}/{}", "maven", artifact_path), + bytes.to_vec(), + Some("application/java-archive".to_string()), + uploaded_files_mutex.as_ref(), + semaphore.clone(), + ).await?; + } + + Ok::(lib) + })).await?; + + let elapsed = now.elapsed(); + info!("Elapsed lib DL: {:.2?}", elapsed); + + let new_profile = PartialVersionInfo { + id: version_info.id, + inherits_from: version_info.inherits_from, + release_time: version_info.release_time, + time: version_info.time, + main_class: version_info.main_class, + minecraft_arguments: version_info.minecraft_arguments, + arguments: version_info.arguments, + libraries: libs, + type_: version_info.type_, + data: Some(profile.data), + processors: Some(profile.processors), + }; + + let version_path = format!( + "neo/v{}/versions/{}.json", + daedalus::modded::CURRENT_NEOFORGE_FORMAT_VERSION, + new_profile.id + ); + + upload_file_to_bucket( + version_path.clone(), + serde_json::to_vec(&new_profile)?, + Some("application/json".to_string()), + uploaded_files_mutex.as_ref(), + semaphore.clone(), + ).await?; + + return Ok(Some(LoaderVersion { + id: loader_version_full, + url: format_url(&version_path), + stable: false + })); + } + + Ok(None) + }.await + }); + + { + let len = loaders_futures.len(); + let mut versions = loaders_futures.into_iter().peekable(); + let mut chunk_index = 0; + while versions.peek().is_some() { + let now = Instant::now(); + + let chunk: Vec<_> = versions.by_ref().take(1).collect(); + let res = futures::future::try_join_all(chunk).await?; + loaders_versions.extend(res.into_iter().flatten()); + + chunk_index += 1; + + let elapsed = now.elapsed(); + info!("Loader Chunk {}/{len} Elapsed: {:.2?}", chunk_index, elapsed); + } + } + } + + versions.lock().await.push(daedalus::modded::Version { + id: minecraft_version, + stable: true, + loaders: loaders_versions + }); + + Ok::<(), Error>(()) + }); + } + } + + { + let len = version_futures.len(); + let mut versions = version_futures.into_iter().peekable(); + let mut chunk_index = 0; + while versions.peek().is_some() { + let now = Instant::now(); + + let chunk: Vec<_> = versions.by_ref().take(1).collect(); + futures::future::try_join_all(chunk).await?; + + chunk_index += 1; + + let elapsed = now.elapsed(); + info!("Chunk {}/{len} Elapsed: {:.2?}", chunk_index, elapsed); + } + } + + if let Ok(versions) = Arc::try_unwrap(versions) { + let mut versions = versions.into_inner(); + + versions.sort_by(|x, y| { + minecraft_versions + .versions + .iter() + .position(|z| x.id == z.id) + .unwrap_or_default() + .cmp( + &minecraft_versions + .versions + .iter() + .position(|z| y.id == z.id) + .unwrap_or_default(), + ) + }); + + for version in &mut versions { + let loader_versions = maven_metadata.get(&version.id); + if let Some(loader_versions) = loader_versions { + version.loaders.sort_by(|x, y| { + loader_versions + .iter() + .position(|z| y.id == z.1) + .unwrap_or_default() + .cmp( + &loader_versions + .iter() + .position(|z| x.id == z.1) + .unwrap_or_default(), + ) + }); + version.loaders.reverse(); + } + } + + upload_file_to_bucket( + format!( + "neo/v{}/manifest.json", + daedalus::modded::CURRENT_NEOFORGE_FORMAT_VERSION, + ), + serde_json::to_vec(&Manifest { + game_versions: versions, + })?, + Some("application/json".to_string()), + uploaded_files_mutex.as_ref(), + semaphore, + ) + .await?; + } + + if let Ok(uploaded_files_mutex) = Arc::try_unwrap(uploaded_files_mutex) { + uploaded_files.extend(uploaded_files_mutex.into_inner()); + } + + Ok(()) +} + +const DEFAULT_MAVEN_METADATA_URL: &str = + "https://maven.neoforged.net/net/neoforged/forge/maven-metadata.xml"; + +#[derive(Debug, Deserialize)] +struct Metadata { + versioning: Versioning, +} + +#[derive(Debug, Deserialize)] +struct Versioning { + versions: Versions, +} + +#[derive(Debug, Deserialize)] +struct Versions { + version: Vec, +} + +pub async fn fetch_maven_metadata( + url: Option<&str>, + semaphore: Arc, +) -> Result>, Error> { + let values = serde_xml_rs::from_str::( + &String::from_utf8( + download_file( + url.unwrap_or(DEFAULT_MAVEN_METADATA_URL), + None, + semaphore, + ) + .await? + .to_vec(), + ) + .unwrap_or_default(), + )?; + + let mut map: HashMap> = HashMap::new(); + + for value in values.versioning.versions.version { + let original = value.clone(); + + let parts: Vec<&str> = value.split('-').collect(); + if parts.len() == 2 { + map.entry(parts[0].to_string()) + .or_insert(Vec::new()) + .push((original, parts[1].to_string())); + } + } + + Ok(map) +} + +#[derive(Serialize, Deserialize, Debug)] +#[serde(rename_all = "camelCase")] +struct ForgeInstallerProfileV2 { + pub spec: i32, + pub profile: String, + pub version: String, + pub json: String, + pub path: Option, + pub minecraft: String, + pub data: HashMap, + pub libraries: Vec, + pub processors: Vec, +} From c1d28381e8da98153f9a665ce4a32fcda524d729 Mon Sep 17 00:00:00 2001 From: Jai A Date: Mon, 21 Aug 2023 14:36:43 -0400 Subject: [PATCH 56/85] actually bump version --- daedalus/Cargo.toml | 2 +- daedalus_client/Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/daedalus/Cargo.toml b/daedalus/Cargo.toml index f00c0ae85..75262c279 100644 --- a/daedalus/Cargo.toml +++ b/daedalus/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "daedalus" -version = "0.1.23" +version = "0.1.25" authors = ["Jai A "] edition = "2018" license = "MIT" diff --git a/daedalus_client/Cargo.toml b/daedalus_client/Cargo.toml index ca19f0147..8cd0fd2fd 100644 --- a/daedalus_client/Cargo.toml +++ b/daedalus_client/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "daedalus_client" -version = "0.1.23" +version = "0.1.25" authors = ["Jai A "] edition = "2018" From 57b1932b5eb64a53f4947bcd8b97fef985e05b2a Mon Sep 17 00:00:00 2001 From: Geometrically <18202329+Geometrically@users.noreply.github.com> Date: Tue, 12 Sep 2023 14:19:24 -0400 Subject: [PATCH 57/85] Fix minecraft meta links (#12) * Fix minecraft meta links * remove debug * bump v * no mut --- daedalus/Cargo.toml | 2 +- daedalus_client/Cargo.toml | 2 +- daedalus_client/src/minecraft.rs | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/daedalus/Cargo.toml b/daedalus/Cargo.toml index 75262c279..6aff4dd4d 100644 --- a/daedalus/Cargo.toml +++ b/daedalus/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "daedalus" -version = "0.1.25" +version = "0.1.26" authors = ["Jai A "] edition = "2018" license = "MIT" diff --git a/daedalus_client/Cargo.toml b/daedalus_client/Cargo.toml index 8cd0fd2fd..87b3d4171 100644 --- a/daedalus_client/Cargo.toml +++ b/daedalus_client/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "daedalus_client" -version = "0.1.25" +version = "0.1.26" authors = ["Jai A "] edition = "2018" diff --git a/daedalus_client/src/minecraft.rs b/daedalus_client/src/minecraft.rs index 6badc9c4a..e284c27c0 100644 --- a/daedalus_client/src/minecraft.rs +++ b/daedalus_client/src/minecraft.rs @@ -25,7 +25,7 @@ pub async fn retrieve_data( let mut manifest = daedalus::minecraft::fetch_version_manifest(None).await?; - let cloned_manifest = Arc::new(Mutex::new(manifest.clone())); + let cloned_manifest = Arc::new(Mutex::new(old_manifest.clone().unwrap_or(manifest.clone()))); let patches = fetch_library_patches()?; let cloned_patches = Arc::new(&patches); From b1ca2cc2b6f558fea82396b1ad8ce954ca6e2d73 Mon Sep 17 00:00:00 2001 From: Geometrically <18202329+Geometrically@users.noreply.github.com> Date: Sat, 21 Oct 2023 13:09:21 -0700 Subject: [PATCH 58/85] Fix quilt + fabric intermediary syncing issues (#13) * Fix quilt + fabric intermediary syncing issues * fix comp * Fix random versions not working/updating meta --- daedalus_client/library-patches.json | 544 ++++++++++++++++++++++----- daedalus_client/src/fabric.rs | 47 ++- daedalus_client/src/main.rs | 47 ++- daedalus_client/src/minecraft.rs | 3 +- daedalus_client/src/neo.rs | 2 +- daedalus_client/src/quilt.rs | 46 ++- docker-compose.yml | 17 + 7 files changed, 559 insertions(+), 147 deletions(-) create mode 100644 docker-compose.yml diff --git a/daedalus_client/library-patches.json b/daedalus_client/library-patches.json index 31a6a8cb8..ebde4ed02 100644 --- a/daedalus_client/library-patches.json +++ b/daedalus_client/library-patches.json @@ -1,4 +1,48 @@ [ + { + "_comment": "Only allow osx-arm64 for existing LWJGL 3.3.2", + "match": [ + "org.lwjgl:lwjgl-glfw-natives-macos-arm64:3.3.2", + "org.lwjgl:lwjgl-jemalloc-natives-macos-arm64:3.3.2", + "org.lwjgl:lwjgl-openal-natives-macos-arm64:3.3.2", + "org.lwjgl:lwjgl-opengl-natives-macos-arm64:3.3.2", + "org.lwjgl:lwjgl-stb-natives-macos-arm64:3.3.2", + "org.lwjgl:lwjgl-tinyfd-natives-macos-arm64:3.3.2", + "org.lwjgl:lwjgl-natives-macos-arm64:3.3.2" + ], + "override": { + "rules": [ + { + "action": "allow", + "os": { + "name": "osx-arm64" + } + } + ] + } + }, + { + "_comment": "Only allow windows-arm64 for existing LWJGL 3.3.2", + "match": [ + "org.lwjgl:lwjgl-glfw-natives-windows-arm64:3.3.2", + "org.lwjgl:lwjgl-jemalloc-natives-windows-arm64:3.3.2", + "org.lwjgl:lwjgl-openal-natives-windows-arm64:3.3.2", + "org.lwjgl:lwjgl-opengl-natives-windows-arm64:3.3.2", + "org.lwjgl:lwjgl-stb-natives-windows-arm64:3.3.2", + "org.lwjgl:lwjgl-tinyfd-natives-windows-arm64:3.3.2", + "org.lwjgl:lwjgl-natives-windows-arm64:3.3.2" + ], + "override": { + "rules": [ + { + "action": "allow", + "os": { + "name": "windows-arm64" + } + } + ] + } + }, { "_comment": "Add missing tinyfd to the broken LWJGL 3.2.2 variant", "match": [ @@ -36,8 +80,7 @@ "natives-windows": { "sha1": "e9115958773644e863332a6a06488d26f9e1fc9f", "size": 208314, - "url": - "https://libraries.minecraft.net/org/lwjgl/lwjgl-tinyfd/3.2.2/lwjgl-tinyfd-3.2.2-natives-windows.jar" + "url": "https://libraries.minecraft.net/org/lwjgl/lwjgl-tinyfd/3.2.2/lwjgl-tinyfd-3.2.2-natives-windows.jar" } } }, @@ -62,8 +105,7 @@ "artifact": { "sha1": "369a83621e3c65496348491e533cb97fe5f2f37d", "size": 91947, - "url": - "https://github.com/MinecraftMachina/Java-Objective-C-Bridge/releases/download/1.1.0-mmachina.1/java-objc-bridge-1.1.jar" + "url": "https://github.com/MinecraftMachina/Java-Objective-C-Bridge/releases/download/1.1.0-mmachina.1/java-objc-bridge-1.1.jar" } }, "name": "ca.weblite:java-objc-bridge:1.1.0-mmachina.1", @@ -94,14 +136,12 @@ "natives-linux-arm64": { "sha1": "42b388ccb7c63cec4e9f24f4dddef33325f8b212", "size": 10932, - "url": - "https://github.com/theofficialgman/lwjgl3-binaries-arm64/raw/lwjgl-2.9.4/jinput-platform-2.0.5-natives-linux.jar" + "url": "https://github.com/theofficialgman/lwjgl3-binaries-arm64/raw/lwjgl-2.9.4/jinput-platform-2.0.5-natives-linux.jar" }, "natives-linux-arm32": { "sha1": "f3c455b71c5146acb5f8a9513247fc06db182fd5", "size": 4521, - "url": - "https://github.com/theofficialgman/lwjgl3-binaries-arm32/raw/lwjgl-2.9.4/jinput-platform-2.0.5-natives-linux.jar" + "url": "https://github.com/theofficialgman/lwjgl3-binaries-arm32/raw/lwjgl-2.9.4/jinput-platform-2.0.5-natives-linux.jar" } } }, @@ -125,9 +165,7 @@ ], "override": { "rules": [ - { - "action": "allow" - }, + { "action": "disallow", "os": { @@ -169,9 +207,7 @@ ], "override": { "rules": [ - { - "action": "allow" - }, + { "action": "disallow", "os": { @@ -198,8 +234,7 @@ "artifact": { "sha1": "697517568c68e78ae0b4544145af031c81082dfe", "size": 1047168, - "url": - "https://libraries.minecraft.net/org/lwjgl/lwjgl/lwjgl/2.9.4-nightly-20150209/lwjgl-2.9.4-nightly-20150209.jar" + "url": "https://libraries.minecraft.net/org/lwjgl/lwjgl/lwjgl/2.9.4-nightly-20150209/lwjgl-2.9.4-nightly-20150209.jar" } }, "name": "org.lwjgl.lwjgl:lwjgl:2.9.4-nightly-20150209", @@ -238,9 +273,7 @@ ], "override": { "rules": [ - { - "action": "allow" - }, + { "action": "disallow", "os": { @@ -267,8 +300,7 @@ "artifact": { "sha1": "d51a7c040a721d13efdfbd34f8b257b2df882ad0", "size": 173887, - "url": - "https://libraries.minecraft.net/org/lwjgl/lwjgl/lwjgl_util/2.9.4-nightly-20150209/lwjgl_util-2.9.4-nightly-20150209.jar" + "url": "https://libraries.minecraft.net/org/lwjgl/lwjgl/lwjgl_util/2.9.4-nightly-20150209/lwjgl_util-2.9.4-nightly-20150209.jar" } }, "name": "org.lwjgl.lwjgl:lwjgl_util:2.9.4-nightly-20150209", @@ -312,20 +344,17 @@ "natives-osx-arm64": { "sha1": "eff546c0b319d6ffc7a835652124c18089c67f36", "size": 488316, - "url": - "https://github.com/MinecraftMachina/lwjgl/releases/download/2.9.4-20150209-mmachina.2/lwjgl-platform-2.9.4-nightly-20150209-natives-osx.jar" + "url": "https://github.com/MinecraftMachina/lwjgl/releases/download/2.9.4-20150209-mmachina.2/lwjgl-platform-2.9.4-nightly-20150209-natives-osx.jar" }, "natives-linux-arm64": { "sha1": "63ac7da0f4a4785c7eadc0f8edc1e9dcc4dd08cb", "size": 579979, - "url": - "https://github.com/theofficialgman/lwjgl3-binaries-arm64/raw/lwjgl-2.9.4/lwjgl-platform-2.9.4-nightly-20150209-natives-linux.jar" + "url": "https://github.com/theofficialgman/lwjgl3-binaries-arm64/raw/lwjgl-2.9.4/lwjgl-platform-2.9.4-nightly-20150209-natives-linux.jar" }, "natives-linux-arm32": { "sha1": "fa483e540a9a753a5ffbb23dcf7879a5bf752611", "size": 475177, - "url": - "https://github.com/theofficialgman/lwjgl3-binaries-arm32/raw/lwjgl-2.9.4/lwjgl-platform-2.9.4-nightly-20150209-natives-linux.jar" + "url": "https://github.com/theofficialgman/lwjgl3-binaries-arm32/raw/lwjgl-2.9.4/lwjgl-platform-2.9.4-nightly-20150209-natives-linux.jar" } } }, @@ -346,9 +375,7 @@ ], "override": { "rules": [ - { - "action": "allow" - }, + { "action": "disallow", "os": { @@ -399,8 +426,7 @@ "natives-linux-arm64": { "sha1": "074ad243761147df0d060fbefc814614d2ff75cc", "size": 85072, - "url": - "https://github.com/theofficialgman/lwjgl3-binaries-arm64/raw/lwjgl-3.2.2/lwjgl-glfw-natives-linux-arm64.jar" + "url": "https://github.com/theofficialgman/lwjgl3-binaries-arm64/raw/lwjgl-3.2.2/lwjgl-glfw-natives-linux-arm64.jar" } } }, @@ -446,8 +472,7 @@ "natives-linux-arm32": { "sha1": "4265f2fbe3b9d642591165165a17cf406cf7b98e", "size": 80186, - "url": - "https://github.com/theofficialgman/lwjgl3-binaries-arm32/raw/lwjgl-3.2.2/lwjgl-glfw-natives-linux-arm32.jar" + "url": "https://github.com/theofficialgman/lwjgl3-binaries-arm32/raw/lwjgl-3.2.2/lwjgl-glfw-natives-linux-arm32.jar" } } }, @@ -493,8 +518,7 @@ "natives-osx-arm64": { "sha1": "71d793d0a5a42e3dfe78eb882abc2523a2c6b496", "size": 129076, - "url": - "https://github.com/MinecraftMachina/lwjgl3/releases/download/3.3.1-mmachina.1/lwjgl-glfw-natives-macos-arm64.jar" + "url": "https://github.com/MinecraftMachina/lwjgl3/releases/download/3.3.1-mmachina.1/lwjgl-glfw-natives-macos-arm64.jar" } } }, @@ -523,9 +547,7 @@ ], "override": { "rules": [ - { - "action": "allow" - }, + { "action": "disallow", "os": { @@ -576,8 +598,7 @@ "natives-linux-arm64": { "sha1": "762d7d80c9cdf3a3f3fc80c8a5f86612255edfe0", "size": 156343, - "url": - "https://github.com/theofficialgman/lwjgl3-binaries-arm64/raw/lwjgl-3.2.2/lwjgl-jemalloc-patched-natives-linux-arm64.jar" + "url": "https://github.com/theofficialgman/lwjgl3-binaries-arm64/raw/lwjgl-3.2.2/lwjgl-jemalloc-patched-natives-linux-arm64.jar" } } }, @@ -623,8 +644,7 @@ "natives-linux-arm32": { "sha1": "9163a2a5559ef87bc13ead8fea84417ea3928748", "size": 134237, - "url": - "https://github.com/theofficialgman/lwjgl3-binaries-arm32/raw/lwjgl-3.2.2/lwjgl-jemalloc-natives-linux-arm32.jar" + "url": "https://github.com/theofficialgman/lwjgl3-binaries-arm32/raw/lwjgl-3.2.2/lwjgl-jemalloc-natives-linux-arm32.jar" } } }, @@ -670,8 +690,7 @@ "natives-osx-arm64": { "sha1": "b0be721188d2e7195798780b1c5fe7eafe8091c1", "size": 103478, - "url": - "https://github.com/MinecraftMachina/lwjgl3/releases/download/3.3.1-mmachina.1/lwjgl-jemalloc-natives-macos-arm64.jar" + "url": "https://github.com/MinecraftMachina/lwjgl3/releases/download/3.3.1-mmachina.1/lwjgl-jemalloc-natives-macos-arm64.jar" } } }, @@ -700,9 +719,7 @@ ], "override": { "rules": [ - { - "action": "allow" - }, + { "action": "disallow", "os": { @@ -753,8 +770,7 @@ "natives-linux-arm64": { "sha1": "948e415b5b2a2c650c25b377a4a9f443b21ce92e", "size": 469432, - "url": - "https://github.com/theofficialgman/lwjgl3-binaries-arm64/raw/lwjgl-3.2.2/lwjgl-openal-natives-linux-arm64.jar" + "url": "https://github.com/theofficialgman/lwjgl3-binaries-arm64/raw/lwjgl-3.2.2/lwjgl-openal-natives-linux-arm64.jar" } } }, @@ -800,8 +816,7 @@ "natives-linux-arm32": { "sha1": "ecbc981fdd996492a1f6334f003ed62e5a8c0cd5", "size": 398418, - "url": - "https://github.com/theofficialgman/lwjgl3-binaries-arm32/raw/lwjgl-3.2.2/lwjgl-openal-natives-linux-arm32.jar" + "url": "https://github.com/theofficialgman/lwjgl3-binaries-arm32/raw/lwjgl-3.2.2/lwjgl-openal-natives-linux-arm32.jar" } } }, @@ -847,8 +862,7 @@ "natives-osx-arm64": { "sha1": "6b80fc0b982a0723b141e88859c42d6f71bd723f", "size": 346131, - "url": - "https://github.com/MinecraftMachina/lwjgl3/releases/download/3.3.1-mmachina.1/lwjgl-openal-natives-macos-arm64.jar" + "url": "https://github.com/MinecraftMachina/lwjgl3/releases/download/3.3.1-mmachina.1/lwjgl-openal-natives-macos-arm64.jar" } } }, @@ -877,9 +891,7 @@ ], "override": { "rules": [ - { - "action": "allow" - }, + { "action": "disallow", "os": { @@ -930,8 +942,7 @@ "natives-linux-arm64": { "sha1": "bd40897077bf7d12f562da898b18ac2c68e1f9d7", "size": 56109, - "url": - "https://github.com/theofficialgman/lwjgl3-binaries-arm64/raw/lwjgl-3.2.2/lwjgl-opengl-natives-linux-arm64.jar" + "url": "https://github.com/theofficialgman/lwjgl3-binaries-arm64/raw/lwjgl-3.2.2/lwjgl-opengl-natives-linux-arm64.jar" } } }, @@ -977,8 +988,7 @@ "natives-linux-arm32": { "sha1": "3af5599c74dd76dd8dbb567b3f9b4963a6abeed5", "size": 56388, - "url": - "https://github.com/theofficialgman/lwjgl3-binaries-arm32/raw/lwjgl-3.2.2/lwjgl-opengl-natives-linux-arm32.jar" + "url": "https://github.com/theofficialgman/lwjgl3-binaries-arm32/raw/lwjgl-3.2.2/lwjgl-opengl-natives-linux-arm32.jar" } } }, @@ -1024,8 +1034,7 @@ "natives-osx-arm64": { "sha1": "bb575058e0372f515587b5d2d04ff7db185f3ffe", "size": 41667, - "url": - "https://github.com/MinecraftMachina/lwjgl3/releases/download/3.3.1-mmachina.1/lwjgl-opengl-natives-macos-arm64.jar" + "url": "https://github.com/MinecraftMachina/lwjgl3/releases/download/3.3.1-mmachina.1/lwjgl-opengl-natives-macos-arm64.jar" } } }, @@ -1054,9 +1063,7 @@ ], "override": { "rules": [ - { - "action": "allow" - }, + { "action": "disallow", "os": { @@ -1107,8 +1114,7 @@ "natives-linux-arm64": { "sha1": "077efa7d7ea41b32df5c6078e912e724cccd06db", "size": 202038, - "url": - "https://github.com/theofficialgman/lwjgl3-binaries-arm64/raw/lwjgl-3.2.2/lwjgl-stb-natives-linux-arm64.jar" + "url": "https://github.com/theofficialgman/lwjgl3-binaries-arm64/raw/lwjgl-3.2.2/lwjgl-stb-natives-linux-arm64.jar" } } }, @@ -1154,8 +1160,7 @@ "natives-linux-arm32": { "sha1": "ec9d70aaebd0ff76dfeecf8f00b56118bf3706b1", "size": 149387, - "url": - "https://github.com/theofficialgman/lwjgl3-binaries-arm32/raw/lwjgl-3.2.2/lwjgl-stb-natives-linux-arm32.jar" + "url": "https://github.com/theofficialgman/lwjgl3-binaries-arm32/raw/lwjgl-3.2.2/lwjgl-stb-natives-linux-arm32.jar" } } }, @@ -1201,8 +1206,7 @@ "natives-osx-arm64": { "sha1": "98f0ad956c754723ef354d50057cc30417ef376a", "size": 178409, - "url": - "https://github.com/MinecraftMachina/lwjgl3/releases/download/3.3.1-mmachina.1/lwjgl-stb-natives-macos-arm64.jar" + "url": "https://github.com/MinecraftMachina/lwjgl3/releases/download/3.3.1-mmachina.1/lwjgl-stb-natives-macos-arm64.jar" } } }, @@ -1231,9 +1235,7 @@ ], "override": { "rules": [ - { - "action": "allow" - }, + { "action": "disallow", "os": { @@ -1284,8 +1286,7 @@ "natives-linux-arm64": { "sha1": "37c744ca289b5d7ae155d79e39029488b3254e5b", "size": 37893, - "url": - "https://github.com/theofficialgman/lwjgl3-binaries-arm64/raw/lwjgl-3.2.2/lwjgl-tinyfd-natives-linux-arm64.jar" + "url": "https://github.com/theofficialgman/lwjgl3-binaries-arm64/raw/lwjgl-3.2.2/lwjgl-tinyfd-natives-linux-arm64.jar" } } }, @@ -1331,8 +1332,7 @@ "natives-linux-arm32": { "sha1": "82d16054ada6633297a3108fb6d8bae98800c76f", "size": 41663, - "url": - "https://github.com/theofficialgman/lwjgl3-binaries-arm32/raw/lwjgl-3.2.2/lwjgl-tinyfd-natives-linux-arm32.jar" + "url": "https://github.com/theofficialgman/lwjgl3-binaries-arm32/raw/lwjgl-3.2.2/lwjgl-tinyfd-natives-linux-arm32.jar" } } }, @@ -1378,8 +1378,7 @@ "natives-osx-arm64": { "sha1": "015b931a2daba8f0c317d84c9d14e8e98ae56e0c", "size": 41384, - "url": - "https://github.com/MinecraftMachina/lwjgl3/releases/download/3.3.1-mmachina.1/lwjgl-tinyfd-natives-macos-arm64.jar" + "url": "https://github.com/MinecraftMachina/lwjgl3/releases/download/3.3.1-mmachina.1/lwjgl-tinyfd-natives-macos-arm64.jar" } } }, @@ -1408,9 +1407,7 @@ ], "override": { "rules": [ - { - "action": "allow" - }, + { "action": "disallow", "os": { @@ -1461,8 +1458,7 @@ "natives-linux-arm64": { "sha1": "612efd57d12b2e48e554858eb35e7e2eb46ebb4c", "size": 87121, - "url": - "https://github.com/theofficialgman/lwjgl3-binaries-arm64/raw/lwjgl-3.2.2/lwjgl-natives-linux-arm64.jar" + "url": "https://github.com/theofficialgman/lwjgl3-binaries-arm64/raw/lwjgl-3.2.2/lwjgl-natives-linux-arm64.jar" } } }, @@ -1508,8 +1504,7 @@ "natives-linux-arm32": { "sha1": "6bd0b37fef777a309936a72dc7f63126e8c79ea5", "size": 90296, - "url": - "https://github.com/theofficialgman/lwjgl3-binaries-arm32/raw/lwjgl-3.2.2/lwjgl-natives-linux-arm32.jar" + "url": "https://github.com/theofficialgman/lwjgl3-binaries-arm32/raw/lwjgl-3.2.2/lwjgl-natives-linux-arm32.jar" } } }, @@ -1555,8 +1550,7 @@ "natives-osx-arm64": { "sha1": "984df31fadaab86838877b112e5b4e4f68a00ccf", "size": 42693, - "url": - "https://github.com/MinecraftMachina/lwjgl3/releases/download/3.3.1-mmachina.1/lwjgl-natives-macos-arm64.jar" + "url": "https://github.com/MinecraftMachina/lwjgl3/releases/download/3.3.1-mmachina.1/lwjgl-natives-macos-arm64.jar" } } }, @@ -1672,8 +1666,7 @@ "artifact": { "sha1": "749be48a9b86ee2c3a2da5fd77511208adcfb33b", "size": 159993, - "url": - "https://github.com/theofficialgman/lwjgl3-binaries-arm64/raw/lwjgl-3.3.1/lwjgl-jemalloc-patched-natives-linux-arm64.jar" + "url": "https://github.com/theofficialgman/lwjgl3-binaries-arm64/raw/lwjgl-3.3.1/lwjgl-jemalloc-patched-natives-linux-arm64.jar" } }, "name": "org.lwjgl:lwjgl-jemalloc-natives-linux-arm64:3.3.1-gman64.1", @@ -2000,6 +1993,370 @@ } ] }, + { + "_comment": "Add linux-arm64 support for LWJGL 3.3.2", + "match": [ + "org.lwjgl:lwjgl-glfw:3.3.2" + ], + "additionalLibraries": [ + { + "downloads": { + "artifact": { + "sha1": "bc49e64bae0f7ff103a312ee8074a34c4eb034c7", + "size": 120168, + "url": "https://build.lwjgl.org/release/3.3.2/bin/lwjgl-glfw/lwjgl-glfw-natives-linux-arm64.jar" + } + }, + "name": "org.lwjgl:lwjgl-glfw-natives-linux-arm64:3.3.2-lwjgl.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm64" + } + } + ] + } + ] + }, + { + "_comment": "Add linux-arm64 support for LWJGL 3.3.2", + "match": [ + "org.lwjgl:lwjgl-jemalloc:3.3.2" + ], + "additionalLibraries": [ + { + "downloads": { + "artifact": { + "sha1": "5249f18a9ae20ea86c5816bc3107a888ce7a17d2", + "size": 206402, + "url": "https://build.lwjgl.org/release/3.3.2/bin/lwjgl-jemalloc/lwjgl-jemalloc-natives-linux-arm64.jar" + } + }, + "name": "org.lwjgl:lwjgl-jemalloc-natives-linux-arm64:3.3.2-lwjgl.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm64" + } + } + ] + } + ] + }, + { + "_comment": "Add linux-arm64 support for LWJGL 3.3.2", + "match": [ + "org.lwjgl:lwjgl-openal:3.3.2" + ], + "additionalLibraries": [ + { + "downloads": { + "artifact": { + "sha1": "22408980cc579709feaf9acb807992d3ebcf693f", + "size": 590865, + "url": "https://build.lwjgl.org/release/3.3.2/bin/lwjgl-openal/lwjgl-openal-natives-linux-arm64.jar" + } + }, + "name": "org.lwjgl:lwjgl-openal-natives-linux-arm64:3.3.2-lwjgl.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm64" + } + } + ] + } + ] + }, + { + "_comment": "Add linux-arm64 support for LWJGL 3.3.2", + "match": [ + "org.lwjgl:lwjgl-opengl:3.3.2" + ], + "additionalLibraries": [ + { + "downloads": { + "artifact": { + "sha1": "bb9eb56da6d1d549d6a767218e675e36bc568eb9", + "size": 58627, + "url": "https://build.lwjgl.org/release/3.3.2/bin/lwjgl-opengl/lwjgl-opengl-natives-linux-arm64.jar" + } + }, + "name": "org.lwjgl:lwjgl-opengl-natives-linux-arm64:3.3.2-lwjgl.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm64" + } + } + ] + } + ] + }, + { + "_comment": "Add linux-arm64 support for LWJGL 3.3.2", + "match": [ + "org.lwjgl:lwjgl-stb:3.3.2" + ], + "additionalLibraries": [ + { + "downloads": { + "artifact": { + "sha1": "11a380c37b0f03cb46db235e064528f84d736ff7", + "size": 207419, + "url": "https://build.lwjgl.org/release/3.3.2/bin/lwjgl-stb/lwjgl-stb-natives-linux-arm64.jar" + } + }, + "name": "org.lwjgl:lwjgl-stb-natives-linux-arm64:3.3.2-lwjgl.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm64" + } + } + ] + } + ] + }, + { + "_comment": "Add linux-arm64 support for LWJGL 3.3.2", + "match": [ + "org.lwjgl:lwjgl-tinyfd:3.3.2" + ], + "additionalLibraries": [ + { + "downloads": { + "artifact": { + "sha1": "93f8c5bc1984963cd79109891fb5a9d1e580373e", + "size": 43381, + "url": "https://build.lwjgl.org/release/3.3.2/bin/lwjgl-tinyfd/lwjgl-tinyfd-natives-linux-arm64.jar" + } + }, + "name": "org.lwjgl:lwjgl-tinyfd-natives-linux-arm64:3.3.2-lwjgl.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm64" + } + } + ] + } + ] + }, + { + "_comment": "Add linux-arm64 support for LWJGL 3.3.2", + "match": [ + "org.lwjgl:lwjgl:3.3.2" + ], + "additionalLibraries": [ + { + "downloads": { + "artifact": { + "sha1": "8bd89332c90a90e6bc4aa997a25c05b7db02c90a", + "size": 90795, + "url": "https://build.lwjgl.org/release/3.3.2/bin/lwjgl/lwjgl-natives-linux-arm64.jar" + } + }, + "name": "org.lwjgl:lwjgl-natives-linux-arm64:3.3.2-lwjgl.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm64" + } + } + ] + } + ] + }, + { + "_comment": "Add linux-arm32 support for LWJGL 3.3.2", + "match": [ + "org.lwjgl:lwjgl-glfw:3.3.2" + ], + "additionalLibraries": [ + { + "downloads": { + "artifact": { + "sha1": "5907d9a6b7c44fb0612a63bb1cff5992588f65be", + "size": 110067, + "url": "https://build.lwjgl.org/release/3.3.2/bin/lwjgl-glfw/lwjgl-glfw-natives-linux-arm32.jar" + } + }, + "name": "org.lwjgl:lwjgl-glfw-natives-linux-arm32:3.3.2-lwjgl.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm32" + } + } + ] + } + ] + }, + { + "_comment": "Add linux-arm32 support for LWJGL 3.3.2", + "match": [ + "org.lwjgl:lwjgl-jemalloc:3.3.2" + ], + "additionalLibraries": [ + { + "downloads": { + "artifact": { + "sha1": "9367437ce192e4d6f5725d53d85520644c0b0d6f", + "size": 177571, + "url": "https://build.lwjgl.org/release/3.3.2/bin/lwjgl-jemalloc/lwjgl-jemalloc-natives-linux-arm32.jar" + } + }, + "name": "org.lwjgl:lwjgl-jemalloc-natives-linux-arm32:3.3.2-lwjgl.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm32" + } + } + ] + } + ] + }, + { + "_comment": "Add linux-arm32 support for LWJGL 3.3.2", + "match": [ + "org.lwjgl:lwjgl-openal:3.3.2" + ], + "additionalLibraries": [ + { + "downloads": { + "artifact": { + "sha1": "7c82bbc33ef49ee4094b216c940db564b2998224", + "size": 503352, + "url": "https://build.lwjgl.org/release/3.3.2/bin/lwjgl-openal/lwjgl-openal-natives-linux-arm32.jar" + } + }, + "name": "org.lwjgl:lwjgl-openal-natives-linux-arm32:3.3.2-lwjgl.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm32" + } + } + ] + } + ] + }, + { + "_comment": "Add linux-arm32 support for LWJGL 3.3.2", + "match": [ + "org.lwjgl:lwjgl-opengl:3.3.2" + ], + "additionalLibraries": [ + { + "downloads": { + "artifact": { + "sha1": "821f9a2d1d583c44893f42b96f6977682b48a99b", + "size": 59265, + "url": "https://build.lwjgl.org/release/3.3.2/bin/lwjgl-opengl/lwjgl-opengl-natives-linux-arm32.jar" + } + }, + "name": "org.lwjgl:lwjgl-opengl-natives-linux-arm32:3.3.2-lwjgl.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm32" + } + } + ] + } + ] + }, + { + "_comment": "Add linux-arm32 support for LWJGL 3.3.2", + "match": [ + "org.lwjgl:lwjgl-stb:3.3.2" + ], + "additionalLibraries": [ + { + "downloads": { + "artifact": { + "sha1": "ca9333da184aade20757151f4615f1e27ca521ae", + "size": 154928, + "url": "https://build.lwjgl.org/release/3.3.2/bin/lwjgl-stb/lwjgl-stb-natives-linux-arm32.jar" + } + }, + "name": "org.lwjgl:lwjgl-stb-natives-linux-arm32:3.3.2-lwjgl.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm32" + } + } + ] + } + ] + }, + { + "_comment": "Add linux-arm32 support for LWJGL 3.3.2", + "match": [ + "org.lwjgl:lwjgl-tinyfd:3.3.2" + ], + "additionalLibraries": [ + { + "downloads": { + "artifact": { + "sha1": "807e220913aa0740449ff90d3b3d825cf5f359ed", + "size": 48788, + "url": "https://build.lwjgl.org/release/3.3.2/bin/lwjgl-tinyfd/lwjgl-tinyfd-natives-linux-arm32.jar" + } + }, + "name": "org.lwjgl:lwjgl-tinyfd-natives-linux-arm32:3.3.2-lwjgl.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm32" + } + } + ] + } + ] + }, + { + "_comment": "Add linux-arm32 support for LWJGL 3.3.2", + "match": [ + "org.lwjgl:lwjgl:3.3.2" + ], + "additionalLibraries": [ + { + "downloads": { + "artifact": { + "sha1": "afcbfaaa46f217e98a6da4208550f71de1f2a225", + "size": 89347, + "url": "https://build.lwjgl.org/release/3.3.2/bin/lwjgl/lwjgl-natives-linux-arm32.jar" + } + }, + "name": "org.lwjgl:lwjgl-natives-linux-arm32:3.3.2-lwjgl.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm32" + } + } + ] + } + ] + }, { "_comment": "Replace glfw from 3.3.1 with version from 3.3.2 to prevent stack smashing", "match": [ @@ -2016,5 +2373,4 @@ "name": "org.lwjgl:lwjgl-glfw-natives-linux:3.3.2-lwjgl.1" } } -] - +] \ No newline at end of file diff --git a/daedalus_client/src/fabric.rs b/daedalus_client/src/fabric.rs index 94d2488f4..c7e55fae7 100644 --- a/daedalus_client/src/fabric.rs +++ b/daedalus_client/src/fabric.rs @@ -31,8 +31,25 @@ pub async fn retrieve_data( { let mut loaders = loaders_mutex.write().await; - for loader in &list.loader { - loaders.push((Box::new(loader.stable), loader.version.clone())) + for (index, loader) in list.loader.iter().enumerate() { + if versions.iter().any(|x| { + x.id == DUMMY_REPLACE_STRING + && x.loaders.iter().any(|x| x.id == loader.version) + }) { + if index == 0 { + loaders.push(( + Box::new(loader.stable), + loader.version.clone(), + Box::new(true), + )) + } + } else { + loaders.push(( + Box::new(loader.stable), + loader.version.clone(), + Box::new(false), + )) + } } list.loader @@ -46,16 +63,7 @@ pub async fn retrieve_data( let loader_versions = futures::future::try_join_all( loaders_mutex.read().await.clone().into_iter().map( - |(stable, loader)| async { - { - if versions.iter().any(|x| { - x.id == DUMMY_REPLACE_STRING - && x.loaders.iter().any(|x| x.id == loader) - }) { - return Ok(None); - } - } - + |(stable, loader, skip_upload)| async { let version = fetch_fabric_version( DUMMY_GAME_VERSION, &loader, @@ -63,8 +71,8 @@ pub async fn retrieve_data( ) .await?; - Ok::, String, PartialVersionInfo)>, Error>( - Some((stable, loader, version)), + Ok::<(Box, String, PartialVersionInfo, Box), Error>( + (stable, loader, version, skip_upload), ) }, ), @@ -73,8 +81,8 @@ pub async fn retrieve_data( let visited_artifacts_mutex = Arc::new(Mutex::new(Vec::new())); futures::future::try_join_all(loader_versions.into_iter() - .flatten().map( - |(stable, loader, version)| async { + .map( + |(stable, loader, version, skip_upload)| async { let libs = futures::future::try_join_all( version.libraries.into_iter().map(|mut lib| async { { @@ -167,6 +175,13 @@ pub async fn retrieve_data( ) .await?; + if async move { + *skip_upload + }.await { + return Ok::<(), Error>(()) + } + + let version_path = format!( "fabric/v{}/versions/{}.json", daedalus::modded::CURRENT_FABRIC_FORMAT_VERSION, diff --git a/daedalus_client/src/main.rs b/daedalus_client/src/main.rs index e706fa078..af34ce3a8 100644 --- a/daedalus_client/src/main.rs +++ b/daedalus_client/src/main.rs @@ -147,26 +147,35 @@ fn check_env_vars() -> bool { } lazy_static::lazy_static! { - static ref CLIENT : Bucket = Bucket::new( - &dotenvy::var("S3_BUCKET_NAME").unwrap(), - if &*dotenvy::var("S3_REGION").unwrap() == "r2" { - Region::R2 { - account_id: dotenvy::var("S3_URL").unwrap(), - } + static ref CLIENT : Bucket = { + let region = dotenvy::var("S3_REGION").unwrap(); + let b = Bucket::new( + &dotenvy::var("S3_BUCKET_NAME").unwrap(), + if &*region == "r2" { + Region::R2 { + account_id: dotenvy::var("S3_URL").unwrap(), + } + } else { + Region::Custom { + region: region.clone(), + endpoint: dotenvy::var("S3_URL").unwrap(), + } + }, + Credentials::new( + Some(&*dotenvy::var("S3_ACCESS_TOKEN").unwrap()), + Some(&*dotenvy::var("S3_SECRET").unwrap()), + None, + None, + None, + ).unwrap(), + ).unwrap(); + + if region == "path-style" { + b.with_path_style() } else { - Region::Custom { - region: dotenvy::var("S3_REGION").unwrap(), - endpoint: dotenvy::var("S3_URL").unwrap(), - } - }, - Credentials::new( - Some(&*dotenvy::var("S3_ACCESS_TOKEN").unwrap()), - Some(&*dotenvy::var("S3_SECRET").unwrap()), - None, - None, - None, - ).unwrap(), - ).unwrap(); + b + } + }; } pub async fn upload_file_to_bucket( diff --git a/daedalus_client/src/minecraft.rs b/daedalus_client/src/minecraft.rs index e284c27c0..99ed51bc0 100644 --- a/daedalus_client/src/minecraft.rs +++ b/daedalus_client/src/minecraft.rs @@ -25,7 +25,8 @@ pub async fn retrieve_data( let mut manifest = daedalus::minecraft::fetch_version_manifest(None).await?; - let cloned_manifest = Arc::new(Mutex::new(old_manifest.clone().unwrap_or(manifest.clone()))); + let cloned_manifest = + Arc::new(Mutex::new(old_manifest.clone().unwrap_or(manifest.clone()))); let patches = fetch_library_patches()?; let cloned_patches = Arc::new(&patches); diff --git a/daedalus_client/src/neo.rs b/daedalus_client/src/neo.rs index 172a56676..fd4c6bb42 100644 --- a/daedalus_client/src/neo.rs +++ b/daedalus_client/src/neo.rs @@ -447,7 +447,7 @@ pub async fn fetch_maven_metadata( let parts: Vec<&str> = value.split('-').collect(); if parts.len() == 2 { map.entry(parts[0].to_string()) - .or_insert(Vec::new()) + .or_default() .push((original, parts[1].to_string())); } } diff --git a/daedalus_client/src/quilt.rs b/daedalus_client/src/quilt.rs index 827ad00e6..caa4a6741 100644 --- a/daedalus_client/src/quilt.rs +++ b/daedalus_client/src/quilt.rs @@ -31,8 +31,25 @@ pub async fn retrieve_data( { let mut loaders = loaders_mutex.write().await; - for loader in &list.loader { - loaders.push((Box::new(false), loader.version.clone())) + for (index, loader) in list.loader.iter().enumerate() { + if versions.iter().any(|x| { + x.id == DUMMY_REPLACE_STRING + && x.loaders.iter().any(|x| x.id == loader.version) + }) { + if index == 0 { + loaders.push(( + Box::new(false), + loader.version.clone(), + Box::new(true), + )) + } + } else { + loaders.push(( + Box::new(false), + loader.version.clone(), + Box::new(false), + )) + } } list.loader @@ -46,16 +63,7 @@ pub async fn retrieve_data( let loader_versions = futures::future::try_join_all( loaders_mutex.read().await.clone().into_iter().map( - |(stable, loader)| async { - { - if versions.iter().any(|x| { - x.id == DUMMY_REPLACE_STRING - && x.loaders.iter().any(|x| x.id == loader) - }) { - return Ok(None); - } - } - + |(stable, loader, skip_upload)| async { let version = fetch_quilt_version( DUMMY_GAME_VERSION, &loader, @@ -63,8 +71,8 @@ pub async fn retrieve_data( ) .await?; - Ok::, String, PartialVersionInfo)>, Error>( - Some((stable, loader, version)), + Ok::<(Box, String, PartialVersionInfo, Box), Error>( + (stable, loader, version, skip_upload), ) }, ), @@ -73,8 +81,8 @@ pub async fn retrieve_data( let visited_artifacts_mutex = Arc::new(Mutex::new(Vec::new())); futures::future::try_join_all(loader_versions.into_iter() - .flatten().map( - |(stable, loader, version)| async { + .map( + |(stable, loader, version, skip_upload)| async { let libs = futures::future::try_join_all( version.libraries.into_iter().map(|mut lib| async { { @@ -167,6 +175,12 @@ pub async fn retrieve_data( ) .await?; + if async move { + *skip_upload + }.await { + return Ok::<(), Error>(()) + } + let version_path = format!( "quilt/v{}/versions/{}.json", daedalus::modded::CURRENT_QUILT_FORMAT_VERSION, diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 000000000..4e74e5ad8 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,17 @@ +version: '3' + +services: + minio: + image: quay.io/minio/minio + volumes: + - minio-data:/data + ports: + - "9000:9000" + - "9001:9001" + environment: + MINIO_ROOT_USER: minioadmin + MINIO_ROOT_PASSWORD: miniosecret + command: server /data --console-address ":9001" + +volumes: + minio-data: \ No newline at end of file From 6fe1fa34556ce7e170f6e0cbeb71dd7500a58219 Mon Sep 17 00:00:00 2001 From: Geometrically <18202329+Geometrically@users.noreply.github.com> Date: Wed, 8 Nov 2023 17:57:37 -0700 Subject: [PATCH 59/85] Fix crash when MC updates (#14) * Fix crash when MC updates * Fix fabric ordering + add neoforge support * run clippy * run clippy --- daedalus/src/modded.rs | 19 +++++++- daedalus_client/src/fabric.rs | 5 +-- daedalus_client/src/minecraft.rs | 39 +++++++++++++---- daedalus_client/src/neo.rs | 75 +++++++++++++++++++++----------- daedalus_client/src/quilt.rs | 5 +-- 5 files changed, 100 insertions(+), 43 deletions(-) diff --git a/daedalus/src/modded.rs b/daedalus/src/modded.rs index 83c579ab5..d678c1e39 100644 --- a/daedalus/src/modded.rs +++ b/daedalus/src/modded.rs @@ -4,7 +4,7 @@ use crate::minecraft::{ Argument, ArgumentType, Library, VersionInfo, VersionType, }; use chrono::{DateTime, Utc}; -use serde::{Deserialize, Serialize}; +use serde::{Deserialize, Deserializer, Serialize}; use std::collections::HashMap; #[cfg(feature = "bincode")] @@ -32,6 +32,21 @@ pub struct SidedDataEntry { pub server: String, } +fn deserialize_date<'de, D>(deserializer: D) -> Result, D::Error> +where + D: Deserializer<'de>, +{ + let s = String::deserialize(deserializer)?; + + DateTime::parse_from_rfc3339(&s) + .map(|dt| dt.with_timezone(&Utc)) + .or_else(|_| { + DateTime::parse_from_str(&s, "%Y-%m-%dT%H:%M:%S%.f") + .map(|dt| dt.with_timezone(&Utc)) + }) + .map_err(serde::de::Error::custom) +} + #[cfg_attr(feature = "bincode", derive(Encode, Decode))] #[derive(Serialize, Deserialize, Debug)] #[serde(rename_all = "camelCase")] @@ -43,9 +58,11 @@ pub struct PartialVersionInfo { pub inherits_from: String, /// The time that the version was released #[cfg_attr(feature = "bincode", bincode(with_serde))] + #[serde(deserialize_with = "deserialize_date")] pub release_time: DateTime, /// The latest time a file in this version was updated #[cfg_attr(feature = "bincode", bincode(with_serde))] + #[serde(deserialize_with = "deserialize_date")] pub time: DateTime, #[serde(skip_serializing_if = "Option::is_none")] /// The classpath to the main class to launch the game diff --git a/daedalus_client/src/fabric.rs b/daedalus_client/src/fabric.rs index c7e55fae7..1a997da77 100644 --- a/daedalus_client/src/fabric.rs +++ b/daedalus_client/src/fabric.rs @@ -12,7 +12,7 @@ pub async fn retrieve_data( uploaded_files: &mut Vec, semaphore: Arc, ) -> Result<(), Error> { - let mut list = fetch_fabric_versions(None, semaphore.clone()).await?; + let list = fetch_fabric_versions(None, semaphore.clone()).await?; let old_manifest = daedalus::modded::fetch_manifest(&format_url(&format!( "fabric/v{}/manifest.json", daedalus::modded::CURRENT_FABRIC_FORMAT_VERSION, @@ -51,9 +51,6 @@ pub async fn retrieve_data( )) } } - - list.loader - .retain(|x| loaders.iter().any(|val| val.1 == x.version)) } const DUMMY_GAME_VERSION: &str = "1.19.4-rc2"; diff --git a/daedalus_client/src/minecraft.rs b/daedalus_client/src/minecraft.rs index 99ed51bc0..1f0e5b7cf 100644 --- a/daedalus_client/src/minecraft.rs +++ b/daedalus_client/src/minecraft.rs @@ -141,18 +141,39 @@ pub async fn retrieve_data( let mut cloned_manifest = cloned_manifest_mutex.lock().await; - let position = cloned_manifest + if let Some(position) = cloned_manifest .versions .iter() .position(|x| version.id == x.id) - .unwrap(); - cloned_manifest.versions[position].url = - format_url(&version_path); - cloned_manifest.versions[position].assets_index_sha1 = - Some(version_info.asset_index.sha1.clone()); - cloned_manifest.versions[position].assets_index_url = - Some(format_url(&assets_path)); - cloned_manifest.versions[position].sha1 = version_info_hash; + { + cloned_manifest.versions[position].url = + format_url(&version_path); + cloned_manifest.versions[position].assets_index_sha1 = + Some(version_info.asset_index.sha1.clone()); + cloned_manifest.versions[position].assets_index_url = + Some(format_url(&assets_path)); + cloned_manifest.versions[position].sha1 = + version_info_hash; + } else { + cloned_manifest.versions.insert( + 0, + daedalus::minecraft::Version { + id: version_info.id.clone(), + type_: version_info.type_.clone(), + url: format_url(&version_path), + time: version_info.time, + release_time: version_info.release_time, + sha1: version_info_hash, + compliance_level: 1, + assets_index_url: Some( + version_info.asset_index.sha1.clone(), + ), + assets_index_sha1: Some( + version_info.asset_index.sha1.clone(), + ), + }, + ) + } } let mut download_assets = false; diff --git a/daedalus_client/src/neo.rs b/daedalus_client/src/neo.rs index fd4c6bb42..ddc075643 100644 --- a/daedalus_client/src/neo.rs +++ b/daedalus_client/src/neo.rs @@ -17,7 +17,7 @@ pub async fn retrieve_data( uploaded_files: &mut Vec, semaphore: Arc, ) -> Result<(), Error> { - let maven_metadata = fetch_maven_metadata(None, semaphore.clone()).await?; + let maven_metadata = fetch_maven_metadata(semaphore.clone()).await?; let old_manifest = daedalus::modded::fetch_manifest(&format_url(&format!( "neo/v{}/manifest.json", daedalus::modded::CURRENT_NEOFORGE_FORMAT_VERSION, @@ -42,10 +42,10 @@ pub async fn retrieve_data( for (minecraft_version, loader_versions) in maven_metadata.clone() { let mut loaders = Vec::new(); - for (full, loader_version) in loader_versions { + for (full, loader_version, new_forge) in loader_versions { let version = Version::parse(&loader_version)?; - loaders.push((full, version)) + loaders.push((full, version, new_forge.to_string())) } if !loaders.is_empty() { @@ -53,7 +53,7 @@ pub async fn retrieve_data( let mut loaders_versions = Vec::new(); { - let loaders_futures = loaders.into_iter().map(|(loader_version_full, _)| async { + let loaders_futures = loaders.into_iter().map(|(loader_version_full, _, new_forge)| async { let versions_mutex = Arc::clone(&old_versions); let visited_assets = Arc::clone(&visited_assets_mutex); let uploaded_files_mutex = Arc::clone(&uploaded_files_mutex); @@ -72,13 +72,13 @@ pub async fn retrieve_data( } info!("Forge - Installer Start {}", loader_version_full.clone()); - let bytes = download_file(&format!("https://maven.neoforged.net/net/neoforged/forge/{0}/forge-{0}-installer.jar", loader_version_full), None, semaphore.clone()).await?; + let bytes = download_file(&format!("https://maven.neoforged.net/net/neoforged/{1}/{0}/{1}-{0}-installer.jar", loader_version_full, if &*new_forge == "true" { "neoforge" } else { "forge" }), None, semaphore.clone()).await?; let reader = std::io::Cursor::new(bytes); if let Ok(archive) = zip::ZipArchive::new(reader) { let mut archive_clone = archive.clone(); - let mut profile = tokio::task::spawn_blocking(move || { + let mut profile = tokio::task::spawn_blocking(move || { let mut install_profile = archive_clone.by_name("install_profile.json")?; let mut contents = String::new(); @@ -404,8 +404,10 @@ pub async fn retrieve_data( Ok(()) } -const DEFAULT_MAVEN_METADATA_URL: &str = +const DEFAULT_MAVEN_METADATA_URL_1: &str = "https://maven.neoforged.net/net/neoforged/forge/maven-metadata.xml"; +const DEFAULT_MAVEN_METADATA_URL_2: &str = + "https://maven.neoforged.net/net/neoforged/neoforge/maven-metadata.xml"; #[derive(Debug, Deserialize)] struct Metadata { @@ -423,32 +425,55 @@ struct Versions { } pub async fn fetch_maven_metadata( - url: Option<&str>, semaphore: Arc, -) -> Result>, Error> { - let values = serde_xml_rs::from_str::( - &String::from_utf8( - download_file( - url.unwrap_or(DEFAULT_MAVEN_METADATA_URL), - None, - semaphore, +) -> Result>, Error> { + async fn fetch_values( + url: &str, + semaphore: Arc, + ) -> Result { + Ok(serde_xml_rs::from_str( + &String::from_utf8( + download_file(url, None, semaphore).await?.to_vec(), ) - .await? - .to_vec(), - ) - .unwrap_or_default(), - )?; + .unwrap_or_default(), + )?) + } + + let forge_values = + fetch_values(DEFAULT_MAVEN_METADATA_URL_1, semaphore.clone()).await?; + let neo_values = + fetch_values(DEFAULT_MAVEN_METADATA_URL_2, semaphore).await?; - let mut map: HashMap> = HashMap::new(); + let mut map: HashMap> = HashMap::new(); - for value in values.versioning.versions.version { + for value in forge_values.versioning.versions.version { let original = value.clone(); let parts: Vec<&str> = value.split('-').collect(); if parts.len() == 2 { - map.entry(parts[0].to_string()) - .or_default() - .push((original, parts[1].to_string())); + map.entry(parts[0].to_string()).or_default().push(( + original, + parts[1].to_string(), + false, + )); + } + } + + for value in neo_values.versioning.versions.version { + let original = value.clone(); + + let mut parts = value.split('.'); + + if let Some(major) = parts.next() { + if let Some(minor) = parts.next() { + let game_version = format!("1.{}.{}", major, minor); + + map.entry(game_version.clone()).or_default().push(( + original.clone(), + format!("{}-{}", game_version, original), + true, + )); + } } } diff --git a/daedalus_client/src/quilt.rs b/daedalus_client/src/quilt.rs index caa4a6741..cd0df2da2 100644 --- a/daedalus_client/src/quilt.rs +++ b/daedalus_client/src/quilt.rs @@ -12,7 +12,7 @@ pub async fn retrieve_data( uploaded_files: &mut Vec, semaphore: Arc, ) -> Result<(), Error> { - let mut list = fetch_quilt_versions(None, semaphore.clone()).await?; + let list = fetch_quilt_versions(None, semaphore.clone()).await?; let old_manifest = daedalus::modded::fetch_manifest(&format_url(&format!( "quilt/v{}/manifest.json", daedalus::modded::CURRENT_QUILT_FORMAT_VERSION, @@ -51,9 +51,6 @@ pub async fn retrieve_data( )) } } - - list.loader - .retain(|x| loaders.iter().any(|val| val.1 == x.version)) } const DUMMY_GAME_VERSION: &str = "1.19.4-rc2"; From ac07ac52348a6257470d7ed182cb65ab2b16a1a5 Mon Sep 17 00:00:00 2001 From: Geometrically <18202329+Geometrically@users.noreply.github.com> Date: Wed, 15 Nov 2023 16:35:33 -0700 Subject: [PATCH 60/85] Fix date parsing for partial versions (#15) --- daedalus/Cargo.toml | 2 +- daedalus/src/modded.rs | 10 +++------- daedalus_client/Cargo.toml | 2 +- 3 files changed, 5 insertions(+), 9 deletions(-) diff --git a/daedalus/Cargo.toml b/daedalus/Cargo.toml index 6aff4dd4d..5c20d6abb 100644 --- a/daedalus/Cargo.toml +++ b/daedalus/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "daedalus" -version = "0.1.26" +version = "0.1.27" authors = ["Jai A "] edition = "2018" license = "MIT" diff --git a/daedalus/src/modded.rs b/daedalus/src/modded.rs index d678c1e39..e72eafead 100644 --- a/daedalus/src/modded.rs +++ b/daedalus/src/modded.rs @@ -3,7 +3,7 @@ use crate::{download_file, Error}; use crate::minecraft::{ Argument, ArgumentType, Library, VersionInfo, VersionType, }; -use chrono::{DateTime, Utc}; +use chrono::{DateTime, TimeZone, Utc}; use serde::{Deserialize, Deserializer, Serialize}; use std::collections::HashMap; @@ -38,12 +38,8 @@ where { let s = String::deserialize(deserializer)?; - DateTime::parse_from_rfc3339(&s) - .map(|dt| dt.with_timezone(&Utc)) - .or_else(|_| { - DateTime::parse_from_str(&s, "%Y-%m-%dT%H:%M:%S%.f") - .map(|dt| dt.with_timezone(&Utc)) - }) + serde_json::from_str::>(&format!("\"{s}\"")) + .or_else(|_| Utc.datetime_from_str(&s, "%Y-%m-%dT%H:%M:%S%.9f")) .map_err(serde::de::Error::custom) } diff --git a/daedalus_client/Cargo.toml b/daedalus_client/Cargo.toml index 87b3d4171..47f8aa9af 100644 --- a/daedalus_client/Cargo.toml +++ b/daedalus_client/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "daedalus_client" -version = "0.1.26" +version = "0.1.27" authors = ["Jai A "] edition = "2018" From 8b16cd1b363b525c17bc9e93e4393389c3677492 Mon Sep 17 00:00:00 2001 From: Geometrically <18202329+Geometrically@users.noreply.github.com> Date: Tue, 25 Jun 2024 15:47:27 -0700 Subject: [PATCH 61/85] Daedalus Rewrite + Code Cleanup (#16) * [wip] rewrite daedalus, vanilla, fabric, and quilt * finish forge + neo * fix docker * fix neoforge 1.21+ * update concurrency limit * finish * remove mac garb --- .env | 16 +- .github/workflows/run.yml | 49 + .gitignore | 1 + .idea/daedalus.iml | 1 + Dockerfile | 2 +- daedalus/Cargo.toml | 10 +- daedalus/src/lib.rs | 121 --- daedalus/src/minecraft.rs | 75 +- daedalus/src/modded.rs | 38 +- daedalus_client/Cargo.toml | 25 +- daedalus_client/library-patches.json | 543 ++++++++++- daedalus_client/src/error.rs | 63 ++ daedalus_client/src/fabric.rs | 562 +++++------ daedalus_client/src/forge.rs | 1312 ++++++++++++++------------ daedalus_client/src/main.rs | 337 +++---- daedalus_client/src/minecraft.rs | 428 ++++----- daedalus_client/src/neo.rs | 495 ---------- daedalus_client/src/quilt.rs | 370 -------- daedalus_client/src/util.rs | 369 ++++++++ 19 files changed, 2311 insertions(+), 2506 deletions(-) create mode 100644 .github/workflows/run.yml create mode 100644 daedalus_client/src/error.rs delete mode 100644 daedalus_client/src/neo.rs delete mode 100644 daedalus_client/src/quilt.rs create mode 100644 daedalus_client/src/util.rs diff --git a/.env b/.env index bbcfb5cbf..9aac220ad 100644 --- a/.env +++ b/.env @@ -1,9 +1,15 @@ -RUST_LOG=info +RUST_LOG=warn,daedalus_client=trace -BASE_URL=https://modrinth-cdn-staging.nyc3.digitaloceanspaces.com +BASE_URL=http://localhost:9000/meta + +CONCURRENCY_LIMIT=10 S3_ACCESS_TOKEN=none S3_SECRET=none -S3_URL=none -S3_REGION=none -S3_BUCKET_NAME=none +S3_URL=http://localhost:9000 +S3_REGION=path-style +S3_BUCKET_NAME=meta + +CLOUDFLARE_INTEGRATION=false +CLOUDFLARE_TOKEN=none +CLOUDFLARE_ZONE_ID=none \ No newline at end of file diff --git a/.github/workflows/run.yml b/.github/workflows/run.yml new file mode 100644 index 000000000..b97d2bdde --- /dev/null +++ b/.github/workflows/run.yml @@ -0,0 +1,49 @@ +name: Run Meta + +on: + schedule: + - cron: '*/5 * * * *' + +jobs: + run-docker: + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v3 + + - name: Log in to GitHub Container Registry + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + + - name: Pull Docker image from GHCR + run: docker pull ghcr.io/modrinth/daedalus:latest + + - name: Run Docker container + env: + BASE_URL: ${{ secrets.BASE_URL }} + S3_ACCESS_TOKEN: ${{ secrets.S3_ACCESS_TOKEN }} + S3_SECRET: ${{ secrets.S3_SECRET }} + S3_URL: ${{ secrets.S3_URL }} + S3_REGION: ${{ secrets.S3_REGION }} + S3_BUCKET_NAME: ${{ secrets.S3_BUCKET_NAME }} + CLOUDFLARE_INTEGRATION: ${{ secrets.CLOUDFLARE_INTEGRATION }} + CLOUDFLARE_TOKEN: ${{ secrets.CLOUDFLARE_TOKEN }} + CLOUDFLARE_ZONE_ID: ${{ secrets.CLOUDFLARE_ZONE_ID }} + run: | + docker run -d \ + --name daedalus \ + -e BASE_URL=$BASE_URL \ + -e S3_ACCESS_TOKEN=$S3_ACCESS_TOKEN \ + -e S3_SECRET=$S3_SECRET \ + -e S3_URL=$S3_URL \ + -e S3_REGION=$S3_REGION \ + -e S3_BUCKET_NAME=$S3_BUCKET_NAME \ + -e CLOUDFLARE_INTEGRATION=$CLOUDFLARE_INTEGRATION \ + -e CLOUDFLARE_TOKEN=$CLOUDFLARE_TOKEN \ + -e CLOUDFLARE_ZONE_ID=$CLOUDFLARE_ZONE_ID \ + ghcr.io/modrinth/daedalus:latest \ No newline at end of file diff --git a/.gitignore b/.gitignore index dae6eff4c..06b153acc 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ ### Intellij ### # Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio, WebStorm and Rider # Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 +caches/ # User-specific stuff .idea/**/workspace.xml diff --git a/.idea/daedalus.iml b/.idea/daedalus.iml index ec7bb0139..cc8620ac8 100644 --- a/.idea/daedalus.iml +++ b/.idea/daedalus.iml @@ -6,6 +6,7 @@ + diff --git a/Dockerfile b/Dockerfile index a2a341da0..c015f6748 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM rust:1.68.2 as build +FROM rust:1.79.0 as build ENV PKG_CONFIG_ALLOW_CROSS=1 WORKDIR /usr/src/daedalus diff --git a/daedalus/Cargo.toml b/daedalus/Cargo.toml index 5c20d6abb..1acc5253b 100644 --- a/daedalus/Cargo.toml +++ b/daedalus/Cargo.toml @@ -1,8 +1,8 @@ [package] name = "daedalus" -version = "0.1.27" -authors = ["Jai A "] -edition = "2018" +version = "0.2.0" +authors = ["Jai A "] +edition = "2021" license = "MIT" description = "Utilities for querying and parsing Minecraft metadata" repository = "https://github.com/modrinth/daedalus/" @@ -14,12 +14,8 @@ readme = "README.md" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -reqwest = { version = "0.11", features = ["json"] } serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" chrono = { version = "0.4", features = ["serde"] } bytes = "1" thiserror = "1.0" -tokio = { version = "1", features = ["full"] } -sha1 = { version = "0.6.1", features = ["std"]} -bincode = {version = "2.0.0-rc.2", features = ["serde"], optional = true} diff --git a/daedalus/src/lib.rs b/daedalus/src/lib.rs index 51221eede..790088890 100644 --- a/daedalus/src/lib.rs +++ b/daedalus/src/lib.rs @@ -12,30 +12,6 @@ pub mod modded; #[derive(thiserror::Error, Debug)] /// An error type representing possible errors when fetching metadata pub enum Error { - #[error("Failed to validate file checksum at url {url} with hash {hash} after {tries} tries")] - /// A checksum was failed to validate for a file - ChecksumFailure { - /// The checksum's hash - hash: String, - /// The URL of the file attempted to be downloaded - url: String, - /// The amount of tries that the file was downloaded until failure - tries: u32, - }, - /// There was an error while deserializing metadata - #[error("Error while deserializing JSON")] - SerdeError(#[from] serde_json::Error), - /// There was a network error when fetching an object - #[error("Unable to fetch {item}")] - FetchError { - /// The internal reqwest error - inner: reqwest::Error, - /// The item that was failed to be fetched - item: String, - }, - /// There was an error when managing async tasks - #[error("Error while managing asynchronous tasks")] - TaskError(#[from] tokio::task::JoinError), /// Error while parsing input #[error("{0}")] ParseError(String), @@ -124,100 +100,3 @@ pub fn get_path_from_artifact(artifact: &str) -> Result { )) } } - -/// Downloads a file from specified mirrors -pub async fn download_file_mirrors( - base: &str, - mirrors: &[&str], - sha1: Option<&str>, -) -> Result { - if mirrors.is_empty() { - return Err(Error::ParseError("No mirrors provided!".to_string())); - } - - for (index, mirror) in mirrors.iter().enumerate() { - let result = download_file(&format!("{}{}", mirror, base), sha1).await; - - if result.is_ok() || (result.is_err() && index == (mirrors.len() - 1)) { - return result; - } - } - - unreachable!() -} - -/// Downloads a file with retry and checksum functionality -pub async fn download_file( - url: &str, - sha1: Option<&str>, -) -> Result { - let mut headers = reqwest::header::HeaderMap::new(); - if let Ok(header) = reqwest::header::HeaderValue::from_str(&format!( - "modrinth/daedalus/{} (support@modrinth.com)", - env!("CARGO_PKG_VERSION") - )) { - headers.insert(reqwest::header::USER_AGENT, header); - } - let client = reqwest::Client::builder() - .tcp_keepalive(Some(std::time::Duration::from_secs(10))) - .timeout(std::time::Duration::from_secs(15)) - .default_headers(headers) - .build() - .map_err(|err| Error::FetchError { - inner: err, - item: url.to_string(), - })?; - - for attempt in 1..=4 { - let result = client.get(url).send().await; - - match result { - Ok(x) => { - let bytes = x.bytes().await; - - if let Ok(bytes) = bytes { - if let Some(sha1) = sha1 { - if &*get_hash(bytes.clone()).await? != sha1 { - if attempt <= 3 { - continue; - } else { - return Err(Error::ChecksumFailure { - hash: sha1.to_string(), - url: url.to_string(), - tries: attempt, - }); - } - } - } - - return Ok(bytes); - } else if attempt <= 3 { - continue; - } else if let Err(err) = bytes { - return Err(Error::FetchError { - inner: err, - item: url.to_string(), - }); - } - } - Err(_) if attempt <= 3 => continue, - Err(err) => { - return Err(Error::FetchError { - inner: err, - item: url.to_string(), - }) - } - } - } - - unreachable!() -} - -/// Computes a checksum of the input bytes -pub async fn get_hash(bytes: bytes::Bytes) -> Result { - let hash = - tokio::task::spawn_blocking(|| sha1::Sha1::from(bytes).hexdigest()) - .await?; - - Ok(hash) -} diff --git a/daedalus/src/minecraft.rs b/daedalus/src/minecraft.rs index 747fa8a06..021c91ab7 100644 --- a/daedalus/src/minecraft.rs +++ b/daedalus/src/minecraft.rs @@ -1,16 +1,11 @@ use crate::modded::{Processor, SidedDataEntry}; -use crate::{download_file, Error}; use chrono::{DateTime, Utc}; use serde::{Deserialize, Serialize}; use std::collections::HashMap; -#[cfg(feature = "bincode")] -use bincode::{Decode, Encode}; - /// The latest version of the format the model structs deserialize to pub const CURRENT_FORMAT_VERSION: usize = 0; -#[cfg_attr(feature = "bincode", derive(Encode, Decode))] #[derive(Serialize, Deserialize, Debug, Clone)] #[serde(rename_all = "snake_case")] /// The version type @@ -37,7 +32,6 @@ impl VersionType { } } -#[cfg_attr(feature = "bincode", derive(Encode, Decode))] #[derive(Serialize, Deserialize, Debug, Clone)] #[serde(rename_all = "camelCase")] /// A game version of Minecraft @@ -50,26 +44,18 @@ pub struct Version { /// A link to additional information about the version pub url: String, /// The latest time a file in this version was updated - #[cfg_attr(feature = "bincode", bincode(with_serde))] pub time: DateTime, /// The time this version was released - #[cfg_attr(feature = "bincode", bincode(with_serde))] pub release_time: DateTime, /// The SHA1 hash of the additional information about the version pub sha1: String, /// Whether the version supports the latest player safety features pub compliance_level: u32, #[serde(skip_serializing_if = "Option::is_none")] - /// (Modrinth Provided) The link to the assets index for this version - /// This is only available when using the Modrinth mirror - pub assets_index_url: Option, - #[serde(skip_serializing_if = "Option::is_none")] - /// (Modrinth Provided) The SHA1 hash of the assets index for this version - /// This is only available when using the Modrinth mirror - pub assets_index_sha1: Option, + /// (Modrinth Provided) The SHA1 hash of the original unmodified Minecraft versions JSON + pub original_sha1: Option, } -#[cfg_attr(feature = "bincode", derive(Encode, Decode))] #[derive(Serialize, Deserialize, Debug, Clone)] /// The latest snapshot and release of the game pub struct LatestVersion { @@ -79,7 +65,6 @@ pub struct LatestVersion { pub snapshot: String, } -#[cfg_attr(feature = "bincode", derive(Encode, Decode))] #[derive(Serialize, Deserialize, Debug, Clone)] /// Data of all game versions of Minecraft pub struct VersionManifest { @@ -93,16 +78,6 @@ pub struct VersionManifest { pub const VERSION_MANIFEST_URL: &str = "https://piston-meta.mojang.com/mc/game/version_manifest_v2.json"; -/// Fetches a version manifest from the specified URL. If no URL is specified, the default is used. -pub async fn fetch_version_manifest( - url: Option<&str>, -) -> Result { - Ok(serde_json::from_slice( - &download_file(url.unwrap_or(VERSION_MANIFEST_URL), None).await?, - )?) -} - -#[cfg_attr(feature = "bincode", derive(Encode, Decode))] #[derive(Serialize, Deserialize, Debug)] #[serde(rename_all = "camelCase")] /// Information about the assets of the game @@ -119,7 +94,6 @@ pub struct AssetIndex { pub url: String, } -#[cfg_attr(feature = "bincode", derive(Encode, Decode))] #[derive(Serialize, Deserialize, Debug, Eq, PartialEq, Hash)] #[serde(rename_all = "snake_case")] /// The type of download @@ -136,7 +110,6 @@ pub enum DownloadType { WindowsServer, } -#[cfg_attr(feature = "bincode", derive(Encode, Decode))] #[derive(Serialize, Deserialize, Debug)] /// Download information of a file pub struct Download { @@ -148,7 +121,6 @@ pub struct Download { pub url: String, } -#[cfg_attr(feature = "bincode", derive(Encode, Decode))] #[derive(Serialize, Deserialize, Debug, Clone)] /// Download information of a library pub struct LibraryDownload { @@ -163,7 +135,6 @@ pub struct LibraryDownload { pub url: String, } -#[cfg_attr(feature = "bincode", derive(Encode, Decode))] #[derive(Serialize, Deserialize, Debug, Clone)] /// A list of files that should be downloaded for libraries pub struct LibraryDownloads { @@ -176,7 +147,6 @@ pub struct LibraryDownloads { pub classifiers: Option>, } -#[cfg_attr(feature = "bincode", derive(Encode, Decode))] #[derive(Serialize, Deserialize, Debug, Clone)] #[serde(rename_all = "snake_case")] /// The action a rule can follow @@ -187,7 +157,6 @@ pub enum RuleAction { Disallow, } -#[cfg_attr(feature = "bincode", derive(Encode, Decode))] #[derive(Serialize, Deserialize, Debug, Eq, PartialEq, Hash, Clone)] #[serde(rename_all = "kebab-case")] /// An enum representing the different types of operating systems @@ -210,7 +179,6 @@ pub enum Os { Unknown, } -#[cfg_attr(feature = "bincode", derive(Encode, Decode))] #[derive(Serialize, Deserialize, Debug, Clone)] /// A rule which depends on what OS the user is on pub struct OsRule { @@ -225,7 +193,6 @@ pub struct OsRule { pub arch: Option, } -#[cfg_attr(feature = "bincode", derive(Encode, Decode))] #[derive(Serialize, Deserialize, Debug, Clone)] /// A rule which depends on the toggled features of the launcher pub struct FeatureRule { @@ -248,7 +215,6 @@ pub struct FeatureRule { pub is_quick_play_realms: Option, } -#[cfg_attr(feature = "bincode", derive(Encode, Decode))] #[derive(Serialize, Deserialize, Debug, Clone)] /// A rule deciding whether a file is downloaded, an argument is used, etc. pub struct Rule { @@ -262,7 +228,6 @@ pub struct Rule { pub features: Option, } -#[cfg_attr(feature = "bincode", derive(Encode, Decode))] #[derive(Serialize, Deserialize, Debug, Clone)] /// Information delegating the extraction of the library pub struct LibraryExtract { @@ -271,7 +236,6 @@ pub struct LibraryExtract { pub exclude: Option>, } -#[cfg_attr(feature = "bincode", derive(Encode, Decode))] #[derive(Serialize, Deserialize, Debug)] #[serde(rename_all = "camelCase")] /// Information about the java version the game needs @@ -282,7 +246,6 @@ pub struct JavaVersion { pub major_version: u32, } -#[cfg_attr(feature = "bincode", derive(Encode, Decode))] #[derive(Serialize, Deserialize, Debug, Clone)] /// A library which the game relies on to run pub struct Library { @@ -309,6 +272,9 @@ pub struct Library { #[serde(default = "default_include_in_classpath")] /// Whether the library should be included in the classpath at the game's launch pub include_in_classpath: bool, + #[serde(default = "default_downloadable")] + /// Whether the library should be downloaded + pub downloadable: bool, } #[derive(Deserialize, Debug, Clone)] @@ -397,8 +363,10 @@ pub fn merge_partial_library( fn default_include_in_classpath() -> bool { true } +fn default_downloadable() -> bool { + true +} -#[cfg_attr(feature = "bincode", derive(Encode, Decode))] #[derive(Serialize, Deserialize, Debug, Clone)] #[serde(untagged)] /// A container for an argument or multiple arguments @@ -409,7 +377,6 @@ pub enum ArgumentValue { Many(Vec), } -#[cfg_attr(feature = "bincode", derive(Encode, Decode))] #[derive(Serialize, Deserialize, Debug, Clone)] #[serde(untagged)] /// A command line argument passed to a program @@ -425,7 +392,6 @@ pub enum Argument { }, } -#[cfg_attr(feature = "bincode", derive(Encode, Decode))] #[derive(Serialize, Deserialize, Debug, Eq, PartialEq, Hash, Clone, Copy)] #[serde(rename_all = "snake_case")] /// The type of argument @@ -436,7 +402,6 @@ pub enum ArgumentType { Jvm, } -#[cfg_attr(feature = "bincode", derive(Encode, Decode))] #[derive(Serialize, Deserialize, Debug)] #[serde(rename_all = "camelCase")] /// Information about a version @@ -481,16 +446,6 @@ pub struct VersionInfo { pub processors: Option>, } -/// Fetches detailed information about a version from the manifest -pub async fn fetch_version_info( - version: &Version, -) -> Result { - Ok(serde_json::from_slice( - &download_file(&version.url, Some(&version.sha1)).await?, - )?) -} - -#[cfg_attr(feature = "bincode", derive(Encode, Decode))] #[derive(Serialize, Deserialize, Debug)] /// An asset of the game pub struct Asset { @@ -500,23 +455,9 @@ pub struct Asset { pub size: u32, } -#[cfg_attr(feature = "bincode", derive(Encode, Decode))] #[derive(Serialize, Deserialize, Debug)] /// An index containing all assets the game needs pub struct AssetsIndex { /// A hashmap containing the filename (key) and asset (value) pub objects: HashMap, } - -/// Fetches the assets index from the version info -pub async fn fetch_assets_index( - version: &VersionInfo, -) -> Result { - Ok(serde_json::from_slice( - &download_file( - &version.asset_index.url, - Some(&version.asset_index.sha1), - ) - .await?, - )?) -} diff --git a/daedalus/src/modded.rs b/daedalus/src/modded.rs index e72eafead..c510fe78b 100644 --- a/daedalus/src/modded.rs +++ b/daedalus/src/modded.rs @@ -1,5 +1,3 @@ -use crate::{download_file, Error}; - use crate::minecraft::{ Argument, ArgumentType, Library, VersionInfo, VersionType, }; @@ -7,9 +5,6 @@ use chrono::{DateTime, TimeZone, Utc}; use serde::{Deserialize, Deserializer, Serialize}; use std::collections::HashMap; -#[cfg(feature = "bincode")] -use bincode::{Decode, Encode}; - /// The latest version of the format the fabric model structs deserialize to pub const CURRENT_FABRIC_FORMAT_VERSION: usize = 0; /// The latest version of the format the fabric model structs deserialize to @@ -23,7 +18,6 @@ pub const CURRENT_NEOFORGE_FORMAT_VERSION: usize = 0; pub const DUMMY_REPLACE_STRING: &str = "${modrinth.gameVersion}"; /// A data variable entry that depends on the side of the installation -#[cfg_attr(feature = "bincode", derive(Encode, Decode))] #[derive(Serialize, Deserialize, Debug)] pub struct SidedDataEntry { /// The value on the client @@ -43,7 +37,6 @@ where .map_err(serde::de::Error::custom) } -#[cfg_attr(feature = "bincode", derive(Encode, Decode))] #[derive(Serialize, Deserialize, Debug)] #[serde(rename_all = "camelCase")] /// A partial version returned by fabric meta @@ -53,11 +46,9 @@ pub struct PartialVersionInfo { /// The version ID this partial version inherits from pub inherits_from: String, /// The time that the version was released - #[cfg_attr(feature = "bincode", bincode(with_serde))] #[serde(deserialize_with = "deserialize_date")] pub release_time: DateTime, /// The latest time a file in this version was updated - #[cfg_attr(feature = "bincode", bincode(with_serde))] #[serde(deserialize_with = "deserialize_date")] pub time: DateTime, #[serde(skip_serializing_if = "Option::is_none")] @@ -83,7 +74,6 @@ pub struct PartialVersionInfo { } /// A processor to be ran after downloading the files -#[cfg_attr(feature = "bincode", derive(Encode, Decode))] #[derive(Serialize, Deserialize, Debug)] pub struct Processor { /// Maven coordinates for the JAR library of this processor. @@ -101,13 +91,6 @@ pub struct Processor { pub sides: Option>, } -/// Fetches the version manifest of a game version's URL -pub async fn fetch_partial_version( - url: &str, -) -> Result { - Ok(serde_json::from_slice(&download_file(url, None).await?)?) -} - /// Merges a partial version into a complete one pub fn merge_partial_version( partial: PartialVersionInfo, @@ -154,15 +137,10 @@ pub fn merge_partial_version( .libraries .into_iter() .chain(merge.libraries) - .map(|x| Library { - downloads: x.downloads, - extract: x.extract, - name: x.name.replace(DUMMY_REPLACE_STRING, &merge_id), - url: x.url, - natives: x.natives, - rules: x.rules, - checksums: x.checksums, - include_in_classpath: x.include_in_classpath, + .map(|mut x| { + x.name = x.name.replace(DUMMY_REPLACE_STRING, &merge_id); + + x }) .collect::>(), main_class: if let Some(main_class) = partial.main_class { @@ -180,7 +158,6 @@ pub fn merge_partial_version( } } -#[cfg_attr(feature = "bincode", derive(Encode, Decode))] #[derive(Serialize, Deserialize, Debug, Clone)] #[serde(rename_all = "camelCase")] /// A manifest containing information about a mod loader's versions @@ -189,7 +166,6 @@ pub struct Manifest { pub game_versions: Vec, } -#[cfg_attr(feature = "bincode", derive(Encode, Decode))] #[derive(Serialize, Deserialize, Debug, Clone)] /// A game version of Minecraft pub struct Version { @@ -201,7 +177,6 @@ pub struct Version { pub loaders: Vec, } -#[cfg_attr(feature = "bincode", derive(Encode, Decode))] #[derive(Serialize, Deserialize, Debug, Clone)] /// A version of a Minecraft mod loader pub struct LoaderVersion { @@ -212,8 +187,3 @@ pub struct LoaderVersion { /// Whether the loader is stable or not pub stable: bool, } - -/// Fetches the manifest of a mod loader -pub async fn fetch_manifest(url: &str) -> Result { - Ok(serde_json::from_slice(&download_file(url, None).await?)?) -} diff --git a/daedalus_client/Cargo.toml b/daedalus_client/Cargo.toml index 47f8aa9af..ce8204fac 100644 --- a/daedalus_client/Cargo.toml +++ b/daedalus_client/Cargo.toml @@ -1,8 +1,8 @@ [package] name = "daedalus_client" -version = "0.1.27" -authors = ["Jai A "] -edition = "2018" +version = "0.2.0" +authors = ["Jai A "] +edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html @@ -11,16 +11,23 @@ daedalus = { path = "../daedalus" } tokio = { version = "1", features = ["full"] } futures = "0.3.25" dotenvy = "0.15.6" -log = "0.4.17" -env_logger= "0.10.0" serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" serde-xml-rs = "0.6.0" lazy_static = "1.4.0" thiserror = "1.0" -reqwest = "0.11.13" -zip = "0.6.3" +reqwest = { version = "0.12.5", features = ["stream", "json"] } +async_zip = { version = "0.0.17", features = ["full"] } semver = "1.0" chrono = { version = "0.4", features = ["serde"] } -bytes = "1.3.0" -rust-s3 = "0.33.0" +bytes = "1.6.0" +rust-s3 = "0.34.0" +dashmap = "5.5.3" +sha1_smol = { version = "1.0.0", features = ["std"] } +indexmap = { version = "2.2.6", features = ["serde"]} +itertools = "0.13.0" +tracing-error = "0.2.0" + +tracing = "0.1" +tracing-subscriber = { version = "0.3", features = ["env-filter"] } +tracing-futures = { version = "0.2.5", features = ["futures", "tokio"] } \ No newline at end of file diff --git a/daedalus_client/library-patches.json b/daedalus_client/library-patches.json index ebde4ed02..268341a96 100644 --- a/daedalus_client/library-patches.json +++ b/daedalus_client/library-patches.json @@ -1,14 +1,23 @@ [ { - "_comment": "Only allow osx-arm64 for existing LWJGL 3.3.2", + "_comment": "Only allow osx-arm64 for existing LWJGL 3.3.2/3.3.3", "match": [ + "org.lwjgl:lwjgl-freetype-natives-macos-arm64:3.3.2", "org.lwjgl:lwjgl-glfw-natives-macos-arm64:3.3.2", "org.lwjgl:lwjgl-jemalloc-natives-macos-arm64:3.3.2", "org.lwjgl:lwjgl-openal-natives-macos-arm64:3.3.2", "org.lwjgl:lwjgl-opengl-natives-macos-arm64:3.3.2", "org.lwjgl:lwjgl-stb-natives-macos-arm64:3.3.2", "org.lwjgl:lwjgl-tinyfd-natives-macos-arm64:3.3.2", - "org.lwjgl:lwjgl-natives-macos-arm64:3.3.2" + "org.lwjgl:lwjgl-natives-macos-arm64:3.3.2", + "org.lwjgl:lwjgl-freetype-natives-macos-arm64:3.3.3", + "org.lwjgl:lwjgl-glfw-natives-macos-arm64:3.3.3", + "org.lwjgl:lwjgl-jemalloc-natives-macos-arm64:3.3.3", + "org.lwjgl:lwjgl-openal-natives-macos-arm64:3.3.3", + "org.lwjgl:lwjgl-opengl-natives-macos-arm64:3.3.3", + "org.lwjgl:lwjgl-stb-natives-macos-arm64:3.3.3", + "org.lwjgl:lwjgl-tinyfd-natives-macos-arm64:3.3.3", + "org.lwjgl:lwjgl-natives-macos-arm64:3.3.3" ], "override": { "rules": [ @@ -22,15 +31,24 @@ } }, { - "_comment": "Only allow windows-arm64 for existing LWJGL 3.3.2", + "_comment": "Only allow windows-arm64 for existing LWJGL 3.3.2/3.3.3", "match": [ + "org.lwjgl:lwjgl-freetype-natives-windows-arm64:3.3.2", "org.lwjgl:lwjgl-glfw-natives-windows-arm64:3.3.2", "org.lwjgl:lwjgl-jemalloc-natives-windows-arm64:3.3.2", "org.lwjgl:lwjgl-openal-natives-windows-arm64:3.3.2", "org.lwjgl:lwjgl-opengl-natives-windows-arm64:3.3.2", "org.lwjgl:lwjgl-stb-natives-windows-arm64:3.3.2", "org.lwjgl:lwjgl-tinyfd-natives-windows-arm64:3.3.2", - "org.lwjgl:lwjgl-natives-windows-arm64:3.3.2" + "org.lwjgl:lwjgl-natives-windows-arm64:3.3.2", + "org.lwjgl:lwjgl-freetype-natives-windows-arm64:3.3.3", + "org.lwjgl:lwjgl-glfw-natives-windows-arm64:3.3.3", + "org.lwjgl:lwjgl-jemalloc-natives-windows-arm64:3.3.3", + "org.lwjgl:lwjgl-openal-natives-windows-arm64:3.3.3", + "org.lwjgl:lwjgl-opengl-natives-windows-arm64:3.3.3", + "org.lwjgl:lwjgl-stb-natives-windows-arm64:3.3.3", + "org.lwjgl:lwjgl-tinyfd-natives-windows-arm64:3.3.3", + "org.lwjgl:lwjgl-natives-windows-arm64:3.3.3" ], "override": { "rules": [ @@ -165,7 +183,9 @@ ], "override": { "rules": [ - + { + "action": "allow" + }, { "action": "disallow", "os": { @@ -202,12 +222,13 @@ "org.lwjgl.lwjgl:lwjgl:2.9.1-nightly-20131120", "org.lwjgl.lwjgl:lwjgl:2.9.1-nightly-20131017", "org.lwjgl.lwjgl:lwjgl:2.9.1-nightly-20130708-debug3", - "org.lwjgl.lwjgl:lwjgl:2.9.1", - "org.lwjgl.lwjgl:lwjgl:2.9.0" + "org.lwjgl.lwjgl:lwjgl:2.9.1" ], "override": { "rules": [ - + { + "action": "allow" + }, { "action": "disallow", "os": { @@ -268,12 +289,13 @@ "org.lwjgl.lwjgl:lwjgl_util:2.9.1-nightly-20131120", "org.lwjgl.lwjgl:lwjgl_util:2.9.1-nightly-20131017", "org.lwjgl.lwjgl:lwjgl_util:2.9.1-nightly-20130708-debug3", - "org.lwjgl.lwjgl:lwjgl_util:2.9.1", - "org.lwjgl.lwjgl:lwjgl_util:2.9.0" + "org.lwjgl.lwjgl:lwjgl_util:2.9.1" ], "override": { "rules": [ - + { + "action": "allow" + }, { "action": "disallow", "os": { @@ -335,8 +357,7 @@ "org.lwjgl.lwjgl:lwjgl-platform:2.9.1-nightly-20131120", "org.lwjgl.lwjgl:lwjgl-platform:2.9.1-nightly-20131017", "org.lwjgl.lwjgl:lwjgl-platform:2.9.1-nightly-20130708-debug3", - "org.lwjgl.lwjgl:lwjgl-platform:2.9.1", - "org.lwjgl.lwjgl:lwjgl-platform:2.9.0" + "org.lwjgl.lwjgl:lwjgl-platform:2.9.1" ], "override": { "downloads": { @@ -375,7 +396,9 @@ ], "override": { "rules": [ - + { + "action": "allow" + }, { "action": "disallow", "os": { @@ -547,7 +570,9 @@ ], "override": { "rules": [ - + { + "action": "allow" + }, { "action": "disallow", "os": { @@ -719,7 +744,9 @@ ], "override": { "rules": [ - + { + "action": "allow" + }, { "action": "disallow", "os": { @@ -891,7 +918,9 @@ ], "override": { "rules": [ - + { + "action": "allow" + }, { "action": "disallow", "os": { @@ -1063,7 +1092,9 @@ ], "override": { "rules": [ - + { + "action": "allow" + }, { "action": "disallow", "os": { @@ -1235,7 +1266,9 @@ ], "override": { "rules": [ - + { + "action": "allow" + }, { "action": "disallow", "os": { @@ -1407,7 +1440,9 @@ ], "override": { "rules": [ - + { + "action": "allow" + }, { "action": "disallow", "os": { @@ -1993,6 +2028,32 @@ } ] }, + { + "_comment": "Add linux-arm64 support for LWJGL 3.3.2", + "match": [ + "org.lwjgl:lwjgl-freetype:3.3.2" + ], + "additionalLibraries": [ + { + "downloads": { + "artifact": { + "sha1": "896e7d9b8f60d7273f3d491c69270afc67ece3ce", + "size": 1073374, + "url": "https://build.lwjgl.org/release/3.3.2/bin/lwjgl-freetype/lwjgl-freetype-natives-linux-arm64.jar" + } + }, + "name": "org.lwjgl:lwjgl-freetype-natives-linux-arm64:3.3.2-lwjgl.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm64" + } + } + ] + } + ] + }, { "_comment": "Add linux-arm64 support for LWJGL 3.3.2", "match": [ @@ -2175,6 +2236,32 @@ } ] }, + { + "_comment": "Add linux-arm32 support for LWJGL 3.3.2", + "match": [ + "org.lwjgl:lwjgl-freetype:3.3.2" + ], + "additionalLibraries": [ + { + "downloads": { + "artifact": { + "sha1": "b7f77ceb951182659fd400437272aa7e96709968", + "size": 924657, + "url": "https://build.lwjgl.org/release/3.3.2/bin/lwjgl-freetype/lwjgl-freetype-natives-linux-arm32.jar" + } + }, + "name": "org.lwjgl:lwjgl-freetype-natives-linux-arm32:3.3.2-lwjgl.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm32" + } + } + ] + } + ] + }, { "_comment": "Add linux-arm32 support for LWJGL 3.3.2", "match": [ @@ -2357,6 +2444,422 @@ } ] }, + { + "_comment": "Add linux-arm64 support for LWJGL 3.3.3", + "match": [ + "org.lwjgl:lwjgl-freetype:3.3.3" + ], + "additionalLibraries": [ + { + "downloads": { + "artifact": { + "sha1": "498965aac06c4a0d42df1fbef6bacd05bde7f974", + "size": 1093516, + "url": "https://build.lwjgl.org/release/3.3.3/bin/lwjgl-freetype/lwjgl-freetype-natives-linux-arm64.jar" + } + }, + "name": "org.lwjgl:lwjgl-freetype-natives-linux-arm64:3.3.3-lwjgl.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm64" + } + } + ] + } + ] + }, + { + "_comment": "Add linux-arm64 support for LWJGL 3.3.3", + "match": [ + "org.lwjgl:lwjgl-glfw:3.3.3" + ], + "additionalLibraries": [ + { + "downloads": { + "artifact": { + "sha1": "492a0f11f85b85899a6568f07511160c1b87cd38", + "size": 122159, + "url": "https://build.lwjgl.org/release/3.3.3/bin/lwjgl-glfw/lwjgl-glfw-natives-linux-arm64.jar" + } + }, + "name": "org.lwjgl:lwjgl-glfw-natives-linux-arm64:3.3.3-lwjgl.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm64" + } + } + ] + } + ] + }, + { + "_comment": "Add linux-arm64 support for LWJGL 3.3.3", + "match": [ + "org.lwjgl:lwjgl-jemalloc:3.3.3" + ], + "additionalLibraries": [ + { + "downloads": { + "artifact": { + "sha1": "eff8b86798191192fe2cba2dc2776109f30c239d", + "size": 209315, + "url": "https://build.lwjgl.org/release/3.3.3/bin/lwjgl-jemalloc/lwjgl-jemalloc-natives-linux-arm64.jar" + } + }, + "name": "org.lwjgl:lwjgl-jemalloc-natives-linux-arm64:3.3.3-lwjgl.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm64" + } + } + ] + } + ] + }, + { + "_comment": "Add linux-arm64 support for LWJGL 3.3.3", + "match": [ + "org.lwjgl:lwjgl-openal:3.3.3" + ], + "additionalLibraries": [ + { + "downloads": { + "artifact": { + "sha1": "ad8f302118a65bb8d615f8a2a680db58fb8f835e", + "size": 592963, + "url": "https://build.lwjgl.org/release/3.3.3/bin/lwjgl-openal/lwjgl-openal-natives-linux-arm64.jar" + } + }, + "name": "org.lwjgl:lwjgl-openal-natives-linux-arm64:3.3.3-lwjgl.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm64" + } + } + ] + } + ] + }, + { + "_comment": "Add linux-arm64 support for LWJGL 3.3.3", + "match": [ + "org.lwjgl:lwjgl-opengl:3.3.3" + ], + "additionalLibraries": [ + { + "downloads": { + "artifact": { + "sha1": "2096f6b94b2d68745d858fbfe53aacf5f0c8074c", + "size": 58625, + "url": "https://build.lwjgl.org/release/3.3.3/bin/lwjgl-opengl/lwjgl-opengl-natives-linux-arm64.jar" + } + }, + "name": "org.lwjgl:lwjgl-opengl-natives-linux-arm64:3.3.3-lwjgl.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm64" + } + } + ] + } + ] + }, + { + "_comment": "Add linux-arm64 support for LWJGL 3.3.3", + "match": [ + "org.lwjgl:lwjgl-stb:3.3.3" + ], + "additionalLibraries": [ + { + "downloads": { + "artifact": { + "sha1": "ddc177afc2be1ee8d93684b11363b80589a13fe1", + "size": 207418, + "url": "https://build.lwjgl.org/release/3.3.3/bin/lwjgl-stb/lwjgl-stb-natives-linux-arm64.jar" + } + }, + "name": "org.lwjgl:lwjgl-stb-natives-linux-arm64:3.3.3-lwjgl.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm64" + } + } + ] + } + ] + }, + { + "_comment": "Add linux-arm64 support for LWJGL 3.3.3", + "match": [ + "org.lwjgl:lwjgl-tinyfd:3.3.3" + ], + "additionalLibraries": [ + { + "downloads": { + "artifact": { + "sha1": "2823a8c955c758d0954d282888075019ef99cec7", + "size": 43864, + "url": "https://build.lwjgl.org/release/3.3.3/bin/lwjgl-tinyfd/lwjgl-tinyfd-natives-linux-arm64.jar" + } + }, + "name": "org.lwjgl:lwjgl-tinyfd-natives-linux-arm64:3.3.3-lwjgl.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm64" + } + } + ] + } + ] + }, + { + "_comment": "Add linux-arm64 support for LWJGL 3.3.3", + "match": [ + "org.lwjgl:lwjgl:3.3.3" + ], + "additionalLibraries": [ + { + "downloads": { + "artifact": { + "sha1": "f35d8b6ffe1ac1e3a5eb1d4e33de80f044ad5fd8", + "size": 91294, + "url": "https://build.lwjgl.org/release/3.3.3/bin/lwjgl/lwjgl-natives-linux-arm64.jar" + } + }, + "name": "org.lwjgl:lwjgl-natives-linux-arm64:3.3.3-lwjgl.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm64" + } + } + ] + } + ] + }, + { + "_comment": "Add linux-arm32 support for LWJGL 3.3.3", + "match": [ + "org.lwjgl:lwjgl-freetype:3.3.3" + ], + "additionalLibraries": [ + { + "downloads": { + "artifact": { + "sha1": "7dd3b1f751571adaf2c4dc882bc675a5d1e796e6", + "size": 942636, + "url": "https://build.lwjgl.org/release/3.3.3/bin/lwjgl-freetype/lwjgl-freetype-natives-linux-arm32.jar" + } + }, + "name": "org.lwjgl:lwjgl-freetype-natives-linux-arm32:3.3.3-lwjgl.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm32" + } + } + ] + } + ] + }, + { + "_comment": "Add linux-arm32 support for LWJGL 3.3.3", + "match": [ + "org.lwjgl:lwjgl-glfw:3.3.3" + ], + "additionalLibraries": [ + { + "downloads": { + "artifact": { + "sha1": "d9af485c32545b37dd5359b163161d42d7534dcf", + "size": 112560, + "url": "https://build.lwjgl.org/release/3.3.3/bin/lwjgl-glfw/lwjgl-glfw-natives-linux-arm32.jar" + } + }, + "name": "org.lwjgl:lwjgl-glfw-natives-linux-arm32:3.3.3-lwjgl.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm32" + } + } + ] + } + ] + }, + { + "_comment": "Add linux-arm32 support for LWJGL 3.3.3", + "match": [ + "org.lwjgl:lwjgl-jemalloc:3.3.3" + ], + "additionalLibraries": [ + { + "downloads": { + "artifact": { + "sha1": "109b6931880d02d4e65ced38928a16e41d19873e", + "size": 178324, + "url": "https://build.lwjgl.org/release/3.3.3/bin/lwjgl-jemalloc/lwjgl-jemalloc-natives-linux-arm32.jar" + } + }, + "name": "org.lwjgl:lwjgl-jemalloc-natives-linux-arm32:3.3.3-lwjgl.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm32" + } + } + ] + } + ] + }, + { + "_comment": "Add linux-arm32 support for LWJGL 3.3.3", + "match": [ + "org.lwjgl:lwjgl-openal:3.3.3" + ], + "additionalLibraries": [ + { + "downloads": { + "artifact": { + "sha1": "e1702aa09d20359d6cf5cb2999fa7685a785eca7", + "size": 505618, + "url": "https://build.lwjgl.org/release/3.3.3/bin/lwjgl-openal/lwjgl-openal-natives-linux-arm32.jar" + } + }, + "name": "org.lwjgl:lwjgl-openal-natives-linux-arm32:3.3.3-lwjgl.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm32" + } + } + ] + } + ] + }, + { + "_comment": "Add linux-arm32 support for LWJGL 3.3.3", + "match": [ + "org.lwjgl:lwjgl-opengl:3.3.3" + ], + "additionalLibraries": [ + { + "downloads": { + "artifact": { + "sha1": "dbba17fc5ac0985d14a57c11f9537617d67b9952", + "size": 59263, + "url": "https://build.lwjgl.org/release/3.3.3/bin/lwjgl-opengl/lwjgl-opengl-natives-linux-arm32.jar" + } + }, + "name": "org.lwjgl:lwjgl-opengl-natives-linux-arm32:3.3.3-lwjgl.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm32" + } + } + ] + } + ] + }, + { + "_comment": "Add linux-arm32 support for LWJGL 3.3.3", + "match": [ + "org.lwjgl:lwjgl-stb:3.3.3" + ], + "additionalLibraries": [ + { + "downloads": { + "artifact": { + "sha1": "1ae28ff044699ff29b0e980ffabd73fba8a664b3", + "size": 154931, + "url": "https://build.lwjgl.org/release/3.3.3/bin/lwjgl-stb/lwjgl-stb-natives-linux-arm32.jar" + } + }, + "name": "org.lwjgl:lwjgl-stb-natives-linux-arm32:3.3.3-lwjgl.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm32" + } + } + ] + } + ] + }, + { + "_comment": "Add linux-arm32 support for LWJGL 3.3.3", + "match": [ + "org.lwjgl:lwjgl-tinyfd:3.3.3" + ], + "additionalLibraries": [ + { + "downloads": { + "artifact": { + "sha1": "c2a0a05c82c4b9f69ded0b6ad5f417addea78ce2", + "size": 49495, + "url": "https://build.lwjgl.org/release/3.3.3/bin/lwjgl-tinyfd/lwjgl-tinyfd-natives-linux-arm32.jar" + } + }, + "name": "org.lwjgl:lwjgl-tinyfd-natives-linux-arm32:3.3.3-lwjgl.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm32" + } + } + ] + } + ] + }, + { + "_comment": "Add linux-arm32 support for LWJGL 3.3.3", + "match": [ + "org.lwjgl:lwjgl:3.3.3" + ], + "additionalLibraries": [ + { + "downloads": { + "artifact": { + "sha1": "2075c51a80f0ef0f22ba616ba54007ac2b0debd4", + "size": 89565, + "url": "https://build.lwjgl.org/release/3.3.3/bin/lwjgl/lwjgl-natives-linux-arm32.jar" + } + }, + "name": "org.lwjgl:lwjgl-natives-linux-arm32:3.3.3-lwjgl.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm32" + } + } + ] + } + ] + }, { "_comment": "Replace glfw from 3.3.1 with version from 3.3.2 to prevent stack smashing", "match": [ diff --git a/daedalus_client/src/error.rs b/daedalus_client/src/error.rs new file mode 100644 index 000000000..d4a0167cb --- /dev/null +++ b/daedalus_client/src/error.rs @@ -0,0 +1,63 @@ +use tracing_error::InstrumentError; + +#[derive(thiserror::Error, Debug)] +pub enum ErrorKind { + #[error("Daedalus Error: {0}")] + Daedalus(#[from] daedalus::Error), + #[error("Invalid input: {0}")] + InvalidInput(String), + #[error("Error while managing asynchronous tasks")] + TaskError(#[from] tokio::task::JoinError), + #[error("Error while deserializing JSON: {0}")] + SerdeJSON(#[from] serde_json::Error), + #[error("Error while deserializing XML: {0}")] + SerdeXML(#[from] serde_xml_rs::Error), + #[error("Failed to validate file checksum at url {url} with hash {hash} after {tries} tries")] + ChecksumFailure { + hash: String, + url: String, + tries: u32, + }, + #[error("Unable to fetch {item}")] + Fetch { inner: reqwest::Error, item: String }, + #[error("Error while uploading file to S3: {file}")] + S3 { + inner: s3::error::S3Error, + file: String, + }, + #[error("Error acquiring semaphore: {0}")] + Acquire(#[from] tokio::sync::AcquireError), + #[error("Tracing error: {0}")] + Tracing(#[from] tracing::subscriber::SetGlobalDefaultError), + #[error("Zip error: {0}")] + Zip(#[from] async_zip::error::ZipError), +} + +#[derive(Debug)] +pub struct Error { + pub source: tracing_error::TracedError, +} + +impl std::fmt::Display for Error { + fn fmt(&self, fmt: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(fmt, "{}", self.source) + } +} + +impl> From for Error { + fn from(source: E) -> Self { + let error = Into::::into(source); + + Self { + source: error.in_current_span(), + } + } +} + +impl ErrorKind { + pub fn as_error(self) -> Error { + self.into() + } +} + +pub type Result = core::result::Result; diff --git a/daedalus_client/src/fabric.rs b/daedalus_client/src/fabric.rs index 1a997da77..8d1d9431b 100644 --- a/daedalus_client/src/fabric.rs +++ b/daedalus_client/src/fabric.rs @@ -1,372 +1,276 @@ -use crate::{download_file, format_url, upload_file_to_bucket, Error}; -use daedalus::minecraft::{Library, VersionManifest}; -use daedalus::modded::{ - LoaderVersion, Manifest, PartialVersionInfo, Version, DUMMY_REPLACE_STRING, -}; -use serde::{Deserialize, Serialize}; +use crate::util::{download_file, fetch_json, format_url}; +use crate::{insert_mirrored_artifact, Error, MirrorArtifact, UploadFile}; +use daedalus::modded::{Manifest, PartialVersionInfo, DUMMY_REPLACE_STRING}; +use dashmap::DashMap; +use serde::Deserialize; use std::sync::Arc; -use tokio::sync::{Mutex, RwLock, Semaphore}; +use tokio::sync::Semaphore; -pub async fn retrieve_data( - minecraft_versions: &VersionManifest, - uploaded_files: &mut Vec, +#[tracing::instrument(skip(semaphore, upload_files, mirror_artifacts))] +pub async fn fetch_fabric( semaphore: Arc, + upload_files: &DashMap, + mirror_artifacts: &DashMap, ) -> Result<(), Error> { - let list = fetch_fabric_versions(None, semaphore.clone()).await?; - let old_manifest = daedalus::modded::fetch_manifest(&format_url(&format!( - "fabric/v{}/manifest.json", + fetch( daedalus::modded::CURRENT_FABRIC_FORMAT_VERSION, - ))) + "fabric", + "https://meta.fabricmc.net/v2", + "https://maven.fabricmc.net/", + semaphore, + upload_files, + mirror_artifacts, + ) .await - .ok(); - - let mut versions = if let Some(old_manifest) = old_manifest { - old_manifest.game_versions - } else { - Vec::new() - }; - - let loaders_mutex = RwLock::new(Vec::new()); - - { - let mut loaders = loaders_mutex.write().await; - - for (index, loader) in list.loader.iter().enumerate() { - if versions.iter().any(|x| { - x.id == DUMMY_REPLACE_STRING - && x.loaders.iter().any(|x| x.id == loader.version) - }) { - if index == 0 { - loaders.push(( - Box::new(loader.stable), - loader.version.clone(), - Box::new(true), - )) - } - } else { - loaders.push(( - Box::new(loader.stable), - loader.version.clone(), - Box::new(false), - )) - } - } - } - - const DUMMY_GAME_VERSION: &str = "1.19.4-rc2"; - - let loader_version_mutex = Mutex::new(Vec::new()); - let uploaded_files_mutex = Arc::new(Mutex::new(Vec::new())); +} - let loader_versions = futures::future::try_join_all( - loaders_mutex.read().await.clone().into_iter().map( - |(stable, loader, skip_upload)| async { - let version = fetch_fabric_version( - DUMMY_GAME_VERSION, - &loader, - semaphore.clone(), - ) - .await?; +#[tracing::instrument(skip(semaphore, upload_files, mirror_artifacts))] +pub async fn fetch_quilt( + semaphore: Arc, + upload_files: &DashMap, + mirror_artifacts: &DashMap, +) -> Result<(), Error> { + fetch( + daedalus::modded::CURRENT_QUILT_FORMAT_VERSION, + "quilt", + "https://meta.quiltmc.org/v3", + "https://meta.quiltmc.org/", + semaphore, + upload_files, + mirror_artifacts, + ) + .await +} - Ok::<(Box, String, PartialVersionInfo, Box), Error>( - (stable, loader, version, skip_upload), - ) - }, - ), +#[tracing::instrument(skip(semaphore, upload_files, mirror_artifacts))] +async fn fetch( + format_version: usize, + mod_loader: &str, + meta_url: &str, + maven_url: &str, + semaphore: Arc, + upload_files: &DashMap, + mirror_artifacts: &DashMap, +) -> Result<(), Error> { + let modrinth_manifest = fetch_json::( + &format_url(&format!("{mod_loader}/v{format_version}/manifest.json",)), + &semaphore, + ) + .await + .ok(); + let fabric_manifest = fetch_json::( + &format!("{meta_url}/versions"), + &semaphore, ) .await?; - let visited_artifacts_mutex = Arc::new(Mutex::new(Vec::new())); - futures::future::try_join_all(loader_versions.into_iter() - .map( - |(stable, loader, version, skip_upload)| async { - let libs = futures::future::try_join_all( - version.libraries.into_iter().map(|mut lib| async { - { - let mut visited_assets = - visited_artifacts_mutex.lock().await; - - if visited_assets.contains(&lib.name) { - lib.name = lib.name.replace(DUMMY_GAME_VERSION, DUMMY_REPLACE_STRING); - lib.url = Some(format_url("maven/")); - - return Ok(lib); - } else { - visited_assets.push(lib.name.clone()) - } - } - - if lib.name.contains(DUMMY_GAME_VERSION) { - lib.name = lib.name.replace(DUMMY_GAME_VERSION, DUMMY_REPLACE_STRING); - futures::future::try_join_all(list.game.clone().into_iter().map(|game_version| async { - let semaphore = semaphore.clone(); - let uploaded_files_mutex = uploaded_files_mutex.clone(); - let lib_name = lib.name.clone(); - let lib_url = lib.url.clone(); - - async move { - let artifact_path = - daedalus::get_path_from_artifact(&lib_name.replace(DUMMY_REPLACE_STRING, &game_version.version))?; - - let artifact = download_file( - &format!( - "{}{}", - lib_url.unwrap_or_else(|| { - "https://maven.fabricmc.net/".to_string() - }), - artifact_path - ), - None, - semaphore.clone(), - ) - .await?; - - upload_file_to_bucket( - format!("{}/{}", "maven", artifact_path), - artifact.to_vec(), - Some("application/java-archive".to_string()), - &uploaded_files_mutex, - semaphore.clone(), - ) - .await?; - - Ok::<(), Error>(()) - }.await?; - - Ok::<(), Error>(()) - })).await?; - lib.url = Some(format_url("maven/")); - - return Ok(lib); - } - - let artifact_path = - daedalus::get_path_from_artifact(&lib.name)?; + // We check Modrinth's fabric version manifest and compare if the fabric version exists in Modrinth's database + // We also check intermediary versions that are newly added to query + let (fetch_fabric_versions, fetch_intermediary_versions) = + if let Some(modrinth_manifest) = modrinth_manifest { + let (mut fetch_versions, mut fetch_intermediary_versions) = + (Vec::new(), Vec::new()); - let artifact = download_file( - &format!( - "{}{}", - lib.url.unwrap_or_else(|| { - "https://maven.fabricmc.net/".to_string() - }), - artifact_path - ), - None, - semaphore.clone(), - ) - .await?; - - lib.url = Some(format_url("maven/")); - - upload_file_to_bucket( - format!("{}/{}", "maven", artifact_path), - artifact.to_vec(), - Some("application/java-archive".to_string()), - &uploaded_files_mutex, - semaphore.clone(), - ) - .await?; - - Ok::(lib) - }), - ) - .await?; - - if async move { - *skip_upload - }.await { - return Ok::<(), Error>(()) + for version in &fabric_manifest.loader { + if !modrinth_manifest + .game_versions + .iter() + .any(|x| x.loaders.iter().any(|x| x.id == version.version)) + { + fetch_versions.push(version); + } } - - let version_path = format!( - "fabric/v{}/versions/{}.json", - daedalus::modded::CURRENT_FABRIC_FORMAT_VERSION, - &loader - ); - - upload_file_to_bucket( - version_path.clone(), - serde_json::to_vec(&PartialVersionInfo { - arguments: version.arguments, - id: version - .id - .replace(DUMMY_GAME_VERSION, DUMMY_REPLACE_STRING), - main_class: version.main_class, - release_time: version.release_time, - time: version.time, - type_: version.type_, - inherits_from: version - .inherits_from - .replace(DUMMY_GAME_VERSION, DUMMY_REPLACE_STRING), - libraries: libs, - minecraft_arguments: version.minecraft_arguments, - processors: None, - data: None, - })?, - Some("application/json".to_string()), - &uploaded_files_mutex, - semaphore.clone(), - ) - .await?; - - { - let mut loader_version_map = loader_version_mutex.lock().await; - async move { - loader_version_map.push(LoaderVersion { - id: loader.to_string(), - url: format_url(&version_path), - stable: *stable, - }); + for version in &fabric_manifest.intermediary { + if !modrinth_manifest + .game_versions + .iter() + .any(|x| x.id == version.version) + && fabric_manifest + .game + .iter() + .any(|x| x.version == version.version) + { + fetch_intermediary_versions.push(version); } - .await; } - Ok::<(), Error>(()) - }, - )) - .await?; - - let mut loader_version_mutex = loader_version_mutex.into_inner(); - if !loader_version_mutex.is_empty() { - if let Some(version) = - versions.iter_mut().find(|x| x.id == DUMMY_REPLACE_STRING) - { - version.loaders.append(&mut loader_version_mutex); + (fetch_versions, fetch_intermediary_versions) } else { - versions.push(Version { - id: DUMMY_REPLACE_STRING.to_string(), - stable: true, - loaders: loader_version_mutex, - }); - } - } + ( + fabric_manifest.loader.iter().collect(), + fabric_manifest.intermediary.iter().collect(), + ) + }; - for version in &list.game { - if !versions.iter().any(|x| x.id == version.version) { - versions.push(Version { - id: version.version.clone(), - stable: version.stable, - loaders: vec![], - }); + const DUMMY_GAME_VERSION: &str = "1.21"; + + if !fetch_intermediary_versions.is_empty() { + for x in &fetch_intermediary_versions { + insert_mirrored_artifact( + &x.maven, + maven_url.to_string(), + mirror_artifacts, + )?; } } - versions.sort_by(|x, y| { - minecraft_versions - .versions + if !fetch_fabric_versions.is_empty() { + let fabric_version_manifest_urls = fetch_fabric_versions .iter() - .position(|z| x.id == z.id) - .unwrap_or_default() - .cmp( - &minecraft_versions - .versions - .iter() - .position(|z| y.id == z.id) - .unwrap_or_default(), - ) - }); - - for version in &mut versions { - version.loaders.sort_by(|x, y| { - list.loader - .iter() - .position(|z| x.id == *z.version) - .unwrap_or_default() - .cmp( - &list - .loader - .iter() - .position(|z| y.id == z.version) - .unwrap_or_default(), + .map(|x| { + format!( + "{}/versions/loader/{}/{}/profile/json", + meta_url, DUMMY_GAME_VERSION, x.version ) - }) - } + }) + .collect::>(); + let fabric_version_manifests = futures::future::try_join_all( + fabric_version_manifest_urls + .iter() + .map(|x| download_file(x, None, &semaphore)), + ) + .await? + .into_iter() + .map(|x| serde_json::from_slice(&x)) + .collect::, serde_json::Error>>()?; + + let patched_version_manifests = fabric_version_manifests + .into_iter() + .map(|mut version_info| { + for lib in &mut version_info.libraries { + let new_name = lib + .name + .replace(DUMMY_GAME_VERSION, DUMMY_REPLACE_STRING); + // If a library is not intermediary, we add it to mirror artifacts to be mirrored + if lib.name == new_name { + insert_mirrored_artifact( + &new_name, + lib.url + .clone() + .unwrap_or_else(|| maven_url.to_string()), + mirror_artifacts, + )?; + } else { + lib.name = new_name; + } - upload_file_to_bucket( - format!( - "fabric/v{}/manifest.json", - daedalus::modded::CURRENT_FABRIC_FORMAT_VERSION, - ), - serde_json::to_vec(&Manifest { - game_versions: versions, - })?, - Some("application/json".to_string()), - &uploaded_files_mutex, - semaphore, - ) - .await?; + lib.url = Some(format_url("maven/")); + } - if let Ok(uploaded_files_mutex) = Arc::try_unwrap(uploaded_files_mutex) { - uploaded_files.extend(uploaded_files_mutex.into_inner()); + version_info.id = version_info + .id + .replace(DUMMY_GAME_VERSION, DUMMY_REPLACE_STRING); + version_info.inherits_from = version_info + .inherits_from + .replace(DUMMY_GAME_VERSION, DUMMY_REPLACE_STRING); + + Ok(version_info) + }) + .collect::, Error>>()?; + let serialized_version_manifests = patched_version_manifests + .iter() + .map(|x| serde_json::to_vec(x).map(bytes::Bytes::from)) + .collect::, serde_json::Error>>()?; + + serialized_version_manifests + .into_iter() + .enumerate() + .for_each(|(index, bytes)| { + let loader = fetch_fabric_versions[index]; + + let version_path = format!( + "{mod_loader}/v{format_version}/versions/{}.json", + loader.version + ); + + upload_files.insert( + version_path, + UploadFile { + file: bytes, + content_type: Some("application/json".to_string()), + }, + ); + }); } - Ok(()) -} - -const FABRIC_META_URL: &str = "https://meta.fabricmc.net/v2"; + if !fetch_fabric_versions.is_empty() + || !fetch_intermediary_versions.is_empty() + { + let fabric_manifest_path = + format!("{mod_loader}/v{format_version}/manifest.json",); + + let loader_versions = daedalus::modded::Version { + id: DUMMY_REPLACE_STRING.to_string(), + stable: true, + loaders: fabric_manifest + .loader + .into_iter() + .map(|x| { + let version_path = format!( + "{mod_loader}/v{format_version}/versions/{}.json", + x.version, + ); + + daedalus::modded::LoaderVersion { + id: x.version, + url: format_url(&version_path), + stable: x.stable, + } + }) + .collect(), + }; + + let manifest = daedalus::modded::Manifest { + game_versions: std::iter::once(loader_versions) + .chain(fabric_manifest.game.into_iter().map(|x| { + daedalus::modded::Version { + id: x.version, + stable: x.stable, + loaders: vec![], + } + })) + .collect(), + }; + + upload_files.insert( + fabric_manifest_path, + UploadFile { + file: bytes::Bytes::from(serde_json::to_vec(&manifest)?), + content_type: Some("application/json".to_string()), + }, + ); + } -async fn fetch_fabric_version( - version_number: &str, - loader_version: &str, - semaphore: Arc, -) -> Result { - Ok(serde_json::from_slice( - &download_file( - &format!( - "{}/versions/loader/{}/{}/profile/json", - FABRIC_META_URL, version_number, loader_version - ), - None, - semaphore, - ) - .await?, - )?) + Ok(()) } -#[derive(Serialize, Deserialize, Debug, Clone)] -/// Versions of fabric components +#[derive(Deserialize, Debug, Clone)] struct FabricVersions { - /// Versions of Minecraft that fabric supports - pub game: Vec, - /// Available versions of the fabric loader pub loader: Vec, + pub game: Vec, + #[serde(alias = "hashed")] + pub intermediary: Vec, } -#[derive(Serialize, Deserialize, Debug, Clone)] -/// A version of Minecraft that fabric supports -struct FabricGameVersion { - /// The version number of the game +#[derive(Deserialize, Debug, Clone)] +struct FabricLoaderVersion { + // pub separator: String, + // pub build: u32, + // pub maven: String, pub version: String, - /// Whether the Minecraft version is stable or not + #[serde(default)] pub stable: bool, } -#[derive(Serialize, Deserialize, Debug, Clone)] -/// A version of the fabric loader -struct FabricLoaderVersion { - /// The separator to get the build number - pub separator: String, - /// The build number - pub build: u32, - /// The maven artifact +#[derive(Deserialize, Debug, Clone)] +struct FabricIntermediaryVersion { pub maven: String, - /// The version number of the fabric loader pub version: String, - /// Whether the loader is stable or not - pub stable: bool, } -/// Fetches the list of fabric versions -async fn fetch_fabric_versions( - url: Option<&str>, - semaphore: Arc, -) -> Result { - Ok(serde_json::from_slice( - &download_file( - url.unwrap_or(&*format!("{}/versions", FABRIC_META_URL)), - None, - semaphore, - ) - .await?, - )?) + +#[derive(Deserialize, Debug, Clone)] +struct FabricGameVersion { + pub version: String, + pub stable: bool, } diff --git a/daedalus_client/src/forge.rs b/daedalus_client/src/forge.rs index fc3bf2f76..99892f225 100644 --- a/daedalus_client/src/forge.rs +++ b/daedalus_client/src/forge.rs @@ -1,650 +1,766 @@ -use crate::{ - download_file, download_file_mirrors, format_url, upload_file_to_bucket, - Error, -}; +use crate::util::{download_file, fetch_json, fetch_xml, format_url}; +use crate::{insert_mirrored_artifact, Error, MirrorArtifact, UploadFile}; use chrono::{DateTime, Utc}; -use daedalus::minecraft::{ - Argument, ArgumentType, Library, VersionManifest, VersionType, -}; -use daedalus::modded::{ - LoaderVersion, Manifest, PartialVersionInfo, Processor, SidedDataEntry, -}; -use lazy_static::lazy_static; -use log::info; -use semver::{Version, VersionReq}; -use serde::{Deserialize, Serialize}; +use daedalus::get_path_from_artifact; +use daedalus::modded::PartialVersionInfo; +use dashmap::DashMap; +use futures::io::Cursor; +use indexmap::IndexMap; +use itertools::Itertools; +use serde::de::DeserializeOwned; +use serde::Deserialize; use std::collections::HashMap; -use std::io::Read; use std::sync::Arc; -use std::time::Instant; -use tokio::sync::{Mutex, Semaphore}; - -lazy_static! { - static ref FORGE_MANIFEST_V1_QUERY: VersionReq = - VersionReq::parse(">=8.0.684, <23.5.2851").unwrap(); - static ref FORGE_MANIFEST_V2_QUERY_P1: VersionReq = - VersionReq::parse(">=23.5.2851, <31.2.52").unwrap(); - static ref FORGE_MANIFEST_V2_QUERY_P2: VersionReq = - VersionReq::parse(">=32.0.1, <37.0.0").unwrap(); - static ref FORGE_MANIFEST_V3_QUERY: VersionReq = - VersionReq::parse(">=37.0.0").unwrap(); -} +use tokio::sync::Semaphore; -pub async fn retrieve_data( - minecraft_versions: &VersionManifest, - uploaded_files: &mut Vec, +#[tracing::instrument(skip(semaphore, upload_files, mirror_artifacts))] +pub async fn fetch_forge( semaphore: Arc, + upload_files: &DashMap, + mirror_artifacts: &DashMap, ) -> Result<(), Error> { - let maven_metadata = fetch_maven_metadata(None, semaphore.clone()).await?; - let old_manifest = daedalus::modded::fetch_manifest(&format_url(&format!( - "forge/v{}/manifest.json", + let forge_manifest = fetch_json::>>( + "https://files.minecraftforge.net/net/minecraftforge/forge/maven-metadata.json", + &semaphore, + ) + .await?; + + let mut format_version = 0; + + let forge_versions = forge_manifest.into_iter().flat_map(|(game_version, versions)| versions.into_iter().map(|loader_version| { + // Forge versions can be in these specific formats: + // 1.10.2-12.18.1.2016-failtests + // 1.9-12.16.0.1886 + // 1.9-12.16.0.1880-1.9 + // 1.14.4-28.1.30 + // This parses them to get the actual Forge version. Ex: 1.15.2-31.1.87 -> 31.1.87 + let version_split = loader_version.split('-').nth(1).unwrap_or(&loader_version).to_string(); + + // Forge has 3 installer formats: + // - Format 0 (Unsupported ATM): Forge Legacy (pre-1.5.2). Uses Binary Patch method to install + // To install: Download patch, download minecraft client JAR. Combine patch and client JAR and delete META-INF/. + // (pre-1.3-2) Client URL: https://maven.minecraftforge.net/net/minecraftforge/forge/{version}/forge-{version}-client.zip + // (pre-1.3-2) Server URL: https://maven.minecraftforge.net/net/minecraftforge/forge/{version}/forge-{version}-server.zip + // (1.3-2-onwards) Universal URL: https://maven.minecraftforge.net/net/minecraftforge/forge/{version}/forge-{version}-universal.zip + // - Format 1: Forge Installer Legacy (1.5.2-1.12.2ish) + // To install: Extract install_profile.json from archive. "versionInfo" is the profile's version info. Convert it to the modern format + // Extract forge library from archive. Path is at "install"."path". + // - Format 2: Forge Installer Modern + // To install: Extract install_profile.json from archive. Extract version.json from archive. Combine the two and extract all libraries + // which are embedded into the installer JAR. + // Then upload. The launcher will need to run processors! + if format_version != 1 && &*version_split == "7.8.0.684" { + format_version = 1; + } else if format_version != 2 && &*version_split == "14.23.5.2851" { + format_version = 2; + } + + ForgeVersion { + format_version, + installer_url: format!("https://maven.minecraftforge.net/net/minecraftforge/forge/{0}/forge-{0}-installer.jar", loader_version), + raw: loader_version, + loader_version: version_split, + game_version: game_version.clone(), + } + }) + .collect::>()) + // TODO: support format version 0 (see above) + .filter(|x| x.format_version != 0) + .filter(|x| { + // These following Forge versions are broken and cannot be installed + const BLACKLIST : &[&str] = &[ + // Not supported due to `data` field being `[]` even though the type is a map + "1.12.2-14.23.5.2851", + // Malformed Archives + "1.6.1-8.9.0.749", + "1.6.1-8.9.0.751", + "1.6.4-9.11.1.960", + "1.6.4-9.11.1.961", + "1.6.4-9.11.1.963", + "1.6.4-9.11.1.964", + ]; + + !BLACKLIST.contains(&&*x.raw) + }) + .collect::>(); + + fetch( daedalus::modded::CURRENT_FORGE_FORMAT_VERSION, - ))) + "forge", + "https://maven.minecraftforge.net/", + forge_versions, + semaphore, + upload_files, + mirror_artifacts, + ) .await - .ok(); +} + +#[tracing::instrument(skip(semaphore, upload_files, mirror_artifacts))] +pub async fn fetch_neo( + semaphore: Arc, + upload_files: &DashMap, + mirror_artifacts: &DashMap, +) -> Result<(), Error> { + #[derive(Debug, Deserialize)] + struct Metadata { + versioning: Versioning, + } - let old_versions = - Arc::new(Mutex::new(if let Some(old_manifest) = old_manifest { - old_manifest.game_versions + #[derive(Debug, Deserialize)] + struct Versioning { + versions: Versions, + } + + #[derive(Debug, Deserialize)] + struct Versions { + version: Vec, + } + + let forge_versions = fetch_xml::( + "https://maven.neoforged.net/net/neoforged/forge/maven-metadata.xml", + &semaphore, + ) + .await?; + let neo_versions = fetch_xml::( + "https://maven.neoforged.net/net/neoforged/neoforge/maven-metadata.xml", + &semaphore, + ) + .await?; + + let parsed_versions = forge_versions.versioning.versions.version.into_iter().map(|loader_version| { + // NeoForge Forge versions can be in these specific formats: + // 1.20.1-47.1.74 + // 47.1.82 + // This parses them to get the actual Forge version. Ex: 1.20.1-47.1.74 -> 47.1.74 + let version_split = loader_version.split('-').nth(1).unwrap_or(&loader_version).to_string(); + + Ok(ForgeVersion { + format_version: 2, + installer_url: format!("https://maven.neoforged.net/net/neoforged/forge/{0}/forge-{0}-installer.jar", loader_version), + raw: loader_version, + loader_version: version_split, + game_version: "1.20.1".to_string(), // All NeoForge Forge versions are for 1.20.1 + }) + }).chain(neo_versions.versioning.versions.version.into_iter().map(|loader_version| { + let mut parts = loader_version.split('.'); + + // NeoForge Forge versions are in this format: 20.2.29-beta, 20.6.119 + // Where the first number is the major MC version, the second is the minor MC version, and the third is the NeoForge version + let major = parts.next().ok_or_else( + || crate::ErrorKind::InvalidInput(format!("Unable to find major game version for NeoForge {loader_version}")) + )?; + + let minor = parts.next().ok_or_else( + || crate::ErrorKind::InvalidInput(format!("Unable to find minor game version for NeoForge {loader_version}")) + )?; + + let game_version = if minor == "0" { + format!("1.{major}") } else { - Vec::new() - })); + format!("1.{major}.{minor}") + }; + + Ok(ForgeVersion { + format_version: 2, + installer_url: format!("https://maven.neoforged.net/net/neoforged/neoforge/{0}/neoforge-{0}-installer.jar", loader_version), + loader_version: loader_version.clone(), + raw: loader_version, + game_version, + }) + })) + .collect::, Error>>()? + .into_iter() + .filter(|x| { + // These following Forge versions are broken and cannot be installed + const BLACKLIST : &[&str] = &[ + // Unreachable / 404 + "1.20.1-47.1.7", + "47.1.82", + ]; + + !BLACKLIST.contains(&&*x.raw) + }).collect(); + + fetch( + daedalus::modded::CURRENT_NEOFORGE_FORMAT_VERSION, + "neo", + "https://maven.neoforged.net/", + parsed_versions, + semaphore, + upload_files, + mirror_artifacts, + ) + .await +} - let versions = Arc::new(Mutex::new(Vec::new())); +#[tracing::instrument(skip( + forge_versions, + semaphore, + upload_files, + mirror_artifacts +))] +async fn fetch( + format_version: usize, + mod_loader: &str, + maven_url: &str, + forge_versions: Vec, + semaphore: Arc, + upload_files: &DashMap, + mirror_artifacts: &DashMap, +) -> Result<(), Error> { + let modrinth_manifest = fetch_json::( + &format_url(&format!("{mod_loader}/v{format_version}/manifest.json",)), + &semaphore, + ) + .await + .ok(); - let visited_assets_mutex = Arc::new(Mutex::new(Vec::new())); - let uploaded_files_mutex = Arc::new(Mutex::new(Vec::new())); + let fetch_versions = if let Some(modrinth_manifest) = modrinth_manifest { + let mut fetch_versions = Vec::new(); - let mut version_futures = Vec::new(); + for version in &forge_versions { + if !modrinth_manifest.game_versions.iter().any(|x| { + x.id == version.game_version + && x.loaders.iter().any(|x| x.id == version.loader_version) + }) { + fetch_versions.push(version); + } + } - for (minecraft_version, loader_versions) in maven_metadata.clone() { - let mut loaders = Vec::new(); + fetch_versions + } else { + forge_versions.iter().collect() + }; - for loader_version_full in loader_versions { - let loader_version = loader_version_full.split('-').nth(1); + if !fetch_versions.is_empty() { + let forge_installers = futures::future::try_join_all( + fetch_versions + .iter() + .map(|x| download_file(&x.installer_url, None, &semaphore)), + ) + .await?; - if let Some(loader_version_raw) = loader_version { - // This is a dirty hack to get around Forge not complying with SemVer, but whatever - // Most of this is a hack anyways :( - // Works for all forge versions! - let split = - loader_version_raw.split('.').collect::>(); - let loader_version = if split.len() >= 4 { - if split[0].parse::().unwrap_or(0) < 6 { - format!("{}.{}.{}", split[0], split[1], split[3]) - } else { - format!("{}.{}.{}", split[1], split[2], split[3]) - } - } else { - loader_version_raw.to_string() - }; + #[tracing::instrument(skip(raw, upload_files, mirror_artifacts))] + async fn read_forge_installer( + raw: bytes::Bytes, + loader: &ForgeVersion, + maven_url: &str, + upload_files: &DashMap, + mirror_artifacts: &DashMap, + ) -> Result { + tracing::trace!( + "Reading forge installer for {}", + loader.loader_version + ); + type ZipFileReader = async_zip::base::read::seek::ZipFileReader< + Cursor, + >; + + let cursor = Cursor::new(raw); + let mut zip = ZipFileReader::new(cursor).await?; + + #[tracing::instrument(skip(zip))] + async fn read_file( + zip: &mut ZipFileReader, + file_name: &str, + ) -> Result>, Error> { + let zip_index_option = + zip.file().entries().iter().position(|f| { + f.filename().as_str().unwrap_or_default() == file_name + }); - let version = Version::parse(&loader_version)?; + if let Some(zip_index) = zip_index_option { + let mut buffer = Vec::new(); + let mut reader = zip.reader_with_entry(zip_index).await?; + reader.read_to_end_checked(&mut buffer).await?; - if FORGE_MANIFEST_V1_QUERY.matches(&version) - || FORGE_MANIFEST_V2_QUERY_P1.matches(&version) - || FORGE_MANIFEST_V2_QUERY_P2.matches(&version) - || FORGE_MANIFEST_V3_QUERY.matches(&version) - { - loaders.push((loader_version_full, version)) + Ok(Some(buffer)) + } else { + Ok(None) } } - } - if !loaders.is_empty() { - version_futures.push(async { - let mut loaders_versions = Vec::new(); + #[tracing::instrument(skip(zip))] + async fn read_json( + zip: &mut ZipFileReader, + file_name: &str, + ) -> Result, Error> { + if let Some(file) = read_file(zip, file_name).await? { + Ok(Some(serde_json::from_slice(&file)?)) + } else { + Ok(None) + } + } - { - let loaders_futures = loaders.into_iter().map(|(loader_version_full, version)| async { - let versions_mutex = Arc::clone(&old_versions); - let visited_assets = Arc::clone(&visited_assets_mutex); - let uploaded_files_mutex = Arc::clone(&uploaded_files_mutex); - let semaphore = Arc::clone(&semaphore); - let minecraft_version = minecraft_version.clone(); - - async move { - /// These forge versions are not worth supporting! - const WHITELIST : &[&str] = &[ - // Not supported due to `data` field being `[]` even though the type is a map - "1.12.2-14.23.5.2851", - // Malformed Archives - "1.6.1-8.9.0.749", - "1.6.1-8.9.0.751", - "1.6.4-9.11.1.960", - "1.6.4-9.11.1.961", - "1.6.4-9.11.1.963", - "1.6.4-9.11.1.964", - ]; - - if WHITELIST.contains(&&*loader_version_full) { - return Ok(None); - } + if loader.format_version == 1 { + #[derive(Deserialize, Debug)] + #[serde(rename_all = "camelCase")] + struct ForgeInstallerProfileInstallDataV1 { + // pub mirror_list: String, + // pub target: String, + /// Path to the Forge universal library + pub file_path: String, + // pub logo: String, + // pub welcome: String, + // pub version: String, + /// Maven coordinates of the Forge universal library + pub path: String, + // pub profile_name: String, + pub minecraft: String, + } - { - let versions = versions_mutex.lock().await; - let version = versions.iter().find(|x| - x.id == minecraft_version).and_then(|x| x.loaders.iter().find(|x| x.id == loader_version_full)); + #[derive(Deserialize, Debug)] + #[serde(rename_all = "camelCase")] + struct ForgeInstallerProfileManifestV1 { + pub id: String, + pub libraries: Vec, + pub main_class: Option, + pub minecraft_arguments: Option, + pub release_time: DateTime, + pub time: DateTime, + pub type_: daedalus::minecraft::VersionType, + // pub assets: Option, + // pub inherits_from: Option, + // pub jar: Option, + } - if let Some(version) = version { - return Ok::, Error>(Some(version.clone())); - } - } + #[derive(Deserialize, Debug)] + #[serde(rename_all = "camelCase")] + struct ForgeInstallerProfileV1 { + pub install: ForgeInstallerProfileInstallDataV1, + pub version_info: ForgeInstallerProfileManifestV1, + } - info!("Forge - Installer Start {}", loader_version_full.clone()); - let bytes = download_file(&format!("https://maven.minecraftforge.net/net/minecraftforge/forge/{0}/forge-{0}-installer.jar", loader_version_full), None, semaphore.clone()).await?; - - let reader = std::io::Cursor::new(bytes); - - if let Ok(archive) = zip::ZipArchive::new(reader) { - if FORGE_MANIFEST_V1_QUERY.matches(&version) { - let mut archive_clone = archive.clone(); - let profile = tokio::task::spawn_blocking(move || { - let mut install_profile = archive_clone.by_name("install_profile.json")?; - - let mut contents = String::new(); - install_profile.read_to_string(&mut contents)?; - - Ok::(serde_json::from_str::(&contents)?) - }).await??; - - let mut archive_clone = archive.clone(); - let file_path = profile.install.file_path.clone(); - let forge_universal_bytes = tokio::task::spawn_blocking(move || { - let mut forge_universal_file = archive_clone.by_name(&file_path)?; - let mut forge_universal = Vec::new(); - forge_universal_file.read_to_end(&mut forge_universal)?; - - - Ok::(bytes::Bytes::from(forge_universal)) - }).await??; - let forge_universal_path = profile.install.path.clone(); - - let now = Instant::now(); - let libs = futures::future::try_join_all(profile.version_info.libraries.into_iter().map(|mut lib| async { - if let Some(url) = lib.url { - { - let mut visited_assets = visited_assets.lock().await; - - if visited_assets.contains(&lib.name) { - lib.url = Some(format_url("maven/")); - - return Ok::(lib); - } else { - visited_assets.push(lib.name.clone()) - } - } - - let artifact_path = - daedalus::get_path_from_artifact(&lib.name)?; - - let artifact = if lib.name == forge_universal_path { - forge_universal_bytes.clone() - } else { - let mirrors = vec![&*url, "https://maven.creeperhost.net/", "https://libraries.minecraft.net/"]; - - download_file_mirrors( - &artifact_path, - &mirrors, - None, - semaphore.clone(), - ) - .await? - }; - - lib.url = Some(format_url("maven/")); - - upload_file_to_bucket( - format!("{}/{}", "maven", artifact_path), - artifact.to_vec(), - Some("application/java-archive".to_string()), - uploaded_files_mutex.as_ref(), - semaphore.clone(), - ).await?; - } - - Ok::(lib) - })).await?; - - let elapsed = now.elapsed(); - info!("Elapsed lib DL: {:.2?}", elapsed); - - let new_profile = PartialVersionInfo { - id: profile.version_info.id, - inherits_from: profile.install.minecraft, - release_time: profile.version_info.release_time, - time: profile.version_info.time, - main_class: profile.version_info.main_class, - minecraft_arguments: profile.version_info.minecraft_arguments.clone(), - arguments: profile.version_info.minecraft_arguments.map(|x| [(ArgumentType::Game, x.split(' ').map(|x| Argument::Normal(x.to_string())).collect())].iter().cloned().collect()), - libraries: libs, - type_: profile.version_info.type_, - data: None, - processors: None - }; - - let version_path = format!( - "forge/v{}/versions/{}.json", - daedalus::modded::CURRENT_FORGE_FORMAT_VERSION, - new_profile.id - ); - - upload_file_to_bucket( - version_path.clone(), - serde_json::to_vec(&new_profile)?, - Some("application/json".to_string()), - uploaded_files_mutex.as_ref(), - semaphore.clone(), - ).await?; - - return Ok(Some(LoaderVersion { - id: loader_version_full, - url: format_url(&version_path), - stable: false - })); - } else if FORGE_MANIFEST_V2_QUERY_P1.matches(&version) || FORGE_MANIFEST_V2_QUERY_P2.matches(&version) || FORGE_MANIFEST_V3_QUERY.matches(&version) { - let mut archive_clone = archive.clone(); - let mut profile = tokio::task::spawn_blocking(move || { - let mut install_profile = archive_clone.by_name("install_profile.json")?; - - let mut contents = String::new(); - install_profile.read_to_string(&mut contents)?; - - Ok::(serde_json::from_str::(&contents)?) - }).await??; - - let mut archive_clone = archive.clone(); - let version_info = tokio::task::spawn_blocking(move || { - let mut install_profile = archive_clone.by_name("version.json")?; - - let mut contents = String::new(); - install_profile.read_to_string(&mut contents)?; - - Ok::(serde_json::from_str::(&contents)?) - }).await??; - - - let mut libs : Vec = version_info.libraries.into_iter().chain(profile.libraries.into_iter().map(|x| Library { - downloads: x.downloads, - extract: x.extract, - name: x.name, - url: x.url, - natives: x.natives, - rules: x.rules, - checksums: x.checksums, - include_in_classpath: false - })).collect(); - - let mut local_libs : HashMap = HashMap::new(); - - for lib in &libs { - if lib.downloads.as_ref().and_then(|x| x.artifact.as_ref().map(|x| x.url.is_empty())).unwrap_or(false) { - let mut archive_clone = archive.clone(); - let lib_name_clone = lib.name.clone(); - - let lib_bytes = tokio::task::spawn_blocking(move || { - let mut lib_file = archive_clone.by_name(&format!("maven/{}", daedalus::get_path_from_artifact(&lib_name_clone)?))?; - let mut lib_bytes = Vec::new(); - lib_file.read_to_end(&mut lib_bytes)?; - - Ok::(bytes::Bytes::from(lib_bytes)) - }).await??; - - local_libs.insert(lib.name.clone(), lib_bytes); - } - } - - let path = profile.path.clone(); - let version = profile.version.clone(); - - for entry in profile.data.values_mut() { - if entry.client.starts_with('/') || entry.server.starts_with('/') { - macro_rules! read_data { - ($value:expr) => { - let mut archive_clone = archive.clone(); - let value_clone = $value.clone(); - let lib_bytes = tokio::task::spawn_blocking(move || { - let mut lib_file = archive_clone.by_name(&value_clone[1..value_clone.len()])?; - let mut lib_bytes = Vec::new(); - lib_file.read_to_end(&mut lib_bytes)?; - - Ok::(bytes::Bytes::from(lib_bytes)) - }).await??; - - let split = $value.split('/').last(); - - if let Some(last) = split { - let mut file = last.split('.'); - - if let Some(file_name) = file.next() { - if let Some(ext) = file.next() { - let path = format!("{}:{}@{}", path.as_deref().unwrap_or(&*format!("net.minecraftforge:forge:{}", version)), file_name, ext); - $value = format!("[{}]", &path); - local_libs.insert(path.clone(), bytes::Bytes::from(lib_bytes)); - - libs.push(Library { - downloads: None, - extract: None, - name: path, - url: Some("".to_string()), - natives: None, - rules: None, - checksums: None, - include_in_classpath: false, - }); - } - } - } - } - } - - if entry.client.starts_with('/') { - read_data!(entry.client); - } - - if entry.server.starts_with('/') { - read_data!(entry.server); - } - } - } - - let now = Instant::now(); - let libs = futures::future::try_join_all(libs.into_iter().map(|mut lib| async { - let artifact_path = - daedalus::get_path_from_artifact(&lib.name)?; - - { - let mut visited_assets = visited_assets.lock().await; - - if visited_assets.contains(&lib.name) { - if let Some(ref mut downloads) = lib.downloads { - if let Some(ref mut artifact) = downloads.artifact { - artifact.url = format_url(&format!("maven/{}", artifact_path)); - } - } else if lib.url.is_some() { - lib.url = Some(format_url("maven/")); - } - - return Ok::(lib); - } else { - visited_assets.push(lib.name.clone()) - } - } - - let artifact_bytes = if let Some(ref mut downloads) = lib.downloads { - if let Some(ref mut artifact) = downloads.artifact { - let res = if artifact.url.is_empty() { - local_libs.get(&lib.name).cloned() - } else { - Some(download_file( - &artifact.url, - Some(&*artifact.sha1), - semaphore.clone(), - ) - .await?) - }; - - if res.is_some() { - artifact.url = format_url(&format!("maven/{}", artifact_path)); - } - - res - } else { None } - } else if let Some(ref mut url) = lib.url { - let res = if url.is_empty() { - local_libs.get(&lib.name).cloned() - } else { - Some(download_file( - url, - None, - semaphore.clone(), - ) - .await?) - }; - - if res.is_some() { - lib.url = Some(format_url("maven/")); - } - - res - } else { None }; - - if let Some(bytes) = artifact_bytes { - upload_file_to_bucket( - format!("{}/{}", "maven", artifact_path), - bytes.to_vec(), - Some("application/java-archive".to_string()), - uploaded_files_mutex.as_ref(), - semaphore.clone(), - ).await?; - } - - Ok::(lib) - })).await?; - - let elapsed = now.elapsed(); - info!("Elapsed lib DL: {:.2?}", elapsed); - - let new_profile = PartialVersionInfo { - id: version_info.id, - inherits_from: version_info.inherits_from, - release_time: version_info.release_time, - time: version_info.time, - main_class: version_info.main_class, - minecraft_arguments: version_info.minecraft_arguments, - arguments: version_info.arguments, - libraries: libs, - type_: version_info.type_, - data: Some(profile.data), - processors: Some(profile.processors), - }; - - let version_path = format!( - "forge/v{}/versions/{}.json", - daedalus::modded::CURRENT_FORGE_FORMAT_VERSION, - new_profile.id - ); - - upload_file_to_bucket( - version_path.clone(), - serde_json::to_vec(&new_profile)?, - Some("application/json".to_string()), - uploaded_files_mutex.as_ref(), - semaphore.clone(), - ).await?; - - return Ok(Some(LoaderVersion { - id: loader_version_full, - url: format_url(&version_path), - stable: false - })); - } + let install_profile = read_json::( + &mut zip, + "install_profile.json", + ) + .await? + .ok_or_else(|| { + crate::ErrorKind::InvalidInput(format!( + "No install_profile.json present for loader {}", + loader.installer_url + )) + })?; + + let forge_library = + read_file(&mut zip, &install_profile.install.file_path) + .await? + .ok_or_else(|| { + crate::ErrorKind::InvalidInput(format!( + "No forge library present for loader {}", + loader.installer_url + )) + })?; + + upload_files.insert( + format!( + "maven/{}", + get_path_from_artifact(&install_profile.install.path)? + ), + UploadFile { + file: bytes::Bytes::from(forge_library), + content_type: None, + }, + ); + + Ok(PartialVersionInfo { + id: install_profile.version_info.id, + inherits_from: install_profile.install.minecraft, + release_time: install_profile.version_info.release_time, + time: install_profile.version_info.time, + main_class: install_profile.version_info.main_class, + minecraft_arguments: install_profile + .version_info + .minecraft_arguments + .clone(), + arguments: install_profile + .version_info + .minecraft_arguments + .map(|x| { + [( + daedalus::minecraft::ArgumentType::Game, + x.split(' ') + .map(|x| { + daedalus::minecraft::Argument::Normal( + x.to_string(), + ) + }) + .collect(), + )] + .iter() + .cloned() + .collect() + }), + libraries: install_profile + .version_info + .libraries + .into_iter() + .map(|mut lib| { + // For all libraries besides the forge lib extracted, we mirror them from maven servers + if lib.name != install_profile.install.path { + // TODO: add mirrors "https://maven.creeperhost.net/", "https://libraries.minecraft.net/" + insert_mirrored_artifact( + &lib.name, + lib.url.clone().unwrap_or_else(|| { + maven_url.to_string() + }), + mirror_artifacts, + )?; } - Ok(None) - }.await - }); + lib.url = Some(format_url("maven/")); - { - let len = loaders_futures.len(); - let mut versions = loaders_futures.into_iter().peekable(); - let mut chunk_index = 0; - while versions.peek().is_some() { - let now = Instant::now(); + Ok(lib) + }) + .collect::, Error>>()?, + type_: install_profile.version_info.type_, + data: None, + processors: None, + }) + } else if loader.format_version == 2 { + #[derive(Deserialize, Debug)] + #[serde(rename_all = "camelCase")] + struct ForgeInstallerProfileV2 { + // pub spec: i32, + // pub profile: String, + // pub version: String, + // pub json: String, + pub path: Option, + // pub minecraft: String, + pub data: HashMap, + pub libraries: Vec, + pub processors: Vec, + } - let chunk: Vec<_> = versions.by_ref().take(1).collect(); - let res = futures::future::try_join_all(chunk).await?; - loaders_versions.extend(res.into_iter().flatten()); + let install_profile = read_json::( + &mut zip, + "install_profile.json", + ) + .await? + .ok_or_else(|| { + crate::ErrorKind::InvalidInput(format!( + "No install_profile.json present for loader {}", + loader.installer_url + )) + })?; + + let mut version_info = + read_json::(&mut zip, "version.json") + .await? + .ok_or_else(|| { + crate::ErrorKind::InvalidInput(format!( + "No version.json present for loader {}", + loader.installer_url + )) + })?; + + version_info.processors = Some(install_profile.processors); + version_info.libraries.extend( + install_profile.libraries.into_iter().map(|mut x| { + x.include_in_classpath = false; + + x + }), + ); + + async fn mirror_forge_library( + mut zip: ZipFileReader, + mut lib: daedalus::minecraft::Library, + upload_files: &DashMap, + mirror_artifacts: &DashMap, + ) -> Result + { + let artifact_path = get_path_from_artifact(&lib.name)?; + + if let Some(ref mut artifact) = + lib.downloads.as_mut().and_then(|x| x.artifact.as_mut()) + { + if !artifact.url.is_empty() { + insert_mirrored_artifact( + &lib.name, + artifact.url.clone(), + mirror_artifacts, + )?; - chunk_index += 1; + artifact.url = + format_url(&format!("maven/{}", artifact_path)); - let elapsed = now.elapsed(); - info!("Loader Chunk {}/{len} Elapsed: {:.2?}", chunk_index, elapsed); + return Ok(lib); + } + } else if let Some(url) = &lib.url { + if !url.is_empty() { + // TODO: add mirrors "https://maven.creeperhost.net/", "https://libraries.minecraft.net/" + insert_mirrored_artifact( + &lib.name, + url.clone(), + mirror_artifacts, + )?; + + lib.url = Some(format_url("maven/")); + + return Ok(lib); } } - //futures::future::try_join_all(loaders_futures).await?; - } - versions.lock().await.push(daedalus::modded::Version { - id: minecraft_version, - stable: true, - loaders: loaders_versions - }); - - Ok::<(), Error>(()) - }); - } - } + // Other libraries are generally available in the "maven" directory of the installer. If they are + // not present here, they will be generated by Forge processors. + let extract_path = format!("maven/{artifact_path}"); + if let Some(file) = + read_file(&mut zip, &extract_path).await? + { + upload_files.insert( + extract_path, + UploadFile { + file: bytes::Bytes::from(file), + content_type: None, + }, + ); + + lib.url = Some(format_url("maven/")); + } else { + lib.downloadable = false; + } - { - let len = version_futures.len(); - let mut versions = version_futures.into_iter().peekable(); - let mut chunk_index = 0; - while versions.peek().is_some() { - let now = Instant::now(); + Ok(lib) + } - let chunk: Vec<_> = versions.by_ref().take(1).collect(); - futures::future::try_join_all(chunk).await?; + version_info.libraries = futures::future::try_join_all( + version_info.libraries.into_iter().map(|lib| { + mirror_forge_library( + zip.clone(), + lib, + upload_files, + mirror_artifacts, + ) + }), + ) + .await?; + + // In Minecraft Forge modern installers, processors are run during the install process. Some processors + // are extracted from the installer JAR. This function finds these files, extracts them, and uploads them + // and registers them as libraries instead. + // Ex: + // "BINPATCH": { + // "client": "/data/client.lzma", + // "server": "/data/server.lzma" + // }, + // Becomes: + // "BINPATCH": { + // "client": "[net.minecraftforge:forge:1.20.3-49.0.1:shim:client@lzma]", + // "server": "[net.minecraftforge:forge:1.20.3-49.0.1:shim:server@lzma]" + // }, + // And the resulting library is added to the profile's libraries + let mut new_data = HashMap::new(); + for (key, entry) in install_profile.data { + async fn extract_data( + zip: &mut ZipFileReader, + key: &str, + value: &str, + upload_files: &DashMap, + libs: &mut Vec, + install_profile_path: Option<&str>, + version: &ForgeVersion, + ) -> Result { + let extract_file = + read_file(zip, &value[1..value.len()]) + .await? + .ok_or_else(|| { + crate::ErrorKind::InvalidInput(format!( + "Unable reading data key {key} at path {value}", + )) + })?; + + let file_name = value.split('/').last() + .ok_or_else(|| { + crate::ErrorKind::InvalidInput(format!( + "Unable reading filename for data key {key} at path {value}", + + )) + })?; + + let mut file = file_name.split('.'); + let file_name = file.next() + .ok_or_else(|| { + crate::ErrorKind::InvalidInput(format!( + "Unable reading filename only for data key {key} at path {value}", + )) + })?; + let ext = file.next() + .ok_or_else(|| { + crate::ErrorKind::InvalidInput(format!( + "Unable reading extension only for data key {key} at path {value}", + )) + })?; + + let path = format!( + "{}:{}@{}", + install_profile_path.unwrap_or(&*format!( + "net.minecraftforge:forge:{}", + version.raw + )), + file_name, + ext + ); + + upload_files.insert( + format!("maven/{}", get_path_from_artifact(&path)?), + UploadFile { + file: bytes::Bytes::from(extract_file), + content_type: None, + }, + ); + + libs.push(daedalus::minecraft::Library { + downloads: None, + extract: None, + name: path.clone(), + url: Some(format_url("maven/")), + natives: None, + rules: None, + checksums: None, + include_in_classpath: false, + downloadable: true, + }); + + Ok(format!("[{path}]")) + } - chunk_index += 1; + let client = if entry.client.starts_with('/') { + extract_data( + &mut zip, + &key, + &entry.client, + upload_files, + &mut version_info.libraries, + install_profile.path.as_deref(), + loader, + ) + .await? + } else { + entry.client.clone() + }; + + let server = if entry.server.starts_with('/') { + extract_data( + &mut zip, + &key, + &entry.server, + upload_files, + &mut version_info.libraries, + install_profile.path.as_deref(), + loader, + ) + .await? + } else { + entry.server.clone() + }; - let elapsed = now.elapsed(); - info!("Chunk {}/{len} Elapsed: {:.2?}", chunk_index, elapsed); - } - } - //futures::future::try_join_all(version_futures).await?; + new_data.insert( + key.clone(), + daedalus::modded::SidedDataEntry { client, server }, + ); + } - if let Ok(versions) = Arc::try_unwrap(versions) { - let mut versions = versions.into_inner(); + version_info.data = Some(new_data); - versions.sort_by(|x, y| { - minecraft_versions - .versions - .iter() - .position(|z| { - x.id.replace("1.7.10_pre4", "1.7.10-pre4") == z.id - }) - .unwrap_or_default() - .cmp( - &minecraft_versions - .versions - .iter() - .position(|z| { - y.id.replace("1.7.10_pre4", "1.7.10-pre4") == z.id - }) - .unwrap_or_default(), - ) - }); - - for version in &mut versions { - let loader_versions = maven_metadata.get(&version.id); - if let Some(loader_versions) = loader_versions { - version.loaders.sort_by(|x, y| { - loader_versions - .iter() - .position(|z| &y.id == z) - .unwrap_or_default() - .cmp( - &loader_versions - .iter() - .position(|z| &x.id == z) - .unwrap_or_default(), - ) - }) + Ok(version_info) + } else { + Err(crate::ErrorKind::InvalidInput(format!( + "Unknown format version {} for loader {}", + loader.format_version, loader.installer_url + )) + .into()) } } - upload_file_to_bucket( - format!( - "forge/v{}/manifest.json", - daedalus::modded::CURRENT_FORGE_FORMAT_VERSION, - ), - serde_json::to_vec(&Manifest { - game_versions: versions, - })?, - Some("application/json".to_string()), - uploaded_files_mutex.as_ref(), - semaphore, + let forge_version_infos = futures::future::try_join_all( + forge_installers + .into_iter() + .enumerate() + .map(|(index, raw)| { + let loader = fetch_versions[index]; + + read_forge_installer( + raw, + loader, + maven_url, + upload_files, + mirror_artifacts, + ) + }), ) .await?; - } - if let Ok(uploaded_files_mutex) = Arc::try_unwrap(uploaded_files_mutex) { - uploaded_files.extend(uploaded_files_mutex.into_inner()); + let serialized_version_manifests = forge_version_infos + .iter() + .map(|x| serde_json::to_vec(x).map(bytes::Bytes::from)) + .collect::, serde_json::Error>>()?; + + serialized_version_manifests + .into_iter() + .enumerate() + .for_each(|(index, bytes)| { + let loader = fetch_versions[index]; + + let version_path = format!( + "{mod_loader}/v{format_version}/versions/{}.json", + loader.loader_version + ); + + upload_files.insert( + version_path, + UploadFile { + file: bytes, + content_type: Some("application/json".to_string()), + }, + ); + }); + + let forge_manifest_path = + format!("{mod_loader}/v{format_version}/manifest.json",); + + let manifest = daedalus::modded::Manifest { + game_versions: forge_versions + .into_iter() + .rev() + .chunk_by(|x| x.game_version.clone()) + .into_iter() + .map(|(game_version, loaders)| daedalus::modded::Version { + id: game_version, + stable: true, + loaders: loaders + .map(|x| daedalus::modded::LoaderVersion { + url: format_url(&format!( + "{mod_loader}/v{format_version}/versions/{}.json", + x.loader_version + )), + id: x.loader_version, + stable: false, + }) + .collect(), + }) + .collect(), + }; + + upload_files.insert( + forge_manifest_path, + UploadFile { + file: bytes::Bytes::from(serde_json::to_vec(&manifest)?), + content_type: Some("application/json".to_string()), + }, + ); } Ok(()) } -const DEFAULT_MAVEN_METADATA_URL: &str = - "https://files.minecraftforge.net/net/minecraftforge/forge/maven-metadata.json"; - -/// Fetches the forge maven metadata from the specified URL. If no URL is specified, the default is used. -/// Returns a hashmap specifying the versions of the forge mod loader -/// The hashmap key is a Minecraft version, and the value is the loader versions that work on -/// the specified Minecraft version -pub async fn fetch_maven_metadata( - url: Option<&str>, - semaphore: Arc, -) -> Result>, Error> { - Ok(serde_json::from_slice( - &download_file( - url.unwrap_or(DEFAULT_MAVEN_METADATA_URL), - None, - semaphore, - ) - .await?, - )?) -} - -#[derive(Serialize, Deserialize, Debug)] -#[serde(rename_all = "camelCase")] -struct ForgeInstallerProfileInstallDataV1 { - pub mirror_list: String, - pub target: String, - /// Path to the Forge universal library - pub file_path: String, - pub logo: String, - pub welcome: String, - pub version: String, - /// Maven coordinates of the Forge universal library - pub path: String, - pub profile_name: String, - pub minecraft: String, -} - -#[derive(Serialize, Deserialize, Debug)] -#[serde(rename_all = "camelCase")] -struct ForgeInstallerProfileManifestV1 { - pub id: String, - pub libraries: Vec, - pub main_class: Option, - pub minecraft_arguments: Option, - pub release_time: DateTime, - pub time: DateTime, - pub type_: VersionType, - pub assets: Option, - pub inherits_from: Option, - pub jar: Option, -} - -#[derive(Serialize, Deserialize, Debug)] -#[serde(rename_all = "camelCase")] -struct ForgeInstallerProfileV1 { - pub install: ForgeInstallerProfileInstallDataV1, - pub version_info: ForgeInstallerProfileManifestV1, -} - -#[derive(Serialize, Deserialize, Debug)] -#[serde(rename_all = "camelCase")] -struct ForgeInstallerProfileV2 { - pub spec: i32, - pub profile: String, - pub version: String, - pub json: String, - pub path: Option, - pub minecraft: String, - pub data: HashMap, - pub libraries: Vec, - pub processors: Vec, +#[derive(Debug)] +struct ForgeVersion { + pub format_version: usize, + pub raw: String, + pub loader_version: String, + pub game_version: String, + pub installer_url: String, } diff --git a/daedalus_client/src/main.rs b/daedalus_client/src/main.rs index af34ce3a8..33df7090e 100644 --- a/daedalus_client/src/main.rs +++ b/daedalus_client/src/main.rs @@ -1,118 +1,147 @@ -use log::{error, info, warn}; -use s3::creds::Credentials; -use s3::error::S3Error; -use s3::{Bucket, Region}; +use crate::util::{ + format_url, upload_file_to_bucket, upload_url_to_bucket_mirrors, + REQWEST_CLIENT, +}; +use daedalus::get_path_from_artifact; +use dashmap::{DashMap, DashSet}; use std::sync::Arc; -use std::time::Duration; use tokio::sync::Semaphore; +use tracing_error::ErrorLayer; +use tracing_subscriber::{fmt, prelude::*, EnvFilter}; +mod error; mod fabric; mod forge; mod minecraft; -mod neo; -mod quilt; - -#[derive(thiserror::Error, Debug)] -pub enum Error { - #[error("{0}")] - DaedalusError(#[from] daedalus::Error), - #[error("Error while deserializing JSON")] - SerdeError(#[from] serde_json::Error), - #[error("Error while deserializing XML")] - XMLError(#[from] serde_xml_rs::Error), - #[error("Unable to fetch {item}")] - FetchError { inner: reqwest::Error, item: String }, - #[error("Error while managing asynchronous tasks")] - TaskError(#[from] tokio::task::JoinError), - #[error("Error while uploading file to S3")] - S3Error { inner: S3Error, file: String }, - #[error("Error while parsing version as semver: {0}")] - SemVerError(#[from] semver::Error), - #[error("Error while reading zip file: {0}")] - ZipError(#[from] zip::result::ZipError), - #[error("Error while reading zip file: {0}")] - IoError(#[from] std::io::Error), - #[error("Error while obtaining strong reference to Arc")] - ArcError, - #[error("Error acquiring semaphore: {0}")] - AcquireError(#[from] tokio::sync::AcquireError), -} +pub mod util; + +pub use error::{Error, ErrorKind, Result}; #[tokio::main] -async fn main() { - env_logger::init(); +async fn main() -> Result<()> { + dotenvy::dotenv().ok(); - if check_env_vars() { - error!("Some environment variables are missing!"); + let subscriber = tracing_subscriber::registry() + .with(fmt::layer()) + .with(EnvFilter::from_default_env()) + .with(ErrorLayer::default()); - return; - } + tracing::subscriber::set_global_default(subscriber)?; - let mut timer = tokio::time::interval(Duration::from_secs(60 * 60)); - let semaphore = Arc::new(Semaphore::new(10)); + tracing::info!("Initialized tracing. Starting Daedalus!"); - loop { - timer.tick().await; + if check_env_vars() { + tracing::error!("Some environment variables are missing!"); - let mut uploaded_files = Vec::new(); + return Ok(()); + } - let versions = match minecraft::retrieve_data( - &mut uploaded_files, - semaphore.clone(), + let semaphore = Arc::new(Semaphore::new( + dotenvy::var("CONCURRENCY_LIMIT") + .ok() + .and_then(|x| x.parse().ok()) + .unwrap_or(10), + )); + + // path, upload file + let upload_files: DashMap = DashMap::new(); + // path, mirror artifact + let mirror_artifacts: DashMap = DashMap::new(); + + minecraft::fetch(semaphore.clone(), &upload_files, &mirror_artifacts) + .await?; + fabric::fetch_fabric(semaphore.clone(), &upload_files, &mirror_artifacts) + .await?; + fabric::fetch_quilt(semaphore.clone(), &upload_files, &mirror_artifacts) + .await?; + forge::fetch_neo(semaphore.clone(), &upload_files, &mirror_artifacts) + .await?; + forge::fetch_forge(semaphore.clone(), &upload_files, &mirror_artifacts) + .await?; + + futures::future::try_join_all(upload_files.iter().map(|x| { + upload_file_to_bucket( + x.key().clone(), + x.value().file.clone(), + x.value().content_type.clone(), + &semaphore, ) - .await - { - Ok(res) => Some(res), - Err(err) => { - error!("{:?}", err); - - None + })) + .await?; + + futures::future::try_join_all(mirror_artifacts.iter().map(|x| { + upload_url_to_bucket_mirrors( + format!("maven/{}", x.key()), + x.value().mirrors.iter().map(|x| x.key().clone()).collect(), + &semaphore, + ) + })) + .await?; + + if let Ok(token) = dotenvy::var("CLOUDFLARE_TOKEN") { + if let Ok(zone_id) = dotenvy::var("CLOUDFLARE_ZONE_ID") { + let cache_clears = upload_files + .into_iter() + .map(|x| format_url(&x.0)) + .chain( + mirror_artifacts + .into_iter() + .map(|x| format_url(&format!("maven/{}", x.0))), + ) + .collect::>(); + + // Cloudflare ratelimits cache clears to 500 files per request + for chunk in cache_clears.chunks(500) { + REQWEST_CLIENT.post(format!("https://api.cloudflare.com/client/v4/zones/{zone_id}/purge_cache")) + .bearer_auth(&token) + .json(&serde_json::json!({ + "files": chunk + })) + .send() + .await + .map_err(|err| { + ErrorKind::Fetch { + inner: err, + item: "cloudflare clear cache".to_string(), + } + })? + .error_for_status() + .map_err(|err| { + ErrorKind::Fetch { + inner: err, + item: "cloudflare clear cache".to_string(), + } + })?; } - }; - - if let Some(manifest) = versions { - match fabric::retrieve_data( - &manifest, - &mut uploaded_files, - semaphore.clone(), - ) - .await - { - Ok(..) => {} - Err(err) => error!("{:?}", err), - }; - match forge::retrieve_data( - &manifest, - &mut uploaded_files, - semaphore.clone(), - ) - .await - { - Ok(..) => {} - Err(err) => error!("{:?}", err), - }; - match quilt::retrieve_data( - &manifest, - &mut uploaded_files, - semaphore.clone(), - ) - .await - { - Ok(..) => {} - Err(err) => error!("{:?}", err), - }; - match neo::retrieve_data( - &manifest, - &mut uploaded_files, - semaphore.clone(), - ) - .await - { - Ok(..) => {} - Err(err) => error!("{:?}", err), - }; } } + + Ok(()) +} + +pub struct UploadFile { + file: bytes::Bytes, + content_type: Option, +} + +pub struct MirrorArtifact { + pub mirrors: DashSet, +} + +pub fn insert_mirrored_artifact( + artifact: &str, + mirror: String, + mirror_artifacts: &DashMap, +) -> Result<()> { + mirror_artifacts + .entry(get_path_from_artifact(artifact)?) + .or_insert(MirrorArtifact { + mirrors: DashSet::new(), + }) + .mirrors + .insert(mirror); + + Ok(()) } fn check_env_vars() -> bool { @@ -124,7 +153,7 @@ fn check_env_vars() -> bool { .and_then(|s| s.parse::().ok()) .is_none() { - warn!( + tracing::warn!( "Variable `{}` missing in dotenvy or not of type `{}`", var, std::any::type_name::() @@ -143,110 +172,14 @@ fn check_env_vars() -> bool { failed |= check_var::("S3_REGION"); failed |= check_var::("S3_BUCKET_NAME"); - failed -} - -lazy_static::lazy_static! { - static ref CLIENT : Bucket = { - let region = dotenvy::var("S3_REGION").unwrap(); - let b = Bucket::new( - &dotenvy::var("S3_BUCKET_NAME").unwrap(), - if &*region == "r2" { - Region::R2 { - account_id: dotenvy::var("S3_URL").unwrap(), - } - } else { - Region::Custom { - region: region.clone(), - endpoint: dotenvy::var("S3_URL").unwrap(), - } - }, - Credentials::new( - Some(&*dotenvy::var("S3_ACCESS_TOKEN").unwrap()), - Some(&*dotenvy::var("S3_SECRET").unwrap()), - None, - None, - None, - ).unwrap(), - ).unwrap(); - - if region == "path-style" { - b.with_path_style() - } else { - b - } - }; -} - -pub async fn upload_file_to_bucket( - path: String, - bytes: Vec, - content_type: Option, - uploaded_files: &tokio::sync::Mutex>, - semaphore: Arc, -) -> Result<(), Error> { - let _permit = semaphore.acquire().await?; - info!("{} started uploading", path); - let key = path.clone(); - - for attempt in 1..=4 { - let result = if let Some(ref content_type) = content_type { - CLIENT - .put_object_with_content_type(key.clone(), &bytes, content_type) - .await - } else { - CLIENT.put_object(key.clone(), &bytes).await - } - .map_err(|err| Error::S3Error { - inner: err, - file: path.clone(), - }); - - match result { - Ok(_) => { - { - info!("{} done uploading", path); - let mut uploaded_files = uploaded_files.lock().await; - uploaded_files.push(key); - } - - return Ok(()); - } - Err(_) if attempt <= 3 => continue, - Err(_) => { - result?; - } - } + if dotenvy::var("CLOUDFLARE_INTEGRATION") + .ok() + .and_then(|x| x.parse::().ok()) + .unwrap_or(false) + { + failed |= check_var::("CLOUDFLARE_TOKEN"); + failed |= check_var::("CLOUDFLARE_ZONE_ID"); } - unreachable!() -} -pub fn format_url(path: &str) -> String { - format!("{}/{}", &*dotenvy::var("BASE_URL").unwrap(), path) -} - -pub async fn download_file( - url: &str, - sha1: Option<&str>, - semaphore: Arc, -) -> Result { - let _permit = semaphore.acquire().await?; - info!("{} started downloading", url); - let val = daedalus::download_file(url, sha1).await?; - info!("{} finished downloading", url); - Ok(val) -} - -pub async fn download_file_mirrors( - base: &str, - mirrors: &[&str], - sha1: Option<&str>, - semaphore: Arc, -) -> Result { - let _permit = semaphore.acquire().await?; - info!("{} started downloading", base); - let val = daedalus::download_file_mirrors(base, mirrors, sha1).await?; - info!("{} finished downloading", base); - - Ok(val) + failed } diff --git a/daedalus_client/src/minecraft.rs b/daedalus_client/src/minecraft.rs index 1f0e5b7cf..e98b12044 100644 --- a/daedalus_client/src/minecraft.rs +++ b/daedalus_client/src/minecraft.rs @@ -1,286 +1,181 @@ -use crate::download_file; -use crate::{format_url, upload_file_to_bucket, Error}; -use daedalus::get_hash; +use crate::util::fetch_json; +use crate::{ + util::download_file, util::format_url, util::sha1_async, Error, + MirrorArtifact, UploadFile, +}; use daedalus::minecraft::{ - merge_partial_library, Library, PartialLibrary, VersionManifest, + merge_partial_library, Library, PartialLibrary, VersionInfo, + VersionManifest, VERSION_MANIFEST_URL, }; -use log::info; +use dashmap::DashMap; use serde::Deserialize; use std::sync::Arc; -use std::time::Instant; -use tokio::sync::{Mutex, Semaphore}; +use tokio::sync::Semaphore; -pub async fn retrieve_data( - uploaded_files: &mut Vec, +#[tracing::instrument(skip(semaphore, upload_files, _mirror_artifacts))] +pub async fn fetch( semaphore: Arc, -) -> Result { - let old_manifest = daedalus::minecraft::fetch_version_manifest(Some( - &*format_url(&format!( + upload_files: &DashMap, + _mirror_artifacts: &DashMap, +) -> Result<(), Error> { + let modrinth_manifest = fetch_json::( + &format_url(&format!( "minecraft/v{}/manifest.json", daedalus::minecraft::CURRENT_FORMAT_VERSION )), - )) + &semaphore, + ) .await .ok(); - - let mut manifest = - daedalus::minecraft::fetch_version_manifest(None).await?; - let cloned_manifest = - Arc::new(Mutex::new(old_manifest.clone().unwrap_or(manifest.clone()))); - - let patches = fetch_library_patches()?; - let cloned_patches = Arc::new(&patches); - - let visited_assets_mutex = Arc::new(Mutex::new(Vec::new())); - let uploaded_files_mutex = Arc::new(Mutex::new(Vec::new())); - - let now = Instant::now(); - - let mut version_futures = Vec::new(); - - for version in manifest.versions.iter_mut() { - version_futures.push(async { - let old_version = if let Some(old_manifest) = &old_manifest { - old_manifest.versions.iter().find(|x| x.id == version.id) - } else { - None - }; - - if old_version.is_some() { - return Ok(()); - } - - let visited_assets_mutex = Arc::clone(&visited_assets_mutex); - let cloned_manifest_mutex = Arc::clone(&cloned_manifest); - let uploaded_files_mutex = Arc::clone(&uploaded_files_mutex); - let semaphore = Arc::clone(&semaphore); - let patches = Arc::clone(&cloned_patches); - - let assets_hash = - old_version.and_then(|x| x.assets_index_sha1.clone()); - - async move { - let mut upload_futures = Vec::new(); - - let mut version_info = - daedalus::minecraft::fetch_version_info(version).await?; - - fn patch_library( - patches: &Vec, - mut library: Library, - ) -> Vec { - let mut val = Vec::new(); - - let actual_patches = patches - .iter() - .filter(|x| x.match_.contains(&library.name)) - .collect::>(); - - if !actual_patches.is_empty() { - for patch in actual_patches { - if let Some(override_) = &patch.override_ { - library = merge_partial_library( - override_.clone(), - library, - ); - } - - if let Some(additional_libraries) = - &patch.additional_libraries - { - for additional_library in additional_libraries { - if patch - .patch_additional_libraries - .unwrap_or(false) - { - let mut libs = patch_library( - patches, - additional_library.clone(), - ); - val.append(&mut libs) - } else { - val.push(additional_library.clone()); - } - } - } - } - - val.push(library); + let mojang_manifest = + fetch_json::(VERSION_MANIFEST_URL, &semaphore).await?; + + // TODO: experimental snapshots: https://github.com/PrismLauncher/meta/blob/main/meta/common/mojang-minecraft-experiments.json + // TODO: old snapshots: https://github.com/PrismLauncher/meta/blob/main/meta/common/mojang-minecraft-old-snapshots.json + + // We check Modrinth's version manifest and compare if the version 1) exists in Modrinth's database and 2) is unchanged + // If they are not, we will fetch them + let (fetch_versions, existing_versions) = + if let Some(mut modrinth_manifest) = modrinth_manifest { + let (mut fetch_versions, mut existing_versions) = + (Vec::new(), Vec::new()); + + for version in mojang_manifest.versions { + if let Some(index) = modrinth_manifest + .versions + .iter() + .position(|x| x.id == version.id) + { + let modrinth_version = + modrinth_manifest.versions.remove(index); + + if modrinth_version + .original_sha1 + .as_ref() + .map(|x| x == &version.sha1) + .unwrap_or(false) + { + existing_versions.push(modrinth_version); } else { - val.push(library); + fetch_versions.push(version); } - - val + } else { + fetch_versions.push(version); } + } - let mut new_libraries = Vec::new(); - for library in version_info.libraries.clone() { - let mut libs = patch_library(&patches, library); - new_libraries.append(&mut libs) + (fetch_versions, existing_versions) + } else { + (mojang_manifest.versions, Vec::new()) + }; + + if !fetch_versions.is_empty() { + let version_manifests = futures::future::try_join_all( + fetch_versions + .iter() + .map(|x| download_file(&x.url, Some(&x.sha1), &semaphore)), + ) + .await? + .into_iter() + .map(|x| serde_json::from_slice(&x)) + .collect::, serde_json::Error>>()?; + + // Patch libraries of Minecraft versions for M-series Mac Support, Better Linux Compatibility, etc + let library_patches = fetch_library_patches()?; + let patched_version_manifests = version_manifests + .into_iter() + .map(|mut x| { + if !library_patches.is_empty() { + let mut new_libraries = Vec::new(); + for library in x.libraries { + let mut libs = patch_library(&library_patches, library); + new_libraries.append(&mut libs) + } + x.libraries = new_libraries } - version_info.libraries = new_libraries; - - let version_info_hash = get_hash(bytes::Bytes::from( - serde_json::to_vec(&version_info)?, - )) - .await?; + x + }) + .collect::>(); + + // serialize + compute hashes + let serialized_version_manifests = patched_version_manifests + .iter() + .map(|x| serde_json::to_vec(x).map(bytes::Bytes::from)) + .collect::, serde_json::Error>>()?; + let hashes_version_manifests = futures::future::try_join_all( + serialized_version_manifests + .iter() + .map(|x| sha1_async(x.clone())), + ) + .await?; + + // We upload the new version manifests and add them to the versions list + let mut new_versions = patched_version_manifests + .into_iter() + .zip(serialized_version_manifests.into_iter()) + .zip(hashes_version_manifests.into_iter()) + .map(|((version, bytes), hash)| { let version_path = format!( "minecraft/v{}/versions/{}.json", daedalus::minecraft::CURRENT_FORMAT_VERSION, version.id ); - let assets_path = format!( - "minecraft/v{}/assets/{}.json", - daedalus::minecraft::CURRENT_FORMAT_VERSION, - version_info.asset_index.id - ); - let assets_index_url = version_info.asset_index.url.clone(); - { - let mut cloned_manifest = - cloned_manifest_mutex.lock().await; + let url = format_url(&version_path); + upload_files.insert( + version_path, + UploadFile { + file: bytes, + content_type: Some("application/json".to_string()), + }, + ); - if let Some(position) = cloned_manifest - .versions + daedalus::minecraft::Version { + original_sha1: fetch_versions .iter() - .position(|x| version.id == x.id) - { - cloned_manifest.versions[position].url = - format_url(&version_path); - cloned_manifest.versions[position].assets_index_sha1 = - Some(version_info.asset_index.sha1.clone()); - cloned_manifest.versions[position].assets_index_url = - Some(format_url(&assets_path)); - cloned_manifest.versions[position].sha1 = - version_info_hash; - } else { - cloned_manifest.versions.insert( - 0, - daedalus::minecraft::Version { - id: version_info.id.clone(), - type_: version_info.type_.clone(), - url: format_url(&version_path), - time: version_info.time, - release_time: version_info.release_time, - sha1: version_info_hash, - compliance_level: 1, - assets_index_url: Some( - version_info.asset_index.sha1.clone(), - ), - assets_index_sha1: Some( - version_info.asset_index.sha1.clone(), - ), - }, - ) - } - } - - let mut download_assets = false; - - { - let mut visited_assets = visited_assets_mutex.lock().await; - - if !visited_assets.contains(&version_info.asset_index.id) { - if let Some(assets_hash) = assets_hash { - if version_info.asset_index.sha1 != assets_hash { - download_assets = true; - } - } else { - download_assets = true; - } - } - - if download_assets { - visited_assets - .push(version_info.asset_index.id.clone()); - } - } - - if download_assets { - let assets_index = download_file( - &assets_index_url, - Some(&version_info.asset_index.sha1), - semaphore.clone(), - ) - .await?; - - { - upload_futures.push(upload_file_to_bucket( - assets_path, - assets_index.to_vec(), - Some("application/json".to_string()), - uploaded_files_mutex.as_ref(), - semaphore.clone(), - )); - } + .find(|x| x.id == version.id) + .map(|x| x.sha1.clone()), + id: version.id, + type_: version.type_, + url, + time: version.time, + release_time: version.release_time, + sha1: hash, + compliance_level: 1, } + }) + .chain(existing_versions.into_iter()) + .collect::>(); - { - upload_futures.push(upload_file_to_bucket( - version_path, - serde_json::to_vec(&version_info)?, - Some("application/json".to_string()), - uploaded_files_mutex.as_ref(), - semaphore.clone(), - )); - } - - futures::future::try_join_all(upload_futures).await?; - - Ok::<(), Error>(()) - } - .await?; - - Ok::<(), Error>(()) - }) - } - - { - let mut versions = version_futures.into_iter().peekable(); - let mut chunk_index = 0; - while versions.peek().is_some() { - let now = Instant::now(); - - let chunk: Vec<_> = versions.by_ref().take(100).collect(); - futures::future::try_join_all(chunk).await?; - - chunk_index += 1; + new_versions.sort_by(|a, b| b.release_time.cmp(&a.release_time)); - let elapsed = now.elapsed(); - info!("Chunk {} Elapsed: {:.2?}", chunk_index, elapsed); - } - } - //futures::future::try_join_all(version_futures).await?; - - upload_file_to_bucket( - format!( + // create and upload the new manifest + let version_manifest_path = format!( "minecraft/v{}/manifest.json", daedalus::minecraft::CURRENT_FORMAT_VERSION - ), - serde_json::to_vec(&*cloned_manifest.lock().await)?, - Some("application/json".to_string()), - uploaded_files_mutex.as_ref(), - semaphore, - ) - .await?; - - if let Ok(uploaded_files_mutex) = Arc::try_unwrap(uploaded_files_mutex) { - uploaded_files.extend(uploaded_files_mutex.into_inner()); + ); + + let new_manifest = VersionManifest { + latest: mojang_manifest.latest, + versions: new_versions, + }; + + upload_files.insert( + version_manifest_path, + UploadFile { + file: bytes::Bytes::from(serde_json::to_vec(&new_manifest)?), + content_type: Some("application/json".to_string()), + }, + ); } - let elapsed = now.elapsed(); - info!("Elapsed: {:.2?}", elapsed); - - Ok(Arc::try_unwrap(cloned_manifest) - .map_err(|_| Error::ArcError)? - .into_inner()) + Ok(()) } #[derive(Deserialize, Debug)] #[serde(rename_all = "camelCase")] -/// A version of the fabric loader -struct LibraryPatch { +pub struct LibraryPatch { #[serde(rename = "_comment")] pub _comment: String, #[serde(rename = "match")] @@ -291,8 +186,45 @@ struct LibraryPatch { pub patch_additional_libraries: Option, } -/// Fetches the list of fabric versions fn fetch_library_patches() -> Result, Error> { let patches = include_bytes!("../library-patches.json"); Ok(serde_json::from_slice(patches)?) } + +pub fn patch_library( + patches: &Vec, + mut library: Library, +) -> Vec { + let mut val = Vec::new(); + + let actual_patches = patches + .iter() + .filter(|x| x.match_.contains(&library.name)) + .collect::>(); + + if !actual_patches.is_empty() { + for patch in actual_patches { + if let Some(override_) = &patch.override_ { + library = merge_partial_library(override_.clone(), library); + } + + if let Some(additional_libraries) = &patch.additional_libraries { + for additional_library in additional_libraries { + if patch.patch_additional_libraries.unwrap_or(false) { + let mut libs = + patch_library(patches, additional_library.clone()); + val.append(&mut libs) + } else { + val.push(additional_library.clone()); + } + } + } + } + + val.push(library); + } else { + val.push(library); + } + + val +} diff --git a/daedalus_client/src/neo.rs b/daedalus_client/src/neo.rs deleted file mode 100644 index ddc075643..000000000 --- a/daedalus_client/src/neo.rs +++ /dev/null @@ -1,495 +0,0 @@ -use crate::{download_file, format_url, upload_file_to_bucket, Error}; -use daedalus::minecraft::{Library, VersionManifest}; -use daedalus::modded::{ - LoaderVersion, Manifest, PartialVersionInfo, Processor, SidedDataEntry, -}; -use log::info; -use semver::Version; -use serde::{Deserialize, Serialize}; -use std::collections::HashMap; -use std::io::Read; -use std::sync::Arc; -use std::time::Instant; -use tokio::sync::{Mutex, Semaphore}; - -pub async fn retrieve_data( - minecraft_versions: &VersionManifest, - uploaded_files: &mut Vec, - semaphore: Arc, -) -> Result<(), Error> { - let maven_metadata = fetch_maven_metadata(semaphore.clone()).await?; - let old_manifest = daedalus::modded::fetch_manifest(&format_url(&format!( - "neo/v{}/manifest.json", - daedalus::modded::CURRENT_NEOFORGE_FORMAT_VERSION, - ))) - .await - .ok(); - - let old_versions = - Arc::new(Mutex::new(if let Some(old_manifest) = old_manifest { - old_manifest.game_versions - } else { - Vec::new() - })); - - let versions = Arc::new(Mutex::new(Vec::new())); - - let visited_assets_mutex = Arc::new(Mutex::new(Vec::new())); - let uploaded_files_mutex = Arc::new(Mutex::new(Vec::new())); - - let mut version_futures = Vec::new(); - - for (minecraft_version, loader_versions) in maven_metadata.clone() { - let mut loaders = Vec::new(); - - for (full, loader_version, new_forge) in loader_versions { - let version = Version::parse(&loader_version)?; - - loaders.push((full, version, new_forge.to_string())) - } - - if !loaders.is_empty() { - version_futures.push(async { - let mut loaders_versions = Vec::new(); - - { - let loaders_futures = loaders.into_iter().map(|(loader_version_full, _, new_forge)| async { - let versions_mutex = Arc::clone(&old_versions); - let visited_assets = Arc::clone(&visited_assets_mutex); - let uploaded_files_mutex = Arc::clone(&uploaded_files_mutex); - let semaphore = Arc::clone(&semaphore); - let minecraft_version = minecraft_version.clone(); - - async move { - { - let versions = versions_mutex.lock().await; - let version = versions.iter().find(|x| - x.id == minecraft_version).and_then(|x| x.loaders.iter().find(|x| x.id == loader_version_full)); - - if let Some(version) = version { - return Ok::, Error>(Some(version.clone())); - } - } - - info!("Forge - Installer Start {}", loader_version_full.clone()); - let bytes = download_file(&format!("https://maven.neoforged.net/net/neoforged/{1}/{0}/{1}-{0}-installer.jar", loader_version_full, if &*new_forge == "true" { "neoforge" } else { "forge" }), None, semaphore.clone()).await?; - - let reader = std::io::Cursor::new(bytes); - - if let Ok(archive) = zip::ZipArchive::new(reader) { - let mut archive_clone = archive.clone(); - let mut profile = tokio::task::spawn_blocking(move || { - let mut install_profile = archive_clone.by_name("install_profile.json")?; - - let mut contents = String::new(); - install_profile.read_to_string(&mut contents)?; - - Ok::(serde_json::from_str::(&contents)?) - }).await??; - - let mut archive_clone = archive.clone(); - let version_info = tokio::task::spawn_blocking(move || { - let mut install_profile = archive_clone.by_name("version.json")?; - - let mut contents = String::new(); - install_profile.read_to_string(&mut contents)?; - - Ok::(serde_json::from_str::(&contents)?) - }).await??; - - - let mut libs : Vec = version_info.libraries.into_iter().chain(profile.libraries.into_iter().map(|x| Library { - downloads: x.downloads, - extract: x.extract, - name: x.name, - url: x.url, - natives: x.natives, - rules: x.rules, - checksums: x.checksums, - include_in_classpath: false - })).collect(); - - let mut local_libs : HashMap = HashMap::new(); - - for lib in &libs { - if lib.downloads.as_ref().and_then(|x| x.artifact.as_ref().map(|x| x.url.is_empty())).unwrap_or(false) { - let mut archive_clone = archive.clone(); - let lib_name_clone = lib.name.clone(); - - let lib_bytes = tokio::task::spawn_blocking(move || { - let mut lib_file = archive_clone.by_name(&format!("maven/{}", daedalus::get_path_from_artifact(&lib_name_clone)?))?; - let mut lib_bytes = Vec::new(); - lib_file.read_to_end(&mut lib_bytes)?; - - Ok::(bytes::Bytes::from(lib_bytes)) - }).await??; - - local_libs.insert(lib.name.clone(), lib_bytes); - } - } - - let path = profile.path.clone(); - let version = profile.version.clone(); - - for entry in profile.data.values_mut() { - if entry.client.starts_with('/') || entry.server.starts_with('/') { - macro_rules! read_data { - ($value:expr) => { - let mut archive_clone = archive.clone(); - let value_clone = $value.clone(); - let lib_bytes = tokio::task::spawn_blocking(move || { - let mut lib_file = archive_clone.by_name(&value_clone[1..value_clone.len()])?; - let mut lib_bytes = Vec::new(); - lib_file.read_to_end(&mut lib_bytes)?; - - Ok::(bytes::Bytes::from(lib_bytes)) - }).await??; - - let split = $value.split('/').last(); - - if let Some(last) = split { - let mut file = last.split('.'); - - if let Some(file_name) = file.next() { - if let Some(ext) = file.next() { - let path = format!("{}:{}@{}", path.as_deref().unwrap_or(&*format!("net.minecraftforge:forge:{}", version)), file_name, ext); - $value = format!("[{}]", &path); - local_libs.insert(path.clone(), bytes::Bytes::from(lib_bytes)); - - libs.push(Library { - downloads: None, - extract: None, - name: path, - url: Some("".to_string()), - natives: None, - rules: None, - checksums: None, - include_in_classpath: false, - }); - } - } - } - } - } - - if entry.client.starts_with('/') { - read_data!(entry.client); - } - - if entry.server.starts_with('/') { - read_data!(entry.server); - } - } - } - - let now = Instant::now(); - let libs = futures::future::try_join_all(libs.into_iter().map(|mut lib| async { - let artifact_path = - daedalus::get_path_from_artifact(&lib.name)?; - - { - let mut visited_assets = visited_assets.lock().await; - - if visited_assets.contains(&lib.name) { - if let Some(ref mut downloads) = lib.downloads { - if let Some(ref mut artifact) = downloads.artifact { - artifact.url = format_url(&format!("maven/{}", artifact_path)); - } - } else if lib.url.is_some() { - lib.url = Some(format_url("maven/")); - } - - return Ok::(lib); - } else { - visited_assets.push(lib.name.clone()) - } - } - - let artifact_bytes = if let Some(ref mut downloads) = lib.downloads { - if let Some(ref mut artifact) = downloads.artifact { - let res = if artifact.url.is_empty() { - local_libs.get(&lib.name).cloned() - } else { - Some(download_file( - &artifact.url, - Some(&*artifact.sha1), - semaphore.clone(), - ) - .await?) - }; - - if res.is_some() { - artifact.url = format_url(&format!("maven/{}", artifact_path)); - } - - res - } else { None } - } else if let Some(ref mut url) = lib.url { - let res = if url.is_empty() { - local_libs.get(&lib.name).cloned() - } else { - Some(download_file( - url, - None, - semaphore.clone(), - ) - .await?) - }; - - if res.is_some() { - lib.url = Some(format_url("maven/")); - } - - res - } else { None }; - - if let Some(bytes) = artifact_bytes { - upload_file_to_bucket( - format!("{}/{}", "maven", artifact_path), - bytes.to_vec(), - Some("application/java-archive".to_string()), - uploaded_files_mutex.as_ref(), - semaphore.clone(), - ).await?; - } - - Ok::(lib) - })).await?; - - let elapsed = now.elapsed(); - info!("Elapsed lib DL: {:.2?}", elapsed); - - let new_profile = PartialVersionInfo { - id: version_info.id, - inherits_from: version_info.inherits_from, - release_time: version_info.release_time, - time: version_info.time, - main_class: version_info.main_class, - minecraft_arguments: version_info.minecraft_arguments, - arguments: version_info.arguments, - libraries: libs, - type_: version_info.type_, - data: Some(profile.data), - processors: Some(profile.processors), - }; - - let version_path = format!( - "neo/v{}/versions/{}.json", - daedalus::modded::CURRENT_NEOFORGE_FORMAT_VERSION, - new_profile.id - ); - - upload_file_to_bucket( - version_path.clone(), - serde_json::to_vec(&new_profile)?, - Some("application/json".to_string()), - uploaded_files_mutex.as_ref(), - semaphore.clone(), - ).await?; - - return Ok(Some(LoaderVersion { - id: loader_version_full, - url: format_url(&version_path), - stable: false - })); - } - - Ok(None) - }.await - }); - - { - let len = loaders_futures.len(); - let mut versions = loaders_futures.into_iter().peekable(); - let mut chunk_index = 0; - while versions.peek().is_some() { - let now = Instant::now(); - - let chunk: Vec<_> = versions.by_ref().take(1).collect(); - let res = futures::future::try_join_all(chunk).await?; - loaders_versions.extend(res.into_iter().flatten()); - - chunk_index += 1; - - let elapsed = now.elapsed(); - info!("Loader Chunk {}/{len} Elapsed: {:.2?}", chunk_index, elapsed); - } - } - } - - versions.lock().await.push(daedalus::modded::Version { - id: minecraft_version, - stable: true, - loaders: loaders_versions - }); - - Ok::<(), Error>(()) - }); - } - } - - { - let len = version_futures.len(); - let mut versions = version_futures.into_iter().peekable(); - let mut chunk_index = 0; - while versions.peek().is_some() { - let now = Instant::now(); - - let chunk: Vec<_> = versions.by_ref().take(1).collect(); - futures::future::try_join_all(chunk).await?; - - chunk_index += 1; - - let elapsed = now.elapsed(); - info!("Chunk {}/{len} Elapsed: {:.2?}", chunk_index, elapsed); - } - } - - if let Ok(versions) = Arc::try_unwrap(versions) { - let mut versions = versions.into_inner(); - - versions.sort_by(|x, y| { - minecraft_versions - .versions - .iter() - .position(|z| x.id == z.id) - .unwrap_or_default() - .cmp( - &minecraft_versions - .versions - .iter() - .position(|z| y.id == z.id) - .unwrap_or_default(), - ) - }); - - for version in &mut versions { - let loader_versions = maven_metadata.get(&version.id); - if let Some(loader_versions) = loader_versions { - version.loaders.sort_by(|x, y| { - loader_versions - .iter() - .position(|z| y.id == z.1) - .unwrap_or_default() - .cmp( - &loader_versions - .iter() - .position(|z| x.id == z.1) - .unwrap_or_default(), - ) - }); - version.loaders.reverse(); - } - } - - upload_file_to_bucket( - format!( - "neo/v{}/manifest.json", - daedalus::modded::CURRENT_NEOFORGE_FORMAT_VERSION, - ), - serde_json::to_vec(&Manifest { - game_versions: versions, - })?, - Some("application/json".to_string()), - uploaded_files_mutex.as_ref(), - semaphore, - ) - .await?; - } - - if let Ok(uploaded_files_mutex) = Arc::try_unwrap(uploaded_files_mutex) { - uploaded_files.extend(uploaded_files_mutex.into_inner()); - } - - Ok(()) -} - -const DEFAULT_MAVEN_METADATA_URL_1: &str = - "https://maven.neoforged.net/net/neoforged/forge/maven-metadata.xml"; -const DEFAULT_MAVEN_METADATA_URL_2: &str = - "https://maven.neoforged.net/net/neoforged/neoforge/maven-metadata.xml"; - -#[derive(Debug, Deserialize)] -struct Metadata { - versioning: Versioning, -} - -#[derive(Debug, Deserialize)] -struct Versioning { - versions: Versions, -} - -#[derive(Debug, Deserialize)] -struct Versions { - version: Vec, -} - -pub async fn fetch_maven_metadata( - semaphore: Arc, -) -> Result>, Error> { - async fn fetch_values( - url: &str, - semaphore: Arc, - ) -> Result { - Ok(serde_xml_rs::from_str( - &String::from_utf8( - download_file(url, None, semaphore).await?.to_vec(), - ) - .unwrap_or_default(), - )?) - } - - let forge_values = - fetch_values(DEFAULT_MAVEN_METADATA_URL_1, semaphore.clone()).await?; - let neo_values = - fetch_values(DEFAULT_MAVEN_METADATA_URL_2, semaphore).await?; - - let mut map: HashMap> = HashMap::new(); - - for value in forge_values.versioning.versions.version { - let original = value.clone(); - - let parts: Vec<&str> = value.split('-').collect(); - if parts.len() == 2 { - map.entry(parts[0].to_string()).or_default().push(( - original, - parts[1].to_string(), - false, - )); - } - } - - for value in neo_values.versioning.versions.version { - let original = value.clone(); - - let mut parts = value.split('.'); - - if let Some(major) = parts.next() { - if let Some(minor) = parts.next() { - let game_version = format!("1.{}.{}", major, minor); - - map.entry(game_version.clone()).or_default().push(( - original.clone(), - format!("{}-{}", game_version, original), - true, - )); - } - } - } - - Ok(map) -} - -#[derive(Serialize, Deserialize, Debug)] -#[serde(rename_all = "camelCase")] -struct ForgeInstallerProfileV2 { - pub spec: i32, - pub profile: String, - pub version: String, - pub json: String, - pub path: Option, - pub minecraft: String, - pub data: HashMap, - pub libraries: Vec, - pub processors: Vec, -} diff --git a/daedalus_client/src/quilt.rs b/daedalus_client/src/quilt.rs deleted file mode 100644 index cd0df2da2..000000000 --- a/daedalus_client/src/quilt.rs +++ /dev/null @@ -1,370 +0,0 @@ -use crate::{download_file, format_url, upload_file_to_bucket, Error}; -use daedalus::minecraft::{Library, VersionManifest}; -use daedalus::modded::{ - LoaderVersion, Manifest, PartialVersionInfo, Version, DUMMY_REPLACE_STRING, -}; -use serde::{Deserialize, Serialize}; -use std::sync::Arc; -use tokio::sync::{Mutex, RwLock, Semaphore}; - -pub async fn retrieve_data( - minecraft_versions: &VersionManifest, - uploaded_files: &mut Vec, - semaphore: Arc, -) -> Result<(), Error> { - let list = fetch_quilt_versions(None, semaphore.clone()).await?; - let old_manifest = daedalus::modded::fetch_manifest(&format_url(&format!( - "quilt/v{}/manifest.json", - daedalus::modded::CURRENT_QUILT_FORMAT_VERSION, - ))) - .await - .ok(); - - let mut versions = if let Some(old_manifest) = old_manifest { - old_manifest.game_versions - } else { - Vec::new() - }; - - let loaders_mutex = RwLock::new(Vec::new()); - - { - let mut loaders = loaders_mutex.write().await; - - for (index, loader) in list.loader.iter().enumerate() { - if versions.iter().any(|x| { - x.id == DUMMY_REPLACE_STRING - && x.loaders.iter().any(|x| x.id == loader.version) - }) { - if index == 0 { - loaders.push(( - Box::new(false), - loader.version.clone(), - Box::new(true), - )) - } - } else { - loaders.push(( - Box::new(false), - loader.version.clone(), - Box::new(false), - )) - } - } - } - - const DUMMY_GAME_VERSION: &str = "1.19.4-rc2"; - - let loader_version_mutex = Mutex::new(Vec::new()); - let uploaded_files_mutex = Arc::new(Mutex::new(Vec::new())); - - let loader_versions = futures::future::try_join_all( - loaders_mutex.read().await.clone().into_iter().map( - |(stable, loader, skip_upload)| async { - let version = fetch_quilt_version( - DUMMY_GAME_VERSION, - &loader, - semaphore.clone(), - ) - .await?; - - Ok::<(Box, String, PartialVersionInfo, Box), Error>( - (stable, loader, version, skip_upload), - ) - }, - ), - ) - .await?; - - let visited_artifacts_mutex = Arc::new(Mutex::new(Vec::new())); - futures::future::try_join_all(loader_versions.into_iter() - .map( - |(stable, loader, version, skip_upload)| async { - let libs = futures::future::try_join_all( - version.libraries.into_iter().map(|mut lib| async { - { - let mut visited_assets = - visited_artifacts_mutex.lock().await; - - if visited_assets.contains(&lib.name) { - lib.name = lib.name.replace(DUMMY_GAME_VERSION, DUMMY_REPLACE_STRING); - lib.url = Some(format_url("maven/")); - - return Ok(lib); - } else { - visited_assets.push(lib.name.clone()) - } - } - - if lib.name.contains(DUMMY_GAME_VERSION) { - lib.name = lib.name.replace(DUMMY_GAME_VERSION, DUMMY_REPLACE_STRING); - futures::future::try_join_all(list.game.clone().into_iter().map(|game_version| async { - let semaphore = semaphore.clone(); - let uploaded_files_mutex = uploaded_files_mutex.clone(); - let lib_name = lib.name.clone(); - let lib_url = lib.url.clone(); - - async move { - let artifact_path = - daedalus::get_path_from_artifact(&lib_name.replace(DUMMY_REPLACE_STRING, &game_version.version))?; - - let artifact = download_file( - &format!( - "{}{}", - lib_url.unwrap_or_else(|| { - "https://maven.quiltmc.org/".to_string() - }), - artifact_path - ), - None, - semaphore.clone(), - ) - .await?; - - upload_file_to_bucket( - format!("{}/{}", "maven", artifact_path), - artifact.to_vec(), - Some("application/java-archive".to_string()), - &uploaded_files_mutex, - semaphore.clone(), - ) - .await?; - - Ok::<(), Error>(()) - }.await?; - - Ok::<(), Error>(()) - })).await?; - lib.url = Some(format_url("maven/")); - - return Ok(lib); - } - - let artifact_path = - daedalus::get_path_from_artifact(&lib.name)?; - - let artifact = download_file( - &format!( - "{}{}", - lib.url.unwrap_or_else(|| { - "https://maven.quiltmc.org/".to_string() - }), - artifact_path - ), - None, - semaphore.clone(), - ) - .await?; - - lib.url = Some(format_url("maven/")); - - upload_file_to_bucket( - format!("{}/{}", "maven", artifact_path), - artifact.to_vec(), - Some("application/java-archive".to_string()), - &uploaded_files_mutex, - semaphore.clone(), - ) - .await?; - - Ok::(lib) - }), - ) - .await?; - - if async move { - *skip_upload - }.await { - return Ok::<(), Error>(()) - } - - let version_path = format!( - "quilt/v{}/versions/{}.json", - daedalus::modded::CURRENT_QUILT_FORMAT_VERSION, - &loader - ); - - upload_file_to_bucket( - version_path.clone(), - serde_json::to_vec(&PartialVersionInfo { - arguments: version.arguments, - id: version - .id - .replace(DUMMY_GAME_VERSION, DUMMY_REPLACE_STRING), - main_class: version.main_class, - release_time: version.release_time, - time: version.time, - type_: version.type_, - inherits_from: version - .inherits_from - .replace(DUMMY_GAME_VERSION, DUMMY_REPLACE_STRING), - libraries: libs, - minecraft_arguments: version.minecraft_arguments, - processors: None, - data: None, - })?, - Some("application/json".to_string()), - &uploaded_files_mutex, - semaphore.clone(), - ) - .await?; - - { - let mut loader_version_map = loader_version_mutex.lock().await; - async move { - loader_version_map.push(LoaderVersion { - id: loader.to_string(), - url: format_url(&version_path), - stable: *stable, - }); - } - .await; - } - - Ok::<(), Error>(()) - }, - )) - .await?; - - let mut loader_version_mutex = loader_version_mutex.into_inner(); - if !loader_version_mutex.is_empty() { - if let Some(version) = - versions.iter_mut().find(|x| x.id == DUMMY_REPLACE_STRING) - { - version.loaders.append(&mut loader_version_mutex); - } else { - versions.push(Version { - id: DUMMY_REPLACE_STRING.to_string(), - stable: true, - loaders: loader_version_mutex, - }); - } - } - - for version in &list.game { - if !versions.iter().any(|x| x.id == version.version) { - versions.push(Version { - id: version.version.clone(), - stable: version.stable, - loaders: vec![], - }); - } - } - - versions.sort_by(|x, y| { - minecraft_versions - .versions - .iter() - .position(|z| x.id == z.id) - .unwrap_or_default() - .cmp( - &minecraft_versions - .versions - .iter() - .position(|z| y.id == z.id) - .unwrap_or_default(), - ) - }); - - for version in &mut versions { - version.loaders.sort_by(|x, y| { - list.loader - .iter() - .position(|z| x.id == *z.version) - .unwrap_or_default() - .cmp( - &list - .loader - .iter() - .position(|z| y.id == z.version) - .unwrap_or_default(), - ) - }) - } - - upload_file_to_bucket( - format!( - "quilt/v{}/manifest.json", - daedalus::modded::CURRENT_QUILT_FORMAT_VERSION, - ), - serde_json::to_vec(&Manifest { - game_versions: versions, - })?, - Some("application/json".to_string()), - &uploaded_files_mutex, - semaphore, - ) - .await?; - - if let Ok(uploaded_files_mutex) = Arc::try_unwrap(uploaded_files_mutex) { - uploaded_files.extend(uploaded_files_mutex.into_inner()); - } - - Ok(()) -} - -const QUILT_META_URL: &str = "https://meta.quiltmc.org/v3"; - -async fn fetch_quilt_version( - version_number: &str, - loader_version: &str, - semaphore: Arc, -) -> Result { - Ok(serde_json::from_slice( - &download_file( - &format!( - "{}/versions/loader/{}/{}/profile/json", - QUILT_META_URL, version_number, loader_version - ), - None, - semaphore, - ) - .await?, - )?) -} - -#[derive(Serialize, Deserialize, Debug, Clone)] -/// Versions of quilt components -struct QuiltVersions { - /// Versions of Minecraft that quilt supports - pub game: Vec, - /// Available versions of the quilt loader - pub loader: Vec, -} - -#[derive(Serialize, Deserialize, Debug, Clone)] -/// A version of Minecraft that quilt supports -struct QuiltGameVersion { - /// The version number of the game - pub version: String, - /// Whether the Minecraft version is stable or not - pub stable: bool, -} - -#[derive(Serialize, Deserialize, Debug, Clone)] -/// A version of the quilt loader -struct QuiltLoaderVersion { - /// The separator to get the build number - pub separator: String, - /// The build number - pub build: u32, - /// The maven artifact - pub maven: String, - /// The version number of the quilt loader - pub version: String, -} - -/// Fetches the list of quilt versions -async fn fetch_quilt_versions( - url: Option<&str>, - semaphore: Arc, -) -> Result { - Ok(serde_json::from_slice( - &download_file( - url.unwrap_or(&*format!("{}/versions", QUILT_META_URL)), - None, - semaphore, - ) - .await?, - )?) -} diff --git a/daedalus_client/src/util.rs b/daedalus_client/src/util.rs new file mode 100644 index 000000000..0b571f0ce --- /dev/null +++ b/daedalus_client/src/util.rs @@ -0,0 +1,369 @@ +use crate::{Error, ErrorKind}; +use bytes::{Bytes, BytesMut}; +use futures::StreamExt; +use s3::creds::Credentials; +use s3::{Bucket, Region}; +use serde::de::DeserializeOwned; +use std::sync::Arc; +use tokio::sync::Semaphore; + +lazy_static::lazy_static! { + static ref BUCKET : Bucket = { + let region = dotenvy::var("S3_REGION").unwrap(); + let b = Bucket::new( + &dotenvy::var("S3_BUCKET_NAME").unwrap(), + if &*region == "r2" { + Region::R2 { + account_id: dotenvy::var("S3_URL").unwrap(), + } + } else { + Region::Custom { + region: region.clone(), + endpoint: dotenvy::var("S3_URL").unwrap(), + } + }, + Credentials::new( + Some(&*dotenvy::var("S3_ACCESS_TOKEN").unwrap()), + Some(&*dotenvy::var("S3_SECRET").unwrap()), + None, + None, + None, + ).unwrap(), + ).unwrap(); + + if region == "path-style" { + b.with_path_style() + } else { + b + } + }; +} + +lazy_static::lazy_static! { + pub static ref REQWEST_CLIENT: reqwest::Client = { + let mut headers = reqwest::header::HeaderMap::new(); + if let Ok(header) = reqwest::header::HeaderValue::from_str(&format!( + "modrinth/daedalus/{} (support@modrinth.com)", + env!("CARGO_PKG_VERSION") + )) { + headers.insert(reqwest::header::USER_AGENT, header); + } + + reqwest::Client::builder() + .tcp_keepalive(Some(std::time::Duration::from_secs(10))) + .timeout(std::time::Duration::from_secs(15)) + .default_headers(headers) + .build() + .unwrap() + }; +} + +#[tracing::instrument(skip(bytes, semaphore))] +pub async fn upload_file_to_bucket( + path: String, + bytes: Bytes, + content_type: Option, + semaphore: &Arc, +) -> Result<(), Error> { + let _permit = semaphore.acquire().await?; + let key = path.clone(); + + const RETRIES: i32 = 3; + for attempt in 1..=(RETRIES + 1) { + tracing::trace!("Attempting file upload, attempt {attempt}"); + let result = if let Some(ref content_type) = content_type { + BUCKET + .put_object_with_content_type(key.clone(), &bytes, content_type) + .await + } else { + BUCKET.put_object(key.clone(), &bytes).await + } + .map_err(|err| ErrorKind::S3 { + inner: err, + file: path.clone(), + }); + + match result { + Ok(_) => return Ok(()), + Err(_) if attempt <= RETRIES => continue, + Err(_) => { + result?; + } + } + } + unreachable!() +} + +pub async fn upload_url_to_bucket_mirrors( + base: String, + mirrors: Vec, + semaphore: &Arc, +) -> Result<(), Error> { + if mirrors.is_empty() { + return Err(ErrorKind::InvalidInput( + "No mirrors provided!".to_string(), + ) + .into()); + } + + for (index, mirror) in mirrors.iter().enumerate() { + let result = upload_url_to_bucket( + &base, + &format!("{}{}", mirror, base), + semaphore, + ) + .await; + + if result.is_ok() || (result.is_err() && index == (mirrors.len() - 1)) { + return result; + } + } + + unreachable!() +} + +#[tracing::instrument(skip(semaphore))] +pub async fn upload_url_to_bucket( + path: &str, + url: &str, + semaphore: &Arc, +) -> Result<(), Error> { + let _permit = semaphore.acquire().await?; + + const RETRIES: i32 = 3; + for attempt in 1..=(RETRIES + 1) { + tracing::trace!("Attempting streaming file upload, attempt {attempt}"); + + let result: Result<(), Error> = { + let response = + REQWEST_CLIENT.get(url).send().await.map_err(|err| { + ErrorKind::Fetch { + inner: err, + item: url.to_string(), + } + })?; + + let content_type = response + .headers() + .get(reqwest::header::CONTENT_TYPE) + .and_then(|ct| ct.to_str().ok()) + .unwrap_or("application/octet-stream") + .to_string(); + + let total_size = response.content_length().unwrap_or(0); + + const MIN_PART_SIZE: usize = 5 * 1024 * 1024; + + if total_size < MIN_PART_SIZE as u64 { + let data = + response.bytes().await.map_err(|err| ErrorKind::Fetch { + inner: err, + item: url.to_string(), + })?; + BUCKET.put_object(&path, &data).await.map_err(|err| { + ErrorKind::S3 { + inner: err, + file: path.to_string(), + } + })?; + } else { + let mut stream = response.bytes_stream(); + + let multipart = BUCKET + .initiate_multipart_upload(path, &content_type) + .await + .map_err(|err| ErrorKind::S3 { + inner: err, + file: path.to_string(), + })?; + + let mut parts = Vec::new(); + let mut buffer = BytesMut::new(); + + async fn upload_part( + parts: &mut Vec, + buffer: Vec, + path: &str, + upload_id: &str, + content_type: &str, + ) -> Result<(), Error> { + let part = BUCKET + .put_multipart_chunk( + buffer, + path, + (parts.len() + 1) as u32, + upload_id, + content_type, + ) + .await + .map_err(|err| ErrorKind::S3 { + inner: err, + file: path.to_string(), + })?; + + parts.push(part); + + Ok(()) + } + + while let Some(chunk) = stream.next().await { + let chunk = chunk.map_err(|err| ErrorKind::Fetch { + inner: err, + item: url.to_string(), + })?; + + buffer.extend_from_slice(&chunk); + + if buffer.len() >= MIN_PART_SIZE { + upload_part( + &mut parts, + buffer.to_vec(), + path, + &multipart.upload_id, + &content_type, + ) + .await?; + buffer.clear(); + } + } + + if !buffer.is_empty() { + let part = BUCKET + .put_multipart_chunk( + buffer.to_vec(), + path, + (parts.len() + 1) as u32, + &multipart.upload_id, + &content_type, + ) + .await + .map_err(|err| ErrorKind::S3 { + inner: err, + file: path.to_string(), + })?; + + parts.push(part); + } + + BUCKET + .complete_multipart_upload( + path, + &multipart.upload_id, + parts, + ) + .await + .map_err(|err| ErrorKind::S3 { + inner: err, + file: path.to_string(), + })?; + } + + Ok(()) + }; + + match result { + Ok(_) => return Ok(()), + Err(_) if attempt <= RETRIES => continue, + Err(_) => { + result?; + } + } + } + unreachable!() +} + +#[tracing::instrument(skip(bytes))] +pub async fn sha1_async(bytes: Bytes) -> Result { + let hash = tokio::task::spawn_blocking(move || { + sha1_smol::Sha1::from(bytes).hexdigest() + }) + .await?; + + Ok(hash) +} + +#[tracing::instrument(skip(semaphore))] +pub async fn download_file( + url: &str, + sha1: Option<&str>, + semaphore: &Arc, +) -> Result { + let _permit = semaphore.acquire().await?; + tracing::trace!("Starting file download"); + + const RETRIES: u32 = 10; + for attempt in 1..=(RETRIES + 1) { + let result = REQWEST_CLIENT + .get(url) + .send() + .await + .and_then(|x| x.error_for_status()); + + match result { + Ok(x) => { + let bytes = x.bytes().await; + + if let Ok(bytes) = bytes { + if let Some(sha1) = sha1 { + if &*sha1_async(bytes.clone()).await? != sha1 { + if attempt <= 3 { + continue; + } else { + return Err( + crate::ErrorKind::ChecksumFailure { + hash: sha1.to_string(), + url: url.to_string(), + tries: attempt, + } + .into(), + ); + } + } + } + + return Ok(bytes); + } else if attempt <= RETRIES { + continue; + } else if let Err(err) = bytes { + return Err(crate::ErrorKind::Fetch { + inner: err, + item: url.to_string(), + } + .into()); + } + } + Err(_) if attempt <= RETRIES => continue, + Err(err) => { + return Err(crate::ErrorKind::Fetch { + inner: err, + item: url.to_string(), + } + .into()) + } + } + } + + unreachable!() +} + +pub async fn fetch_json( + url: &str, + semaphore: &Arc, +) -> Result { + Ok(serde_json::from_slice( + &download_file(url, None, semaphore).await?, + )?) +} + +pub async fn fetch_xml( + url: &str, + semaphore: &Arc, +) -> Result { + Ok(serde_xml_rs::from_reader( + &*download_file(url, None, semaphore).await?, + )?) +} + +pub fn format_url(path: &str) -> String { + format!("{}/{}", &*dotenvy::var("BASE_URL").unwrap(), path) +} From 4274a8ed6840129fb0caf1e28f9c5da02d97d34b Mon Sep 17 00:00:00 2001 From: Geometrically <18202329+Geometrically@users.noreply.github.com> Date: Fri, 28 Jun 2024 15:44:17 -0700 Subject: [PATCH 62/85] Fix forge install issues (#18) * Fix forge install issues * remove mac garb --- daedalus/Cargo.toml | 2 +- daedalus_client/Cargo.toml | 2 +- daedalus_client/src/fabric.rs | 34 +++++-- daedalus_client/src/forge.rs | 68 +++++++++----- daedalus_client/src/main.rs | 111 +++++++++++++++-------- daedalus_client/src/util.rs | 161 +++------------------------------- 6 files changed, 163 insertions(+), 215 deletions(-) diff --git a/daedalus/Cargo.toml b/daedalus/Cargo.toml index 1acc5253b..82e16108e 100644 --- a/daedalus/Cargo.toml +++ b/daedalus/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "daedalus" -version = "0.2.0" +version = "0.2.1" authors = ["Jai A "] edition = "2021" license = "MIT" diff --git a/daedalus_client/Cargo.toml b/daedalus_client/Cargo.toml index ce8204fac..c0bdbd2b0 100644 --- a/daedalus_client/Cargo.toml +++ b/daedalus_client/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "daedalus_client" -version = "0.2.0" +version = "0.2.1" authors = ["Jai A "] edition = "2021" diff --git a/daedalus_client/src/fabric.rs b/daedalus_client/src/fabric.rs index 8d1d9431b..a7e352b24 100644 --- a/daedalus_client/src/fabric.rs +++ b/daedalus_client/src/fabric.rs @@ -17,6 +17,7 @@ pub async fn fetch_fabric( "fabric", "https://meta.fabricmc.net/v2", "https://maven.fabricmc.net/", + &[], semaphore, upload_files, mirror_artifacts, @@ -34,7 +35,11 @@ pub async fn fetch_quilt( daedalus::modded::CURRENT_QUILT_FORMAT_VERSION, "quilt", "https://meta.quiltmc.org/v3", - "https://meta.quiltmc.org/", + "https://maven.quiltmc.org/repository/release/", + &[ + // This version is broken as it contains invalid library coordinates + "0.17.5-beta.4", + ], semaphore, upload_files, mirror_artifacts, @@ -48,6 +53,7 @@ async fn fetch( mod_loader: &str, meta_url: &str, maven_url: &str, + skip_versions: &[&str], semaphore: Arc, upload_files: &DashMap, mirror_artifacts: &DashMap, @@ -76,6 +82,7 @@ async fn fetch( .game_versions .iter() .any(|x| x.loaders.iter().any(|x| x.id == version.version)) + && !skip_versions.contains(&&*version.version) { fetch_versions.push(version); } @@ -98,7 +105,11 @@ async fn fetch( (fetch_versions, fetch_intermediary_versions) } else { ( - fabric_manifest.loader.iter().collect(), + fabric_manifest + .loader + .iter() + .filter(|x| !skip_versions.contains(&&*x.version)) + .collect(), fabric_manifest.intermediary.iter().collect(), ) }; @@ -109,7 +120,9 @@ async fn fetch( for x in &fetch_intermediary_versions { insert_mirrored_artifact( &x.maven, - maven_url.to_string(), + None, + vec![maven_url.to_string()], + false, mirror_artifacts, )?; } @@ -142,13 +155,24 @@ async fn fetch( let new_name = lib .name .replace(DUMMY_GAME_VERSION, DUMMY_REPLACE_STRING); + + // Hard-code: This library is not present on fabric's maven, so we fetch it from MC libraries + if &*lib.name == "net.minecraft:launchwrapper:1.12" { + lib.url = Some( + "https://libraries.minecraft.net/".to_string(), + ); + } + // If a library is not intermediary, we add it to mirror artifacts to be mirrored if lib.name == new_name { insert_mirrored_artifact( &new_name, - lib.url + None, + vec![lib + .url .clone() - .unwrap_or_else(|| maven_url.to_string()), + .unwrap_or_else(|| maven_url.to_string())], + false, mirror_artifacts, )?; } else { diff --git a/daedalus_client/src/forge.rs b/daedalus_client/src/forge.rs index 99892f225..09c8e0aa6 100644 --- a/daedalus_client/src/forge.rs +++ b/daedalus_client/src/forge.rs @@ -1,4 +1,6 @@ -use crate::util::{download_file, fetch_json, fetch_xml, format_url}; +use crate::util::{ + download_file, fetch_json, fetch_xml, format_url, sha1_async, +}; use crate::{insert_mirrored_artifact, Error, MirrorArtifact, UploadFile}; use chrono::{DateTime, Utc}; use daedalus::get_path_from_artifact; @@ -246,6 +248,7 @@ async fn fetch( raw: bytes::Bytes, loader: &ForgeVersion, maven_url: &str, + mod_loader: &str, upload_files: &DashMap, mirror_artifacts: &DashMap, ) -> Result { @@ -399,15 +402,27 @@ async fn fetch( .into_iter() .map(|mut lib| { // For all libraries besides the forge lib extracted, we mirror them from maven servers - if lib.name != install_profile.install.path { - // TODO: add mirrors "https://maven.creeperhost.net/", "https://libraries.minecraft.net/" - insert_mirrored_artifact( - &lib.name, - lib.url.clone().unwrap_or_else(|| { - maven_url.to_string() - }), - mirror_artifacts, - )?; + // unless the URL is empty/null or available on Minecraft's servers + if let Some(url) = lib.url { + if lib.name != install_profile.install.path + && !url.is_empty() + && !url.contains( + "https://libraries.minecraft.net/", + ) + { + insert_mirrored_artifact( + &lib.name, + None, + vec![ + url, + "https://maven.creeperhost.net/" + .to_string(), + maven_url.to_string(), + ], + false, + mirror_artifacts, + )?; + } } lib.url = Some(format_url("maven/")); @@ -468,6 +483,7 @@ async fn fetch( async fn mirror_forge_library( mut zip: ZipFileReader, mut lib: daedalus::minecraft::Library, + maven_url: &str, upload_files: &DashMap, mirror_artifacts: &DashMap, ) -> Result @@ -480,7 +496,9 @@ async fn fetch( if !artifact.url.is_empty() { insert_mirrored_artifact( &lib.name, - artifact.url.clone(), + Some(artifact.sha1.clone()), + vec![artifact.url.clone()], + true, mirror_artifacts, )?; @@ -491,10 +509,18 @@ async fn fetch( } } else if let Some(url) = &lib.url { if !url.is_empty() { - // TODO: add mirrors "https://maven.creeperhost.net/", "https://libraries.minecraft.net/" insert_mirrored_artifact( &lib.name, - url.clone(), + None, + vec![ + url.clone(), + "https://libraries.minecraft.net/" + .to_string(), + "https://maven.creeperhost.net/" + .to_string(), + maven_url.to_string(), + ], + false, mirror_artifacts, )?; @@ -531,6 +557,7 @@ async fn fetch( mirror_forge_library( zip.clone(), lib, + maven_url, upload_files, mirror_artifacts, ) @@ -560,7 +587,7 @@ async fn fetch( value: &str, upload_files: &DashMap, libs: &mut Vec, - install_profile_path: Option<&str>, + mod_loader: &str, version: &ForgeVersion, ) -> Result { let extract_file = @@ -595,11 +622,9 @@ async fn fetch( })?; let path = format!( - "{}:{}@{}", - install_profile_path.unwrap_or(&*format!( - "net.minecraftforge:forge:{}", - version.raw - )), + "com.modrinth.daedalus:{}-installer-extracts:{}:{}@{}", + mod_loader, + version.raw, file_name, ext ); @@ -634,7 +659,7 @@ async fn fetch( &entry.client, upload_files, &mut version_info.libraries, - install_profile.path.as_deref(), + mod_loader, loader, ) .await? @@ -649,7 +674,7 @@ async fn fetch( &entry.server, upload_files, &mut version_info.libraries, - install_profile.path.as_deref(), + mod_loader, loader, ) .await? @@ -686,6 +711,7 @@ async fn fetch( raw, loader, maven_url, + mod_loader, upload_files, mirror_artifacts, ) diff --git a/daedalus_client/src/main.rs b/daedalus_client/src/main.rs index 33df7090e..8e934cbd5 100644 --- a/daedalus_client/src/main.rs +++ b/daedalus_client/src/main.rs @@ -72,46 +72,63 @@ async fn main() -> Result<()> { futures::future::try_join_all(mirror_artifacts.iter().map(|x| { upload_url_to_bucket_mirrors( format!("maven/{}", x.key()), - x.value().mirrors.iter().map(|x| x.key().clone()).collect(), + x.value() + .mirrors + .iter() + .map(|mirror| { + if mirror.entire_url { + mirror.path.clone() + } else { + format!("{}{}", mirror.path, x.key()) + } + }) + .collect(), + x.sha1.clone(), &semaphore, ) })) .await?; - if let Ok(token) = dotenvy::var("CLOUDFLARE_TOKEN") { - if let Ok(zone_id) = dotenvy::var("CLOUDFLARE_ZONE_ID") { - let cache_clears = upload_files - .into_iter() - .map(|x| format_url(&x.0)) - .chain( - mirror_artifacts - .into_iter() - .map(|x| format_url(&format!("maven/{}", x.0))), - ) - .collect::>(); - - // Cloudflare ratelimits cache clears to 500 files per request - for chunk in cache_clears.chunks(500) { - REQWEST_CLIENT.post(format!("https://api.cloudflare.com/client/v4/zones/{zone_id}/purge_cache")) - .bearer_auth(&token) - .json(&serde_json::json!({ + if dotenvy::var("CLOUDFLARE_INTEGRATION") + .ok() + .and_then(|x| x.parse::().ok()) + .unwrap_or(false) + { + if let Ok(token) = dotenvy::var("CLOUDFLARE_TOKEN") { + if let Ok(zone_id) = dotenvy::var("CLOUDFLARE_ZONE_ID") { + let cache_clears = upload_files + .into_iter() + .map(|x| format_url(&x.0)) + .chain( + mirror_artifacts + .into_iter() + .map(|x| format_url(&format!("maven/{}", x.0))), + ) + .collect::>(); + + // Cloudflare ratelimits cache clears to 500 files per request + for chunk in cache_clears.chunks(500) { + REQWEST_CLIENT.post(format!("https://api.cloudflare.com/client/v4/zones/{zone_id}/purge_cache")) + .bearer_auth(&token) + .json(&serde_json::json!({ "files": chunk })) - .send() - .await - .map_err(|err| { - ErrorKind::Fetch { - inner: err, - item: "cloudflare clear cache".to_string(), - } - })? - .error_for_status() - .map_err(|err| { - ErrorKind::Fetch { - inner: err, - item: "cloudflare clear cache".to_string(), - } - })?; + .send() + .await + .map_err(|err| { + ErrorKind::Fetch { + inner: err, + item: "cloudflare clear cache".to_string(), + } + })? + .error_for_status() + .map_err(|err| { + ErrorKind::Fetch { + inner: err, + item: "cloudflare clear cache".to_string(), + } + })?; + } } } } @@ -125,21 +142,37 @@ pub struct UploadFile { } pub struct MirrorArtifact { - pub mirrors: DashSet, + pub sha1: Option, + pub mirrors: DashSet, } +#[derive(Eq, PartialEq, Hash)] +pub struct Mirror { + path: String, + entire_url: bool, +} + +#[tracing::instrument(skip(mirror_artifacts))] pub fn insert_mirrored_artifact( artifact: &str, - mirror: String, + sha1: Option, + mirrors: Vec, + entire_url: bool, mirror_artifacts: &DashMap, ) -> Result<()> { - mirror_artifacts + let mut val = mirror_artifacts .entry(get_path_from_artifact(artifact)?) .or_insert(MirrorArtifact { + sha1, mirrors: DashSet::new(), - }) - .mirrors - .insert(mirror); + }); + + for mirror in mirrors { + val.mirrors.insert(Mirror { + path: mirror, + entire_url, + }); + } Ok(()) } diff --git a/daedalus_client/src/util.rs b/daedalus_client/src/util.rs index 0b571f0ce..72ff25575 100644 --- a/daedalus_client/src/util.rs +++ b/daedalus_client/src/util.rs @@ -1,6 +1,5 @@ use crate::{Error, ErrorKind}; -use bytes::{Bytes, BytesMut}; -use futures::StreamExt; +use bytes::Bytes; use s3::creds::Credentials; use s3::{Bucket, Region}; use serde::de::DeserializeOwned; @@ -95,8 +94,9 @@ pub async fn upload_file_to_bucket( } pub async fn upload_url_to_bucket_mirrors( - base: String, + upload_path: String, mirrors: Vec, + sha1: Option, semaphore: &Arc, ) -> Result<(), Error> { if mirrors.is_empty() { @@ -108,8 +108,9 @@ pub async fn upload_url_to_bucket_mirrors( for (index, mirror) in mirrors.iter().enumerate() { let result = upload_url_to_bucket( - &base, - &format!("{}{}", mirror, base), + upload_path.clone(), + mirror.clone(), + sha1.clone(), semaphore, ) .await; @@ -124,152 +125,16 @@ pub async fn upload_url_to_bucket_mirrors( #[tracing::instrument(skip(semaphore))] pub async fn upload_url_to_bucket( - path: &str, - url: &str, + path: String, + url: String, + sha1: Option, semaphore: &Arc, ) -> Result<(), Error> { - let _permit = semaphore.acquire().await?; - - const RETRIES: i32 = 3; - for attempt in 1..=(RETRIES + 1) { - tracing::trace!("Attempting streaming file upload, attempt {attempt}"); - - let result: Result<(), Error> = { - let response = - REQWEST_CLIENT.get(url).send().await.map_err(|err| { - ErrorKind::Fetch { - inner: err, - item: url.to_string(), - } - })?; - - let content_type = response - .headers() - .get(reqwest::header::CONTENT_TYPE) - .and_then(|ct| ct.to_str().ok()) - .unwrap_or("application/octet-stream") - .to_string(); - - let total_size = response.content_length().unwrap_or(0); - - const MIN_PART_SIZE: usize = 5 * 1024 * 1024; - - if total_size < MIN_PART_SIZE as u64 { - let data = - response.bytes().await.map_err(|err| ErrorKind::Fetch { - inner: err, - item: url.to_string(), - })?; - BUCKET.put_object(&path, &data).await.map_err(|err| { - ErrorKind::S3 { - inner: err, - file: path.to_string(), - } - })?; - } else { - let mut stream = response.bytes_stream(); - - let multipart = BUCKET - .initiate_multipart_upload(path, &content_type) - .await - .map_err(|err| ErrorKind::S3 { - inner: err, - file: path.to_string(), - })?; - - let mut parts = Vec::new(); - let mut buffer = BytesMut::new(); - - async fn upload_part( - parts: &mut Vec, - buffer: Vec, - path: &str, - upload_id: &str, - content_type: &str, - ) -> Result<(), Error> { - let part = BUCKET - .put_multipart_chunk( - buffer, - path, - (parts.len() + 1) as u32, - upload_id, - content_type, - ) - .await - .map_err(|err| ErrorKind::S3 { - inner: err, - file: path.to_string(), - })?; - - parts.push(part); + let data = download_file(&url, sha1.as_deref(), semaphore).await?; - Ok(()) - } - - while let Some(chunk) = stream.next().await { - let chunk = chunk.map_err(|err| ErrorKind::Fetch { - inner: err, - item: url.to_string(), - })?; - - buffer.extend_from_slice(&chunk); - - if buffer.len() >= MIN_PART_SIZE { - upload_part( - &mut parts, - buffer.to_vec(), - path, - &multipart.upload_id, - &content_type, - ) - .await?; - buffer.clear(); - } - } - - if !buffer.is_empty() { - let part = BUCKET - .put_multipart_chunk( - buffer.to_vec(), - path, - (parts.len() + 1) as u32, - &multipart.upload_id, - &content_type, - ) - .await - .map_err(|err| ErrorKind::S3 { - inner: err, - file: path.to_string(), - })?; - - parts.push(part); - } - - BUCKET - .complete_multipart_upload( - path, - &multipart.upload_id, - parts, - ) - .await - .map_err(|err| ErrorKind::S3 { - inner: err, - file: path.to_string(), - })?; - } - - Ok(()) - }; + upload_file_to_bucket(path, data, None, semaphore).await?; - match result { - Ok(_) => return Ok(()), - Err(_) if attempt <= RETRIES => continue, - Err(_) => { - result?; - } - } - } - unreachable!() + Ok(()) } #[tracing::instrument(skip(bytes))] @@ -294,7 +159,7 @@ pub async fn download_file( const RETRIES: u32 = 10; for attempt in 1..=(RETRIES + 1) { let result = REQWEST_CLIENT - .get(url) + .get(&url.replace("http://", "https://")) .send() .await .and_then(|x| x.error_for_status()); From fc3056b0e011e65ef83a851fa655f26061e16dd4 Mon Sep 17 00:00:00 2001 From: Jai A Date: Fri, 28 Jun 2024 15:53:59 -0700 Subject: [PATCH 63/85] add rust log env --- .github/workflows/run.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/run.yml b/.github/workflows/run.yml index b97d2bdde..c5f0b827c 100644 --- a/.github/workflows/run.yml +++ b/.github/workflows/run.yml @@ -37,6 +37,7 @@ jobs: run: | docker run -d \ --name daedalus \ + -e RUST_LOG=warn,daedalus_client=trace -e BASE_URL=$BASE_URL \ -e S3_ACCESS_TOKEN=$S3_ACCESS_TOKEN \ -e S3_SECRET=$S3_SECRET \ From 6e8e053b88c1370ca8862d7508fce1ffa1ef0164 Mon Sep 17 00:00:00 2001 From: Jai A Date: Fri, 28 Jun 2024 16:40:12 -0700 Subject: [PATCH 64/85] fix newline --- .github/workflows/run.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/run.yml b/.github/workflows/run.yml index c5f0b827c..dbd8b8471 100644 --- a/.github/workflows/run.yml +++ b/.github/workflows/run.yml @@ -37,7 +37,7 @@ jobs: run: | docker run -d \ --name daedalus \ - -e RUST_LOG=warn,daedalus_client=trace + -e RUST_LOG=warn,daedalus_client=trace \ -e BASE_URL=$BASE_URL \ -e S3_ACCESS_TOKEN=$S3_ACCESS_TOKEN \ -e S3_SECRET=$S3_SECRET \ From 02ebe59d2f30e6a568ba9246c451d2f3ce67acea Mon Sep 17 00:00:00 2001 From: Jai A Date: Fri, 28 Jun 2024 16:45:06 -0700 Subject: [PATCH 65/85] add dispatch support for meta --- .github/workflows/run.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/run.yml b/.github/workflows/run.yml index dbd8b8471..e01d317ab 100644 --- a/.github/workflows/run.yml +++ b/.github/workflows/run.yml @@ -3,6 +3,7 @@ name: Run Meta on: schedule: - cron: '*/5 * * * *' + workflow_dispatch: jobs: run-docker: From 8a0329b23da5e81492622183b3a8ebade21d5cbc Mon Sep 17 00:00:00 2001 From: Jai A Date: Fri, 28 Jun 2024 16:48:03 -0700 Subject: [PATCH 66/85] don't run in detached mode --- .github/workflows/run.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/run.yml b/.github/workflows/run.yml index e01d317ab..4255e2228 100644 --- a/.github/workflows/run.yml +++ b/.github/workflows/run.yml @@ -36,7 +36,7 @@ jobs: CLOUDFLARE_TOKEN: ${{ secrets.CLOUDFLARE_TOKEN }} CLOUDFLARE_ZONE_ID: ${{ secrets.CLOUDFLARE_ZONE_ID }} run: | - docker run -d \ + docker run \ --name daedalus \ -e RUST_LOG=warn,daedalus_client=trace \ -e BASE_URL=$BASE_URL \ From 88db79188c1f773985e63ff7506eb8637e8a2a79 Mon Sep 17 00:00:00 2001 From: Jai A Date: Fri, 28 Jun 2024 16:50:47 -0700 Subject: [PATCH 67/85] use rustls --- .github/workflows/run.yml | 2 +- daedalus/Cargo.toml | 2 +- daedalus_client/Cargo.toml | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/run.yml b/.github/workflows/run.yml index 4255e2228..fa9e51a99 100644 --- a/.github/workflows/run.yml +++ b/.github/workflows/run.yml @@ -48,4 +48,4 @@ jobs: -e CLOUDFLARE_INTEGRATION=$CLOUDFLARE_INTEGRATION \ -e CLOUDFLARE_TOKEN=$CLOUDFLARE_TOKEN \ -e CLOUDFLARE_ZONE_ID=$CLOUDFLARE_ZONE_ID \ - ghcr.io/modrinth/daedalus:latest \ No newline at end of file + ghcr.io/modrinth/daedalus:latest diff --git a/daedalus/Cargo.toml b/daedalus/Cargo.toml index 82e16108e..1b28e72ee 100644 --- a/daedalus/Cargo.toml +++ b/daedalus/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "daedalus" -version = "0.2.1" +version = "0.2.2" authors = ["Jai A "] edition = "2021" license = "MIT" diff --git a/daedalus_client/Cargo.toml b/daedalus_client/Cargo.toml index c0bdbd2b0..de3b52c92 100644 --- a/daedalus_client/Cargo.toml +++ b/daedalus_client/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "daedalus_client" -version = "0.2.1" +version = "0.2.2" authors = ["Jai A "] edition = "2021" @@ -16,7 +16,7 @@ serde_json = "1.0" serde-xml-rs = "0.6.0" lazy_static = "1.4.0" thiserror = "1.0" -reqwest = { version = "0.12.5", features = ["stream", "json"] } +reqwest = { version = "0.12.5", features = ["stream", "json", "rustls-tls"] } async_zip = { version = "0.0.17", features = ["full"] } semver = "1.0" chrono = { version = "0.4", features = ["serde"] } From 7be02318e029c8e73d51535108549a6c00b0f98c Mon Sep 17 00:00:00 2001 From: Jai A Date: Fri, 28 Jun 2024 16:55:54 -0700 Subject: [PATCH 68/85] add openssl --- .github/workflows/run.yml | 2 +- Dockerfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/run.yml b/.github/workflows/run.yml index fa9e51a99..e50497a56 100644 --- a/.github/workflows/run.yml +++ b/.github/workflows/run.yml @@ -22,7 +22,7 @@ jobs: - name: Pull Docker image from GHCR - run: docker pull ghcr.io/modrinth/daedalus:latest + run: docker pull ghcr.io/modrinth/daedalus:master - name: Run Docker container env: diff --git a/Dockerfile b/Dockerfile index c015f6748..a54439de7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,7 +9,7 @@ RUN cargo build --release FROM debian:bullseye-slim RUN apt-get update \ - && apt-get install -y --no-install-recommends ca-certificates \ + && apt-get install -y --no-install-recommends ca-certificates openssl \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* From 721365578ae72a4e479bfd7660877d51c27303ea Mon Sep 17 00:00:00 2001 From: Jai A Date: Fri, 28 Jun 2024 16:59:06 -0700 Subject: [PATCH 69/85] update action to use master branch --- .github/workflows/run.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/run.yml b/.github/workflows/run.yml index e50497a56..1409fb47f 100644 --- a/.github/workflows/run.yml +++ b/.github/workflows/run.yml @@ -48,4 +48,4 @@ jobs: -e CLOUDFLARE_INTEGRATION=$CLOUDFLARE_INTEGRATION \ -e CLOUDFLARE_TOKEN=$CLOUDFLARE_TOKEN \ -e CLOUDFLARE_ZONE_ID=$CLOUDFLARE_ZONE_ID \ - ghcr.io/modrinth/daedalus:latest + ghcr.io/modrinth/daedalus:master From 34d931573cac307b3685d3dcc9f47cf88f347ff1 Mon Sep 17 00:00:00 2001 From: Jai A Date: Fri, 28 Jun 2024 17:02:30 -0700 Subject: [PATCH 70/85] try with bookworm --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index a54439de7..c926fa8d4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,7 +6,7 @@ COPY . . RUN cargo build --release -FROM debian:bullseye-slim +FROM debian:bookwormg-slim RUN apt-get update \ && apt-get install -y --no-install-recommends ca-certificates openssl \ From 0ae1e40d79685a045360acf9a76ee38849392502 Mon Sep 17 00:00:00 2001 From: Jai A Date: Fri, 28 Jun 2024 17:03:54 -0700 Subject: [PATCH 71/85] fix typo --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index c926fa8d4..19c88febe 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,7 +6,7 @@ COPY . . RUN cargo build --release -FROM debian:bookwormg-slim +FROM debian:bookworm-slim RUN apt-get update \ && apt-get install -y --no-install-recommends ca-certificates openssl \ From 9763a43943f3a97f52c572f6fb47a000f3eba227 Mon Sep 17 00:00:00 2001 From: Norbiros Date: Wed, 31 Jul 2024 22:14:30 +0200 Subject: [PATCH 72/85] fix: Correctly replace linux natives for LWJGL 3.3.1 (#20) --- daedalus_client/library-patches.json | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/daedalus_client/library-patches.json b/daedalus_client/library-patches.json index 268341a96..0acd32e09 100644 --- a/daedalus_client/library-patches.json +++ b/daedalus_client/library-patches.json @@ -2863,7 +2863,8 @@ { "_comment": "Replace glfw from 3.3.1 with version from 3.3.2 to prevent stack smashing", "match": [ - "org.lwjgl:lwjgl-glfw-natives-linux:3.3.1" + "org.lwjgl:lwjgl-glfw-natives-linux:3.3.1", + "org.lwjgl:lwjgl-glfw:3.3.1:natives-linux" ], "override": { "downloads": { @@ -2876,4 +2877,4 @@ "name": "org.lwjgl:lwjgl-glfw-natives-linux:3.3.2-lwjgl.1" } } -] \ No newline at end of file +] From 6de8d2684adc7d699daa4c9f3b944b84cb8c4528 Mon Sep 17 00:00:00 2001 From: Norbiros Date: Wed, 31 Jul 2024 22:14:39 +0200 Subject: [PATCH 73/85] fix: Disable `Run Meta` on forks (#19) * fix: Disable `Run Meta` on forks * fix: Also don't run `docker` action --- .github/workflows/docker.yml | 1 + .github/workflows/run.yml | 1 + 2 files changed, 2 insertions(+) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index fb101422a..274085fff 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -9,6 +9,7 @@ on: pull_request: jobs: docker: + if: github.repository_owner == 'modrinth' runs-on: ubuntu-latest steps: - name: Checkout diff --git a/.github/workflows/run.yml b/.github/workflows/run.yml index 1409fb47f..635c6119c 100644 --- a/.github/workflows/run.yml +++ b/.github/workflows/run.yml @@ -7,6 +7,7 @@ on: jobs: run-docker: + if: github.repository_owner == 'modrinth' runs-on: ubuntu-latest steps: From 93ae24e7075f992e0855bd7875f437507e51cd12 Mon Sep 17 00:00:00 2001 From: Jai A Date: Sat, 17 Aug 2024 13:28:00 -0700 Subject: [PATCH 74/85] Fix forge format version 1 --- daedalus/src/minecraft.rs | 2 -- daedalus_client/src/forge.rs | 17 +++++++++-------- daedalus_client/src/main.rs | 2 +- daedalus_client/src/util.rs | 2 +- 4 files changed, 11 insertions(+), 12 deletions(-) diff --git a/daedalus/src/minecraft.rs b/daedalus/src/minecraft.rs index 021c91ab7..e117052ad 100644 --- a/daedalus/src/minecraft.rs +++ b/daedalus/src/minecraft.rs @@ -430,10 +430,8 @@ pub struct VersionInfo { /// The minimum version of the Minecraft Launcher that can run this version of the game pub minimum_launcher_version: u32, /// The time that the version was released - #[cfg_attr(feature = "bincode", bincode(with_serde))] pub release_time: DateTime, /// The latest time a file in this version was updated - #[cfg_attr(feature = "bincode", bincode(with_serde))] pub time: DateTime, #[serde(rename = "type")] /// The type of version diff --git a/daedalus_client/src/forge.rs b/daedalus_client/src/forge.rs index 09c8e0aa6..a560b81aa 100644 --- a/daedalus_client/src/forge.rs +++ b/daedalus_client/src/forge.rs @@ -1,5 +1,5 @@ use crate::util::{ - download_file, fetch_json, fetch_xml, format_url, sha1_async, + download_file, fetch_json, fetch_xml, format_url, }; use crate::{insert_mirrored_artifact, Error, MirrorArtifact, UploadFile}; use chrono::{DateTime, Utc}; @@ -403,9 +403,10 @@ async fn fetch( .map(|mut lib| { // For all libraries besides the forge lib extracted, we mirror them from maven servers // unless the URL is empty/null or available on Minecraft's servers - if let Some(url) = lib.url { - if lib.name != install_profile.install.path - && !url.is_empty() + if let Some(ref url) = lib.url { + if lib.name == install_profile.install.path { + lib.url = Some(format_url("maven/")); + } else if !url.is_empty() && !url.contains( "https://libraries.minecraft.net/", ) @@ -414,7 +415,7 @@ async fn fetch( &lib.name, None, vec![ - url, + url.clone(), "https://maven.creeperhost.net/" .to_string(), maven_url.to_string(), @@ -422,11 +423,11 @@ async fn fetch( false, mirror_artifacts, )?; + + lib.url = Some(format_url("maven/")); } } - lib.url = Some(format_url("maven/")); - Ok(lib) }) .collect::, Error>>()?, @@ -442,7 +443,7 @@ async fn fetch( // pub profile: String, // pub version: String, // pub json: String, - pub path: Option, + // pub path: Option, // pub minecraft: String, pub data: HashMap, pub libraries: Vec, diff --git a/daedalus_client/src/main.rs b/daedalus_client/src/main.rs index 8e934cbd5..870a55ec0 100644 --- a/daedalus_client/src/main.rs +++ b/daedalus_client/src/main.rs @@ -160,7 +160,7 @@ pub fn insert_mirrored_artifact( entire_url: bool, mirror_artifacts: &DashMap, ) -> Result<()> { - let mut val = mirror_artifacts + let val = mirror_artifacts .entry(get_path_from_artifact(artifact)?) .or_insert(MirrorArtifact { sha1, diff --git a/daedalus_client/src/util.rs b/daedalus_client/src/util.rs index 72ff25575..b5e4b360a 100644 --- a/daedalus_client/src/util.rs +++ b/daedalus_client/src/util.rs @@ -159,7 +159,7 @@ pub async fn download_file( const RETRIES: u32 = 10; for attempt in 1..=(RETRIES + 1) { let result = REQWEST_CLIENT - .get(&url.replace("http://", "https://")) + .get(url.replace("http://", "https://")) .send() .await .and_then(|x| x.error_for_status()); From 679ffbcce75660410569e0538525b87b972a2fb1 Mon Sep 17 00:00:00 2001 From: Jai A Date: Wed, 21 Aug 2024 18:42:02 -0700 Subject: [PATCH 75/85] Fix Fabric Loader 0.16.0, old forge versions --- daedalus/Cargo.toml | 2 +- daedalus/src/modded.rs | 27 +++++++++++++++++++++++---- daedalus_client/src/fabric.rs | 1 + 3 files changed, 25 insertions(+), 5 deletions(-) diff --git a/daedalus/Cargo.toml b/daedalus/Cargo.toml index 1b28e72ee..a57f4c612 100644 --- a/daedalus/Cargo.toml +++ b/daedalus/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "daedalus" -version = "0.2.2" +version = "0.2.3" authors = ["Jai A "] edition = "2021" license = "MIT" diff --git a/daedalus/src/modded.rs b/daedalus/src/modded.rs index c510fe78b..e7043a4c0 100644 --- a/daedalus/src/modded.rs +++ b/daedalus/src/modded.rs @@ -98,6 +98,27 @@ pub fn merge_partial_version( ) -> VersionInfo { let merge_id = merge.id.clone(); + let mut libraries = vec![]; + + // We skip duplicate libraries that exist already in the partial version + for mut lib in merge.libraries { + let lib_artifact = lib.name.rsplit_once(':').map(|x| x.0); + + if let Some(lib_artifact) = lib_artifact { + if !partial.libraries.iter().any(|x| { + let target_artifact = x.name.rsplit_once(':').map(|x| x.0); + + target_artifact == Some(lib_artifact) && x.include_in_classpath + }) { + libraries.push(lib); + } else { + lib.include_in_classpath = false; + } + } else { + libraries.push(lib); + } + } + VersionInfo { arguments: if let Some(partial_args) = partial.arguments { if let Some(merge_args) = merge.arguments { @@ -133,10 +154,8 @@ pub fn merge_partial_version( downloads: merge.downloads, id: partial.id.replace(DUMMY_REPLACE_STRING, &merge_id), java_version: merge.java_version, - libraries: partial - .libraries - .into_iter() - .chain(merge.libraries) + libraries: libraries.into_iter() + .chain(partial.libraries) .map(|mut x| { x.name = x.name.replace(DUMMY_REPLACE_STRING, &merge_id); diff --git a/daedalus_client/src/fabric.rs b/daedalus_client/src/fabric.rs index a7e352b24..2c394bc23 100644 --- a/daedalus_client/src/fabric.rs +++ b/daedalus_client/src/fabric.rs @@ -47,6 +47,7 @@ pub async fn fetch_quilt( .await } +#[allow(clippy::too_many_arguments)] #[tracing::instrument(skip(semaphore, upload_files, mirror_artifacts))] async fn fetch( format_version: usize, From f212fcf892d20a4de7a21542be6aff835a01e4c3 Mon Sep 17 00:00:00 2001 From: Jai A Date: Sat, 19 Oct 2024 14:40:58 -0700 Subject: [PATCH 76/85] Start monorepo migration --- .../{docker.yml => daedalus-docker.yml} | 18 ++- .../workflows/{run.yml => daedalus-run.yml} | 2 +- .github/workflows/lint.yml | 30 ----- .github/workflows/publish.yml | 19 --- .github/workflows/rust.yml | 29 ----- .gitignore | 122 ------------------ .idea/daedalus.iml | 4 +- .idea/libraries/KotlinJavaRuntime.xml | 26 ++++ Cargo.toml | 6 - .env => apps/daedalus_client/.env | 0 .../daedalus_client}/Cargo.toml | 0 Dockerfile => apps/daedalus_client/Dockerfile | 0 LICENSE => apps/daedalus_client/LICENSE | 2 +- README.md => apps/daedalus_client/README.md | 0 .../daedalus_client/docker-compose.yml | 0 .../daedalus_client}/library-patches.json | 0 apps/daedalus_client/package.json | 10 ++ .../daedalus_client}/src/error.rs | 0 .../daedalus_client}/src/fabric.rs | 0 .../daedalus_client}/src/forge.rs | 0 .../daedalus_client}/src/main.rs | 0 .../daedalus_client}/src/minecraft.rs | 0 .../daedalus_client}/src/util.rs | 0 {daedalus => packages/daedalus}/Cargo.toml | 0 packages/daedalus/LICENSE | 7 + {daedalus => packages/daedalus}/README.md | 2 - packages/daedalus/package.json | 10 ++ {daedalus => packages/daedalus}/src/lib.rs | 0 .../daedalus}/src/minecraft.rs | 0 {daedalus => packages/daedalus}/src/modded.rs | 0 rustfmt.toml | 2 - 31 files changed, 69 insertions(+), 220 deletions(-) rename .github/workflows/{docker.yml => daedalus-docker.yml} (73%) rename .github/workflows/{run.yml => daedalus-run.yml} (96%) delete mode 100644 .github/workflows/lint.yml delete mode 100644 .github/workflows/publish.yml delete mode 100644 .github/workflows/rust.yml delete mode 100644 .gitignore create mode 100644 .idea/libraries/KotlinJavaRuntime.xml delete mode 100644 Cargo.toml rename .env => apps/daedalus_client/.env (100%) rename {daedalus_client => apps/daedalus_client}/Cargo.toml (100%) rename Dockerfile => apps/daedalus_client/Dockerfile (100%) rename LICENSE => apps/daedalus_client/LICENSE (97%) rename README.md => apps/daedalus_client/README.md (100%) rename docker-compose.yml => apps/daedalus_client/docker-compose.yml (100%) rename {daedalus_client => apps/daedalus_client}/library-patches.json (100%) create mode 100644 apps/daedalus_client/package.json rename {daedalus_client => apps/daedalus_client}/src/error.rs (100%) rename {daedalus_client => apps/daedalus_client}/src/fabric.rs (100%) rename {daedalus_client => apps/daedalus_client}/src/forge.rs (100%) rename {daedalus_client => apps/daedalus_client}/src/main.rs (100%) rename {daedalus_client => apps/daedalus_client}/src/minecraft.rs (100%) rename {daedalus_client => apps/daedalus_client}/src/util.rs (100%) rename {daedalus => packages/daedalus}/Cargo.toml (100%) create mode 100644 packages/daedalus/LICENSE rename {daedalus => packages/daedalus}/README.md (86%) create mode 100644 packages/daedalus/package.json rename {daedalus => packages/daedalus}/src/lib.rs (100%) rename {daedalus => packages/daedalus}/src/minecraft.rs (100%) rename {daedalus => packages/daedalus}/src/modded.rs (100%) delete mode 100644 rustfmt.toml diff --git a/.github/workflows/docker.yml b/.github/workflows/daedalus-docker.yml similarity index 73% rename from .github/workflows/docker.yml rename to .github/workflows/daedalus-docker.yml index 274085fff..f3e5a2d71 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/daedalus-docker.yml @@ -1,15 +1,21 @@ -name: docker-build +name: daedalus-docker-build on: push: - branches: - - '**' - tags: - - 'v*' + branches: [ "main" ] + paths: + - .github/workflows/daedalus-docker.yml + - 'apps/daedalus/**' pull_request: + types: [ opened, synchronize ] + paths: + - .github/workflows/daedalus-docker.yml + - 'apps/daedalus/**' + merge_group: + types: [ checks_requested ] + jobs: docker: - if: github.repository_owner == 'modrinth' runs-on: ubuntu-latest steps: - name: Checkout diff --git a/.github/workflows/run.yml b/.github/workflows/daedalus-run.yml similarity index 96% rename from .github/workflows/run.yml rename to .github/workflows/daedalus-run.yml index 635c6119c..dcdbd0496 100644 --- a/.github/workflows/run.yml +++ b/.github/workflows/daedalus-run.yml @@ -23,7 +23,7 @@ jobs: - name: Pull Docker image from GHCR - run: docker pull ghcr.io/modrinth/daedalus:master + run: docker pull ghcr.io/modrinth/daedalus:main - name: Run Docker container env: diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml deleted file mode 100644 index 8bf19521d..000000000 --- a/.github/workflows/lint.yml +++ /dev/null @@ -1,30 +0,0 @@ -name: Rust lint - -on: - push: - branches: [ master ] - pull_request: -env: - CARGO_TERM_COLOR: always -jobs: - lint: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - name: Install toolchain - uses: actions-rs/toolchain@v1 - with: - toolchain: stable - override: true - components: rustfmt, clippy - - name: Cache build artifacts - id: cache-build - uses: actions/cache@v2 - with: - path: target/** - key: ${{ runner.os }}-build-cache - - name: Annotate commit with clippy warnings - uses: actions-rs/clippy-check@v1 - with: - token: ${{ secrets.GITHUB_TOKEN }} - args: --all-features diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml deleted file mode 100644 index 9e224e2ae..000000000 --- a/.github/workflows/publish.yml +++ /dev/null @@ -1,19 +0,0 @@ -name: Publish to crates.io - -on: - push: - tags: - - 'v*' -env: - CARGO_TERM_COLOR: always -jobs: - publish: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v1 - - run: cargo login ${CRATES_IO_TOKEN} - working-directory: ./daedalus - env: - CRATES_IO_TOKEN: ${{ secrets.CRATES_IO_TOKEN }} - - run: cargo publish - working-directory: ./daedalus diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml deleted file mode 100644 index a55aaf6e1..000000000 --- a/.github/workflows/rust.yml +++ /dev/null @@ -1,29 +0,0 @@ -name: Rust building - -on: - push: - branches: [ master ] - pull_request: -env: - CARGO_TERM_COLOR: always -jobs: - build: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - name: Get build cache - id: cache-build - uses: actions/cache@v2 - with: - path: target/** - key: ${{ runner.os }}-build-cache - - name: Install toolchain - uses: actions-rs/toolchain@v1 - with: - toolchain: stable - env: - SQLX_OFFLINE: true - - uses: actions-rs/cargo@v1 - name: Build program - with: - command: build diff --git a/.gitignore b/.gitignore deleted file mode 100644 index 06b153acc..000000000 --- a/.gitignore +++ /dev/null @@ -1,122 +0,0 @@ -### Intellij ### -# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio, WebStorm and Rider -# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 -caches/ - -# User-specific stuff -.idea/**/workspace.xml -.idea/**/tasks.xml -.idea/**/usage.statistics.xml -.idea/**/dictionaries -.idea/**/shelf - -# AWS User-specific -.idea/**/aws.xml - -# Generated files -.idea/**/contentModel.xml - -# Sensitive or high-churn files -.idea/**/dataSources/ -.idea/**/dataSources.ids -.idea/**/dataSources.local.xml -.idea/**/sqlDataSources.xml -.idea/**/dynamic.xml -.idea/**/uiDesigner.xml -.idea/**/dbnavigator.xml - -# Gradle -.idea/**/gradle.xml -.idea/**/libraries - -# Gradle and Maven with auto-import -# When using Gradle or Maven with auto-import, you should exclude module files, -# since they will be recreated, and may cause churn. Uncomment if using -# auto-import. -# .idea/artifacts -# .idea/compiler.xml -# .idea/jarRepositories.xml -# .idea/modules.xml -# .idea/*.iml -# .idea/modules -# *.iml -# *.ipr - -# CMake -cmake-build-*/ - -# Mongo Explorer plugin -.idea/**/mongoSettings.xml - -# File-based project format -*.iws - -# IntelliJ -out/ - -# mpeltonen/sbt-idea plugin -.idea_modules/ - -# JIRA plugin -atlassian-ide-plugin.xml - -# Cursive Clojure plugin -.idea/replstate.xml - -# Crashlytics plugin (for Android Studio and IntelliJ) -com_crashlytics_export_strings.xml -crashlytics.properties -crashlytics-build.properties -fabric.properties - -# Editor-based Rest Client -.idea/httpRequests - -# Android studio 3.1+ serialized cache file -.idea/caches/build_file_checksums.ser - -### Intellij Patch ### -# Comment Reason: https://github.com/joeblau/gitignore.io/issues/186#issuecomment-215987721 - -# *.iml -# modules.xml -# .idea/misc.xml -# *.ipr - -# Sonarlint plugin -# https://plugins.jetbrains.com/plugin/7973-sonarlint -.idea/**/sonarlint/ - -# SonarQube Plugin -# https://plugins.jetbrains.com/plugin/7238-sonarqube-community-plugin -.idea/**/sonarIssues.xml - -# Markdown Navigator plugin -# https://plugins.jetbrains.com/plugin/7896-markdown-navigator-enhanced -.idea/**/markdown-navigator.xml -.idea/**/markdown-navigator-enh.xml -.idea/**/markdown-navigator/ - -# Cache file creation bug -# See https://youtrack.jetbrains.com/issue/JBR-2257 -.idea/$CACHE_FILE$ - -# CodeStream plugin -# https://plugins.jetbrains.com/plugin/12206-codestream -.idea/codestream.xml - -### Rust ### -# Generated by Cargo -# will have compiled files and executables -debug/ -target/ - -# Remove Cargo.lock from gitignore if creating an executable, leave it for libraries -# More information here https://doc.rust-lang.org/cargo/guide/cargo-toml-vs-cargo-lock.html -Cargo.lock - -# These are backup files generated by rustfmt -**/*.rs.bk - -# MSVC Windows builds of rustc generate these, which store debugging information -*.pdb diff --git a/.idea/daedalus.iml b/.idea/daedalus.iml index cc8620ac8..3d715289c 100644 --- a/.idea/daedalus.iml +++ b/.idea/daedalus.iml @@ -3,10 +3,10 @@ - - + + diff --git a/.idea/libraries/KotlinJavaRuntime.xml b/.idea/libraries/KotlinJavaRuntime.xml new file mode 100644 index 000000000..78d18789b --- /dev/null +++ b/.idea/libraries/KotlinJavaRuntime.xml @@ -0,0 +1,26 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/Cargo.toml b/Cargo.toml deleted file mode 100644 index dc360c2e5..000000000 --- a/Cargo.toml +++ /dev/null @@ -1,6 +0,0 @@ -[workspace] - -members = [ - "daedalus", - "daedalus_client" -] \ No newline at end of file diff --git a/.env b/apps/daedalus_client/.env similarity index 100% rename from .env rename to apps/daedalus_client/.env diff --git a/daedalus_client/Cargo.toml b/apps/daedalus_client/Cargo.toml similarity index 100% rename from daedalus_client/Cargo.toml rename to apps/daedalus_client/Cargo.toml diff --git a/Dockerfile b/apps/daedalus_client/Dockerfile similarity index 100% rename from Dockerfile rename to apps/daedalus_client/Dockerfile diff --git a/LICENSE b/apps/daedalus_client/LICENSE similarity index 97% rename from LICENSE rename to apps/daedalus_client/LICENSE index 75e4f7799..bc179be0a 100644 --- a/LICENSE +++ b/apps/daedalus_client/LICENSE @@ -1,4 +1,4 @@ -Copyright © 2023 Rinth, Inc. +Copyright © 2024 Rinth, Inc. Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: diff --git a/README.md b/apps/daedalus_client/README.md similarity index 100% rename from README.md rename to apps/daedalus_client/README.md diff --git a/docker-compose.yml b/apps/daedalus_client/docker-compose.yml similarity index 100% rename from docker-compose.yml rename to apps/daedalus_client/docker-compose.yml diff --git a/daedalus_client/library-patches.json b/apps/daedalus_client/library-patches.json similarity index 100% rename from daedalus_client/library-patches.json rename to apps/daedalus_client/library-patches.json diff --git a/apps/daedalus_client/package.json b/apps/daedalus_client/package.json new file mode 100644 index 000000000..41c8b0ae3 --- /dev/null +++ b/apps/daedalus_client/package.json @@ -0,0 +1,10 @@ +{ + "name": "@modrinth/daedalus_client", + "scripts": { + "build": "cargo build --release", + "lint": "cargo fmt --check && cargo clippy --all-targets --all-features -- -D warnings", + "fix": "cargo fmt && cargo clippy --fix", + "dev": "cargo run", + "test": "cargo test" + } +} \ No newline at end of file diff --git a/daedalus_client/src/error.rs b/apps/daedalus_client/src/error.rs similarity index 100% rename from daedalus_client/src/error.rs rename to apps/daedalus_client/src/error.rs diff --git a/daedalus_client/src/fabric.rs b/apps/daedalus_client/src/fabric.rs similarity index 100% rename from daedalus_client/src/fabric.rs rename to apps/daedalus_client/src/fabric.rs diff --git a/daedalus_client/src/forge.rs b/apps/daedalus_client/src/forge.rs similarity index 100% rename from daedalus_client/src/forge.rs rename to apps/daedalus_client/src/forge.rs diff --git a/daedalus_client/src/main.rs b/apps/daedalus_client/src/main.rs similarity index 100% rename from daedalus_client/src/main.rs rename to apps/daedalus_client/src/main.rs diff --git a/daedalus_client/src/minecraft.rs b/apps/daedalus_client/src/minecraft.rs similarity index 100% rename from daedalus_client/src/minecraft.rs rename to apps/daedalus_client/src/minecraft.rs diff --git a/daedalus_client/src/util.rs b/apps/daedalus_client/src/util.rs similarity index 100% rename from daedalus_client/src/util.rs rename to apps/daedalus_client/src/util.rs diff --git a/daedalus/Cargo.toml b/packages/daedalus/Cargo.toml similarity index 100% rename from daedalus/Cargo.toml rename to packages/daedalus/Cargo.toml diff --git a/packages/daedalus/LICENSE b/packages/daedalus/LICENSE new file mode 100644 index 000000000..bc179be0a --- /dev/null +++ b/packages/daedalus/LICENSE @@ -0,0 +1,7 @@ +Copyright © 2024 Rinth, Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. \ No newline at end of file diff --git a/daedalus/README.md b/packages/daedalus/README.md similarity index 86% rename from daedalus/README.md rename to packages/daedalus/README.md index fb57d0f03..aa5602590 100644 --- a/daedalus/README.md +++ b/packages/daedalus/README.md @@ -2,5 +2,3 @@ Daedalus (the rust library) is a library providing model structs and methods for requesting and parsing things from Minecraft and other mod loaders meta APIs. - -This is a work in progress! \ No newline at end of file diff --git a/packages/daedalus/package.json b/packages/daedalus/package.json new file mode 100644 index 000000000..41c8b0ae3 --- /dev/null +++ b/packages/daedalus/package.json @@ -0,0 +1,10 @@ +{ + "name": "@modrinth/daedalus_client", + "scripts": { + "build": "cargo build --release", + "lint": "cargo fmt --check && cargo clippy --all-targets --all-features -- -D warnings", + "fix": "cargo fmt && cargo clippy --fix", + "dev": "cargo run", + "test": "cargo test" + } +} \ No newline at end of file diff --git a/daedalus/src/lib.rs b/packages/daedalus/src/lib.rs similarity index 100% rename from daedalus/src/lib.rs rename to packages/daedalus/src/lib.rs diff --git a/daedalus/src/minecraft.rs b/packages/daedalus/src/minecraft.rs similarity index 100% rename from daedalus/src/minecraft.rs rename to packages/daedalus/src/minecraft.rs diff --git a/daedalus/src/modded.rs b/packages/daedalus/src/modded.rs similarity index 100% rename from daedalus/src/modded.rs rename to packages/daedalus/src/modded.rs diff --git a/rustfmt.toml b/rustfmt.toml deleted file mode 100644 index f5a8b8674..000000000 --- a/rustfmt.toml +++ /dev/null @@ -1,2 +0,0 @@ -edition = "2018" -max_width = 80 \ No newline at end of file From 9be9658ffb2391aee0eb8517c543fd275f36a40a Mon Sep 17 00:00:00 2001 From: Jai A Date: Sat, 19 Oct 2024 15:14:30 -0700 Subject: [PATCH 77/85] Add daedalus --- Cargo.lock | 1085 ++++++++++++++++++++++++------- Cargo.toml | 4 +- apps/app/Cargo.toml | 2 +- apps/daedalus_client/Cargo.toml | 2 +- packages/app-lib/Cargo.toml | 2 +- 5 files changed, 866 insertions(+), 229 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ddc5ad543..fe2d879aa 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9,12 +9,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5f7b0a21988c1bf877cf4759ef5ddaac04c1c9fe808c9142ecb78ba97d97a28a" dependencies = [ "bitflags 2.6.0", - "bytes", + "bytes 1.7.2", "futures-core", "futures-sink", "memchr", "pin-project-lite", - "tokio", + "tokio 1.40.0", "tokio-util", "tracing", ] @@ -31,7 +31,7 @@ dependencies = [ "futures-util", "log", "once_cell", - "smallvec", + "smallvec 1.13.2", ] [[package]] @@ -45,7 +45,7 @@ dependencies = [ "actix-utils", "actix-web", "bitflags 2.6.0", - "bytes", + "bytes 1.7.2", "derive_more", "futures-core", "http-range", @@ -71,7 +71,7 @@ dependencies = [ "base64 0.22.1", "bitflags 2.6.0", "brotli", - "bytes", + "bytes 1.7.2", "bytestring", "derive_more", "encoding_rs", @@ -89,8 +89,8 @@ dependencies = [ "pin-project-lite", "rand 0.8.5", "sha1 0.10.6", - "smallvec", - "tokio", + "smallvec 1.13.2", + "tokio 1.40.0", "tokio-util", "tracing", "zstd 0.13.2", @@ -115,7 +115,7 @@ dependencies = [ "actix-multipart-derive", "actix-utils", "actix-web", - "bytes", + "bytes 1.7.2", "derive_more", "futures-core", "futures-util", @@ -129,7 +129,7 @@ dependencies = [ "serde_json", "serde_plain", "tempfile", - "tokio", + "tokio 1.40.0", ] [[package]] @@ -152,7 +152,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13d324164c51f63867b57e73ba5936ea151b8a41a1d23d1031eeb9f70d0236f8" dependencies = [ "bytestring", - "cfg-if", + "cfg-if 1.0.0", "http 0.2.12", "regex", "regex-lite", @@ -168,7 +168,7 @@ checksum = "24eda4e2a6e042aa4e55ac438a2ae052d3b5da0ecf83d7411e1a368946925208" dependencies = [ "actix-macros", "futures-core", - "tokio", + "tokio 1.40.0", ] [[package]] @@ -184,7 +184,7 @@ dependencies = [ "futures-util", "mio 1.0.2", "socket2", - "tokio", + "tokio 1.40.0", "tracing", ] @@ -225,9 +225,9 @@ dependencies = [ "actix-utils", "actix-web-codegen", "ahash 0.8.11", - "bytes", + "bytes 1.7.2", "bytestring", - "cfg-if", + "cfg-if 1.0.0", "cookie", "derive_more", "encoding_rs", @@ -245,7 +245,7 @@ dependencies = [ "serde", "serde_json", "serde_urlencoded", - "smallvec", + "smallvec 1.13.2", "socket2", "time", "url", @@ -288,7 +288,7 @@ dependencies = [ "actix-http", "actix-web", "futures-core", - "tokio", + "tokio 1.40.0", ] [[package]] @@ -318,7 +318,7 @@ version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "cipher", "cpufeatures", ] @@ -340,7 +340,7 @@ version = "0.8.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e89da841a80418a9b391ebaea17f5c112ffaaa96f621d2c285b5174da76b9011" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "getrandom 0.2.15", "once_cell", "version_check", @@ -449,7 +449,7 @@ dependencies = [ "rand 0.8.5", "serde", "serde_repr", - "tokio", + "tokio 1.40.0", "url", "zbus", ] @@ -503,7 +503,8 @@ dependencies = [ "futures-io", "memchr", "pin-project-lite", - "tokio", + "tokio 1.40.0", + "xz2", "zstd 0.13.2", "zstd-safe 7.2.1", ] @@ -539,7 +540,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "444b0228950ee6501b3568d3c93bf1176a1fdbc3b758dcd9475046d30f4dc7e8" dependencies = [ "async-lock", - "cfg-if", + "cfg-if 1.0.0", "concurrent-queue", "futures-io", "futures-lite 2.3.0", @@ -574,7 +575,7 @@ dependencies = [ "async-signal", "async-task", "blocking", - "cfg-if", + "cfg-if 1.0.0", "event-listener 5.3.1", "futures-lite 2.3.0", "rustix", @@ -601,7 +602,7 @@ dependencies = [ "async-io", "async-lock", "atomic-waker", - "cfg-if", + "cfg-if 1.0.0", "futures-core", "futures-io", "rustix", @@ -631,7 +632,7 @@ dependencies = [ "smart-default", "smol_str", "thiserror", - "tokio", + "tokio 1.40.0", "uuid 0.8.2", ] @@ -663,7 +664,7 @@ dependencies = [ "log", "pin-project-lite", "rustls-pki-types", - "tokio", + "tokio 1.40.0", "tokio-rustls 0.26.0", "tungstenite", "webpki-roots", @@ -681,7 +682,7 @@ dependencies = [ "futures-lite 2.3.0", "pin-project", "thiserror", - "tokio", + "tokio 1.40.0", "tokio-util", ] @@ -737,6 +738,20 @@ dependencies = [ "url", ] +[[package]] +name = "attohttpc" +version = "0.26.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f77d243921b0979fbbd728dd2d5162e68ac8252976797c24eb5b3a6af9090dc" +dependencies = [ + "http 0.2.12", + "log", + "native-tls", + "serde", + "serde_json", + "url", +] + [[package]] name = "autocfg" version = "1.4.0" @@ -749,7 +764,7 @@ version = "0.34.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3776743bb68d4ad02ba30ba8f64373f1be4e082fe47651767171ce75bb2f6cf5" dependencies = [ - "attohttpc", + "attohttpc 0.22.0", "dirs 4.0.0", "log", "quick-xml 0.26.0", @@ -760,6 +775,23 @@ dependencies = [ "url", ] +[[package]] +name = "aws-creds" +version = "0.36.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "390ad3b77f3e21e01a4a0355865853b681daf1988510b0b15e31c0c4ae7eb0f6" +dependencies = [ + "attohttpc 0.26.1", + "home", + "log", + "quick-xml 0.30.0", + "rust-ini 0.19.0", + "serde", + "thiserror", + "time", + "url", +] + [[package]] name = "aws-region" version = "0.25.5" @@ -776,7 +808,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8d82cb332cdfaed17ae235a638438ac4d4839913cc2af585c3c6746e8f8bee1a" dependencies = [ "addr2line", - "cfg-if", + "cfg-if 1.0.0", "libc", "miniz_oxide 0.8.0", "object", @@ -1034,6 +1066,16 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f1fe948ff07f4bd06c30984e69f5b4899c516a3ef74f34df92a2df2ab535495" +[[package]] +name = "bytes" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "206fdffcfa2df7cbe15601ef46c813fce0965eb3286db6b56c583b814b51c81c" +dependencies = [ + "byteorder", + "iovec", +] + [[package]] name = "bytes" version = "1.7.2" @@ -1049,7 +1091,7 @@ version = "1.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "74d80203ea6b29df88012294f62733de21cfeab47f17b41af3a38bc30a03ee72" dependencies = [ - "bytes", + "bytes 1.7.2", ] [[package]] @@ -1124,7 +1166,7 @@ checksum = "2d886547e41f740c616ae73108f6eb70afe6d940c7bc697cb30f13daec073037" dependencies = [ "camino", "cargo-platform", - "semver", + "semver 1.0.23", "serde", "serde_json", "thiserror", @@ -1189,10 +1231,16 @@ version = "0.15.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d067ad48b8650848b989a59a86c6c36a995d02d2bf778d45c3c5d57bc2718f02" dependencies = [ - "smallvec", + "smallvec 1.13.2", "target-lexicon", ] +[[package]] +name = "cfg-if" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822" + [[package]] name = "cfg-if" version = "1.0.0" @@ -1247,10 +1295,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a0875e527e299fc5f4faba42870bf199a39ab0bb2dbba1b8aef0a2151451130f" dependencies = [ "bstr", - "bytes", + "bytes 1.7.2", "clickhouse-derive", "clickhouse-rs-cityhash-sys", - "futures", + "futures 0.3.30", "hyper 0.14.31", "hyper-tls 0.5.0", "lz4", @@ -1259,7 +1307,7 @@ dependencies = [ "static_assertions", "thiserror", "time", - "tokio", + "tokio 1.40.0", "url", "uuid 1.10.0", ] @@ -1285,6 +1333,15 @@ dependencies = [ "cc", ] +[[package]] +name = "cloudabi" +version = "0.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddfc5b9aa5d4507acaf872de71051dfd0e309860e88966e1051e462a077aac4f" +dependencies = [ + "bitflags 1.3.2", +] + [[package]] name = "cocoa" version = "0.25.0" @@ -1366,11 +1423,11 @@ version = "4.6.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ba5a308b75df32fe02788e748662718f03fde005016435c444eea572398219fd" dependencies = [ - "bytes", + "bytes 1.7.2", "futures-core", "memchr", "pin-project-lite", - "tokio", + "tokio 1.40.0", "tokio-util", ] @@ -1380,7 +1437,7 @@ version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4ca0197aee26d1ae37445ee532fefce43251d24cc7c166799f4d46817f1d3973" dependencies = [ - "crossbeam-utils", + "crossbeam-utils 0.8.20", ] [[package]] @@ -1564,7 +1621,7 @@ version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a97769d94ddab943e4510d138150169a2758b5ef3eb191a9ee688de3e23ef7b3" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", ] [[package]] @@ -1573,7 +1630,18 @@ version = "0.5.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "33480d6946193aa8033910124896ca395333cae7e2d1113d1fef6c3272217df2" dependencies = [ - "crossbeam-utils", + "crossbeam-utils 0.8.20", +] + +[[package]] +name = "crossbeam-deque" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c20ff29ded3204c5106278a81a38f4b482636ed4fa1e6cfbeef193291beb29ed" +dependencies = [ + "crossbeam-epoch 0.8.2", + "crossbeam-utils 0.7.2", + "maybe-uninit", ] [[package]] @@ -1582,8 +1650,23 @@ version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "613f8cc01fe9cf1a3eb3d7f488fd2fa8388403e97039e2f73692932e291a770d" dependencies = [ - "crossbeam-epoch", - "crossbeam-utils", + "crossbeam-epoch 0.9.18", + "crossbeam-utils 0.8.20", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "058ed274caafc1f60c4997b5fc07bf7dc7cca454af7c6e81edffe5f33f70dace" +dependencies = [ + "autocfg", + "cfg-if 0.1.10", + "crossbeam-utils 0.7.2", + "lazy_static", + "maybe-uninit", + "memoffset 0.5.6", + "scopeguard", ] [[package]] @@ -1592,7 +1675,18 @@ version = "0.9.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" dependencies = [ - "crossbeam-utils", + "crossbeam-utils 0.8.20", +] + +[[package]] +name = "crossbeam-queue" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "774ba60a54c213d409d5353bda12d49cd68d14e45036a285234c8d6f91f92570" +dependencies = [ + "cfg-if 0.1.10", + "crossbeam-utils 0.7.2", + "maybe-uninit", ] [[package]] @@ -1601,7 +1695,18 @@ version = "0.3.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "df0346b5d5e76ac2fe4e327c5fd1118d6be7c51dfb18f9b7922923f287471e35" dependencies = [ - "crossbeam-utils", + "crossbeam-utils 0.8.20", +] + +[[package]] +name = "crossbeam-utils" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3c7c73a2d1e9fc0886a08b93e98eb643461230d5f1925e4036204d5f2e261a8" +dependencies = [ + "autocfg", + "cfg-if 0.1.10", + "lazy_static", ] [[package]] @@ -1661,7 +1766,7 @@ dependencies = [ "phf 0.8.0", "proc-macro2", "quote", - "smallvec", + "smallvec 1.13.2", "syn 1.0.109", ] @@ -1740,16 +1845,43 @@ dependencies = [ [[package]] name = "daedalus" version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "314dac850cbf02b728fb531c1f3b1bec5b4ccdef6564db274470363792a6d302" dependencies = [ - "bytes", + "bytes 1.7.2", "chrono", "serde", "serde_json", "thiserror", ] +[[package]] +name = "daedalus_client" +version = "0.2.2" +dependencies = [ + "async_zip", + "bytes 1.7.2", + "chrono", + "daedalus", + "dashmap 5.5.3", + "dotenvy", + "futures 0.3.30", + "indexmap 2.5.0", + "itertools 0.13.0", + "lazy_static", + "reqwest 0.12.7", + "rust-s3 0.34.0", + "semver 1.0.23", + "serde", + "serde-xml-rs", + "serde_json", + "sha1_smol", + "thiserror", + "tokio 1.40.0", + "tracing", + "tracing-error", + "tracing-futures", + "tracing-subscriber", +] + [[package]] name = "darling" version = "0.14.4" @@ -1826,11 +1958,11 @@ version = "5.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "978747c1d849a7d2ee5e8adc0159961c48fb7e5db2f06af6723b80123bb53856" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "hashbrown 0.14.5", - "lock_api", + "lock_api 0.4.12", "once_cell", - "parking_lot_core", + "parking_lot_core 0.9.10", ] [[package]] @@ -1839,12 +1971,12 @@ version = "6.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5041cc499144891f3790297212f32a74fb938e5136a14943f338ef9e0ae276cf" dependencies = [ - "cfg-if", - "crossbeam-utils", + "cfg-if 1.0.0", + "crossbeam-utils 0.8.20", "hashbrown 0.14.5", - "lock_api", + "lock_api 0.4.12", "once_cell", - "parking_lot_core", + "parking_lot_core 0.9.10", "serde", ] @@ -1862,7 +1994,7 @@ checksum = "1bb21987b9fb1613058ba3843121dd18b163b254d8a6e797e144cbac14d96d1b" dependencies = [ "libc", "libdbus-sys", - "winapi", + "winapi 0.3.9", ] [[package]] @@ -1873,7 +2005,7 @@ checksum = "6541a3916932fe57768d4be0b1ffb5ec7cbf74ca8c903fdfd5c0fe8aa958f0ed" dependencies = [ "deadpool-runtime", "num_cpus", - "tokio", + "tokio 1.40.0", ] [[package]] @@ -1892,7 +2024,7 @@ version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "092966b41edc516079bdf31ec78a2e0588d1d0c08f78b91d8307215928642b2b" dependencies = [ - "tokio", + "tokio 1.40.0", ] [[package]] @@ -1994,7 +2126,7 @@ dependencies = [ "convert_case 0.4.0", "proc-macro2", "quote", - "rustc_version", + "rustc_version 0.4.1", "syn 2.0.79", ] @@ -2043,7 +2175,7 @@ version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b98cf8ebf19c3d1b223e151f99a4f9f0690dca41414773390fc824184ac833e1" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "dirs-sys-next", ] @@ -2055,7 +2187,7 @@ checksum = "1b1d1d91c932ef41c0f2663aa8b0ca0342d444d842c06914aa0a7e352d0bada6" dependencies = [ "libc", "redox_users", - "winapi", + "winapi 0.3.9", ] [[package]] @@ -2078,7 +2210,7 @@ checksum = "4ebda144c4fe02d1f7ea1a7d9641b6fc6b580adcfa024ae48797ecdeb6825b4d" dependencies = [ "libc", "redox_users", - "winapi", + "winapi 0.3.9", ] [[package]] @@ -2120,7 +2252,7 @@ dependencies = [ "dlopen2_derive", "libc", "once_cell", - "winapi", + "winapi 0.3.9", ] [[package]] @@ -2258,7 +2390,7 @@ checksum = "f4e24052d7be71f0efb50c201557f6fe7d237cfd5a64fd5bcd7fd8fe32dbbffa" dependencies = [ "cc", "memchr", - "rustc_version", + "rustc_version 0.4.1", "toml 0.8.19", "vswhom", "winreg 0.52.0", @@ -2288,7 +2420,7 @@ version = "0.8.34" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b45de904aa0b010bce2ab45264d0631681847fa7b6f2eaa7dab7619943bc4f59" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", ] [[package]] @@ -2363,7 +2495,7 @@ version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "136d1b5283a1ab77bd9257427ffd09d8667ced0570b6f938942bc7568ed5b943" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "home", "windows-sys 0.48.0", ] @@ -2407,7 +2539,7 @@ dependencies = [ "lebe", "miniz_oxide 0.7.4", "rayon-core", - "smallvec", + "smallvec 1.13.2", "zune-inflate", ] @@ -2461,8 +2593,8 @@ version = "0.3.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "38e2275cc4e4fc009b0669731a1e5ab7ebf11f469eaede2bab9309a5b4d6057f" dependencies = [ - "memoffset", - "rustc_version", + "memoffset 0.9.1", + "rustc_version 0.4.1", ] [[package]] @@ -2471,7 +2603,7 @@ version = "0.2.25" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "35c0522e981e68cbfa8c3f978441a5f34b30b96e146b33cd3359176b50fe8586" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "libc", "libredox", "windows-sys 0.59.0", @@ -2486,7 +2618,7 @@ dependencies = [ "cc", "lazy_static", "libc", - "winapi", + "winapi 0.3.9", ] [[package]] @@ -2585,6 +2717,22 @@ dependencies = [ "libc", ] +[[package]] +name = "fuchsia-zircon" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e9763c69ebaae630ba35f74888db465e49e259ba1bc0eda7d06f4a067615d82" +dependencies = [ + "bitflags 1.3.2", + "fuchsia-zircon-sys", +] + +[[package]] +name = "fuchsia-zircon-sys" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3dcaa9ae7725d12cdb85b3ad99a434db70b468c09ded17e012d86b5c1010f7a7" + [[package]] name = "funty" version = "2.0.0" @@ -2601,6 +2749,12 @@ dependencies = [ "new_debug_unreachable", ] +[[package]] +name = "futures" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a471a38ef8ed83cd6e40aa59c1ffe17db6855c18e3604d9c4ed8c08ebc28678" + [[package]] name = "futures" version = "0.3.30" @@ -2650,8 +2804,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d930c203dd0b6ff06e0201a4a2fe9149b43c684fd4420555b26d21b1a02956f" dependencies = [ "futures-core", - "lock_api", - "parking_lot", + "lock_api 0.4.12", + "parking_lot 0.12.3", ] [[package]] @@ -2870,7 +3024,7 @@ version = "0.1.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8fc3cb4d91f53b50155bdcfd23f6a4c39ae1969c2ae85982b135750cccaf5fce" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "libc", "wasi 0.9.0+wasi-snapshot-preview1", ] @@ -2881,7 +3035,7 @@ version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "libc", "wasi 0.11.0+wasi-snapshot-preview1", ] @@ -2917,7 +3071,7 @@ dependencies = [ "libc", "once_cell", "pin-project-lite", - "smallvec", + "smallvec 1.13.2", "thiserror", ] @@ -2931,7 +3085,7 @@ dependencies = [ "gobject-sys", "libc", "system-deps", - "winapi", + "winapi 0.3.9", ] [[package]] @@ -2953,7 +3107,7 @@ dependencies = [ "libc", "memchr", "once_cell", - "smallvec", + "smallvec 1.13.2", "thiserror", ] @@ -3004,17 +3158,17 @@ version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68a7f542ee6b35af73b06abc0dad1c1bae89964e4e253bc4b587b91c9637867b" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "dashmap 5.5.3", - "futures", + "futures 0.3.30", "futures-timer", "no-std-compat", "nonzero_ext", - "parking_lot", + "parking_lot 0.12.3", "portable-atomic", "quanta", "rand 0.8.5", - "smallvec", + "smallvec 1.13.2", "spinning_top", ] @@ -3087,7 +3241,7 @@ version = "0.3.26" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "81fe527a889e1532da5c525686d96d4c2e74cdd345badf8dfef9f6b39dd5f5e8" dependencies = [ - "bytes", + "bytes 1.7.2", "fnv", "futures-core", "futures-sink", @@ -3095,7 +3249,7 @@ dependencies = [ "http 0.2.12", "indexmap 2.5.0", "slab", - "tokio", + "tokio 1.40.0", "tokio-util", "tracing", ] @@ -3107,14 +3261,14 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "524e8ac6999421f49a846c2d4411f337e53497d8ec55d67753beffa43c5d9205" dependencies = [ "atomic-waker", - "bytes", + "bytes 1.7.2", "fnv", "futures-core", "futures-sink", "http 1.1.0", "indexmap 2.5.0", "slab", - "tokio", + "tokio 1.40.0", "tokio-util", "tracing", ] @@ -3125,7 +3279,7 @@ version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6dd08c532ae367adf81c312a4580bc67f1d0fe8bc9c460520283f4c0ff277888" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "crunchy", ] @@ -3138,6 +3292,12 @@ dependencies = [ "ahash 0.7.8", ] +[[package]] +name = "hashbrown" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43a3c133739dddd0d2990f9a4bdf8eb4b21ef50e4851ca85ab661199821d510e" + [[package]] name = "hashbrown" version = "0.14.5" @@ -3239,7 +3399,7 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f9c7c7c8ac16c798734b8a24560c1362120597c40d5e1459f09498f8f6c8f2ba" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "libc", "windows 0.52.0", ] @@ -3264,7 +3424,7 @@ version = "0.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "601cbb57e577e2f5ef5be8e7b83f0f63994f25aa94d673e54a92d5c516d101f1" dependencies = [ - "bytes", + "bytes 1.7.2", "fnv", "itoa 1.0.11", ] @@ -3275,7 +3435,7 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "21b9ddb458710bc376481b842f5da65cdf31522de232c1ca8146abce2a358258" dependencies = [ - "bytes", + "bytes 1.7.2", "fnv", "itoa 1.0.11", ] @@ -3286,7 +3446,7 @@ version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7ceab25649e9960c0311ea418d17bee82c0dcec1bd053b5f9a66e265a693bed2" dependencies = [ - "bytes", + "bytes 1.7.2", "http 0.2.12", "pin-project-lite", ] @@ -3297,7 +3457,7 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" dependencies = [ - "bytes", + "bytes 1.7.2", "http 1.1.0", ] @@ -3307,7 +3467,7 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "793429d76616a256bcb62c2a2ec2bed781c8307e797e2598c50010f2bee2544f" dependencies = [ - "bytes", + "bytes 1.7.2", "futures-util", "http 1.1.0", "http-body 1.0.1", @@ -3365,7 +3525,7 @@ version = "0.14.31" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8c08302e8fa335b151b788c775ff56e7a03ae64ff85c548ee820fecb70356e85" dependencies = [ - "bytes", + "bytes 1.7.2", "futures-channel", "futures-core", "futures-util", @@ -3377,7 +3537,7 @@ dependencies = [ "itoa 1.0.11", "pin-project-lite", "socket2", - "tokio", + "tokio 1.40.0", "tower-service", "tracing", "want", @@ -3389,7 +3549,7 @@ version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "50dfd22e0e76d0f662d429a5f80fcaf3855009297eab6a0a9f8543834744ba05" dependencies = [ - "bytes", + "bytes 1.7.2", "futures-channel", "futures-util", "h2 0.4.6", @@ -3398,8 +3558,8 @@ dependencies = [ "httparse", "itoa 1.0.11", "pin-project-lite", - "smallvec", - "tokio", + "smallvec 1.13.2", + "tokio 1.40.0", "want", ] @@ -3415,7 +3575,7 @@ dependencies = [ "log", "rustls 0.21.12", "rustls-native-certs", - "tokio", + "tokio 1.40.0", "tokio-rustls 0.24.1", ] @@ -3431,7 +3591,7 @@ dependencies = [ "hyper-util", "rustls 0.23.13", "rustls-pki-types", - "tokio", + "tokio 1.40.0", "tokio-rustls 0.26.0", "tower-service", "webpki-roots", @@ -3443,10 +3603,10 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" dependencies = [ - "bytes", + "bytes 1.7.2", "hyper 0.14.31", "native-tls", - "tokio", + "tokio 1.40.0", "tokio-native-tls", ] @@ -3456,12 +3616,12 @@ version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0" dependencies = [ - "bytes", + "bytes 1.7.2", "http-body-util", "hyper 1.4.1", "hyper-util", "native-tls", - "tokio", + "tokio 1.40.0", "tokio-native-tls", "tower-service", ] @@ -3472,7 +3632,7 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "41296eb09f183ac68eec06e03cdbea2e759633d4067b2f6552fc2e009bcad08b" dependencies = [ - "bytes", + "bytes 1.7.2", "futures-channel", "futures-util", "http 1.1.0", @@ -3480,7 +3640,7 @@ dependencies = [ "hyper 1.4.1", "pin-project-lite", "socket2", - "tokio", + "tokio 1.40.0", "tower-service", "tracing", ] @@ -3574,7 +3734,7 @@ dependencies = [ "icu_normalizer_data", "icu_properties", "icu_provider", - "smallvec", + "smallvec 1.13.2", "utf16_iter", "utf8_iter", "write16", @@ -3670,7 +3830,7 @@ checksum = "bd69211b9b519e98303c015e21a007e293db403b6c85b9b124e133d25e242cdd" dependencies = [ "icu_normalizer", "icu_properties", - "smallvec", + "smallvec 1.13.2", "utf8_iter", ] @@ -3800,7 +3960,16 @@ version = "0.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e0242819d153cba4b4b05a5a8f2a7e9bbf97b6055b2a002b395c96b5ff3c0222" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", +] + +[[package]] +name = "iovec" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2b3ea6ff95e175473f8ffe6a7eb7c00d054240321b84c57051175fe3c1e075e" +dependencies = [ + "libc", ] [[package]] @@ -3856,7 +4025,7 @@ checksum = "334e04b4d781f436dc315cb1e7515bd96826426345d498149e4bde36b67f8ee9" dependencies = [ "async-channel 1.9.0", "castaway", - "crossbeam-utils", + "crossbeam-utils 0.8.20", "curl", "curl-sys", "encoding_rs", @@ -3911,6 +4080,15 @@ dependencies = [ "either", ] +[[package]] +name = "itertools" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "0.4.8" @@ -3973,7 +4151,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1a87aa2bb7d2af34197c04845522473242e1aa17c12f4935d5856491a7fb8c97" dependencies = [ "cesu8", - "cfg-if", + "cfg-if 1.0.0", "combine", "jni-sys", "log", @@ -4050,6 +4228,16 @@ dependencies = [ "serde_json", ] +[[package]] +name = "kernel32-sys" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7507624b29483431c0ba2d82aece8ca6cdba9382bff4ddd0f7490560c056098d" +dependencies = [ + "winapi 0.2.8", + "winapi-build", +] + [[package]] name = "keyboard-types" version = "0.7.0" @@ -4111,7 +4299,7 @@ dependencies = [ "async-trait", "base64 0.21.7", "bitflags 2.6.0", - "bytes", + "bytes 1.7.2", "censor", "chrono", "clickhouse", @@ -4122,7 +4310,7 @@ dependencies = [ "dotenvy", "env_logger", "flate2", - "futures", + "futures 0.3.30", "futures-timer", "futures-util", "governor", @@ -4145,7 +4333,7 @@ dependencies = [ "redis", "regex", "reqwest 0.11.27", - "rust-s3", + "rust-s3 0.33.0", "rust_decimal", "rust_iso3166", "rusty-money", @@ -4160,7 +4348,7 @@ dependencies = [ "sqlx", "tar", "thiserror", - "tokio", + "tokio 1.40.0", "tokio-stream", "totp-rs", "url", @@ -4218,7 +4406,7 @@ dependencies = [ "percent-encoding", "quoted_printable", "socket2", - "tokio", + "tokio 1.40.0", "url", ] @@ -4268,8 +4456,8 @@ version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b67380fd3b2fbe7527a606e18729d21c6f3951633d0500574c4dc22d2d638b9f" dependencies = [ - "cfg-if", - "winapi", + "cfg-if 1.0.0", + "winapi 0.3.9", ] [[package]] @@ -4296,7 +4484,7 @@ checksum = "c0ff37bd590ca25063e35af745c343cb7a0271906fb7b37e4813e8f79f00268d" dependencies = [ "bitflags 2.6.0", "libc", - "redox_syscall", + "redox_syscall 0.5.6", ] [[package]] @@ -4367,6 +4555,15 @@ version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4d873d7c67ce09b42110d801813efbc9364414e356be9935700d368351657487" +[[package]] +name = "lock_api" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4da24a77a3d8a6d4862d95f72e6fdb9c09a643ecdb402d754004a557f2bec75" +dependencies = [ + "scopeguard", +] + [[package]] name = "lock_api" version = "0.4.12" @@ -4411,6 +4608,17 @@ dependencies = [ "libc", ] +[[package]] +name = "lzma-sys" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fda04ab3764e6cde78b9974eec4f779acaba7c4e84b36eca3cf77c581b85d27" +dependencies = [ + "cc", + "libc", + "pkg-config", +] + [[package]] name = "mac" version = "0.1.1" @@ -4478,13 +4686,19 @@ dependencies = [ "syn 2.0.79", ] +[[package]] +name = "maybe-uninit" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60302e4db3a61da70c0cb7991976248362f30319e88850c487b9b95bbf059e00" + [[package]] name = "md-5" version = "0.10.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "digest 0.10.7", ] @@ -4514,7 +4728,7 @@ checksum = "2257ea8ed24b079c21570f473e58cccc3de23b46cee331fc513fccdc3f1ae5a1" dependencies = [ "async-trait", "either", - "futures", + "futures 0.3.30", "futures-io", "isahc", "iso8601", @@ -4539,6 +4753,15 @@ version = "2.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" +[[package]] +name = "memoffset" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "043175f069eda7b85febe4a74abbaeff828d9f8b448515d3151a14a3542811aa" +dependencies = [ + "autocfg", +] + [[package]] name = "memoffset" version = "0.9.1" @@ -4604,6 +4827,25 @@ dependencies = [ "simd-adler32", ] +[[package]] +name = "mio" +version = "0.6.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4afd66f5b91bf2a3bc13fad0e21caedac168ca4c707504e75585648ae80e4cc4" +dependencies = [ + "cfg-if 0.1.10", + "fuchsia-zircon", + "fuchsia-zircon-sys", + "iovec", + "kernel32-sys", + "libc", + "log", + "miow", + "net2", + "slab", + "winapi 0.2.8", +] + [[package]] name = "mio" version = "0.8.11" @@ -4629,6 +4871,29 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "mio-uds" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "afcb699eb26d4332647cc848492bbc15eafb26f08d0304550d5aa1f612e066f0" +dependencies = [ + "iovec", + "libc", + "mio 0.6.23", +] + +[[package]] +name = "miow" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebd808424166322d4a38da87083bfddd3ac4c131334ed55856112eb06d46944d" +dependencies = [ + "kernel32-sys", + "net2", + "winapi 0.2.8", + "ws2_32-sys", +] + [[package]] name = "muda" version = "0.15.1" @@ -4675,7 +4940,7 @@ dependencies = [ "versions", "wfd", "which", - "winapi", + "winapi 0.3.9", ] [[package]] @@ -4725,6 +4990,17 @@ dependencies = [ "jni-sys", ] +[[package]] +name = "net2" +version = "0.2.39" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b13b648036a2339d06de780866fbdfda0dde886de7b3af2ddeba8b14f4ee34ac" +dependencies = [ + "cfg-if 0.1.10", + "libc", + "winapi 0.3.9", +] + [[package]] name = "new_debug_unreachable" version = "1.0.6" @@ -4738,10 +5014,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "71e2746dc3a24dd78b3cfcb7be93368c6de9963d30f43a6a73998a9cf4b17b46" dependencies = [ "bitflags 2.6.0", - "cfg-if", + "cfg-if 1.0.0", "cfg_aliases", "libc", - "memoffset", + "memoffset 0.9.1", ] [[package]] @@ -4816,7 +5092,7 @@ version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e8a3895c6391c39d7fe7ebc444a87eb2991b2a0bc718fdabd071eec617fc68e4" dependencies = [ - "winapi", + "winapi 0.3.9", ] [[package]] @@ -4826,7 +5102,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "77a8165726e8236064dbb45459242600304b42a5ea24ee2948e18e023bf7ba84" dependencies = [ "overload", - "winapi", + "winapi 0.3.9", ] [[package]] @@ -4852,7 +5128,7 @@ dependencies = [ "num-iter", "num-traits", "rand 0.8.5", - "smallvec", + "smallvec 1.13.2", "zeroize", ] @@ -5128,7 +5404,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9529f4786b70a3e8c61e11179af17ab6188ad8d0ded78c5529441ed39d4bd9c1" dependencies = [ "bitflags 2.6.0", - "cfg-if", + "cfg-if 1.0.0", "foreign-types 0.3.2", "libc", "once_cell", @@ -5191,6 +5467,16 @@ dependencies = [ "hashbrown 0.12.3", ] +[[package]] +name = "ordered-multimap" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ed8acf08e98e744e5384c8bc63ceb0364e68a6854187221c18df61c4797690e" +dependencies = [ + "dlv-list 0.5.2", + "hashbrown 0.13.2", +] + [[package]] name = "ordered-multimap" version = "0.7.3" @@ -5281,14 +5567,40 @@ version = "2.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f38d5652c16fde515bb1ecef450ab0f6a219d619a7274976324d5e377f7dceba" +[[package]] +name = "parking_lot" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f842b1982eb6c2fe34036a4fbfb06dd185a3f5c8edfaacdf7d1ea10b07de6252" +dependencies = [ + "lock_api 0.3.4", + "parking_lot_core 0.6.3", + "rustc_version 0.2.3", +] + [[package]] name = "parking_lot" version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f1bf18183cf54e8d6059647fc3063646a1801cf30896933ec2311622cc4b9a27" dependencies = [ - "lock_api", - "parking_lot_core", + "lock_api 0.4.12", + "parking_lot_core 0.9.10", +] + +[[package]] +name = "parking_lot_core" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bda66b810a62be75176a80873726630147a5ca780cd33921e0b5709033e66b0a" +dependencies = [ + "cfg-if 0.1.10", + "cloudabi", + "libc", + "redox_syscall 0.1.57", + "rustc_version 0.2.3", + "smallvec 0.6.14", + "winapi 0.3.9", ] [[package]] @@ -5297,10 +5609,10 @@ version = "0.9.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e401f977ab385c9e4e3ab30627d6f26d00e2c73eef317493c4ec6d468726cf8" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "libc", - "redox_syscall", - "smallvec", + "redox_syscall 0.5.6", + "smallvec 1.13.2", "windows-targets 0.52.6", ] @@ -5630,7 +5942,7 @@ checksum = "4b2d323e8ca7996b3e23126511a523f7e62924d93ecd5ae73b333815b0eb3dce" dependencies = [ "autocfg", "bitflags 1.3.2", - "cfg-if", + "cfg-if 1.0.0", "concurrent-queue", "libc", "log", @@ -5644,7 +5956,7 @@ version = "3.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cc2790cd301dec6cd3b7a025e4815cf825724a51c98dccfe6a3e55f05ffb6511" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "concurrent-queue", "hermit-abi 0.4.0", "pin-project-lite", @@ -5799,12 +6111,12 @@ version = "0.13.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3d33c28a30771f7f96db69893f78b857f7450d7e0237e9c8fc6427a81bae7ed1" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "fnv", "lazy_static", "libc", "memchr", - "parking_lot", + "parking_lot 0.12.3", "procfs", "thiserror", ] @@ -5853,13 +6165,13 @@ version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8e5167a477619228a0b284fac2674e3c388cba90631d7b7de620e6f1fcd08da5" dependencies = [ - "crossbeam-utils", + "crossbeam-utils 0.8.20", "libc", "once_cell", "raw-cpuid", "wasi 0.11.0+wasi-snapshot-preview1", "web-sys", - "winapi", + "winapi 0.3.9", ] [[package]] @@ -5878,6 +6190,16 @@ dependencies = [ "serde", ] +[[package]] +name = "quick-xml" +version = "0.30.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eff6510e86862b57b210fd8cbe8ed3f0d7d600b9c2863cd4549a2e033c66e956" +dependencies = [ + "memchr", + "serde", +] + [[package]] name = "quick-xml" version = "0.31.0" @@ -5902,7 +6224,7 @@ version = "0.11.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8c7c5fdde3cdae7203427dc4f0a68fe0ed09833edc525a03456b153b79828684" dependencies = [ - "bytes", + "bytes 1.7.2", "pin-project-lite", "quinn-proto", "quinn-udp", @@ -5910,7 +6232,7 @@ dependencies = [ "rustls 0.23.13", "socket2", "thiserror", - "tokio", + "tokio 1.40.0", "tracing", ] @@ -5920,7 +6242,7 @@ version = "0.11.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fadfaed2cd7f389d0161bb73eeb07b7b78f8691047a6f3e73caaeae55310a4a6" dependencies = [ - "bytes", + "bytes 1.7.2", "rand 0.8.5", "ring 0.17.8", "rustc-hash", @@ -5966,7 +6288,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "51de85fb3fb6524929c8a2eb85e6b6d363de4e8c48f9e2c2eac4944abc181c93" dependencies = [ "log", - "parking_lot", + "parking_lot 0.12.3", "scheduled-thread-pool", ] @@ -6094,8 +6416,8 @@ version = "1.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1465873a3dfdaa8ae7cb14b4383657caab0b3e8a0aa9ae8e04b044854c8dfce2" dependencies = [ - "crossbeam-deque", - "crossbeam-utils", + "crossbeam-deque 0.8.5", + "crossbeam-utils 0.8.20", ] [[package]] @@ -6107,7 +6429,7 @@ dependencies = [ "ahash 0.8.11", "arc-swap", "async-trait", - "bytes", + "bytes 1.7.2", "combine", "futures-util", "itoa 1.0.11", @@ -6118,11 +6440,17 @@ dependencies = [ "ryu", "sha1_smol", "socket2", - "tokio", + "tokio 1.40.0", "tokio-util", "url", ] +[[package]] +name = "redox_syscall" +version = "0.1.57" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41cc0f7e4d5d4544e8861606a285bb08d3e70712ccc7d2b84d7c0ccfaf4b05ce" + [[package]] name = "redox_syscall" version = "0.5.6" @@ -6221,7 +6549,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dd67538700a17451e7cba03ac727fb961abb7607553461627b97de0b89cf4a62" dependencies = [ "base64 0.21.7", - "bytes", + "bytes 1.7.2", "encoding_rs", "futures-core", "futures-util", @@ -6245,7 +6573,7 @@ dependencies = [ "serde_urlencoded", "sync_wrapper 0.1.2", "system-configuration 0.5.1", - "tokio", + "tokio 1.40.0", "tokio-native-tls", "tokio-util", "tower-service", @@ -6265,7 +6593,7 @@ checksum = "f8f4955649ef5c38cc7f9e8aa41761d48fb9677197daea9984dc54f56aad5e63" dependencies = [ "async-compression", "base64 0.22.1", - "bytes", + "bytes 1.7.2", "encoding_rs", "futures-channel", "futures-core", @@ -6295,7 +6623,7 @@ dependencies = [ "serde_urlencoded", "sync_wrapper 1.0.1", "system-configuration 0.6.1", - "tokio", + "tokio 1.40.0", "tokio-native-tls", "tokio-rustls 0.26.0", "tokio-util", @@ -6370,7 +6698,7 @@ dependencies = [ "spin 0.5.2", "untrusted 0.7.1", "web-sys", - "winapi", + "winapi 0.3.9", ] [[package]] @@ -6380,7 +6708,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c17fa4cb658e3583423e915b9f3acc01cceaee1860e33d59ebae66adc3a2dc0d" dependencies = [ "cc", - "cfg-if", + "cfg-if 1.0.0", "getrandom 0.2.15", "libc", "spin 0.9.8", @@ -6396,7 +6724,7 @@ checksum = "9008cd6385b9e161d8229e1f6549dd23c3d022f132a2ea37ac3a10ac4935779b" dependencies = [ "bitvec", "bytecheck", - "bytes", + "bytes 1.7.2", "hashbrown 0.12.3", "ptr_meta", "rend", @@ -6443,17 +6771,27 @@ version = "0.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f6d5f2436026b4f6e79dc829837d467cc7e9a55ee40e750d716713540715a2df" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "ordered-multimap 0.4.3", ] +[[package]] +name = "rust-ini" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e2a3bcec1f113553ef1c88aae6c020a369d03d55b58de9869a0908930385091" +dependencies = [ + "cfg-if 1.0.0", + "ordered-multimap 0.6.0", +] + [[package]] name = "rust-ini" version = "0.21.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4e310ef0e1b6eeb79169a1171daf9abcb87a2e17c03bee2c4bb100b55c75409f" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "ordered-multimap 0.7.3", "trim-in-place", ] @@ -6465,12 +6803,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1b2ac5ff6acfbe74226fa701b5ef793aaa054055c13ebb7060ad36942956e027" dependencies = [ "async-trait", - "aws-creds", + "aws-creds 0.34.1", "aws-region", "base64 0.13.1", - "bytes", - "cfg-if", - "futures", + "bytes 1.7.2", + "cfg-if 1.0.0", + "futures 0.3.30", "hex", "hmac 0.12.1", "http 0.2.12", @@ -6486,7 +6824,44 @@ dependencies = [ "sha2 0.10.8", "thiserror", "time", - "tokio", + "tokio 1.40.0", + "tokio-stream", + "url", +] + +[[package]] +name = "rust-s3" +version = "0.34.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6679da8efaf4c6f0c161de0961dfe95fb6e9049c398d6fbdada2639f053aedb" +dependencies = [ + "async-trait", + "aws-creds 0.36.0", + "aws-region", + "base64 0.21.7", + "bytes 1.7.2", + "cfg-if 1.0.0", + "futures 0.3.30", + "hex", + "hmac 0.12.1", + "http 0.2.12", + "hyper 0.14.31", + "hyper-tls 0.5.0", + "log", + "maybe-async", + "md5", + "minidom", + "native-tls", + "percent-encoding", + "quick-xml 0.30.0", + "serde", + "serde_derive", + "serde_json", + "sha2 0.10.8", + "thiserror", + "time", + "tokio 1.40.0", + "tokio-native-tls", "tokio-stream", "url", ] @@ -6499,7 +6874,7 @@ checksum = "b082d80e3e3cc52b2ed634388d436fe1f4de6af5786cc2de9ba9737527bdf555" dependencies = [ "arrayvec", "borsh", - "bytes", + "bytes 1.7.2", "num-traits", "rand 0.8.5", "rkyv", @@ -6541,13 +6916,22 @@ version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "583034fd73374156e66797ed8e5b0d5690409c9226b22d87cb7f19821c05d152" +[[package]] +name = "rustc_version" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "138e3e0acb6c9fb258b19b67cb8abd63c00679d2851805ea151465464fe9030a" +dependencies = [ + "semver 0.9.0", +] + [[package]] name = "rustc_version" version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" dependencies = [ - "semver", + "semver 1.0.23", ] [[package]] @@ -6669,7 +7053,7 @@ version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a98f186c7a2f3abbffb802984b7f1dfd65dac8be1aafdaabbca4137f53f0dff7" dependencies = [ - "bytes", + "bytes 1.7.2", "rxml_validation", "smartstring", ] @@ -6710,7 +7094,7 @@ version = "0.2.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3cbc66816425a074528352f5789333ecff06ca41b36b0b0efdfbb29edc391a19" dependencies = [ - "parking_lot", + "parking_lot 0.12.3", ] [[package]] @@ -6827,10 +7211,19 @@ dependencies = [ "phf_codegen 0.8.0", "precomputed-hash", "servo_arc", - "smallvec", + "smallvec 1.13.2", "thin-slice", ] +[[package]] +name = "semver" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d7eb9ef2c18661902cc47e535f9bc51b78acd254da71d375c2f6720d9a40403" +dependencies = [ + "semver-parser", +] + [[package]] name = "semver" version = "1.0.23" @@ -6840,6 +7233,12 @@ dependencies = [ "serde", ] +[[package]] +name = "semver-parser" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" + [[package]] name = "sentry" version = "0.32.3" @@ -6855,7 +7254,7 @@ dependencies = [ "sentry-debug-images", "sentry-panic", "sentry-tracing", - "tokio", + "tokio 1.40.0", "ureq", ] @@ -6891,7 +7290,7 @@ dependencies = [ "hostname", "libc", "os_info", - "rustc_version", + "rustc_version 0.4.1", "sentry-core", "uname", ] @@ -6979,6 +7378,18 @@ dependencies = [ "typeid", ] +[[package]] +name = "serde-xml-rs" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb3aa78ecda1ebc9ec9847d5d3aba7d618823446a049ba2491940506da6e2782" +dependencies = [ + "log", + "serde", + "thiserror", + "xml-rs", +] + [[package]] name = "serde_derive" version = "1.0.210" @@ -7185,7 +7596,7 @@ version = "0.10.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "cpufeatures", "digest 0.10.7", ] @@ -7203,7 +7614,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4d58a1e1bf39749807d89cf2d98ac2dfa0ff1cb3faa38fbb64dd88ac8013d800" dependencies = [ "block-buffer 0.9.0", - "cfg-if", + "cfg-if 1.0.0", "cpufeatures", "digest 0.9.0", "opaque-debug", @@ -7215,7 +7626,7 @@ version = "0.10.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "793db75ad2bcafc3ffa7c68b215fee268f537982cd901d132f89c6343f3a3dc8" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "cpufeatures", "digest 0.10.7", ] @@ -7302,6 +7713,15 @@ dependencies = [ "futures-io", ] +[[package]] +name = "smallvec" +version = "0.6.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b97fcaeba89edba30f044a10c6a3cc39df9c3f17d7cd829dd1446cab35f890e0" +dependencies = [ + "maybe-uninit", +] + [[package]] name = "smallvec" version = "1.13.2" @@ -7368,7 +7788,7 @@ dependencies = [ "objc2-foundation", "objc2-quartz-core", "raw-window-handle 0.6.2", - "redox_syscall", + "redox_syscall 0.5.6", "wasm-bindgen", "web-sys", "windows-sys 0.59.0", @@ -7406,7 +7826,7 @@ version = "0.10.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "47317bbaf63785b53861e1ae2d11b80d6b624211d42cb20efcd210ee6f8a14bc" dependencies = [ - "smallvec", + "smallvec 1.13.2", ] [[package]] @@ -7421,7 +7841,7 @@ version = "0.9.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" dependencies = [ - "lock_api", + "lock_api 0.4.12", ] [[package]] @@ -7430,7 +7850,7 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d96d2d1d716fb500937168cc09353ffdc7a012be8475ac7308e1bdf0e3923300" dependencies = [ - "lock_api", + "lock_api 0.4.12", ] [[package]] @@ -7474,10 +7894,10 @@ checksum = "d4d8060b456358185f7d50c55d9b5066ad956956fddec42ee2e8567134a8936e" dependencies = [ "atoi", "byteorder", - "bytes", + "bytes 1.7.2", "chrono", "crc", - "crossbeam-queue", + "crossbeam-queue 0.3.11", "either", "event-listener 5.3.1", "futures-channel", @@ -7500,10 +7920,10 @@ dependencies = [ "serde", "serde_json", "sha2 0.10.8", - "smallvec", + "smallvec 1.13.2", "sqlformat", "thiserror", - "tokio", + "tokio 1.40.0", "tokio-stream", "tracing", "url", @@ -7545,7 +7965,7 @@ dependencies = [ "sqlx-sqlite", "syn 2.0.79", "tempfile", - "tokio", + "tokio 1.40.0", "url", ] @@ -7559,7 +7979,7 @@ dependencies = [ "base64 0.22.1", "bitflags 2.6.0", "byteorder", - "bytes", + "bytes 1.7.2", "chrono", "crc", "digest 0.10.7", @@ -7585,7 +8005,7 @@ dependencies = [ "serde", "sha1 0.10.6", "sha2 0.10.8", - "smallvec", + "smallvec 1.13.2", "sqlx-core", "stringprep", "thiserror", @@ -7625,7 +8045,7 @@ dependencies = [ "serde", "serde_json", "sha2 0.10.8", - "smallvec", + "smallvec 1.13.2", "sqlx-core", "stringprep", "thiserror", @@ -7670,7 +8090,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "799c883d55abdb5e98af1a7b3f23b9b6de8ecada0ecac058672d7635eb48ca7b" dependencies = [ "cc", - "cfg-if", + "cfg-if 1.0.0", "libc", "psm", "windows-sys 0.59.0", @@ -7696,7 +8116,7 @@ checksum = "f91138e76242f575eb1d3b38b4f1362f10d3a43f47d182a5b359af488a02293b" dependencies = [ "new_debug_unreachable", "once_cell", - "parking_lot", + "parking_lot 0.12.3", "phf_shared 0.10.0", "precomputed-hash", "serde", @@ -7861,7 +8281,7 @@ version = "0.30.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0a5b4ddaee55fb2bea2bf0e5000747e5f5c0de765e5a5ff87f4cd106439f4bb3" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "core-foundation-sys", "libc", "ntapi", @@ -7952,7 +8372,7 @@ dependencies = [ "ndk-sys", "objc", "once_cell", - "parking_lot", + "parking_lot 0.12.3", "raw-window-handle 0.6.2", "scopeguard", "tao-macros", @@ -8005,7 +8425,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "246bd333561c5601241b7a09f19957d5f659667f3c1191c869a066fb309e1841" dependencies = [ "anyhow", - "bytes", + "bytes 1.7.2", "dirs 5.0.1", "dunce", "embed_plist", @@ -8039,7 +8459,7 @@ dependencies = [ "tauri-runtime-wry", "tauri-utils", "thiserror", - "tokio", + "tokio 1.40.0", "tray-icon", "url", "urlpattern", @@ -8063,7 +8483,7 @@ dependencies = [ "json-patch", "quote", "schemars", - "semver", + "semver 1.0.23", "serde", "serde_json", "tauri-codegen", @@ -8087,7 +8507,7 @@ dependencies = [ "png", "proc-macro2", "quote", - "semver", + "semver 1.0.23", "serde", "serde_json", "sha2 0.10.8", @@ -8226,7 +8646,7 @@ dependencies = [ "tauri", "tauri-plugin", "thiserror", - "tokio", + "tokio 1.40.0", ] [[package]] @@ -8258,7 +8678,7 @@ dependencies = [ "infer 0.16.0", "minisign-verify", "reqwest 0.12.7", - "semver", + "semver 1.0.23", "serde", "serde_json", "tar", @@ -8267,7 +8687,7 @@ dependencies = [ "tempfile", "thiserror", "time", - "tokio", + "tokio 1.40.0", "url", "windows-sys 0.59.0", "zip 2.2.0", @@ -8355,7 +8775,7 @@ dependencies = [ "quote", "regex", "schemars", - "semver", + "semver 1.0.23", "serde", "serde-untagged", "serde_json", @@ -8385,7 +8805,7 @@ version = "3.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0f2c9fc62d0beef6951ccffd757e241266a2c833136efbe35af6cd2567dca5b" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "fastrand 2.1.1", "once_cell", "rustix", @@ -8411,7 +8831,7 @@ checksum = "c59df8ac95d96ff9bede18eb7300b0fda5e5d8d90960e76f8e14ae765eedbf1f" dependencies = [ "dirs-next", "rustversion", - "winapi", + "winapi 0.3.9", ] [[package]] @@ -8432,7 +8852,7 @@ dependencies = [ "async_zip", "base64 0.22.1", "byteorder", - "bytes", + "bytes 1.7.2", "chrono", "daedalus", "dashmap 6.1.0", @@ -8440,7 +8860,7 @@ dependencies = [ "discord-rich-presence", "dunce", "flate2", - "futures", + "futures 0.3.30", "indicatif", "lazy_static", "notify", @@ -8461,7 +8881,7 @@ dependencies = [ "tauri", "tempfile", "thiserror", - "tokio", + "tokio 1.40.0", "toml 0.8.19", "tracing", "tracing-error", @@ -8483,7 +8903,7 @@ dependencies = [ "daedalus", "dashmap 6.1.0", "dirs 5.0.1", - "futures", + "futures 0.3.30", "lazy_static", "native-dialog", "objc", @@ -8505,7 +8925,7 @@ dependencies = [ "tauri-plugin-window-state", "theseus", "thiserror", - "tokio", + "tokio 1.40.0", "tracing", "tracing-error", "url", @@ -8518,12 +8938,12 @@ name = "theseus_playground" version = "0.0.0" dependencies = [ "dunce", - "futures", + "futures 0.3.30", "serde", "serde_json", "theseus", "thiserror", - "tokio", + "tokio 1.40.0", "tracing", "tracing-error", "tracing-subscriber", @@ -8564,7 +8984,7 @@ version = "1.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b9ef9bad013ada3808854ceac7b46812a6465ba368859a37e2100283d2d719c" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "once_cell", ] @@ -8644,6 +9064,30 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" +[[package]] +name = "tokio" +version = "0.1.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a09c0b5bb588872ab2f09afa13ee6e9dac11e10a0ec9e8e3ba39a5a5d530af6" +dependencies = [ + "bytes 0.4.12", + "futures 0.1.31", + "mio 0.6.23", + "num_cpus", + "tokio-codec", + "tokio-current-thread", + "tokio-executor", + "tokio-fs", + "tokio-io", + "tokio-reactor", + "tokio-sync", + "tokio-tcp", + "tokio-threadpool", + "tokio-timer", + "tokio-udp", + "tokio-uds", +] + [[package]] name = "tokio" version = "1.40.0" @@ -8651,10 +9095,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e2b070231665d27ad9ec9b8df639893f46727666c6767db40317fbe920a5d998" dependencies = [ "backtrace", - "bytes", + "bytes 1.7.2", "libc", "mio 1.0.2", - "parking_lot", + "parking_lot 0.12.3", "pin-project-lite", "signal-hook-registry", "socket2", @@ -8663,6 +9107,59 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "tokio-codec" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25b2998660ba0e70d18684de5d06b70b70a3a747469af9dea7618cc59e75976b" +dependencies = [ + "bytes 0.4.12", + "futures 0.1.31", + "tokio-io", +] + +[[package]] +name = "tokio-current-thread" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1de0e32a83f131e002238d7ccde18211c0a5397f60cbfffcb112868c2e0e20e" +dependencies = [ + "futures 0.1.31", + "tokio-executor", +] + +[[package]] +name = "tokio-executor" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb2d1b8f4548dbf5e1f7818512e9c406860678f29c300cdf0ebac72d1a3a1671" +dependencies = [ + "crossbeam-utils 0.7.2", + "futures 0.1.31", +] + +[[package]] +name = "tokio-fs" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "297a1206e0ca6302a0eed35b700d292b275256f596e2f3fea7729d5e629b6ff4" +dependencies = [ + "futures 0.1.31", + "tokio-io", + "tokio-threadpool", +] + +[[package]] +name = "tokio-io" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57fc868aae093479e3131e3d165c93b1c7474109d13c90ec0dda2a1bbfff0674" +dependencies = [ + "bytes 0.4.12", + "futures 0.1.31", + "log", +] + [[package]] name = "tokio-macros" version = "2.4.0" @@ -8681,7 +9178,26 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" dependencies = [ "native-tls", - "tokio", + "tokio 1.40.0", +] + +[[package]] +name = "tokio-reactor" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09bc590ec4ba8ba87652da2068d150dcada2cfa2e07faae270a5e0409aa51351" +dependencies = [ + "crossbeam-utils 0.7.2", + "futures 0.1.31", + "lazy_static", + "log", + "mio 0.6.23", + "num_cpus", + "parking_lot 0.9.0", + "slab", + "tokio-executor", + "tokio-io", + "tokio-sync", ] [[package]] @@ -8691,7 +9207,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081" dependencies = [ "rustls 0.21.12", - "tokio", + "tokio 1.40.0", ] [[package]] @@ -8702,7 +9218,7 @@ checksum = "0c7bc40d0e5a97695bb96e27995cd3a08538541b0a846f65bba7a359f36700d4" dependencies = [ "rustls 0.23.13", "rustls-pki-types", - "tokio", + "tokio 1.40.0", ] [[package]] @@ -8713,7 +9229,93 @@ checksum = "4f4e6ce100d0eb49a2734f8c0812bcd324cf357d21810932c5df6b96ef2b86f1" dependencies = [ "futures-core", "pin-project-lite", - "tokio", + "tokio 1.40.0", +] + +[[package]] +name = "tokio-sync" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edfe50152bc8164fcc456dab7891fa9bf8beaf01c5ee7e1dd43a397c3cf87dee" +dependencies = [ + "fnv", + "futures 0.1.31", +] + +[[package]] +name = "tokio-tcp" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "98df18ed66e3b72e742f185882a9e201892407957e45fbff8da17ae7a7c51f72" +dependencies = [ + "bytes 0.4.12", + "futures 0.1.31", + "iovec", + "mio 0.6.23", + "tokio-io", + "tokio-reactor", +] + +[[package]] +name = "tokio-threadpool" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df720b6581784c118f0eb4310796b12b1d242a7eb95f716a8367855325c25f89" +dependencies = [ + "crossbeam-deque 0.7.4", + "crossbeam-queue 0.2.3", + "crossbeam-utils 0.7.2", + "futures 0.1.31", + "lazy_static", + "log", + "num_cpus", + "slab", + "tokio-executor", +] + +[[package]] +name = "tokio-timer" +version = "0.2.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93044f2d313c95ff1cb7809ce9a7a05735b012288a888b62d4434fd58c94f296" +dependencies = [ + "crossbeam-utils 0.7.2", + "futures 0.1.31", + "slab", + "tokio-executor", +] + +[[package]] +name = "tokio-udp" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2a0b10e610b39c38b031a2fcab08e4b82f16ece36504988dcbd81dbba650d82" +dependencies = [ + "bytes 0.4.12", + "futures 0.1.31", + "log", + "mio 0.6.23", + "tokio-codec", + "tokio-io", + "tokio-reactor", +] + +[[package]] +name = "tokio-uds" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab57a4ac4111c8c9dbcf70779f6fc8bc35ae4b2454809febac840ad19bd7e4e0" +dependencies = [ + "bytes 0.4.12", + "futures 0.1.31", + "iovec", + "libc", + "log", + "mio 0.6.23", + "mio-uds", + "tokio-codec", + "tokio-io", + "tokio-reactor", ] [[package]] @@ -8722,12 +9324,12 @@ version = "0.7.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "61e7c3654c13bcd040d4a03abee2c75b1d14a37b423cf5a813ceae1cc903ec6a" dependencies = [ - "bytes", + "bytes 1.7.2", "futures-core", "futures-io", "futures-sink", "pin-project-lite", - "tokio", + "tokio 1.40.0", ] [[package]] @@ -8869,7 +9471,9 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "97d095ae15e245a057c8e8451bab9b3ee1e1f68e9ba2b4fbc18d0ac5237835f2" dependencies = [ + "futures 0.3.30", "pin-project", + "tokio 0.1.22", "tracing", ] @@ -8896,7 +9500,7 @@ dependencies = [ "once_cell", "regex", "sharded-slab", - "smallvec", + "smallvec 1.13.2", "thread_local", "tracing", "tracing-core", @@ -8943,7 +9547,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6e2e2ce1e47ed2994fd43b04c8f618008d4cabdd5ee34027cf14f9d918edd9c8" dependencies = [ "byteorder", - "bytes", + "bytes 1.7.2", "data-encoding", "http 1.1.0", "httparse", @@ -8974,9 +9578,9 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "89daebc3e6fd160ac4aa9fc8b3bf71e1f74fbf92367ae71fb83a037e8bf164b9" dependencies = [ - "memoffset", + "memoffset 0.9.1", "tempfile", - "winapi", + "winapi 0.3.9", ] [[package]] @@ -9334,7 +9938,7 @@ version = "0.2.93" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a82edfc16a6c469f5f44dc7b571814045d60404b55a0ee849f9bcfa2e63dd9b5" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "once_cell", "wasm-bindgen-macro", ] @@ -9360,7 +9964,7 @@ version = "0.4.43" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "61e9300f63a621e96ed275155c108eb6f843b6a26d053f122ab69724559dc8ed" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "js-sys", "wasm-bindgen", "web-sys", @@ -9547,7 +10151,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e713040b67aae5bf1a0ae3e1ebba8cc29ab2b90da9aa1bff6e09031a8a41d7a8" dependencies = [ "libc", - "winapi", + "winapi 0.3.9", ] [[package]] @@ -9568,11 +10172,17 @@ version = "1.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "372d5b87f58ec45c384ba03563b03544dc5fadc3983e434b286913f5b4a9bb6d" dependencies = [ - "redox_syscall", + "redox_syscall 0.5.6", "wasite", "web-sys", ] +[[package]] +name = "winapi" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "167dc9d6949a9b857f3451275e911c3f44255842c1f7a76f33c55103a909087a" + [[package]] name = "winapi" version = "0.3.9" @@ -9583,6 +10193,12 @@ dependencies = [ "winapi-x86_64-pc-windows-gnu", ] +[[package]] +name = "winapi-build" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d315eee3b34aca4797b2da6b13ed88266e6d612562a0c46390af8299fc699bc" + [[package]] name = "winapi-i686-pc-windows-gnu" version = "0.4.0" @@ -9971,7 +10587,7 @@ version = "0.50.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "524e57b2c537c0f9b1e69f1965311ec12182b4122e45035b1508cd24d2adadb1" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "windows-sys 0.48.0", ] @@ -9981,7 +10597,7 @@ version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a277a57398d4bfa075df44f501a17cfdf8542d224f0d36095a2adc7aee4ef0a5" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "windows-sys 0.48.0", ] @@ -10046,6 +10662,16 @@ dependencies = [ "x11-dl", ] +[[package]] +name = "ws2_32-sys" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d59cefebd0c892fa2dd6de581e937301d8552cb44489cdff035c6187cb63fa5e" +dependencies = [ + "winapi 0.2.8", + "winapi-build", +] + [[package]] name = "wyz" version = "0.5.1" @@ -10103,6 +10729,15 @@ version = "0.8.22" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "af4e2e2f7cba5a093896c1e150fbfe177d1883e7448200efb81d40b9d339ef26" +[[package]] +name = "xz2" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "388c44dc09d76f1536602ead6d325eb532f5c122f17782bd57fb47baeeb767e2" +dependencies = [ + "lzma-sys", +] + [[package]] name = "yaserde" version = "0.8.0" @@ -10190,7 +10825,7 @@ dependencies = [ "serde_repr", "sha1 0.10.6", "static_assertions", - "tokio", + "tokio 1.40.0", "tracing", "uds_windows", "windows-sys 0.52.0", @@ -10305,7 +10940,7 @@ dependencies = [ "bzip2", "constant_time_eq 0.1.5", "crc32fast", - "crossbeam-utils", + "crossbeam-utils 0.8.20", "flate2", "hmac 0.12.1", "pbkdf2", @@ -10322,7 +10957,7 @@ checksum = "dc5e4288ea4057ae23afc69a4472434a87a2495cafce6632fd1c4ec9f5cf3494" dependencies = [ "arbitrary", "crc32fast", - "crossbeam-utils", + "crossbeam-utils 0.8.20", "displaydoc", "indexmap 2.5.0", "memchr", diff --git a/Cargo.toml b/Cargo.toml index 2edb492b0..abed47af3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -4,7 +4,9 @@ members = [ './packages/app-lib', './apps/app-playground', './apps/app', - './apps/labrinth' + './apps/labrinth', + './apps/daedalus_client', + './packages/daedalus', ] # Optimize for speed and reduce size on release builds diff --git a/apps/app/Cargo.toml b/apps/app/Cargo.toml index 409038c07..d4120e602 100644 --- a/apps/app/Cargo.toml +++ b/apps/app/Cargo.toml @@ -28,7 +28,7 @@ tauri-plugin-single-instance = { version = "2.0.0-rc" } tokio = { version = "1", features = ["full"] } thiserror = "1.0" futures = "0.3" -daedalus = "0.2.3" +daedalus = { path = "../../packages/daedalus" } chrono = "0.4.26" dirs = "5.0.1" diff --git a/apps/daedalus_client/Cargo.toml b/apps/daedalus_client/Cargo.toml index de3b52c92..37acaeb10 100644 --- a/apps/daedalus_client/Cargo.toml +++ b/apps/daedalus_client/Cargo.toml @@ -7,7 +7,7 @@ edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -daedalus = { path = "../daedalus" } +daedalus = { path = "../../packages/daedalus" } tokio = { version = "1", features = ["full"] } futures = "0.3.25" dotenvy = "0.15.6" diff --git a/packages/app-lib/Cargo.toml b/packages/app-lib/Cargo.toml index a7eff7284..04100f4ac 100644 --- a/packages/app-lib/Cargo.toml +++ b/packages/app-lib/Cargo.toml @@ -22,7 +22,7 @@ urlencoding = "2.1.3" dashmap = { version = "6.0.1", features = ["serde"] } chrono = { version = "0.4.19", features = ["serde"] } -daedalus = { version = "0.2.3" } +daedalus = { path = "../../packages/daedalus" } dirs = "5.0.1" regex = "1.5" From 999dc640bc11c463a0494f8ce00598b4215d260e Mon Sep 17 00:00:00 2001 From: Jai A Date: Sat, 19 Oct 2024 15:44:37 -0700 Subject: [PATCH 78/85] fix package --- .github/workflows/daedalus-docker.yml | 6 ++++-- apps/app/package.json | 3 ++- apps/daedalus_client/package.json | 3 +++ packages/daedalus/package.json | 2 +- pnpm-lock.yaml | 17 ++++++++++++++--- 5 files changed, 24 insertions(+), 7 deletions(-) diff --git a/.github/workflows/daedalus-docker.yml b/.github/workflows/daedalus-docker.yml index f3e5a2d71..8f5e08207 100644 --- a/.github/workflows/daedalus-docker.yml +++ b/.github/workflows/daedalus-docker.yml @@ -17,6 +17,9 @@ on: jobs: docker: runs-on: ubuntu-latest + defaults: + run: + working-directory: ./apps/daedalus steps: - name: Checkout uses: actions/checkout@v2 @@ -37,8 +40,7 @@ jobs: id: docker_build uses: docker/build-push-action@v2 with: - context: . - file: ./Dockerfile + context: ./apps/daedalus push: ${{ github.event_name != 'pull_request' }} tags: ${{ steps.docker_meta.outputs.tags }} labels: ${{ steps.docker_meta.outputs.labels }} diff --git a/apps/app/package.json b/apps/app/package.json index 9ed027b2a..8a5259531 100644 --- a/apps/app/package.json +++ b/apps/app/package.json @@ -13,6 +13,7 @@ }, "dependencies": { "@modrinth/app-frontend": "workspace:*", - "@modrinth/app-lib": "workspace:*" + "@modrinth/app-lib": "workspace:*", + "@modrinth/daedalus": "workspace:*" } } \ No newline at end of file diff --git a/apps/daedalus_client/package.json b/apps/daedalus_client/package.json index 41c8b0ae3..71fb2ebb1 100644 --- a/apps/daedalus_client/package.json +++ b/apps/daedalus_client/package.json @@ -6,5 +6,8 @@ "fix": "cargo fmt && cargo clippy --fix", "dev": "cargo run", "test": "cargo test" + }, + "dependencies": { + "@modrinth/daedalus": "workspace:*" } } \ No newline at end of file diff --git a/packages/daedalus/package.json b/packages/daedalus/package.json index 41c8b0ae3..f01a24e94 100644 --- a/packages/daedalus/package.json +++ b/packages/daedalus/package.json @@ -1,5 +1,5 @@ { - "name": "@modrinth/daedalus_client", + "name": "@modrinth/daedalus", "scripts": { "build": "cargo build --release", "lint": "cargo fmt --check && cargo clippy --all-targets --all-features -- -D warnings", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 8e11b168a..a76e4287d 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -31,6 +31,9 @@ importers: '@modrinth/app-lib': specifier: workspace:* version: link:../../packages/app-lib + '@modrinth/daedalus': + specifier: workspace:* + version: link:../../packages/daedalus devDependencies: '@tauri-apps/cli': specifier: 2.0.0-rc.16 @@ -150,6 +153,12 @@ importers: apps/app-playground: {} + apps/daedalus_client: + dependencies: + '@modrinth/daedalus': + specifier: workspace:* + version: link:../../packages/daedalus + apps/docs: dependencies: '@astrojs/check': @@ -311,6 +320,8 @@ importers: specifier: ^3.4.31 version: 3.4.31(typescript@5.5.4) + packages/daedalus: {} + packages/eslint-config-custom: devDependencies: '@nuxtjs/eslint-config-typescript': @@ -10393,12 +10404,12 @@ snapshots: '@types/eslint-scope@3.7.7': dependencies: '@types/eslint': 9.6.0 - '@types/estree': 1.0.5 + '@types/estree': 1.0.6 optional: true '@types/eslint@9.6.0': dependencies: - '@types/estree': 1.0.5 + '@types/estree': 1.0.6 '@types/json-schema': 7.0.15 optional: true @@ -17169,7 +17180,7 @@ snapshots: webpack@5.92.1: dependencies: '@types/eslint-scope': 3.7.7 - '@types/estree': 1.0.5 + '@types/estree': 1.0.6 '@webassemblyjs/ast': 1.12.1 '@webassemblyjs/wasm-edit': 1.12.1 '@webassemblyjs/wasm-parser': 1.12.1 From b3fa2fa6d230c15066f48e1810d51ee85f191e88 Mon Sep 17 00:00:00 2001 From: Jai A Date: Sat, 19 Oct 2024 19:38:36 -0700 Subject: [PATCH 79/85] Fix docker build --- .github/workflows/daedalus-docker.yml | 2 +- packages/daedalus/src/modded.rs | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/daedalus-docker.yml b/.github/workflows/daedalus-docker.yml index 8f5e08207..1d50ac1d6 100644 --- a/.github/workflows/daedalus-docker.yml +++ b/.github/workflows/daedalus-docker.yml @@ -19,7 +19,7 @@ jobs: runs-on: ubuntu-latest defaults: run: - working-directory: ./apps/daedalus + working-directory: ./apps/daedalus_client steps: - name: Checkout uses: actions/checkout@v2 diff --git a/packages/daedalus/src/modded.rs b/packages/daedalus/src/modded.rs index e7043a4c0..fb945289f 100644 --- a/packages/daedalus/src/modded.rs +++ b/packages/daedalus/src/modded.rs @@ -26,6 +26,7 @@ pub struct SidedDataEntry { pub server: String, } +#[allow(deprecated)] fn deserialize_date<'de, D>(deserializer: D) -> Result, D::Error> where D: Deserializer<'de>, From 5ea0f7d4c7bed7ee529d091203006c20a19f3d90 Mon Sep 17 00:00:00 2001 From: Jai A Date: Sat, 19 Oct 2024 19:42:52 -0700 Subject: [PATCH 80/85] Fix docker build again --- .github/workflows/daedalus-docker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/daedalus-docker.yml b/.github/workflows/daedalus-docker.yml index 1d50ac1d6..1564d4e04 100644 --- a/.github/workflows/daedalus-docker.yml +++ b/.github/workflows/daedalus-docker.yml @@ -40,7 +40,7 @@ jobs: id: docker_build uses: docker/build-push-action@v2 with: - context: ./apps/daedalus + context: ./apps/daedalus_client push: ${{ github.event_name != 'pull_request' }} tags: ${{ steps.docker_meta.outputs.tags }} labels: ${{ steps.docker_meta.outputs.labels }} From 86a64ca92915c13cb9598701c778c8cfc32ee6c4 Mon Sep 17 00:00:00 2001 From: Jai A Date: Sat, 19 Oct 2024 19:46:40 -0700 Subject: [PATCH 81/85] Update readmes --- README.md | 2 +- apps/daedalus_client/README.md | 5 +---- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 9bf4cc6ae..74def556e 100644 --- a/README.md +++ b/README.md @@ -22,7 +22,7 @@ This repository contains two primary packages. For detailed development informat ## Contributing -We welcome contributions! Before submitting any contributions, please read our [contributing guidelines](https://support.modrinth.com/en/articles/8802215-contributing-to-modrinth). +We welcome contributions! Before submitting any contributions, please read our [contributing guidelines](https://docs.modrinth.com/contributing/getting-started/). If you plan to fork this repository for your own purposes, please review our [copying guidelines](COPYING.md). diff --git a/apps/daedalus_client/README.md b/apps/daedalus_client/README.md index f0d5bd82f..7dfca0b3f 100644 --- a/apps/daedalus_client/README.md +++ b/apps/daedalus_client/README.md @@ -6,7 +6,4 @@ and mod loaders for: - Ease for Launcher Devs (Metadata is served in an easy to query and use format) - Reliability (Provides a versioning system which ensures no breakage with updates) -Daedalus is currently a work in progress, but will support the original Minecraft data and reposting for all Forge and -Fabric artifacts. - -Once Daedalus is done, Modrinth will provide full documentation for how to query from it and use it for your own launcher! \ No newline at end of file +Daedalus supports the original Minecraft data and reposting for the Forge, Fabric, Quilt, and NeoForge loaders. \ No newline at end of file From d39db02a7324b065e05b4d2c686454d3f35c1c5c Mon Sep 17 00:00:00 2001 From: Jai A Date: Sat, 19 Oct 2024 20:04:21 -0700 Subject: [PATCH 82/85] Fix docker build --- .github/workflows/daedalus-docker.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/daedalus-docker.yml b/.github/workflows/daedalus-docker.yml index 1564d4e04..4fc174b72 100644 --- a/.github/workflows/daedalus-docker.yml +++ b/.github/workflows/daedalus-docker.yml @@ -17,9 +17,6 @@ on: jobs: docker: runs-on: ubuntu-latest - defaults: - run: - working-directory: ./apps/daedalus_client steps: - name: Checkout uses: actions/checkout@v2 @@ -40,7 +37,7 @@ jobs: id: docker_build uses: docker/build-push-action@v2 with: - context: ./apps/daedalus_client + file: ./apps/daedalus_client/Dockerfile push: ${{ github.event_name != 'pull_request' }} tags: ${{ steps.docker_meta.outputs.tags }} labels: ${{ steps.docker_meta.outputs.labels }} From e88ca8430ebcac367d4933db1dac020b328aac8f Mon Sep 17 00:00:00 2001 From: Jai A Date: Sat, 19 Oct 2024 20:11:18 -0700 Subject: [PATCH 83/85] rust update --- apps/daedalus_client/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/daedalus_client/Dockerfile b/apps/daedalus_client/Dockerfile index 19c88febe..75ffd717f 100644 --- a/apps/daedalus_client/Dockerfile +++ b/apps/daedalus_client/Dockerfile @@ -1,4 +1,4 @@ -FROM rust:1.79.0 as build +FROM rust:1.82.0 as build ENV PKG_CONFIG_ALLOW_CROSS=1 WORKDIR /usr/src/daedalus From 5605910ac80529ad906c1848706119ce128eb116 Mon Sep 17 00:00:00 2001 From: Jai A Date: Mon, 21 Oct 2024 12:11:14 -0700 Subject: [PATCH 84/85] Build spec package --- apps/daedalus_client/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/daedalus_client/Dockerfile b/apps/daedalus_client/Dockerfile index 75ffd717f..f47e750ea 100644 --- a/apps/daedalus_client/Dockerfile +++ b/apps/daedalus_client/Dockerfile @@ -3,7 +3,7 @@ ENV PKG_CONFIG_ALLOW_CROSS=1 WORKDIR /usr/src/daedalus COPY . . -RUN cargo build --release +RUN cargo build --release --package daedalus_client FROM debian:bookworm-slim From 70bf61a645f9f84754548b1f164d9eb31cf112e7 Mon Sep 17 00:00:00 2001 From: Jai A Date: Mon, 21 Oct 2024 13:21:56 -0700 Subject: [PATCH 85/85] Fix fmt --- apps/daedalus_client/src/forge.rs | 4 +--- packages/daedalus/src/modded.rs | 3 ++- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/apps/daedalus_client/src/forge.rs b/apps/daedalus_client/src/forge.rs index a560b81aa..b36266556 100644 --- a/apps/daedalus_client/src/forge.rs +++ b/apps/daedalus_client/src/forge.rs @@ -1,6 +1,4 @@ -use crate::util::{ - download_file, fetch_json, fetch_xml, format_url, -}; +use crate::util::{download_file, fetch_json, fetch_xml, format_url}; use crate::{insert_mirrored_artifact, Error, MirrorArtifact, UploadFile}; use chrono::{DateTime, Utc}; use daedalus::get_path_from_artifact; diff --git a/packages/daedalus/src/modded.rs b/packages/daedalus/src/modded.rs index fb945289f..4016870ff 100644 --- a/packages/daedalus/src/modded.rs +++ b/packages/daedalus/src/modded.rs @@ -155,7 +155,8 @@ pub fn merge_partial_version( downloads: merge.downloads, id: partial.id.replace(DUMMY_REPLACE_STRING, &merge_id), java_version: merge.java_version, - libraries: libraries.into_iter() + libraries: libraries + .into_iter() .chain(partial.libraries) .map(|mut x| { x.name = x.name.replace(DUMMY_REPLACE_STRING, &merge_id);