diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 7f88f3f6ce..6fcc35a587 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -1,9 +1,10 @@ // For format details, see https://aka.ms/devcontainer.json. For config options, see the -// README at: https://github.com/devcontainers/templates/tree/main/src/rust +// README at: https://github.com/devcontainers/templates/tree/main/src/cpp { "name": "qsharp", - "image": "mcr.microsoft.com/devcontainers/python:3", + "image": "mcr.microsoft.com/devcontainers/cpp", "features": { + "ghcr.io/devcontainers/features/python:1": {}, "ghcr.io/devcontainers/features/node:1": { "nodeGypDependencies": true, "version": "lts" diff --git a/.gitignore b/.gitignore index 1b35b03118..8cbb2b6615 100644 --- a/.gitignore +++ b/.gitignore @@ -15,3 +15,5 @@ __pycache__/ /fuzz/artifacts /fuzz/coverage /fuzz/Cargo.lock +.mypy_cache/ +.pytest_cache/ diff --git a/Cargo.lock b/Cargo.lock index 829b025fe2..e1d142a86a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -26,6 +26,13 @@ dependencies = [ "memchr", ] +[[package]] +name = "allocator" +version = "0.0.0" +dependencies = [ + "mimalloc-sys", +] + [[package]] name = "anes" version = "0.1.6" @@ -234,6 +241,15 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "98cc8fbded0c607b7ba9dd60cd98df59af97e84d24e49c8557331cfc26d301ce" +[[package]] +name = "cmake" +version = "0.1.50" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a31c789563b815f77f4250caee12365734369f942439b7defd71e18a48197130" +dependencies = [ + "cc", +] + [[package]] name = "colorchoice" version = "1.0.0" @@ -434,6 +450,7 @@ dependencies = [ name = "fuzz" version = "0.0.0" dependencies = [ + "allocator", "libfuzzer-sys", "qsc", ] @@ -660,6 +677,14 @@ dependencies = [ "syn", ] +[[package]] +name = "mimalloc-sys" +version = "0.0.0" +dependencies = [ + "cc", + "cmake", +] + [[package]] name = "miniz_oxide" version = "0.7.2" @@ -891,6 +916,7 @@ dependencies = [ name = "qsc" version = "0.0.0" dependencies = [ + "allocator", "clap", "criterion", "env_logger", @@ -1096,6 +1122,7 @@ dependencies = [ name = "qsharp" version = "0.0.0" dependencies = [ + "allocator", "miette", "num-bigint", "num-complex", diff --git a/Cargo.toml b/Cargo.toml index 1c51a58ac8..5e96d1009d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,5 +1,7 @@ [workspace] members = [ + "allocator", + "allocator/mimalloc-sys", "compiler/qsc", "compiler/qsc_ast", "compiler/qsc_codegen", diff --git a/README.md b/README.md index 113ce041c6..341f27a229 100644 --- a/README.md +++ b/README.md @@ -23,12 +23,13 @@ Code from this repository powers the Q# development experience on ) - Rust () - Node.js () - wasm-pack () +- cmake () and a C compiler The build script will check these dependencies and their versions and fail if not met. (Or run `python ./prereqs.py` directly to check if the minimum required versions are installed). diff --git a/allocator/Cargo.toml b/allocator/Cargo.toml new file mode 100644 index 0000000000..e75459172f --- /dev/null +++ b/allocator/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "allocator" +authors.workspace = true +homepage.workspace = true +repository.workspace = true +edition.workspace = true +license.workspace = true +version.workspace = true + +[dependencies] +mimalloc-sys = { path = "./mimalloc-sys" } + +[lints] +workspace = true + diff --git a/allocator/mimalloc-sys/CMakeLists.txt b/allocator/mimalloc-sys/CMakeLists.txt new file mode 100644 index 0000000000..0d745774a4 --- /dev/null +++ b/allocator/mimalloc-sys/CMakeLists.txt @@ -0,0 +1,20 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +cmake_minimum_required(VERSION 3.10.0) + + +project(allocator_external) +include(ExternalProject) + +ExternalProject_Add(mimalloc + GIT_REPOSITORY https://github.com/microsoft/mimalloc.git + GIT_TAG $ENV{ALLOCATOR_MIMALLOC_TAG} + GIT_SHALLOW TRUE + GIT_PROGRESS TRUE + CONFIGURE_COMMAND "" + BUILD_COMMAND "" + INSTALL_COMMAND "" + TEST_COMMAND "" + USES_TERMINAL_DOWNLOAD TRUE +) diff --git a/allocator/mimalloc-sys/Cargo.toml b/allocator/mimalloc-sys/Cargo.toml new file mode 100644 index 0000000000..cbf2546887 --- /dev/null +++ b/allocator/mimalloc-sys/Cargo.toml @@ -0,0 +1,20 @@ +[package] +name = "mimalloc-sys" +build = "build.rs" +links = "mimalloc" +authors.workspace = true +homepage.workspace = true +repository.workspace = true +edition.workspace = true +license.workspace = true +version.workspace = true + +[dependencies] + +[lints] +workspace = true + +[build-dependencies] +cmake = "0.1" +cc = "1.0" + diff --git a/allocator/mimalloc-sys/build.rs b/allocator/mimalloc-sys/build.rs new file mode 100644 index 0000000000..1cbba80827 --- /dev/null +++ b/allocator/mimalloc-sys/build.rs @@ -0,0 +1,84 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use std::boxed::Box; +use std::env; +use std::error::Error; +use std::fs; +use std::path::{Path, PathBuf}; + +use cmake::Config; + +// 1.8.2 +//static ALLOCATOR_MIMALLOC_TAG: &str = "b66e3214d8a104669c2ec05ae91ebc26a8f5ab78"; +// 2.1.2 +static ALLOCATOR_MIMALLOC_TAG: &str = "43ce4bd7fd34bcc730c1c7471c99995597415488"; + +fn main() -> Result<(), Box> { + let dst = download_mimalloc()?; + compile_mimalloc(&dst); + println!("cargo:rerun-if-changed=build.rs"); + println!("cargo:rerun-if-changed=CMakeLists.txt"); + Ok(()) +} + +// Compile mimalloc source code and link it to the crate. +// The cc crate is used to compile the source code into a static library. +// The cmake crate is used to download the source code and stage it in the build directory. +// We don't use the cmake crate to compile the source code because the mimalloc build system +// loads extra libraries, changes the name and path around, and does other things that are +// difficult to handle. The cc crate is much simpler and more predictable. +fn compile_mimalloc(dst: &Path) { + let src_dir = dst + .join("build") + .join("mimalloc-prefix") + .join("src") + .join("mimalloc"); + + let mut build = cc::Build::new(); + + build.include(src_dir.join("include")); + build.include(src_dir.join("src")); + build.file(src_dir.join("src/static.c")); + + if build.get_compiler().is_like_msvc() { + build.cpp(true); + build.static_crt(true); + } + // turn off debug mode + build.define("MI_DEBUG", "0"); + + // turning on optimizations doesn't seem to make a difference + //build.opt_level(3); + + build.compile("mimalloc"); + + println!( + "cargo:rustc-link-search=native={}", + dst.join("lib").display() + ); + println!("cargo:rustc-link-lib=static=mimalloc"); +} + +// Use cmake to download mimalloc source code and stage +// it in the build directory. +fn download_mimalloc() -> Result> { + let build_dir = get_build_dir()?; + let mut config = Config::new(build_dir); + + config + .no_build_target(true) + .env("ALLOCATOR_MIMALLOC_TAG", ALLOCATOR_MIMALLOC_TAG) + .very_verbose(true); + + let dst = config.build(); + + Ok(dst) +} + +fn get_build_dir() -> Result> { + let manifest_dir = env::var("CARGO_MANIFEST_DIR")?; + let build_dir = PathBuf::from(manifest_dir.as_str()); + let normalized_build_dir = fs::canonicalize(build_dir)?; + Ok(normalized_build_dir) +} diff --git a/allocator/mimalloc-sys/src/lib.rs b/allocator/mimalloc-sys/src/lib.rs new file mode 100644 index 0000000000..528ba7ef2e --- /dev/null +++ b/allocator/mimalloc-sys/src/lib.rs @@ -0,0 +1,48 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use core::ffi::c_void; +pub static MI_ALIGNMENT_MAX: usize = 1024 * 1024; // 1 MiB + +extern "C" { + /// Allocate size bytes aligned by alignment. + /// size: the number of bytes to allocate + /// alignment: the minimal alignment of the allocated memory. Must be less than MI_ALIGNMENT_MAX + /// returns: a pointer to the allocated memory, or null if out of memory. The returned pointer is aligned by alignment + pub fn mi_malloc_aligned(size: usize, alignment: usize) -> *mut c_void; + pub fn mi_zalloc_aligned(size: usize, alignment: usize) -> *mut c_void; + + /// Free previously allocated memory. + /// The pointer p must have been allocated before (or be nullptr). + /// p: the pointer to the memory to free or nullptr + pub fn mi_free(p: *mut c_void); + pub fn mi_realloc_aligned(p: *mut c_void, newsize: usize, alignment: usize) -> *mut c_void; +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn memory_can_be_allocated_and_freed() { + let ptr = unsafe { mi_malloc_aligned(8, 8) }.cast::(); + assert!(!ptr.cast::().is_null()); + unsafe { mi_free(ptr.cast::()) }; + } + + #[test] + fn memory_can_be_allocated_zeroed_and_freed() { + let ptr = unsafe { mi_zalloc_aligned(8, 8) }.cast::(); + assert!(!ptr.cast::().is_null()); + unsafe { mi_free(ptr.cast::()) }; + } + + #[test] + fn memory_can_be_reallocated_and_freed() { + let ptr = unsafe { mi_malloc_aligned(8, 8) }.cast::(); + assert!(!ptr.cast::().is_null()); + let realloc_ptr = unsafe { mi_realloc_aligned(ptr.cast::(), 8, 8) }.cast::(); + assert!(!realloc_ptr.cast::().is_null()); + unsafe { mi_free(ptr.cast::()) }; + } +} diff --git a/allocator/src/lib.rs b/allocator/src/lib.rs new file mode 100644 index 0000000000..8e6c5cae64 --- /dev/null +++ b/allocator/src/lib.rs @@ -0,0 +1,15 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +#[cfg(not(target_family = "wasm"))] +pub mod mimalloc; + +/// Declare a global allocator if the platform supports it. +#[macro_export] +macro_rules! assign_global { + () => { + #[cfg(not(target_family = "wasm"))] + #[global_allocator] + static GLOBAL: allocator::mimalloc::Mimalloc = allocator::mimalloc::Mimalloc; + }; +} diff --git a/allocator/src/mimalloc.rs b/allocator/src/mimalloc.rs new file mode 100644 index 0000000000..f76418cb9e --- /dev/null +++ b/allocator/src/mimalloc.rs @@ -0,0 +1,79 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use core::alloc::{GlobalAlloc, Layout}; +use core::ffi::c_void; + +use mimalloc_sys::{mi_free, mi_malloc_aligned, mi_realloc_aligned, mi_zalloc_aligned}; + +pub struct Mimalloc; + +unsafe impl GlobalAlloc for Mimalloc { + #[inline] + unsafe fn alloc(&self, layout: Layout) -> *mut u8 { + debug_assert!(layout.align() < mimalloc_sys::MI_ALIGNMENT_MAX); + mi_malloc_aligned(layout.size(), layout.align()).cast::() + } + + #[inline] + unsafe fn dealloc(&self, ptr: *mut u8, _layout: Layout) { + mi_free(ptr.cast::()); + } + + #[inline] + unsafe fn alloc_zeroed(&self, layout: Layout) -> *mut u8 { + debug_assert!(layout.align() < mimalloc_sys::MI_ALIGNMENT_MAX); + mi_zalloc_aligned(layout.size(), layout.align()).cast::() + } + + #[inline] + unsafe fn realloc(&self, ptr: *mut u8, layout: Layout, new_size: usize) -> *mut u8 { + debug_assert!(layout.align() < mimalloc_sys::MI_ALIGNMENT_MAX); + mi_realloc_aligned(ptr.cast::(), new_size, layout.align()).cast::() + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::error::Error; + + #[test] + fn memory_can_be_allocated_and_freed() -> Result<(), Box> { + let layout = Layout::from_size_align(8, 8)?; + let alloc = Mimalloc; + + unsafe { + let ptr = alloc.alloc(layout); + assert!(!ptr.cast::().is_null()); + alloc.dealloc(ptr, layout); + } + Ok(()) + } + + #[test] + fn memory_can_be_alloc_zeroed_and_freed() -> Result<(), Box> { + let layout = Layout::from_size_align(8, 8)?; + let alloc = Mimalloc; + + unsafe { + let ptr = alloc.alloc_zeroed(layout); + assert!(!ptr.cast::().is_null()); + alloc.dealloc(ptr, layout); + } + Ok(()) + } + + #[test] + fn large_chunks_of_memory_can_be_allocated_and_freed() -> Result<(), Box> { + let layout = Layout::from_size_align(2 * 1024 * 1024 * 1024, 8)?; + let alloc = Mimalloc; + + unsafe { + let ptr = alloc.alloc(layout); + assert!(!ptr.cast::().is_null()); + alloc.dealloc(ptr, layout); + } + Ok(()) + } +} diff --git a/compiler/qsc/Cargo.toml b/compiler/qsc/Cargo.toml index 35d09eb087..37698069f5 100644 --- a/compiler/qsc/Cargo.toml +++ b/compiler/qsc/Cargo.toml @@ -29,6 +29,9 @@ qsc_project = { path = "../qsc_project", features = ["fs"] } rustc-hash = { workspace = true } thiserror = { workspace = true } +[target.'cfg(not(any(target_family = "wasm")))'.dependencies] +allocator = { path = "../../allocator" } + [dev-dependencies] criterion = { workspace = true, features = ["cargo_bench_support"] } expect-test = { workspace = true } diff --git a/compiler/qsc/benches/eval.rs b/compiler/qsc/benches/eval.rs index b2698de3b2..3dbee4548d 100644 --- a/compiler/qsc/benches/eval.rs +++ b/compiler/qsc/benches/eval.rs @@ -1,6 +1,8 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. +allocator::assign_global!(); + use criterion::{criterion_group, criterion_main, Criterion}; use indoc::indoc; use qsc::{interpret::Interpreter, PackageType}; diff --git a/compiler/qsc/benches/large.rs b/compiler/qsc/benches/large.rs index 1d3ca57eb5..0102d1a5d9 100644 --- a/compiler/qsc/benches/large.rs +++ b/compiler/qsc/benches/large.rs @@ -1,6 +1,8 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. +allocator::assign_global!(); + use criterion::{criterion_group, criterion_main, Criterion}; use qsc::compile::{self, compile}; use qsc_data_structures::language_features::LanguageFeatures; diff --git a/compiler/qsc/benches/library.rs b/compiler/qsc/benches/library.rs index c60ceeb6fc..a1da2bdc98 100644 --- a/compiler/qsc/benches/library.rs +++ b/compiler/qsc/benches/library.rs @@ -1,6 +1,8 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. +allocator::assign_global!(); + use criterion::{criterion_group, criterion_main, Criterion}; use qsc::compile; use qsc_frontend::compile::{PackageStore, RuntimeCapabilityFlags}; diff --git a/compiler/qsc/src/bin/qsc.rs b/compiler/qsc/src/bin/qsc.rs index 9b47d8f039..ee7a6c50d0 100644 --- a/compiler/qsc/src/bin/qsc.rs +++ b/compiler/qsc/src/bin/qsc.rs @@ -1,6 +1,8 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. +allocator::assign_global!(); + use clap::{crate_version, ArgGroup, Parser, ValueEnum}; use log::info; use miette::{Context, IntoDiagnostic, Report}; diff --git a/compiler/qsc/src/bin/qsi.rs b/compiler/qsc/src/bin/qsi.rs index 5e6d2dbfef..6f3c8649b1 100644 --- a/compiler/qsc/src/bin/qsi.rs +++ b/compiler/qsc/src/bin/qsi.rs @@ -1,6 +1,8 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. +allocator::assign_global!(); + use clap::{crate_version, Parser}; use miette::{Context, IntoDiagnostic, Report, Result}; use num_bigint::BigUint; diff --git a/fuzz/Cargo.toml b/fuzz/Cargo.toml index 16dc9292a7..34b9ba6609 100644 --- a/fuzz/Cargo.toml +++ b/fuzz/Cargo.toml @@ -16,6 +16,9 @@ cargo-fuzz = true libfuzzer-sys = { workspace = true, optional = true } qsc = { path = "../compiler/qsc" } +[target.'cfg(not(any(target_family = "wasm")))'.dependencies] +allocator = { path = "../allocator" } + [features] do_fuzz = [ "dep:libfuzzer-sys" ] diff --git a/fuzz/fuzz_targets/compile.rs b/fuzz/fuzz_targets/compile.rs index 05c5ee7f55..feb9901fbd 100644 --- a/fuzz/fuzz_targets/compile.rs +++ b/fuzz/fuzz_targets/compile.rs @@ -3,6 +3,8 @@ #![no_main] +allocator::assign_global!(); + #[cfg(feature = "do_fuzz")] use libfuzzer_sys::fuzz_target; use qsc::{hir::PackageId, target::Profile, LanguageFeatures, PackageStore, SourceMap}; diff --git a/pip/Cargo.toml b/pip/Cargo.toml index 369a679b62..fe4467e44d 100644 --- a/pip/Cargo.toml +++ b/pip/Cargo.toml @@ -20,6 +20,9 @@ rustc-hash = { workspace = true } [lints] workspace = true +[target.'cfg(not(any(target_family = "wasm")))'.dependencies] +allocator = { path = "../allocator" } + [target.'cfg(not(any(target_os = "windows")))'.dependencies] pyo3 = { workspace = true, features = ["abi3-py37", "extension-module", "num-bigint"] } diff --git a/pip/src/lib.rs b/pip/src/lib.rs index f126254403..88fcd735df 100644 --- a/pip/src/lib.rs +++ b/pip/src/lib.rs @@ -1,6 +1,8 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. +allocator::assign_global!(); + mod displayable_output; mod fs; mod interpreter; diff --git a/prereqs.py b/prereqs.py index 548cd42ed0..772a21aa72 100755 --- a/prereqs.py +++ b/prereqs.py @@ -19,6 +19,7 @@ 17, ) wasmpack_ver = (0, 12, 1) # Latest tested wasm-pack version +cmake_ver = (3, 10) # Ensure CMake version 3.10 or later is installed rust_fmt_ver = (1, 7, 0) # Current version when Rust 1.76 shipped clippy_ver = (0, 1, 76) @@ -181,6 +182,30 @@ def check_prereqs(install=False): else: print("Unable to determine the wasm-pack version") + ### Check the cmake version ### + try: + cmake_version = subprocess.check_output(["cmake", "--version"]) + except FileNotFoundError: + print("CMake not found. Install from https://cmake.org/") + exit(1) + + version_match = re.search( + r"cmake version\s(\d+)\.(\d+).\d+", cmake_version.decode() + ) + if version_match: + cmake_major = int(version_match.group(1)) + cmake_minor = int(version_match.group(2)) + print(f"Detected CMake version: {cmake_major}.{cmake_minor}") + if cmake_major < cmake_ver[0] or ( + cmake_major == cmake_ver[0] and cmake_minor < cmake_ver[1] + ): + print( + f"CMake v{cmake_ver[0]}.{cmake_ver[1]} or later is required. Please update." + ) + exit(1) + else: + raise Exception("Unable to determine the CMake version.") + if __name__ == "__main__": if len(sys.argv) > 1 and sys.argv[1] == "--install":