From 6d32584b19551ee44eaf4c2f3c5180149e6193a0 Mon Sep 17 00:00:00 2001 From: cindytsai Date: Thu, 22 Feb 2024 12:38:39 -0600 Subject: [PATCH 01/27] Prepare to PR to yt-project/main. --- .github/workflows/cmake-build-test.yml | 6 +----- README.md | 2 +- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/.github/workflows/cmake-build-test.yml b/.github/workflows/cmake-build-test.yml index 22ea22ed..fa3e5a58 100644 --- a/.github/workflows/cmake-build-test.yml +++ b/.github/workflows/cmake-build-test.yml @@ -53,11 +53,7 @@ jobs: - name: Install yt, mpi4py, and yt_libyt run: | pip install mpi4py yt - cd ${{ github.workspace }} - cd .. - git clone --depth 1 -b libyt-dev --single-branch "https://github.com/cindytsai/yt_libyt.git" yt_libyt - cd yt_libyt - pip install . + pip install yt-libyt - name: Update GitHub Environment Variables run: | diff --git a/README.md b/README.md index c38dbd4d..d6dc68f1 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ # libyt -[![CMake Build Test](https://github.com/cindytsai/libyt/actions/workflows/cmake-build-test.yml/badge.svg?branch=main)](https://github.com/cindytsai/libyt/actions/workflows/cmake-build-test.yml) +[![CMake Build Test](https://github.com/yt-project/libyt/actions/workflows/cmake-build-test.yml/badge.svg)](https://github.com/yt-project/libyt/actions/workflows/cmake-build-test.yml) [![Documentation Status](https://readthedocs.org/projects/libyt/badge/?version=latest)](https://libyt.readthedocs.io/en/latest/?badge=latest) **_This repo is only for development purpose._** From c2a12f32a4f863cf8fe8eead97d7b32a4fdfc3ef Mon Sep 17 00:00:00 2001 From: cindytsai Date: Thu, 22 Feb 2024 12:41:33 -0600 Subject: [PATCH 02/27] Remove line in readme. --- README.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/README.md b/README.md index d6dc68f1..d57e69d9 100644 --- a/README.md +++ b/README.md @@ -2,8 +2,6 @@ [![CMake Build Test](https://github.com/yt-project/libyt/actions/workflows/cmake-build-test.yml/badge.svg)](https://github.com/yt-project/libyt/actions/workflows/cmake-build-test.yml) [![Documentation Status](https://readthedocs.org/projects/libyt/badge/?version=latest)](https://libyt.readthedocs.io/en/latest/?badge=latest) -**_This repo is only for development purpose._** - `libyt` is an open source C library for simulation, that allows researchers to analyze and visualize data using [`yt`](https://yt-project.org/) or other Python packages in parallel during simulation runtime. In this way, we can skip the step of writing data to local disk before doing analysis using Python. This greatly reduce the disk usage, and increase the temporal resolution. - **Documents**: https://libyt.readthedocs.io/ From e42c4abd47be7c851bd9947bba7a8466d6391692 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 12 Mar 2024 19:52:50 +0000 Subject: [PATCH 03/27] [pre-commit.ci] pre-commit autoupdate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/pre-commit/mirrors-clang-format: v17.0.6 → v18.1.1](https://github.com/pre-commit/mirrors-clang-format/compare/v17.0.6...v18.1.1) --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index b01a5709..cefb159f 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -12,7 +12,7 @@ ci: repos: - repo: https://github.com/pre-commit/mirrors-clang-format - rev: v17.0.6 + rev: v18.1.1 hooks: - id: clang-format types_or: ['c++', 'c'] From 63343be816a0d549153856b67def6f13fd380306 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 12 Mar 2024 19:52:56 +0000 Subject: [PATCH 04/27] [pre-commit.ci] auto fixes from pre-commit.com hooks --- src/TimerControl.cpp | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/TimerControl.cpp b/src/TimerControl.cpp index 1ce62c2e..981e9b3b 100644 --- a/src/TimerControl.cpp +++ b/src/TimerControl.cpp @@ -27,10 +27,8 @@ void TimerControl::CreateFile(const char* filename, int rank) { // Write heading and basic info if (m_MPIRank == 0) { - file_out << "{\"otherData\": {" - << "\"version\": \"" << LIBYT_MAJOR_VERSION << "." << LIBYT_MINOR_VERSION << "." << LIBYT_MICRO_VERSION - << "\"," - << "\"mode\": " + file_out << "{\"otherData\": {" << "\"version\": \"" << LIBYT_MAJOR_VERSION << "." << LIBYT_MINOR_VERSION << "." + << LIBYT_MICRO_VERSION << "\"," << "\"mode\": " #if defined(INTERACTIVE_MODE) << "\"interactive_mode\"" #elif defined(JUPYTER_KERNEL) From 9006161454307b40398e54845b4f96f1da4b01e3 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 25 Mar 2024 17:08:41 +0000 Subject: [PATCH 05/27] [pre-commit.ci] pre-commit autoupdate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/pre-commit/mirrors-clang-format: v18.1.1 → v18.1.2](https://github.com/pre-commit/mirrors-clang-format/compare/v18.1.1...v18.1.2) --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index cefb159f..b71b9a4c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -12,7 +12,7 @@ ci: repos: - repo: https://github.com/pre-commit/mirrors-clang-format - rev: v18.1.1 + rev: v18.1.2 hooks: - id: clang-format types_or: ['c++', 'c'] From df1713e4d8065d22b015c3af2517718f9d1cba4a Mon Sep 17 00:00:00 2001 From: cindytsai Date: Thu, 2 May 2024 00:49:01 -0500 Subject: [PATCH 06/27] Get yt_libyt from PyPI. --- .github/workflows/example-test-run.yml | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/.github/workflows/example-test-run.yml b/.github/workflows/example-test-run.yml index 260a5490..31973d40 100644 --- a/.github/workflows/example-test-run.yml +++ b/.github/workflows/example-test-run.yml @@ -52,12 +52,7 @@ jobs: - name: Install yt, mpi4py, and yt_libyt run: | - pip install mpi4py yt - cd ${{ github.workspace }} - cd .. - git clone --depth 1 -b libyt-dev --single-branch "https://github.com/cindytsai/yt_libyt.git" yt_libyt - cd yt_libyt - pip install . + pip install mpi4py yt yt_libyt - name: Update GitHub Environment Variables run: | From 1a4f71ca65b7dfdab7f1fe5c0d123fa6408744fa Mon Sep 17 00:00:00 2001 From: cindytsai Date: Tue, 7 May 2024 13:21:31 -0500 Subject: [PATCH 07/27] Add horizontal lines in quick start and example. --- doc/example.md | 2 ++ doc/quick-start.md | 2 ++ 2 files changed, 4 insertions(+) diff --git a/doc/example.md b/doc/example.md index a940ef16..24da9028 100644 --- a/doc/example.md +++ b/doc/example.md @@ -134,6 +134,8 @@ The example initializes `libyt`, loads data to `libyt` in every simulation time After loading simulation data using libyt API, we call Python functions defined in `inline_script.py` using [`yt_run_Function`](./libyt-api/run-python-function.md#yt_run_function)/[`yt_run_FunctionArguments`](./libyt-api/run-python-function.md#yt_run_functionarguments) and use `yt` to analyze the data and plot figures. For how to use `yt` see: - [Using `yt`](./in-situ-python-analysis/using-yt.md) +--- + ## What's Next ###### Change Python Script Name diff --git a/doc/quick-start.md b/doc/quick-start.md index 7a76e0f7..ed5e7342 100644 --- a/doc/quick-start.md +++ b/doc/quick-start.md @@ -101,6 +101,8 @@ After reading quick start, we will have an overview of how to use `libyt` for in >>> %libyt exit ``` +--- + ### What's Next ###### Loading Data using libyt API and Do Analysis using yt and Python From 7355a55bc7abff9d08af201b42ac7e0748d3e347 Mon Sep 17 00:00:00 2001 From: cindytsai Date: Tue, 7 May 2024 13:41:41 -0500 Subject: [PATCH 08/27] Try setup python v5 --- .github/workflows/cmake-build-test.yml | 2 +- .github/workflows/example-test-run.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/cmake-build-test.yml b/.github/workflows/cmake-build-test.yml index 6c1daecc..86b9562a 100644 --- a/.github/workflows/cmake-build-test.yml +++ b/.github/workflows/cmake-build-test.yml @@ -38,7 +38,7 @@ jobs: run: cmake --version - name: Setup Python ${{ matrix.python-version }} environment - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} architecture: 'x64' diff --git a/.github/workflows/example-test-run.yml b/.github/workflows/example-test-run.yml index 31973d40..f63b3ba1 100644 --- a/.github/workflows/example-test-run.yml +++ b/.github/workflows/example-test-run.yml @@ -36,7 +36,7 @@ jobs: run: cmake --version - name: Setup Python ${{ matrix.python-version }} environment - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} architecture: 'x64' From 9ac3ac3adb4f4209880ade53f0062f888b73d9cf Mon Sep 17 00:00:00 2001 From: cindytsai Date: Tue, 7 May 2024 14:00:32 -0500 Subject: [PATCH 09/27] Try macOS-latest. --- .github/workflows/cmake-build-test.yml | 2 +- .github/workflows/example-test-run.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/cmake-build-test.yml b/.github/workflows/cmake-build-test.yml index 86b9562a..9754a562 100644 --- a/.github/workflows/cmake-build-test.yml +++ b/.github/workflows/cmake-build-test.yml @@ -25,7 +25,7 @@ jobs: - os: ubuntu-latest mpi: 'openmpi' check_shared_lib: ldd lib/libyt.so - - os: macos-latest + - os: macOS-latest mpi: 'mpich' check_shared_lib: otool -l lib/libyt.dylib python-version: ['3.7', '3.8', '3.9', '3.10', '3.11', '3.12'] diff --git a/.github/workflows/example-test-run.yml b/.github/workflows/example-test-run.yml index f63b3ba1..effc4905 100644 --- a/.github/workflows/example-test-run.yml +++ b/.github/workflows/example-test-run.yml @@ -24,7 +24,7 @@ jobs: platform: - os: ubuntu-latest mpi: 'openmpi' - - os: macos-latest + - os: macOS-latest mpi: 'mpich' python-version: ['3.7', '3.8', '3.9', '3.10', '3.11', '3.12'] From ee242a94edc1b7861e2a183390e67ff5bd2d4843 Mon Sep 17 00:00:00 2001 From: cindytsai Date: Tue, 7 May 2024 14:38:57 -0500 Subject: [PATCH 10/27] Create FAQs page to collect all the FAQs. --- doc/FAQs.md | 2 ++ doc/index.md | 10 +++++++++- 2 files changed, 11 insertions(+), 1 deletion(-) create mode 100644 doc/FAQs.md diff --git a/doc/FAQs.md b/doc/FAQs.md new file mode 100644 index 00000000..e8846a7b --- /dev/null +++ b/doc/FAQs.md @@ -0,0 +1,2 @@ +# FAQs + diff --git a/doc/index.md b/doc/index.md index 669edb02..fb91bb14 100644 --- a/doc/index.md +++ b/doc/index.md @@ -10,13 +10,21 @@ It is an in situ analysis tool that allows researchers to analyze and visualize ```{toctree} :hidden: +:caption: For users quick-start example how-to-install how-it-works -libyt-api/index in-situ-python-analysis/index +FAQs +``` + +```{toctree} +:hidden: +:caption: For developers + +libyt-api/index debug-and-profiling/index ``` From fd058086b3adbb851bd3fe6f5d969a9574cdadf8 Mon Sep 17 00:00:00 2001 From: cindytsai Date: Tue, 7 May 2024 17:02:08 -0500 Subject: [PATCH 11/27] Separate how to install page to how to install and details. --- .../details.md} | 148 +++++------------- doc/how-to-install/how-to-install.md | 104 ++++++++++++ 2 files changed, 142 insertions(+), 110 deletions(-) rename doc/{how-to-install.md => how-to-install/details.md} (78%) create mode 100644 doc/how-to-install/how-to-install.md diff --git a/doc/how-to-install.md b/doc/how-to-install/details.md similarity index 78% rename from doc/how-to-install.md rename to doc/how-to-install/details.md index 6c2cdda4..8fc9e3c0 100644 --- a/doc/how-to-install.md +++ b/doc/how-to-install/details.md @@ -1,68 +1,11 @@ -# How to Install - -> {octicon}`info;1em;sd-text-info;` Go through [Install](#install) and get all [Python Dependencies](#python-dependencies) to get full feature of `libyt`. -> Ignore [Details](#details) unless we want to tweak `libyt` based on our needs. - -## libyt C Library - -### Install -- **CMake** (>=3.15) -- **pkg-config**: Generally, Linux and macOS already have pkg-config installed. -- **GCC compiler** (>4.8): It should be able to support `c++14`. - - `CXX`: Path to `g++` compiler. - - `CC`: Path to `gcc` compiler. -- **Python** (>=3.7): The Python environment we want to use when doing in situ analysis. - - `PYTHON_PATH`: Python installation prefix, the path contains folders `include`, `lib` etc. - - `NumPy`: Should have `NumPy` installed. - - Other [Python Dependencies](#python-dependencies) -- **MPI**: MPI used for compiling simulations and `libyt` needs to be the same. - - `MPI_PATH`: MPI installation prefix, the path contains folders `include`, `lib` etc. -- **Readline**: [GNU `readline` library](https://tiswww.case.edu/php/chet/readline/rltop.html) is already installed on Linux and macOS generally. If not, we can get through system package manager or compile from source ourselves. (Use `--with-curses` when configuring if we compile from source.) - - `READLINE_PATH`: `readline` installation prefix. Provide the path if it is not in system search path. - -**Follow the steps to install `libyt` that is fault-tolerant to Python code, and supports interactive Python prompt, reloading script, and Jupyter Notebook access:** - -1. Shallow clone `libyt` and enter the folder: - ```bash - git clone --depth 1 https://github.com/yt-project/libyt "libyt" - cd libyt - ``` -2. [Optional] Set `gcc` and `g++` compiler: - ```bash - export CC= - export CXX= - ``` -3. Generate build files in `build` folder: - - **Serial Mode (using GCC)**: - ```bash - rm -rf build - cmake -S . -B build -DSERIAL_MODE=ON \ - -DINTERACTIVE_MODE=ON \ - -DJUPYTER_KERNEL=ON \ - -DPYTHON_PATH= - ``` - - **Parallel Mode (using MPI)**: - ```bash - rm -rf build - cmake -S . -B build -DINTERACTIVE_MODE=ON \ - -DJUPYTER_KERNEL=ON \ - -DPYTHON_PATH= \ - -DMPI_PATH= - ``` -4. Build project and install `libyt`: - ```bash - cmake --build build - cmake --install build --prefix - ``` - -### Details +# Details This section lists all options and its related dependencies. -#### Options (=Default Value) -The options are mutually independent to each other. +## Options (=Default Value) +The options are mutually independent to each other. -###### `-DSERIAL_MODE` (=`OFF`) +### `-DSERIAL_MODE` (=`OFF`) :::{table} :width: 100% @@ -75,30 +18,30 @@ The options are mutually independent to each other. > {octicon}`alert;1em;sd-text-danger;` Make sure Python bindings for MPI (`mpi4py`), MPI used for compiling simulation and `libyt` are the same. Check how to install `mpi4py` [here](https://mpi4py.readthedocs.io/en/stable/install.html#installation). -###### `-DINTERACTIVE_MODE` (=`OFF`) +### `-DINTERACTIVE_MODE` (=`OFF`) | | Notes | Dependency | |---------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-----------------| | **Normal Mode** (OFF) | Shut down and terminate all the processes including simulation, if error occurs during in situ analysis. | | -| **Interactive Mode** (ON) | Will not terminate the processes if error occurs while doing in situ analysis. Support [Interactive Python Prompt](./in-situ-python-analysis/interactive-python-prompt.md#interactive-python-prompt) and [Reloading Script](./in-situ-python-analysis/reloading-script.md#reloading-script). | `READLINE_PATH` | +| **Interactive Mode** (ON) | Will not terminate the processes if error occurs while doing in situ analysis. Support [Interactive Python Prompt](../in-situ-python-analysis/interactive-python-prompt.md#interactive-python-prompt) and [Reloading Script](../in-situ-python-analysis/reloading-script.md#reloading-script). | `READLINE_PATH` | -###### `-DJUPYTER_KERNEL` (=`OFF`) +### `-DJUPYTER_KERNEL` (=`OFF`) | | Notes | Dependency | Python dependency | |------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|------------------------------------------------------------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| **Jupyter Kernel Mode** (ON) | Activate Jupyter kernel and enable JupyterLab UI. (See [Jupyter Notebook Access](./in-situ-python-analysis/jupyter-notebook/jupyter-notebook-access.md#jupyter-notebook-access)) | `nlohmann_json_DIR`
`cppzmq_DIR`
`xtl_DIR`
`xeus_DIR`
`xeus-zmq_DIR`
`ZeroMQ_DIR` | [`jupyter_libyt`](https://github.com/yt-project/jupyter_libyt)
[`jupyter-client`](https://jupyter-client.readthedocs.io/en/stable/index.html)
[`jedi`](https://jedi.readthedocs.io/en/latest/)(Optional) | +| **Jupyter Kernel Mode** (ON) | Activate Jupyter kernel and enable JupyterLab UI. (See [Jupyter Notebook Access](../in-situ-python-analysis/jupyter-notebook/jupyter-notebook-access.md#jupyter-notebook-access)) | `nlohmann_json_DIR`
`cppzmq_DIR`
`xtl_DIR`
`xeus_DIR`
`xeus-zmq_DIR`
`ZeroMQ_DIR` | [`jupyter_libyt`](https://github.com/yt-project/jupyter_libyt)
[`jupyter-client`](https://jupyter-client.readthedocs.io/en/stable/index.html)
[`jedi`](https://jedi.readthedocs.io/en/latest/)(Optional) | -###### `-DSUPPORT_TIMER` (=`OFF`) +### `-DSUPPORT_TIMER` (=`OFF`) :::{table} :width: 100% | | Notes | |--------------------------|--------------------------------------------------------------------------------------------------------| -| **Time Profiling** (ON) | Support time profiling. (See [Time Profiling](./debug-and-profiling/time-profiling.md#time-profiling)) | +| **Time Profiling** (ON) | Support time profiling. (See [Time Profiling](../debug-and-profiling/time-profiling.md#time-profiling)) | ::: -#### Dependencies +## Dependencies - **Dependency path** indicates the required path variable name in CMake, and the required version. - **Notes** are things worth notice. @@ -119,7 +62,32 @@ The options are mutually independent to each other. > {octicon}`info;1em;sd-text-info;` If our system doesn't have `readline` installed, use system package manager (ex: `brew`, `apt`) to install. If we want to compile and install from the source code ourselves, make sure `--with-curses` is used when configuring. -#### Step-by-Step Instructions +## Python Dependencies + +- **Python package** required when performing in situ analysis. +- **Notes** are things worth notice. +- **Option** indicates under what circumstances will we need this package. + +:::{table} +:width: 100% + +| Python package | Notes | Option | +|-----------------------------------------------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-----------------------| +| [`NumPy`](https://numpy.org/)
(<2.0) | The fundamental package for scientific computing with Python. | Always | +| [`yt`](https://yt-project.org/) | The core analytic tool. | Always | +| [`yt_libyt`](https://github.com/data-exp-lab/yt_libyt) | `yt` frontend for `libyt`. | Always | +| [`mpi4py`](https://mpi4py.readthedocs.io/en/stable/) | Python bindings for the Message Passing Interface (MPI) standard.
{octicon}`alert;1em;sd-text-danger;` Make sure `mpi4py` used in Python and MPI used in simulation are matched. (Check how to install `mpi4py` [here](https://mpi4py.readthedocs.io/en/stable/install.html#installation).) | `-DSERIAL_MODE=OFF` | +| [`jedi`](https://jedi.readthedocs.io/en/latest/) | Support auto-completion in Jupyter Notebook and JupyterLab. (We will have this if IPython is already installed.) | `-DJUPYTER_KERNEL=ON` | +| [`jupyter-client`](https://jupyter-client.readthedocs.io/en/latest/index.html)
(>=8.0.0) | Jupyter Client. | `-DJUPYTER_KERNEL=ON` | +| [`jupyter_libyt`](https://github.com/yt-project/jupyter_libyt) | Jupyter kernel provisioner for `libyt`. | `-DJUPYTER_KERNEL=ON` | +::: + +> {octicon}`alert;1em;sd-text-danger;` `jupyter-client` and `jupyter_libyt` are used for launching Jupyter Notebook and JupyterLab. Make sure the Python environment used for launching the notebook have them installed. +> +> The Python used in in situ analysis which is also for compiling `libyt` and the Python for launching Jupyter Notebook/JupyterLab might be different. +> For example, when running `libyt` in HPC cluster and connecting to it through your local laptop. (See [Jupyter Notebook Access](../in-situ-python-analysis/jupyter-notebook/jupyter-notebook-access.md)) + +## Step-by-Step Instructions 1. Toggle options, set paths and generate files to build the project. This can be done through either (a) or (b): (a) Set it through editing `CMakeLists.txt` at root directory. For example, this uses option [`-DSERIAL_MODE=OFF`](#-dserial_mode-off) and provides `MPI_PATH`: @@ -156,7 +124,7 @@ The options are mutually independent to each other. cmake --install --prefix ``` -#### Example +### Examples - The following builds `libyt` in serial mode using user designated GCC compiler and then installs the library in `/home/user/softwares/libyt`: ```bash cd libyt # go to project root directory @@ -176,43 +144,3 @@ The options are mutually independent to each other. cmake --build build # build the project cmake --install build --prefix /home/user/softwares/libyt # install ``` - -## Python Dependencies - -- **Python package** required when performing in situ analysis. -- **Notes** are things worth notice. -- **Option** indicates under what circumstances will we need this package. - -:::{table} -:width: 100% - -| Python package | Notes | Option | -|-----------------------------------------------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-----------------------| -| [`NumPy`](https://numpy.org/)
(<2.0) | The fundamental package for scientific computing with Python. | Always | -| [`yt`](https://yt-project.org/) | The core analytic tool. | Always | -| [`yt_libyt`](https://github.com/data-exp-lab/yt_libyt) | `yt` frontend for `libyt`. | Always | -| [`mpi4py`](https://mpi4py.readthedocs.io/en/stable/) | Python bindings for the Message Passing Interface (MPI) standard.
{octicon}`alert;1em;sd-text-danger;` Make sure `mpi4py` used in Python and MPI used in simulation are matched. (Check how to install `mpi4py` [here](https://mpi4py.readthedocs.io/en/stable/install.html#installation).) | `-DSERIAL_MODE=OFF` | -| [`jedi`](https://jedi.readthedocs.io/en/latest/) | Support auto-completion in Jupyter Notebook and JupyterLab. (We will have this if IPython is already installed.) | `-DJUPYTER_KERNEL=ON` | -| [`jupyter-client`](https://jupyter-client.readthedocs.io/en/latest/index.html)
(>=8.0.0) | Jupyter Client. | `-DJUPYTER_KERNEL=ON` | -| [`jupyter_libyt`](https://github.com/yt-project/jupyter_libyt) | Jupyter kernel provisioner for `libyt`. | `-DJUPYTER_KERNEL=ON` | -::: - -> {octicon}`alert;1em;sd-text-danger;` `jupyter-client` and `jupyter_libyt` are used for launching Jupyter Notebook and JupyterLab. Make sure the Python environment used for launching the notebook have them installed. -> -> The Python used in in situ analysis which is also for compiling `libyt` and the Python for launching Jupyter Notebook/JupyterLab might be different. -> For example, when running `libyt` in HPC cluster and connecting to it through your local laptop. (See [Jupyter Notebook Access](./in-situ-python-analysis/jupyter-notebook/jupyter-notebook-access.md)) - -## FAQs - -### Get Errors when Using CMake - -Make sure the folder where CMake generates build files to is empty or not exist yet by removing the folder: -```bash -cd libyt -rm -rf -cmake -S . -B -``` - -### Unable to Link to Dependencies Fetched by libyt After Installation - -Keep `libyt` project repo after installation. `libyt` fetches and stores dependencies under `libyt/vendor` folder, so that the content can be reused in different builds. diff --git a/doc/how-to-install/how-to-install.md b/doc/how-to-install/how-to-install.md new file mode 100644 index 00000000..72e3577d --- /dev/null +++ b/doc/how-to-install/how-to-install.md @@ -0,0 +1,104 @@ +# How to Install + +```{toctree} +:hidden: + +details +``` + +> {octicon}`info;1em;sd-text-info;` Go through [Install](#install) and get all [Python Dependencies](#python-dependencies) to get full feature of `libyt`. +> Ignore [Details](#details) unless we want to tweak `libyt` based on our needs. + +## libyt C Library + +### Install +- **CMake** (>=3.15) +- **pkg-config**: Generally, Linux and macOS already have pkg-config installed. +- **GCC compiler** (>4.8): It should be able to support `c++14`. + - `CXX`: Path to `g++` compiler. + - `CC`: Path to `gcc` compiler. +- **Python** (>=3.7): The Python environment we want to use when doing in situ analysis. + - `PYTHON_PATH`: Python installation prefix, the path contains folders `include`, `lib` etc. + - `NumPy`: Should have `NumPy` installed. + - Other [Python Dependencies](#python-dependencies) +- **MPI**: MPI used for compiling simulations and `libyt` needs to be the same. + - `MPI_PATH`: MPI installation prefix, the path contains folders `include`, `lib` etc. +- **Readline**: [GNU `readline` library](https://tiswww.case.edu/php/chet/readline/rltop.html) is already installed on Linux and macOS generally. If not, we can get through system package manager or compile from source ourselves. (Use `--with-curses` when configuring if we compile from source.) + - `READLINE_PATH`: `readline` installation prefix. Provide the path if it is not in system search path. + +**Follow the steps to install `libyt` that is fault-tolerant to Python code, and supports interactive Python prompt, reloading script, and Jupyter Notebook access:** + +1. Shallow clone `libyt` and enter the folder: + ```bash + git clone --depth 1 https://github.com/yt-project/libyt "libyt" + cd libyt + ``` +2. [Optional] Set `gcc` and `g++` compiler: + ```bash + export CC= + export CXX= + ``` +3. Generate build files in `build` folder: + - **Serial Mode (using GCC)**: + ```bash + rm -rf build + cmake -S . -B build -DSERIAL_MODE=ON \ + -DINTERACTIVE_MODE=ON \ + -DJUPYTER_KERNEL=ON \ + -DPYTHON_PATH= + ``` + - **Parallel Mode (using MPI)**: + ```bash + rm -rf build + cmake -S . -B build -DINTERACTIVE_MODE=ON \ + -DJUPYTER_KERNEL=ON \ + -DPYTHON_PATH= \ + -DMPI_PATH= + ``` +4. Build project and install `libyt`: + ```bash + cmake --build build + cmake --install build --prefix + ``` +5. `libyt` is now installed at `` folder. + + +## Python Dependencies + +- **Python package** required when performing in situ analysis. +- **Notes** are things worth notice. +- **Option** indicates under what circumstances will we need this package. + +:::{table} +:width: 100% + +| Python package | Notes | Option | +|-----------------------------------------------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-----------------------| +| [`NumPy`](https://numpy.org/)
(<2.0) | The fundamental package for scientific computing with Python. | Always | +| [`yt`](https://yt-project.org/) | The core analytic tool. | Always | +| [`yt_libyt`](https://github.com/data-exp-lab/yt_libyt) | `yt` frontend for `libyt`. | Always | +| [`mpi4py`](https://mpi4py.readthedocs.io/en/stable/) | Python bindings for the Message Passing Interface (MPI) standard.
{octicon}`alert;1em;sd-text-danger;` Make sure `mpi4py` used in Python and MPI used in simulation are matched. (Check how to install `mpi4py` [here](https://mpi4py.readthedocs.io/en/stable/install.html#installation).) | `-DSERIAL_MODE=OFF` | +| [`jedi`](https://jedi.readthedocs.io/en/latest/) | Support auto-completion in Jupyter Notebook and JupyterLab. (We will have this if IPython is already installed.) | `-DJUPYTER_KERNEL=ON` | +| [`jupyter-client`](https://jupyter-client.readthedocs.io/en/latest/index.html)
(>=8.0.0) | Jupyter Client. | `-DJUPYTER_KERNEL=ON` | +| [`jupyter_libyt`](https://github.com/yt-project/jupyter_libyt) | Jupyter kernel provisioner for `libyt`. | `-DJUPYTER_KERNEL=ON` | +::: + +> {octicon}`alert;1em;sd-text-danger;` `jupyter-client` and `jupyter_libyt` are used for launching Jupyter Notebook and JupyterLab. Make sure the Python environment used for launching the notebook have them installed. +> +> The Python used in in situ analysis which is also for compiling `libyt` and the Python for launching Jupyter Notebook/JupyterLab might be different. +> For example, when running `libyt` in HPC cluster and connecting to it through your local laptop. (See [Jupyter Notebook Access](../in-situ-python-analysis/jupyter-notebook/jupyter-notebook-access.md)) + +## FAQs + +### Get Errors when Using CMake + +Make sure the folder where CMake generates build files to is empty or not exist yet by removing the folder: +```bash +cd libyt +rm -rf +cmake -S . -B +``` + +### Unable to Link to Dependencies Fetched by libyt After Installation + +Keep `libyt` project repo after installation. `libyt` fetches and stores dependencies under `libyt/vendor` folder, so that the content can be reused in different builds. From ba944533f619178de2c27a2f6d4691b24d5ad015 Mon Sep 17 00:00:00 2001 From: cindytsai Date: Tue, 7 May 2024 17:05:46 -0500 Subject: [PATCH 12/27] Refactor after moving how-to-install to a folder. --- doc/debug-and-profiling/time-profiling.md | 2 +- doc/example.md | 4 ++-- doc/in-situ-python-analysis/index.md | 6 +++--- doc/in-situ-python-analysis/inline-python-script.md | 6 +++--- doc/in-situ-python-analysis/interactive-python-prompt.md | 2 +- .../jupyter-notebook/jupyter-notebook-access.md | 4 ++-- doc/in-situ-python-analysis/reloading-script.md | 4 ++-- doc/in-situ-python-analysis/using-yt.md | 2 +- doc/index.md | 4 ++-- doc/libyt-api/yt_run_interactivemode.md | 2 +- doc/libyt-api/yt_run_jupyterkernel.md | 2 +- doc/libyt-api/yt_run_reloadscript.md | 2 +- doc/quick-start.md | 2 +- 13 files changed, 21 insertions(+), 21 deletions(-) diff --git a/doc/debug-and-profiling/time-profiling.md b/doc/debug-and-profiling/time-profiling.md index d750e98b..f4011e35 100644 --- a/doc/debug-and-profiling/time-profiling.md +++ b/doc/debug-and-profiling/time-profiling.md @@ -2,7 +2,7 @@ ## How to Configure -Compile `libyt` with [`-DSUPPORT_TIMER=ON`](../how-to-install.md#-dsupport_timer-off). +Compile `libyt` with [`-DSUPPORT_TIMER=ON`](../how-to-install/how-to-install.md#-dsupport_timer-off). ## Chrome Tracing -- Visualizing the Profile 1. Since each process dumps its profile `libytTimeProfile_MPI*.json` separately, we run the following to concatenate all of them: diff --git a/doc/example.md b/doc/example.md index 24da9028..9e34f48f 100644 --- a/doc/example.md +++ b/doc/example.md @@ -21,7 +21,7 @@ The example initializes `libyt`, loads data to `libyt` in every simulation time ### Using CMake -1. Follow [Install](./how-to-install.md#install). +1. Follow [Install](./how-to-install/how-to-install.md#install). 2. Enter example folder. Assume we build the project under `libyt/build`: ```bash cd libyt # go to libyt project root folder @@ -30,7 +30,7 @@ The example initializes `libyt`, loads data to `libyt` in every simulation time ### Using Make -1. Follow [Install](./how-to-install.md#install). +1. Follow [Install](./how-to-install/how-to-install.md#install). 2. Go to `libyt/example/amr-example` folder. ```bash cd libyt/example/amr-example diff --git a/doc/in-situ-python-analysis/index.md b/doc/in-situ-python-analysis/index.md index 53b615d1..ce051036 100644 --- a/doc/in-situ-python-analysis/index.md +++ b/doc/in-situ-python-analysis/index.md @@ -23,9 +23,9 @@ faq ## User Interface - [**libyt Defined Commands**](./libyt-defined-command.md#libyt-defined-commands): the commands allow users to load script, export history, set function to run or idle, etc. -- [**Interactive Python Prompt**](./interactive-python-prompt.md#interactive-python-prompt): the prompt can process Python statements and give feedback instantly. ([`-DINTERACTIVE_MODE=ON`](../how-to-install.md#-dinteractive_mode-off)) -- [**Reloading Script**](./reloading-script.md#reloading-script): this allows reloading Python script. ([`-DINTERACTIVE_MODE=ON`](../how-to-install.md#-dinteractive_mode-off)) -- [**Jupyter Notebook Access**](./jupyter-notebook/jupyter-notebook-access.md#jupyter-notebook-access): this allows accessing and analyzing simulation data through JupyterLab/Jupyter Notebook. ([`-DJUPYTER_KERNEL=ON`](../how-to-install.md#-djupyter_kernel-off)) +- [**Interactive Python Prompt**](./interactive-python-prompt.md#interactive-python-prompt): the prompt can process Python statements and give feedback instantly. ([`-DINTERACTIVE_MODE=ON`](../how-to-install/how-to-install.md#-dinteractive_mode-off)) +- [**Reloading Script**](./reloading-script.md#reloading-script): this allows reloading Python script. ([`-DINTERACTIVE_MODE=ON`](../how-to-install/how-to-install.md#-dinteractive_mode-off)) +- [**Jupyter Notebook Access**](./jupyter-notebook/jupyter-notebook-access.md#jupyter-notebook-access): this allows accessing and analyzing simulation data through JupyterLab/Jupyter Notebook. ([`-DJUPYTER_KERNEL=ON`](../how-to-install/how-to-install.md#-djupyter_kernel-off)) ## Limitations diff --git a/doc/in-situ-python-analysis/inline-python-script.md b/doc/in-situ-python-analysis/inline-python-script.md index 25e8c39c..483847b0 100644 --- a/doc/in-situ-python-analysis/inline-python-script.md +++ b/doc/in-situ-python-analysis/inline-python-script.md @@ -11,13 +11,13 @@ The namespace contains function objects in the script. We can use [`yt_run_Funct and [`yt_run_FunctionArguments`](../libyt-api/run-python-function.md#yt_run_functionarguments) to call them during simulation process. ## What Happens if the Python Function Crashed During the Analysis? -If `libyt` is compiled in **normal mode** ([`-DINTERACTIVE_MODE=OFF`](../how-to-install.md#-dinteractive_mode-off)), it is not fault-tolerant to Python, +If `libyt` is compiled in **normal mode** ([`-DINTERACTIVE_MODE=OFF`](../how-to-install/how-to-install.md#-dinteractive_mode-off)), it is not fault-tolerant to Python, so the whole simulation will shut down. -Use **interactive mode** ([`-DINTERACTIVE_MODE=ON`](../how-to-install.md#-dinteractive_mode-off)) or **jupyter kernel mode** ([`-DJUPYTER_KERNEL=ON`](../how-to-install.md#-djupyter_kernel-off)) if we want our in situ Python analysis to be fault-tolerant. +Use **interactive mode** ([`-DINTERACTIVE_MODE=ON`](../how-to-install/how-to-install.md#-dinteractive_mode-off)) or **jupyter kernel mode** ([`-DJUPYTER_KERNEL=ON`](../how-to-install/how-to-install.md#-djupyter_kernel-off)) if we want our in situ Python analysis to be fault-tolerant. ## Can I Update Python Functions Defined in the Script? -We can only update Python functions in **interactive mode** ([`-DINTERACTIVE_MODE=ON`](../how-to-install.md#-dinteractive_mode-off)) or **jupyter kernel mode** ([`-DJUPYTER_KERNEL=ON`](../how-to-install.md#-djupyter_kernel-off)). +We can only update Python functions in **interactive mode** ([`-DINTERACTIVE_MODE=ON`](../how-to-install/how-to-install.md#-dinteractive_mode-off)) or **jupyter kernel mode** ([`-DJUPYTER_KERNEL=ON`](../how-to-install/how-to-install.md#-djupyter_kernel-off)). Since every new added Python object is maintained inside inline Python script's namespace, you can update a Python function by re-define the function again, so that the old function is overwritten. diff --git a/doc/in-situ-python-analysis/interactive-python-prompt.md b/doc/in-situ-python-analysis/interactive-python-prompt.md index 052f7c6c..3f265c0a 100644 --- a/doc/in-situ-python-analysis/interactive-python-prompt.md +++ b/doc/in-situ-python-analysis/interactive-python-prompt.md @@ -2,7 +2,7 @@ ## Requirements -- Compile `libyt` in **interactive mode** ([`-DINTERACTIVE_MODE=ON`](../how-to-install.md#-dinteractive_mode-off)). +- Compile `libyt` in **interactive mode** ([`-DINTERACTIVE_MODE=ON`](../how-to-install/how-to-install.md#-dinteractive_mode-off)). - Call libyt API [`yt_run_InteractiveMode`](../libyt-api/yt_run_interactivemode.md#yt_run_interactivemode). > {octicon}`info;1em;sd-text-info;` Interactive mode only works on local machine or submit the job to HPC platform using interactive jobs like `qsub -I` on PBS scheduler. diff --git a/doc/in-situ-python-analysis/jupyter-notebook/jupyter-notebook-access.md b/doc/in-situ-python-analysis/jupyter-notebook/jupyter-notebook-access.md index d90ac532..ed8864cc 100644 --- a/doc/in-situ-python-analysis/jupyter-notebook/jupyter-notebook-access.md +++ b/doc/in-situ-python-analysis/jupyter-notebook/jupyter-notebook-access.md @@ -9,7 +9,7 @@ remote-cluster ## Requirements -- Compile `libyt` in **jupyter kernel mode** ([`-DJUPYTER_KERNEL=ON`](../../how-to-install.md#-djupyter_kernel-off)). +- Compile `libyt` in **jupyter kernel mode** ([`-DJUPYTER_KERNEL=ON`](../../how-to-install/how-to-install.md#-djupyter_kernel-off)). - Call libyt API [`yt_run_JupyterKernel`](../../libyt-api/yt_run_jupyterkernel.md#yt_run_jupyterkernel). - Python package [`jupyter_libyt`](https://github.com/yt-project/jupyter_libyt), [`jupyter-client`](https://jupyter-client.readthedocs.io/en/stable/index.html), and [`jedi`](https://jedi.readthedocs.io/en/latest/). @@ -45,7 +45,7 @@ We need another process to start Jupyter Notebook/JupyterLab and connect to liby ### Running Simulation -1. Compile `libyt` in **jupyter kernel mode** ([`-DJUPYTER_KERNEL=ON`](../../how-to-install.md#-djupyter_kernel-off)). +1. Compile `libyt` in **jupyter kernel mode** ([`-DJUPYTER_KERNEL=ON`](../../how-to-install/how-to-install.md#-djupyter_kernel-off)). 2. Call libyt API [`yt_run_JupyterKernel`](../../libyt-api/yt_run_jupyterkernel.md#yt_run_jupyterkernel). When flag file is detected, `libyt` will activate libyt kernel. 3. libyt API [`yt_run_JupyterKernel`](../../libyt-api/yt_run_jupyterkernel.md#yt_run_jupyterkernel) returns `YT_SUCCESS` after libyt kernel shuts down (see [How to Exit](#how-to-exit)). 4. Simulation can continue its process. diff --git a/doc/in-situ-python-analysis/reloading-script.md b/doc/in-situ-python-analysis/reloading-script.md index 3ca1e142..45f1ff86 100644 --- a/doc/in-situ-python-analysis/reloading-script.md +++ b/doc/in-situ-python-analysis/reloading-script.md @@ -2,7 +2,7 @@ ## Requirements -- Compile `libyt` in **interactive mode** ([`-DINTERACTIVE_MODE=ON`](../how-to-install.md#-dinteractive_mode-off)). +- Compile `libyt` in **interactive mode** ([`-DINTERACTIVE_MODE=ON`](../how-to-install/how-to-install.md#-dinteractive_mode-off)). - Call libyt API [`yt_run_ReloadScript`](../libyt-api/yt_run_reloadscript.md#yt_run_reloadscript). ## Reloading Script @@ -61,5 +61,5 @@ def func(): ## FAQs ### When Can I Reload Script? -`libyt` supports reloading script feature if it is compiled with [`-DINTERACTIVE_MODE=ON`](../how-to-install.md#-dinteractive_mode-off). +`libyt` supports reloading script feature if it is compiled with [`-DINTERACTIVE_MODE=ON`](../how-to-install/how-to-install.md#-dinteractive_mode-off). The root process reads the file, so it would work on your local desktop and in HPC clusters. diff --git a/doc/in-situ-python-analysis/using-yt.md b/doc/in-situ-python-analysis/using-yt.md index 9ebd7ecc..573328fe 100644 --- a/doc/in-situ-python-analysis/using-yt.md +++ b/doc/in-situ-python-analysis/using-yt.md @@ -2,7 +2,7 @@ ## Requirements - Python package [`yt`](https://yt-project.org/) and [`yt_libyt`](https://github.com/data-exp-lab/yt_libyt). -- If it is for parallel computing in **parallel mode** ([`-DSERIAL_MODE=OFF`](../how-to-install.md#-dserial_mode-off)), it needs [`mpi4py`](https://mpi4py.readthedocs.io/en/stable/install.html#installation). +- If it is for parallel computing in **parallel mode** ([`-DSERIAL_MODE=OFF`](../how-to-install/how-to-install.md#-dserial_mode-off)), it needs [`mpi4py`](https://mpi4py.readthedocs.io/en/stable/install.html#installation). ## Use yt for Parallel Computing > {octicon}`info;1em;sd-text-info;` `libyt` directly borrows parallel computation feature in `yt` using `mpi4py`. Please refer to [**Parallel Computation With yt**](https://yt-project.org/doc/analyzing/parallel_computation.html#parallel-computation-with-yt). diff --git a/doc/index.md b/doc/index.md index fb91bb14..3a388bc7 100644 --- a/doc/index.md +++ b/doc/index.md @@ -14,7 +14,7 @@ It is an in situ analysis tool that allows researchers to analyze and visualize quick-start example -how-to-install +how-to-install/how-to-install how-it-works in-situ-python-analysis/index FAQs @@ -30,7 +30,7 @@ debug-and-profiling/index #### Contents - [**Quick Start**](./quick-start.md) -- [**How to Install**](./how-to-install.md#how-to-install): how to get `libyt`? +- [**How to Install**](./how-to-install/how-to-install.md#how-to-install): how to get `libyt`? - [**How it Works**](./how-it-works.md): what is happening behind the scene? - [**Example**](./example.md): this demonstrates how to implement `libyt` in simulation step by step. - [**libyt API**](./libyt-api/index.md): this is for simulation developers that wish to implement `libyt`. diff --git a/doc/libyt-api/yt_run_interactivemode.md b/doc/libyt-api/yt_run_interactivemode.md index e9a80175..c3cd8fbb 100644 --- a/doc/libyt-api/yt_run_interactivemode.md +++ b/doc/libyt-api/yt_run_interactivemode.md @@ -9,7 +9,7 @@ int yt_run_InteractiveMode(const char* flag_file_name); - `YT_SUCCESS` - `YT_FAIL`: When `libyt` is not compiled with `-DINTERACTIVE_MODE`, it returns `YT_FAIL`. -> {octicon}`info;1em;sd-text-info;` Must compile `libyt` with [`-DINTERACTIVE_MODE=ON`](../how-to-install.md#-dinteractive_mode-off). +> {octicon}`info;1em;sd-text-info;` Must compile `libyt` with [`-DINTERACTIVE_MODE=ON`](../how-to-install/how-to-install.md#-dinteractive_mode-off). ## Example ```cpp diff --git a/doc/libyt-api/yt_run_jupyterkernel.md b/doc/libyt-api/yt_run_jupyterkernel.md index c386d4df..6d387bcc 100644 --- a/doc/libyt-api/yt_run_jupyterkernel.md +++ b/doc/libyt-api/yt_run_jupyterkernel.md @@ -9,7 +9,7 @@ int yt_run_JupyterKernel(const char* flag_file_name, bool use_connection_file); - `YT_SUCCESS` - `YT_FAIL`: When `libyt` is not compiled with `-DJUPYTER_KERNEL=ON`, it returns `YT_FAIL`. -> {octicon}`info;1em;sd-text-info;` Must compile `libyt` with [`-DJUPYTER_KERNEL=ON`](../how-to-install.md#-djupyter_kernel-off). +> {octicon}`info;1em;sd-text-info;` Must compile `libyt` with [`-DJUPYTER_KERNEL=ON`](../how-to-install/how-to-install.md#-djupyter_kernel-off). > {octicon}`info;1em;sd-text-info;` The API only launches a Jupyter kernel, which is an entry point for Jupyter Notebook / JupyterLab to connect to. For how to connect to the kernel, please go to [Jupyter Notebook Access](../in-situ-python-analysis/jupyter-notebook/jupyter-notebook-access.md#jupyter-notebook-access). diff --git a/doc/libyt-api/yt_run_reloadscript.md b/doc/libyt-api/yt_run_reloadscript.md index f1668c9c..bf91811b 100644 --- a/doc/libyt-api/yt_run_reloadscript.md +++ b/doc/libyt-api/yt_run_reloadscript.md @@ -9,7 +9,7 @@ int yt_run_ReloadScript(const char* flag_file_name, const char* reload_file_name - `YT_SUCCESS` - `YT_FAIL`: When `libyt` is not compiled with `-DINTERACTIVE_MODE`, it returns `YT_FAIL`. -> {octicon}`info;1em;sd-text-info;` Must compile `libyt` with [`-DINTERACTIVE_MODE=ON`](../how-to-install.md#-dinteractive_mode-off). +> {octicon}`info;1em;sd-text-info;` Must compile `libyt` with [`-DINTERACTIVE_MODE=ON`](../how-to-install/how-to-install.md#-dinteractive_mode-off). ## Example The code will stop and enter reloading script phase, if `LIBYT_STOP` is detected or an inline function failed. diff --git a/doc/quick-start.md b/doc/quick-start.md index ed5e7342..9d13e613 100644 --- a/doc/quick-start.md +++ b/doc/quick-start.md @@ -9,7 +9,7 @@ After reading quick start, we will have an overview of how to use `libyt` for in > {octicon}`info;1em;sd-text-info;` This demonstration assumes the operating system is either Linux or macOS. -1. Follow [Install](./how-to-install.md#install). +1. Follow [Install](./how-to-install/how-to-install.md#install). 2. Enter quick start folder. Assume we build the project under `libyt/build`: ```bash cd libyt # go to libyt project root folder From 73719a51bc9abbf1a87bfb4fd8d9da24a4c3bb1e Mon Sep 17 00:00:00 2001 From: cindytsai Date: Tue, 7 May 2024 17:10:09 -0500 Subject: [PATCH 13/27] Fix the anchor after splitting details and how-to-install. --- doc/debug-and-profiling/time-profiling.md | 2 +- doc/in-situ-python-analysis/index.md | 6 +++--- doc/in-situ-python-analysis/inline-python-script.md | 6 +++--- doc/in-situ-python-analysis/interactive-python-prompt.md | 2 +- .../jupyter-notebook/jupyter-notebook-access.md | 4 ++-- doc/in-situ-python-analysis/reloading-script.md | 4 ++-- doc/in-situ-python-analysis/using-yt.md | 2 +- doc/libyt-api/yt_run_interactivemode.md | 2 +- doc/libyt-api/yt_run_jupyterkernel.md | 2 +- doc/libyt-api/yt_run_reloadscript.md | 2 +- 10 files changed, 16 insertions(+), 16 deletions(-) diff --git a/doc/debug-and-profiling/time-profiling.md b/doc/debug-and-profiling/time-profiling.md index f4011e35..84d655ea 100644 --- a/doc/debug-and-profiling/time-profiling.md +++ b/doc/debug-and-profiling/time-profiling.md @@ -2,7 +2,7 @@ ## How to Configure -Compile `libyt` with [`-DSUPPORT_TIMER=ON`](../how-to-install/how-to-install.md#-dsupport_timer-off). +Compile `libyt` with [`-DSUPPORT_TIMER=ON`](../how-to-install/details.md#-dsupport_timer-off). ## Chrome Tracing -- Visualizing the Profile 1. Since each process dumps its profile `libytTimeProfile_MPI*.json` separately, we run the following to concatenate all of them: diff --git a/doc/in-situ-python-analysis/index.md b/doc/in-situ-python-analysis/index.md index ce051036..f3c0db92 100644 --- a/doc/in-situ-python-analysis/index.md +++ b/doc/in-situ-python-analysis/index.md @@ -23,9 +23,9 @@ faq ## User Interface - [**libyt Defined Commands**](./libyt-defined-command.md#libyt-defined-commands): the commands allow users to load script, export history, set function to run or idle, etc. -- [**Interactive Python Prompt**](./interactive-python-prompt.md#interactive-python-prompt): the prompt can process Python statements and give feedback instantly. ([`-DINTERACTIVE_MODE=ON`](../how-to-install/how-to-install.md#-dinteractive_mode-off)) -- [**Reloading Script**](./reloading-script.md#reloading-script): this allows reloading Python script. ([`-DINTERACTIVE_MODE=ON`](../how-to-install/how-to-install.md#-dinteractive_mode-off)) -- [**Jupyter Notebook Access**](./jupyter-notebook/jupyter-notebook-access.md#jupyter-notebook-access): this allows accessing and analyzing simulation data through JupyterLab/Jupyter Notebook. ([`-DJUPYTER_KERNEL=ON`](../how-to-install/how-to-install.md#-djupyter_kernel-off)) +- [**Interactive Python Prompt**](./interactive-python-prompt.md#interactive-python-prompt): the prompt can process Python statements and give feedback instantly. ([`-DINTERACTIVE_MODE=ON`](../how-to-install/details.md#-dinteractive_mode-off)) +- [**Reloading Script**](./reloading-script.md#reloading-script): this allows reloading Python script. ([`-DINTERACTIVE_MODE=ON`](../how-to-install/details.md#-dinteractive_mode-off)) +- [**Jupyter Notebook Access**](./jupyter-notebook/jupyter-notebook-access.md#jupyter-notebook-access): this allows accessing and analyzing simulation data through JupyterLab/Jupyter Notebook. ([`-DJUPYTER_KERNEL=ON`](../how-to-install/details.md#-djupyter_kernel-off)) ## Limitations diff --git a/doc/in-situ-python-analysis/inline-python-script.md b/doc/in-situ-python-analysis/inline-python-script.md index 483847b0..8f1f6dc3 100644 --- a/doc/in-situ-python-analysis/inline-python-script.md +++ b/doc/in-situ-python-analysis/inline-python-script.md @@ -11,13 +11,13 @@ The namespace contains function objects in the script. We can use [`yt_run_Funct and [`yt_run_FunctionArguments`](../libyt-api/run-python-function.md#yt_run_functionarguments) to call them during simulation process. ## What Happens if the Python Function Crashed During the Analysis? -If `libyt` is compiled in **normal mode** ([`-DINTERACTIVE_MODE=OFF`](../how-to-install/how-to-install.md#-dinteractive_mode-off)), it is not fault-tolerant to Python, +If `libyt` is compiled in **normal mode** ([`-DINTERACTIVE_MODE=OFF`](../how-to-install/details.md#-dinteractive_mode-off)), it is not fault-tolerant to Python, so the whole simulation will shut down. -Use **interactive mode** ([`-DINTERACTIVE_MODE=ON`](../how-to-install/how-to-install.md#-dinteractive_mode-off)) or **jupyter kernel mode** ([`-DJUPYTER_KERNEL=ON`](../how-to-install/how-to-install.md#-djupyter_kernel-off)) if we want our in situ Python analysis to be fault-tolerant. +Use **interactive mode** ([`-DINTERACTIVE_MODE=ON`](../how-to-install/details.md#-dinteractive_mode-off)) or **jupyter kernel mode** ([`-DJUPYTER_KERNEL=ON`](../how-to-install/details.md#-djupyter_kernel-off)) if we want our in situ Python analysis to be fault-tolerant. ## Can I Update Python Functions Defined in the Script? -We can only update Python functions in **interactive mode** ([`-DINTERACTIVE_MODE=ON`](../how-to-install/how-to-install.md#-dinteractive_mode-off)) or **jupyter kernel mode** ([`-DJUPYTER_KERNEL=ON`](../how-to-install/how-to-install.md#-djupyter_kernel-off)). +We can only update Python functions in **interactive mode** ([`-DINTERACTIVE_MODE=ON`](../how-to-install/details.md#-dinteractive_mode-off)) or **jupyter kernel mode** ([`-DJUPYTER_KERNEL=ON`](../how-to-install/details.md#-djupyter_kernel-off)). Since every new added Python object is maintained inside inline Python script's namespace, you can update a Python function by re-define the function again, so that the old function is overwritten. diff --git a/doc/in-situ-python-analysis/interactive-python-prompt.md b/doc/in-situ-python-analysis/interactive-python-prompt.md index 3f265c0a..c1073e30 100644 --- a/doc/in-situ-python-analysis/interactive-python-prompt.md +++ b/doc/in-situ-python-analysis/interactive-python-prompt.md @@ -2,7 +2,7 @@ ## Requirements -- Compile `libyt` in **interactive mode** ([`-DINTERACTIVE_MODE=ON`](../how-to-install/how-to-install.md#-dinteractive_mode-off)). +- Compile `libyt` in **interactive mode** ([`-DINTERACTIVE_MODE=ON`](../how-to-install/details.md#-dinteractive_mode-off)). - Call libyt API [`yt_run_InteractiveMode`](../libyt-api/yt_run_interactivemode.md#yt_run_interactivemode). > {octicon}`info;1em;sd-text-info;` Interactive mode only works on local machine or submit the job to HPC platform using interactive jobs like `qsub -I` on PBS scheduler. diff --git a/doc/in-situ-python-analysis/jupyter-notebook/jupyter-notebook-access.md b/doc/in-situ-python-analysis/jupyter-notebook/jupyter-notebook-access.md index ed8864cc..23311ba5 100644 --- a/doc/in-situ-python-analysis/jupyter-notebook/jupyter-notebook-access.md +++ b/doc/in-situ-python-analysis/jupyter-notebook/jupyter-notebook-access.md @@ -9,7 +9,7 @@ remote-cluster ## Requirements -- Compile `libyt` in **jupyter kernel mode** ([`-DJUPYTER_KERNEL=ON`](../../how-to-install/how-to-install.md#-djupyter_kernel-off)). +- Compile `libyt` in **jupyter kernel mode** ([`-DJUPYTER_KERNEL=ON`](../../how-to-install/details.md#-djupyter_kernel-off)). - Call libyt API [`yt_run_JupyterKernel`](../../libyt-api/yt_run_jupyterkernel.md#yt_run_jupyterkernel). - Python package [`jupyter_libyt`](https://github.com/yt-project/jupyter_libyt), [`jupyter-client`](https://jupyter-client.readthedocs.io/en/stable/index.html), and [`jedi`](https://jedi.readthedocs.io/en/latest/). @@ -45,7 +45,7 @@ We need another process to start Jupyter Notebook/JupyterLab and connect to liby ### Running Simulation -1. Compile `libyt` in **jupyter kernel mode** ([`-DJUPYTER_KERNEL=ON`](../../how-to-install/how-to-install.md#-djupyter_kernel-off)). +1. Compile `libyt` in **jupyter kernel mode** ([`-DJUPYTER_KERNEL=ON`](../../how-to-install/details.md#-djupyter_kernel-off)). 2. Call libyt API [`yt_run_JupyterKernel`](../../libyt-api/yt_run_jupyterkernel.md#yt_run_jupyterkernel). When flag file is detected, `libyt` will activate libyt kernel. 3. libyt API [`yt_run_JupyterKernel`](../../libyt-api/yt_run_jupyterkernel.md#yt_run_jupyterkernel) returns `YT_SUCCESS` after libyt kernel shuts down (see [How to Exit](#how-to-exit)). 4. Simulation can continue its process. diff --git a/doc/in-situ-python-analysis/reloading-script.md b/doc/in-situ-python-analysis/reloading-script.md index 45f1ff86..7e40a4a4 100644 --- a/doc/in-situ-python-analysis/reloading-script.md +++ b/doc/in-situ-python-analysis/reloading-script.md @@ -2,7 +2,7 @@ ## Requirements -- Compile `libyt` in **interactive mode** ([`-DINTERACTIVE_MODE=ON`](../how-to-install/how-to-install.md#-dinteractive_mode-off)). +- Compile `libyt` in **interactive mode** ([`-DINTERACTIVE_MODE=ON`](../how-to-install/details.md#-dinteractive_mode-off)). - Call libyt API [`yt_run_ReloadScript`](../libyt-api/yt_run_reloadscript.md#yt_run_reloadscript). ## Reloading Script @@ -61,5 +61,5 @@ def func(): ## FAQs ### When Can I Reload Script? -`libyt` supports reloading script feature if it is compiled with [`-DINTERACTIVE_MODE=ON`](../how-to-install/how-to-install.md#-dinteractive_mode-off). +`libyt` supports reloading script feature if it is compiled with [`-DINTERACTIVE_MODE=ON`](../how-to-install/details.md#-dinteractive_mode-off). The root process reads the file, so it would work on your local desktop and in HPC clusters. diff --git a/doc/in-situ-python-analysis/using-yt.md b/doc/in-situ-python-analysis/using-yt.md index 573328fe..b49d3a89 100644 --- a/doc/in-situ-python-analysis/using-yt.md +++ b/doc/in-situ-python-analysis/using-yt.md @@ -2,7 +2,7 @@ ## Requirements - Python package [`yt`](https://yt-project.org/) and [`yt_libyt`](https://github.com/data-exp-lab/yt_libyt). -- If it is for parallel computing in **parallel mode** ([`-DSERIAL_MODE=OFF`](../how-to-install/how-to-install.md#-dserial_mode-off)), it needs [`mpi4py`](https://mpi4py.readthedocs.io/en/stable/install.html#installation). +- If it is for parallel computing in **parallel mode** ([`-DSERIAL_MODE=OFF`](../how-to-install/details.md#-dserial_mode-off)), it needs [`mpi4py`](https://mpi4py.readthedocs.io/en/stable/install.html#installation). ## Use yt for Parallel Computing > {octicon}`info;1em;sd-text-info;` `libyt` directly borrows parallel computation feature in `yt` using `mpi4py`. Please refer to [**Parallel Computation With yt**](https://yt-project.org/doc/analyzing/parallel_computation.html#parallel-computation-with-yt). diff --git a/doc/libyt-api/yt_run_interactivemode.md b/doc/libyt-api/yt_run_interactivemode.md index c3cd8fbb..979505b7 100644 --- a/doc/libyt-api/yt_run_interactivemode.md +++ b/doc/libyt-api/yt_run_interactivemode.md @@ -9,7 +9,7 @@ int yt_run_InteractiveMode(const char* flag_file_name); - `YT_SUCCESS` - `YT_FAIL`: When `libyt` is not compiled with `-DINTERACTIVE_MODE`, it returns `YT_FAIL`. -> {octicon}`info;1em;sd-text-info;` Must compile `libyt` with [`-DINTERACTIVE_MODE=ON`](../how-to-install/how-to-install.md#-dinteractive_mode-off). +> {octicon}`info;1em;sd-text-info;` Must compile `libyt` with [`-DINTERACTIVE_MODE=ON`](../how-to-install/details.md#-dinteractive_mode-off). ## Example ```cpp diff --git a/doc/libyt-api/yt_run_jupyterkernel.md b/doc/libyt-api/yt_run_jupyterkernel.md index 6d387bcc..7c7c7eae 100644 --- a/doc/libyt-api/yt_run_jupyterkernel.md +++ b/doc/libyt-api/yt_run_jupyterkernel.md @@ -9,7 +9,7 @@ int yt_run_JupyterKernel(const char* flag_file_name, bool use_connection_file); - `YT_SUCCESS` - `YT_FAIL`: When `libyt` is not compiled with `-DJUPYTER_KERNEL=ON`, it returns `YT_FAIL`. -> {octicon}`info;1em;sd-text-info;` Must compile `libyt` with [`-DJUPYTER_KERNEL=ON`](../how-to-install/how-to-install.md#-djupyter_kernel-off). +> {octicon}`info;1em;sd-text-info;` Must compile `libyt` with [`-DJUPYTER_KERNEL=ON`](../how-to-install/details.md#-djupyter_kernel-off). > {octicon}`info;1em;sd-text-info;` The API only launches a Jupyter kernel, which is an entry point for Jupyter Notebook / JupyterLab to connect to. For how to connect to the kernel, please go to [Jupyter Notebook Access](../in-situ-python-analysis/jupyter-notebook/jupyter-notebook-access.md#jupyter-notebook-access). diff --git a/doc/libyt-api/yt_run_reloadscript.md b/doc/libyt-api/yt_run_reloadscript.md index bf91811b..ea63fb05 100644 --- a/doc/libyt-api/yt_run_reloadscript.md +++ b/doc/libyt-api/yt_run_reloadscript.md @@ -9,7 +9,7 @@ int yt_run_ReloadScript(const char* flag_file_name, const char* reload_file_name - `YT_SUCCESS` - `YT_FAIL`: When `libyt` is not compiled with `-DINTERACTIVE_MODE`, it returns `YT_FAIL`. -> {octicon}`info;1em;sd-text-info;` Must compile `libyt` with [`-DINTERACTIVE_MODE=ON`](../how-to-install/how-to-install.md#-dinteractive_mode-off). +> {octicon}`info;1em;sd-text-info;` Must compile `libyt` with [`-DINTERACTIVE_MODE=ON`](../how-to-install/details.md#-dinteractive_mode-off). ## Example The code will stop and enter reloading script phase, if `LIBYT_STOP` is detected or an inline function failed. From 7a8db740be19fe8865b15c5e70872873364ab46c Mon Sep 17 00:00:00 2001 From: cindytsai Date: Tue, 7 May 2024 17:46:32 -0500 Subject: [PATCH 14/27] Rename install section to libyt and add admontion --- doc/example.md | 4 ++-- doc/how-to-install/how-to-install.md | 25 +++++++++++++++---------- doc/quick-start.md | 2 +- 3 files changed, 18 insertions(+), 13 deletions(-) diff --git a/doc/example.md b/doc/example.md index 9e34f48f..ced97ae4 100644 --- a/doc/example.md +++ b/doc/example.md @@ -21,7 +21,7 @@ The example initializes `libyt`, loads data to `libyt` in every simulation time ### Using CMake -1. Follow [Install](./how-to-install/how-to-install.md#install). +1. Follow [Install](./how-to-install/how-to-install.md#libyt-c-library). 2. Enter example folder. Assume we build the project under `libyt/build`: ```bash cd libyt # go to libyt project root folder @@ -30,7 +30,7 @@ The example initializes `libyt`, loads data to `libyt` in every simulation time ### Using Make -1. Follow [Install](./how-to-install/how-to-install.md#install). +1. Follow [Install](./how-to-install/how-to-install.md#libyt-c-library). 2. Go to `libyt/example/amr-example` folder. ```bash cd libyt/example/amr-example diff --git a/doc/how-to-install/how-to-install.md b/doc/how-to-install/how-to-install.md index 72e3577d..de43454e 100644 --- a/doc/how-to-install/how-to-install.md +++ b/doc/how-to-install/how-to-install.md @@ -6,24 +6,29 @@ details ``` -> {octicon}`info;1em;sd-text-info;` Go through [Install](#install) and get all [Python Dependencies](#python-dependencies) to get full feature of `libyt`. -> Ignore [Details](#details) unless we want to tweak `libyt` based on our needs. +```{note} +Go through this page to install `libyt` that is + fault-tolerant to Python code, supports interactive Python prompt, supports reloading script, and supports Jupyter Notebook access. + +To turn off some of the features, go to [Details](./details.md). +``` + +## libyt -## libyt C Library +**Requirements:** -### Install -- **CMake** (>=3.15) -- **pkg-config**: Generally, Linux and macOS already have pkg-config installed. -- **GCC compiler** (>4.8): It should be able to support `c++14`. +- **[CMake](https://cmake.org/)** (>=3.15) +- **[pkg-config](https://www.freedesktop.org/wiki/Software/pkg-config/)**: Generally, Linux and macOS already have pkg-config installed. +- **Compiler** (>4.8): It should be able to support `c++14`. - `CXX`: Path to `g++` compiler. - `CC`: Path to `gcc` compiler. -- **Python** (>=3.7): The Python environment we want to use when doing in situ analysis. +- **[Python](https://www.python.org/)** (>=3.7): The Python environment we want to use when doing in situ analysis. - `PYTHON_PATH`: Python installation prefix, the path contains folders `include`, `lib` etc. - `NumPy`: Should have `NumPy` installed. - Other [Python Dependencies](#python-dependencies) -- **MPI**: MPI used for compiling simulations and `libyt` needs to be the same. +- **[OpenMPI](https://www.open-mpi.org/)** or **[MPICH](https://www.mpich.org/)**: MPI used for compiling simulations and `libyt` needs to be the same. - `MPI_PATH`: MPI installation prefix, the path contains folders `include`, `lib` etc. -- **Readline**: [GNU `readline` library](https://tiswww.case.edu/php/chet/readline/rltop.html) is already installed on Linux and macOS generally. If not, we can get through system package manager or compile from source ourselves. (Use `--with-curses` when configuring if we compile from source.) +- **[Readline](https://tiswww.case.edu/php/chet/readline/rltop.html)**: Readline library is already installed on Linux and macOS generally. If not, we can get through system package manager or compile from source ourselves. (Use `--with-curses` when configuring if we compile from source.) - `READLINE_PATH`: `readline` installation prefix. Provide the path if it is not in system search path. **Follow the steps to install `libyt` that is fault-tolerant to Python code, and supports interactive Python prompt, reloading script, and Jupyter Notebook access:** diff --git a/doc/quick-start.md b/doc/quick-start.md index 9d13e613..58c6e3b8 100644 --- a/doc/quick-start.md +++ b/doc/quick-start.md @@ -9,7 +9,7 @@ After reading quick start, we will have an overview of how to use `libyt` for in > {octicon}`info;1em;sd-text-info;` This demonstration assumes the operating system is either Linux or macOS. -1. Follow [Install](./how-to-install/how-to-install.md#install). +1. Follow [Install](./how-to-install/how-to-install.md#libyt-c-library). 2. Enter quick start folder. Assume we build the project under `libyt/build`: ```bash cd libyt # go to libyt project root folder From 9a965ba5c5061e3bee4e491f1dac453bdcc19216 Mon Sep 17 00:00:00 2001 From: cindytsai Date: Tue, 7 May 2024 17:47:55 -0500 Subject: [PATCH 15/27] cont. last commit. --- doc/example.md | 4 ++-- doc/quick-start.md | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/example.md b/doc/example.md index ced97ae4..81db0cef 100644 --- a/doc/example.md +++ b/doc/example.md @@ -21,7 +21,7 @@ The example initializes `libyt`, loads data to `libyt` in every simulation time ### Using CMake -1. Follow [Install](./how-to-install/how-to-install.md#libyt-c-library). +1. Follow [Install](./how-to-install/how-to-install.md#libyt). 2. Enter example folder. Assume we build the project under `libyt/build`: ```bash cd libyt # go to libyt project root folder @@ -30,7 +30,7 @@ The example initializes `libyt`, loads data to `libyt` in every simulation time ### Using Make -1. Follow [Install](./how-to-install/how-to-install.md#libyt-c-library). +1. Follow [Install](./how-to-install/how-to-install.md#libyt). 2. Go to `libyt/example/amr-example` folder. ```bash cd libyt/example/amr-example diff --git a/doc/quick-start.md b/doc/quick-start.md index 58c6e3b8..64da1415 100644 --- a/doc/quick-start.md +++ b/doc/quick-start.md @@ -9,7 +9,7 @@ After reading quick start, we will have an overview of how to use `libyt` for in > {octicon}`info;1em;sd-text-info;` This demonstration assumes the operating system is either Linux or macOS. -1. Follow [Install](./how-to-install/how-to-install.md#libyt-c-library). +1. Follow [Install](./how-to-install/how-to-install.md#libyt). 2. Enter quick start folder. Assume we build the project under `libyt/build`: ```bash cd libyt # go to libyt project root folder From 7b597e3e27fdabbd81818b42ab0bb033b19ce2ca Mon Sep 17 00:00:00 2001 From: cindytsai Date: Tue, 7 May 2024 18:19:54 -0500 Subject: [PATCH 16/27] Single out required versions to a column. --- doc/how-to-install/details.md | 44 +++++++++++++--------------- doc/how-to-install/how-to-install.md | 22 +++++++------- 2 files changed, 30 insertions(+), 36 deletions(-) diff --git a/doc/how-to-install/details.md b/doc/how-to-install/details.md index 8fc9e3c0..ee347d8a 100644 --- a/doc/how-to-install/details.md +++ b/doc/how-to-install/details.md @@ -43,43 +43,39 @@ The options are mutually independent to each other. ## Dependencies -- **Dependency path** indicates the required path variable name in CMake, and the required version. -- **Notes** are things worth notice. - **Option** indicates under what circumstances will we need this dependency. - **Get by libyt** indicates whether libyt will fetch and build the dependency itself, if the paths aren't provided. The downloaded content will be stored under `libyt/vendor`, and `libyt` will link to dependencies inside this folder. -| Dependency path | Notes | Option | Get by libyt | -|--------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-------------------------|:------------:| -| `PYTHON_PATH`
(>=3.7) | Python installation prefix, the path contains folders include, lib etc.
{octicon}`info;1em;sd-text-info;` The Python environemnt will be used in in situ analysis. | Always | No | -| `MPI_PATH` | MPI installation prefix. The path contains folders `include` and `lib`.
{octicon}`alert;1em;sd-text-danger;` Make sure you are using the same MPI to compile `libyt` and your simulation code. | `-DSERIAL_MODE=OFF` | No | -| `READLINE_PATH` | [GNU `readline` library](https://tiswww.case.edu/php/chet/readline/rltop.html) installation prefix. The path contains folders `include` and `lib`.
{octicon}`info;1em;sd-text-info;` Generally, this library exists in Linux and macOS, we don't need to explicitly provide the path. | `-DINTERACTIVE_MODE=ON` | No | -| `nlohmann_json_DIR`
(>=3.2.0, <4.0.0) | Path to `nlohmann_jsonConfig.cmake` after installing [`nlohmann_json`](https://github.com/nlohmann/json). | `-DJUPYTER_KERNEL=ON` | Yes | -| `xtl_DIR`
(>=0.7.0, <0.8.0) | Path to `xtlConfig.cmake` after installing [`xtl`](https://github.com/xtensor-stack/xtl). | `-DJUPYTER_KERNEL=ON` | Yes | -| `ZeroMQ_DIR`
(>=4.2.5, <5.0.0) | Path to `ZeroMQConfig.cmake` after installing [`ZeroMQ`](https://github.com/zeromq/libzmq). (Some system may already have ZeroMQ installed.) | `-DJUPYTER_KERNEL=ON` | Yes | -| `cppzmq_DIR`
(>=4.8.1, <5.0.0) | Path to `cppzmqConfig.cmake` after installing [`cppzmq`](https://github.com/zeromq/cppzmq). | `-DJUPYTER_KERNEL=ON` | Yes | -| `xeus_DIR`
(>=3.0.0, <4.0.0) | Path to `xeusConfig.cmake` after installing [`xeus`](https://github.com/jupyter-xeus/xeus). | `-DJUPYTER_KERNEL=ON` | Yes | -| `xeus-zmq_DIR`
(1.x release) | Path to `xeus-zmqConfig.cmake` after installing [`xeus-zmq`](https://github.com/jupyter-xeus/xeus-zmq). | `-DJUPYTER_KERNEL=ON` | Yes | +| Dependency path | Required version | Notes | Option | Get by libyt | +|---------------------|------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-------------------------|:------------:| +| `PYTHON_PATH` | >=3.7 | Python installation prefix, the path contains folders include, lib etc.
{octicon}`info;1em;sd-text-info;` The Python environemnt will be used in in situ analysis. | Always | No | +| `MPI_PATH` | | MPI installation prefix. The path contains folders `include` and `lib`.
{octicon}`alert;1em;sd-text-danger;` Make sure you are using the same MPI to compile `libyt` and your simulation code. | `-DSERIAL_MODE=OFF` | No | +| `READLINE_PATH` | | [GNU `readline` library](https://tiswww.case.edu/php/chet/readline/rltop.html) installation prefix. The path contains folders `include` and `lib`.
{octicon}`info;1em;sd-text-info;` Generally, this library exists in Linux and macOS, we don't need to explicitly provide the path. | `-DINTERACTIVE_MODE=ON` | No | +| `nlohmann_json_DIR` | >=3.2.0, <4.0.0 | Path to `nlohmann_jsonConfig.cmake` after installing [`nlohmann_json`](https://github.com/nlohmann/json). | `-DJUPYTER_KERNEL=ON` | Yes | +| `xtl_DIR` | >=0.7.0, <0.8.0 | Path to `xtlConfig.cmake` after installing [`xtl`](https://github.com/xtensor-stack/xtl). | `-DJUPYTER_KERNEL=ON` | Yes | +| `ZeroMQ_DIR` | >=4.2.5, <5.0.0 | Path to `ZeroMQConfig.cmake` after installing [`ZeroMQ`](https://github.com/zeromq/libzmq). (Some system may already have ZeroMQ installed.) | `-DJUPYTER_KERNEL=ON` | Yes | +| `cppzmq_DIR` | >=4.8.1, <5.0.0 | Path to `cppzmqConfig.cmake` after installing [`cppzmq`](https://github.com/zeromq/cppzmq). | `-DJUPYTER_KERNEL=ON` | Yes | +| `xeus_DIR` | >=3.0.0, <4.0.0 | Path to `xeusConfig.cmake` after installing [`xeus`](https://github.com/jupyter-xeus/xeus). | `-DJUPYTER_KERNEL=ON` | Yes | +| `xeus-zmq_DIR` | 1.x release | Path to `xeus-zmqConfig.cmake` after installing [`xeus-zmq`](https://github.com/jupyter-xeus/xeus-zmq). | `-DJUPYTER_KERNEL=ON` | Yes | > {octicon}`info;1em;sd-text-info;` If our system doesn't have `readline` installed, use system package manager (ex: `brew`, `apt`) to install. If we want to compile and install from the source code ourselves, make sure `--with-curses` is used when configuring. ## Python Dependencies -- **Python package** required when performing in situ analysis. -- **Notes** are things worth notice. - **Option** indicates under what circumstances will we need this package. :::{table} :width: 100% -| Python package | Notes | Option | -|-----------------------------------------------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-----------------------| -| [`NumPy`](https://numpy.org/)
(<2.0) | The fundamental package for scientific computing with Python. | Always | -| [`yt`](https://yt-project.org/) | The core analytic tool. | Always | -| [`yt_libyt`](https://github.com/data-exp-lab/yt_libyt) | `yt` frontend for `libyt`. | Always | -| [`mpi4py`](https://mpi4py.readthedocs.io/en/stable/) | Python bindings for the Message Passing Interface (MPI) standard.
{octicon}`alert;1em;sd-text-danger;` Make sure `mpi4py` used in Python and MPI used in simulation are matched. (Check how to install `mpi4py` [here](https://mpi4py.readthedocs.io/en/stable/install.html#installation).) | `-DSERIAL_MODE=OFF` | -| [`jedi`](https://jedi.readthedocs.io/en/latest/) | Support auto-completion in Jupyter Notebook and JupyterLab. (We will have this if IPython is already installed.) | `-DJUPYTER_KERNEL=ON` | -| [`jupyter-client`](https://jupyter-client.readthedocs.io/en/latest/index.html)
(>=8.0.0) | Jupyter Client. | `-DJUPYTER_KERNEL=ON` | -| [`jupyter_libyt`](https://github.com/yt-project/jupyter_libyt) | Jupyter kernel provisioner for `libyt`. | `-DJUPYTER_KERNEL=ON` | +| Python package | Required version | Notes | Option | +|--------------------------------------------------------------------------------|------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-----------------------| +| [`NumPy`](https://numpy.org/) | <2.0 | The fundamental package for scientific computing with Python. | Always | +| [`yt`](https://yt-project.org/) | | The core analytic tool. | Always | +| [`yt_libyt`](https://github.com/data-exp-lab/yt_libyt) | | `yt` frontend for `libyt`. | Always | +| [`mpi4py`](https://mpi4py.readthedocs.io/en/stable/) | | Python bindings for the Message Passing Interface (MPI) standard.
{octicon}`alert;1em;sd-text-danger;` Make sure `mpi4py` used in Python and MPI used in simulation are matched. (Check how to install `mpi4py` [here](https://mpi4py.readthedocs.io/en/stable/install.html#installation).) | `-DSERIAL_MODE=OFF` | +| [`jedi`](https://jedi.readthedocs.io/en/latest/) | | Support auto-completion in Jupyter Notebook and JupyterLab. (We will have this if IPython is already installed.) | `-DJUPYTER_KERNEL=ON` | +| [`jupyter-client`](https://jupyter-client.readthedocs.io/en/latest/index.html) | >=8.0.0 | Jupyter Client. | `-DJUPYTER_KERNEL=ON` | +| [`jupyter_libyt`](https://github.com/yt-project/jupyter_libyt) | | Jupyter kernel provisioner for `libyt`. | `-DJUPYTER_KERNEL=ON` | ::: > {octicon}`alert;1em;sd-text-danger;` `jupyter-client` and `jupyter_libyt` are used for launching Jupyter Notebook and JupyterLab. Make sure the Python environment used for launching the notebook have them installed. diff --git a/doc/how-to-install/how-to-install.md b/doc/how-to-install/how-to-install.md index de43454e..579d94a6 100644 --- a/doc/how-to-install/how-to-install.md +++ b/doc/how-to-install/how-to-install.md @@ -6,7 +6,7 @@ details ``` -```{note} +```{seealso} Go through this page to install `libyt` that is fault-tolerant to Python code, supports interactive Python prompt, supports reloading script, and supports Jupyter Notebook access. @@ -70,22 +70,20 @@ To turn off some of the features, go to [Details](./details.md). ## Python Dependencies -- **Python package** required when performing in situ analysis. -- **Notes** are things worth notice. - **Option** indicates under what circumstances will we need this package. :::{table} :width: 100% -| Python package | Notes | Option | -|-----------------------------------------------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-----------------------| -| [`NumPy`](https://numpy.org/)
(<2.0) | The fundamental package for scientific computing with Python. | Always | -| [`yt`](https://yt-project.org/) | The core analytic tool. | Always | -| [`yt_libyt`](https://github.com/data-exp-lab/yt_libyt) | `yt` frontend for `libyt`. | Always | -| [`mpi4py`](https://mpi4py.readthedocs.io/en/stable/) | Python bindings for the Message Passing Interface (MPI) standard.
{octicon}`alert;1em;sd-text-danger;` Make sure `mpi4py` used in Python and MPI used in simulation are matched. (Check how to install `mpi4py` [here](https://mpi4py.readthedocs.io/en/stable/install.html#installation).) | `-DSERIAL_MODE=OFF` | -| [`jedi`](https://jedi.readthedocs.io/en/latest/) | Support auto-completion in Jupyter Notebook and JupyterLab. (We will have this if IPython is already installed.) | `-DJUPYTER_KERNEL=ON` | -| [`jupyter-client`](https://jupyter-client.readthedocs.io/en/latest/index.html)
(>=8.0.0) | Jupyter Client. | `-DJUPYTER_KERNEL=ON` | -| [`jupyter_libyt`](https://github.com/yt-project/jupyter_libyt) | Jupyter kernel provisioner for `libyt`. | `-DJUPYTER_KERNEL=ON` | +| Python package | Required version | Notes | Option | +|--------------------------------------------------------------------------------|------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-----------------------| +| [`NumPy`](https://numpy.org/) | <2.0 | The fundamental package for scientific computing with Python. | Always | +| [`yt`](https://yt-project.org/) | | The core analytic tool. | Always | +| [`yt_libyt`](https://github.com/data-exp-lab/yt_libyt) | | `yt` frontend for `libyt`. | Always | +| [`mpi4py`](https://mpi4py.readthedocs.io/en/stable/) | | Python bindings for the Message Passing Interface (MPI) standard.
{octicon}`alert;1em;sd-text-danger;` Make sure `mpi4py` used in Python and MPI used in simulation are matched. (Check how to install `mpi4py` [here](https://mpi4py.readthedocs.io/en/stable/install.html#installation).) | `-DSERIAL_MODE=OFF` | +| [`jedi`](https://jedi.readthedocs.io/en/latest/) | | Support auto-completion in Jupyter Notebook and JupyterLab. (We will have this if IPython is already installed.) | `-DJUPYTER_KERNEL=ON` | +| [`jupyter-client`](https://jupyter-client.readthedocs.io/en/latest/index.html) | >=8.0.0 | Jupyter Client. | `-DJUPYTER_KERNEL=ON` | +| [`jupyter_libyt`](https://github.com/yt-project/jupyter_libyt) | | Jupyter kernel provisioner for `libyt`. | `-DJUPYTER_KERNEL=ON` | ::: > {octicon}`alert;1em;sd-text-danger;` `jupyter-client` and `jupyter_libyt` are used for launching Jupyter Notebook and JupyterLab. Make sure the Python environment used for launching the notebook have them installed. From 257709426f273fcced50eecf743976a37ad012d4 Mon Sep 17 00:00:00 2001 From: cindytsai Date: Wed, 8 May 2024 16:08:46 -0500 Subject: [PATCH 17/27] Move FAQs to a single page and them link them. --- doc/FAQs.md | 69 +++++++++++++++++++ doc/how-to-install/how-to-install.md | 16 +---- doc/in-situ-python-analysis/faq.md | 24 ------- doc/in-situ-python-analysis/index.md | 4 -- .../interactive-python-prompt.md | 11 +-- .../reloading-script.md | 7 +- doc/in-situ-python-analysis/using-yt.md | 16 +---- 7 files changed, 76 insertions(+), 71 deletions(-) delete mode 100644 doc/in-situ-python-analysis/faq.md diff --git a/doc/FAQs.md b/doc/FAQs.md index e8846a7b..98e5a97d 100644 --- a/doc/FAQs.md +++ b/doc/FAQs.md @@ -1,2 +1,71 @@ # FAQs +## When Installing + +### Get Errors when Using CMake + +Make sure the folder where CMake generates build files is empty or not exist yet by removing the folder: +```bash +cd libyt +rm -rf +cmake -S . -B +``` + +### Unable to Link to Dependencies Fetched by libyt After Installation + +Keep `libyt` project repo after installation. `libyt` fetches and stores dependencies under `libyt/vendor` folder, so that the content can be reused in different builds. + +--- + +## When Running Applications + +### How Does libyt Run Python Script? +`libyt` runs Python script synchronously, which means every MPI process runs the same piece of Python code. +They do the job together under the process space of MPI tasks using [`mpi4py`](https://mpi4py.readthedocs.io/en/stable/index.html). + +### Have Problems when Running libyt in Parallel Mode + +If we happen to get errors related to one-sided MPI, for example: +```text +*** An error occurred in MPI_Win_attach +*** reported by process [3353411585,1] +*** on win rdma window 3 +*** MPI_ERR_RMA_ATTACH: Could not attach RMA segment +*** MPI_ERRORS_ARE_FATAL (processes in this win will now abort, +*** and potentially your MPI job) +3 more processes have sent help message help-mpi-errors.txt / mpi_errors_are_fatal +Set MCA parameter "orte_base_help_aggregate" to 0 to see all help / error messages +``` +Remember to set `OMPI_MCA_osc=sm,pt2pt` before running the application. It is for one-sided MPI communication. For example: + +```bash +OMPI_MCA_osc=sm,pt2pt mpirun -np 3 ./example +``` + +### Why Does my Program Hang and How Do I Solve It? +Though `libyt` can execute any Python module, when it comes to reading simulation data, it requires every MPI process to participate. +The program hanging problem is due to only some MPI processes are accessing the data, but not all of them. + +**Please do**: +1. Check if there is an `if` statements that makes MPI processes non-symmetric. For example, only root process runs the statement: + ```python + def func(): + if yt.is_root(): + ... # <-- This statement only executes in MPI root rank + ``` +2. Move the statement out of `if yt.is_root()` (for the case here). + +> {octicon}`calendar;1em;sd-text-secondary;` When accessing simulation data, `libyt` requires every process to participate. +> We are working on this in both `yt` and `libyt`. + +### Have Problems when Using Interactive Prompt + +#### Why Can't I Find the Prompt `>>>`? +`>>> ` is probably immersed inside the output. +We can hit enter again, which is to provide an empty statement, and it will come out. + +We can make prompt more smoothly by setting [`YT_VERBOSE`](./libyt-api/yt_initialize.md#yt_param_libyt) to `YT_VERBOSE_INFO`. + +#### Where Can I Use Interactive Mode? +`libyt` interactive Python prompt only works on local machine or submit the job to HPC platforms using interactive jobs like `qsub -I` in PBS scheduler. +The reason is that the user interface is exposed in the terminal. diff --git a/doc/how-to-install/how-to-install.md b/doc/how-to-install/how-to-install.md index 579d94a6..52821e62 100644 --- a/doc/how-to-install/how-to-install.md +++ b/doc/how-to-install/how-to-install.md @@ -6,7 +6,7 @@ details ``` -```{seealso} +```{note} Go through this page to install `libyt` that is fault-tolerant to Python code, supports interactive Python prompt, supports reloading script, and supports Jupyter Notebook access. @@ -93,15 +93,5 @@ To turn off some of the features, go to [Details](./details.md). ## FAQs -### Get Errors when Using CMake - -Make sure the folder where CMake generates build files to is empty or not exist yet by removing the folder: -```bash -cd libyt -rm -rf -cmake -S . -B -``` - -### Unable to Link to Dependencies Fetched by libyt After Installation - -Keep `libyt` project repo after installation. `libyt` fetches and stores dependencies under `libyt/vendor` folder, so that the content can be reused in different builds. +- [Get Errors when Using CMake](../FAQs.md#get-errors-when-using-cmake) +- [Unable to Link to Dependencies Fetched by libyt After Installation](../FAQs.md#unable-to-link-to-dependencies-fetched-by-libyt-after-installation) diff --git a/doc/in-situ-python-analysis/faq.md b/doc/in-situ-python-analysis/faq.md deleted file mode 100644 index c7335828..00000000 --- a/doc/in-situ-python-analysis/faq.md +++ /dev/null @@ -1,24 +0,0 @@ -# FAQs - -## How Does libyt Run Python Script? -`libyt` runs Python script synchronously, which means every MPI process runs the same piece of Python code. -They do the job together under the process space of MPI tasks using [`mpi4py`](https://mpi4py.readthedocs.io/en/stable/index.html). - -## Have Problems when Running libyt in Parallel - -If we happen to get errors related to MPI, for example: -```text -*** An error occurred in MPI_Win_attach -*** reported by process [3353411585,1] -*** on win rdma window 3 -*** MPI_ERR_RMA_ATTACH: Could not attach RMA segment -*** MPI_ERRORS_ARE_FATAL (processes in this win will now abort, -*** and potentially your MPI job) -3 more processes have sent help message help-mpi-errors.txt / mpi_errors_are_fatal -Set MCA parameter "orte_base_help_aggregate" to 0 to see all help / error messages -``` -Remember to set `OMPI_MCA_osc=sm,pt2pt` before running the application. It is for one-sided MPI communication. For example: - -```bash -OMPI_MCA_osc=sm,pt2pt mpirun -np 3 ./example -``` \ No newline at end of file diff --git a/doc/in-situ-python-analysis/index.md b/doc/in-situ-python-analysis/index.md index f3c0db92..cc150f26 100644 --- a/doc/in-situ-python-analysis/index.md +++ b/doc/in-situ-python-analysis/index.md @@ -11,7 +11,6 @@ interactive-python-prompt reloading-script jupyter-notebook/jupyter-notebook-access limitation -faq ``` ## Using Python for In Situ Analysis in libyt @@ -31,6 +30,3 @@ faq - [**Limitations in MPI Related Python Tasks**](./limitation.md#limitations-in-mpi-related-python-tasks) -## FAQs - -- [**How Does libyt Run Python Script?**](./faq.md#how-does-libyt-run-python-script) diff --git a/doc/in-situ-python-analysis/interactive-python-prompt.md b/doc/in-situ-python-analysis/interactive-python-prompt.md index c1073e30..9ee94164 100644 --- a/doc/in-situ-python-analysis/interactive-python-prompt.md +++ b/doc/in-situ-python-analysis/interactive-python-prompt.md @@ -44,12 +44,5 @@ Then it will exit the API [`yt_run_InteractiveMode`](../libyt-api/yt_run_interac ## FAQs -### Why Can't I Find the Prompt `>>>`? -`>>> ` is probably immersed inside the output. -We can hit enter again, which is to provide an empty statement, and it will come out. - -We can make prompt more smoothly by setting [`YT_VERBOSE`](../libyt-api/yt_initialize.md#yt_param_libyt) to `YT_VERBOSE_INFO`. - -### Where Can I Use Interactive Mode? -`libyt` interactive Python prompt only works on local machine or submit the job to HPC platforms using interactive jobs like `qsub -I` in PBS scheduler. -The reason is that the user interface is exposed in the terminal. +- [Why Can't I Find the Prompt `>>>`?](../FAQs.md#why-cant-i-find-the-prompt-) +- [Where Can I Use Interactive Mode?](../FAQs.md#where-can-i-use-interactive-mode) \ No newline at end of file diff --git a/doc/in-situ-python-analysis/reloading-script.md b/doc/in-situ-python-analysis/reloading-script.md index 7e40a4a4..9dfbe643 100644 --- a/doc/in-situ-python-analysis/reloading-script.md +++ b/doc/in-situ-python-analysis/reloading-script.md @@ -56,10 +56,5 @@ def func(): ``` ## Known Limitations -- See [Limitations in MPI Related Python Tasks](./limitation.md#limitations-in-mpi-related-python-tasks). - -## FAQs -### When Can I Reload Script? -`libyt` supports reloading script feature if it is compiled with [`-DINTERACTIVE_MODE=ON`](../how-to-install/details.md#-dinteractive_mode-off). -The root process reads the file, so it would work on your local desktop and in HPC clusters. +- See [Limitations in MPI Related Python Tasks](./limitation.md#limitations-in-mpi-related-python-tasks). diff --git a/doc/in-situ-python-analysis/using-yt.md b/doc/in-situ-python-analysis/using-yt.md index b49d3a89..babb5e65 100644 --- a/doc/in-situ-python-analysis/using-yt.md +++ b/doc/in-situ-python-analysis/using-yt.md @@ -97,21 +97,7 @@ libyt inherits field information (ex: units, name aliases) defined in yt fronten ## FAQs -### Why Does my Program Hang and How Do I Solve It? -Though `libyt` can execute any Python module, when it comes to reading simulation data, it requires every MPI process to participate. -The program hanging problem is due to only some MPI processes are accessing the data, but not all of them. - -**Please do**: -1. Check if there is an `if` statements that makes MPI processes non-symmetric. For example, only root process runs the statement: - ```python - def func(): - if yt.is_root(): - ... # <-- This statement only executes in MPI root rank - ``` -2. Move the statement out of `if yt.is_root()` (for the case here). - -> {octicon}`calendar;1em;sd-text-secondary;` When accessing simulation data, `libyt` requires every process to participate. -> We are working on this in both `yt` and `libyt`. +- [Why Does my Program Hang and How Do I Solve It?](../FAQs.md#why-does-my-program-hang-and-how-do-i-solve-it) [^1]: `annotate_quiver`, `annotate_cquiver`, `annotate_velocity`, `annotate_line_integral_convolution`, `annotate_magnetic_field`, and `annotate_particles` From 8722941bcbe32891f4c00919e2144e90acfe3f60 Mon Sep 17 00:00:00 2001 From: cindytsai Date: Wed, 8 May 2024 18:36:02 -0500 Subject: [PATCH 18/27] Update python dependency notes and admonition. Still thinking whether should I add admonition. --- doc/how-to-install/details.md | 10 ++++++---- doc/how-to-install/how-to-install.md | 12 +++++++----- doc/quick-start.md | 4 +++- 3 files changed, 16 insertions(+), 10 deletions(-) diff --git a/doc/how-to-install/details.md b/doc/how-to-install/details.md index ee347d8a..4f1a3913 100644 --- a/doc/how-to-install/details.md +++ b/doc/how-to-install/details.md @@ -78,10 +78,12 @@ The options are mutually independent to each other. | [`jupyter_libyt`](https://github.com/yt-project/jupyter_libyt) | | Jupyter kernel provisioner for `libyt`. | `-DJUPYTER_KERNEL=ON` | ::: -> {octicon}`alert;1em;sd-text-danger;` `jupyter-client` and `jupyter_libyt` are used for launching Jupyter Notebook and JupyterLab. Make sure the Python environment used for launching the notebook have them installed. -> -> The Python used in in situ analysis which is also for compiling `libyt` and the Python for launching Jupyter Notebook/JupyterLab might be different. -> For example, when running `libyt` in HPC cluster and connecting to it through your local laptop. (See [Jupyter Notebook Access](../in-situ-python-analysis/jupyter-notebook/jupyter-notebook-access.md)) +```{note} +`jupyter-client` and `jupyter_libyt` are used for launching Jupyter Notebook and JupyterLab. Make sure the Python environment used for launching the notebook have them installed. + +The Python used in in situ analysis which is also for compiling `libyt` and the Python used for launching Jupyter Notebook/JupyterLab might be different. +For example, when running `libyt` in HPC cluster, the Python environment is different from the one starting Jupyter Notebook on your local laptop. (See [Jupyter Notebook Access](../in-situ-python-analysis/jupyter-notebook/jupyter-notebook-access.md)) +``` ## Step-by-Step Instructions 1. Toggle options, set paths and generate files to build the project. This can be done through either (a) or (b): diff --git a/doc/how-to-install/how-to-install.md b/doc/how-to-install/how-to-install.md index 52821e62..ad80ba7e 100644 --- a/doc/how-to-install/how-to-install.md +++ b/doc/how-to-install/how-to-install.md @@ -6,7 +6,7 @@ details ``` -```{note} +```{seealso} Go through this page to install `libyt` that is fault-tolerant to Python code, supports interactive Python prompt, supports reloading script, and supports Jupyter Notebook access. @@ -86,10 +86,12 @@ To turn off some of the features, go to [Details](./details.md). | [`jupyter_libyt`](https://github.com/yt-project/jupyter_libyt) | | Jupyter kernel provisioner for `libyt`. | `-DJUPYTER_KERNEL=ON` | ::: -> {octicon}`alert;1em;sd-text-danger;` `jupyter-client` and `jupyter_libyt` are used for launching Jupyter Notebook and JupyterLab. Make sure the Python environment used for launching the notebook have them installed. -> -> The Python used in in situ analysis which is also for compiling `libyt` and the Python for launching Jupyter Notebook/JupyterLab might be different. -> For example, when running `libyt` in HPC cluster and connecting to it through your local laptop. (See [Jupyter Notebook Access](../in-situ-python-analysis/jupyter-notebook/jupyter-notebook-access.md)) +```{note} +`jupyter-client` and `jupyter_libyt` are used for launching Jupyter Notebook and JupyterLab. Make sure the Python environment used for launching the notebook have them installed. + +The Python used in in situ analysis which is also for compiling `libyt` and the Python used for launching Jupyter Notebook/JupyterLab might be different. +For example, when running `libyt` in HPC cluster, the Python environment is different from the one starting Jupyter Notebook on your local laptop. (See [Jupyter Notebook Access](../in-situ-python-analysis/jupyter-notebook/jupyter-notebook-access.md)) +``` ## FAQs diff --git a/doc/quick-start.md b/doc/quick-start.md index 64da1415..73853cc1 100644 --- a/doc/quick-start.md +++ b/doc/quick-start.md @@ -7,7 +7,9 @@ After reading quick start, we will have an overview of how to use `libyt` for in ### Building and Running the Project -> {octicon}`info;1em;sd-text-info;` This demonstration assumes the operating system is either Linux or macOS. +```{note} +The quick start assumes the operating system is either Linux or macOS. +``` 1. Follow [Install](./how-to-install/how-to-install.md#libyt). 2. Enter quick start folder. Assume we build the project under `libyt/build`: From f52d3d987293feb61b46e88a68322a09d5d0f202 Mon Sep 17 00:00:00 2001 From: cindytsai Date: Thu, 9 May 2024 10:33:59 -0500 Subject: [PATCH 19/27] Add minimum required version of Python. --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 2c9a4053..1d969e57 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -35,7 +35,7 @@ option(BUILD_SHARED_LIBS "Building using shared libraries" ## find dependencies ## set(Python_ROOT_DIR ${PYTHON_PATH}) -find_package(Python COMPONENTS Development NumPy REQUIRED) +find_package(Python 3.7 COMPONENTS Development NumPy REQUIRED) if (NOT SERIAL_MODE) set(MPI_HOME ${MPI_PATH}) From 9a325264d5fb075ee9e1153cf69a5821a3ff7ecc Mon Sep 17 00:00:00 2001 From: cindytsai Date: Thu, 9 May 2024 11:20:03 -0500 Subject: [PATCH 20/27] Note --enable-shared if compile Python from source, and add FAQs. --- doc/FAQs.md | 25 +++++++++++++------ doc/how-to-install/how-to-install.md | 15 ++++++----- .../interactive-python-prompt.md | 4 +-- doc/in-situ-python-analysis/using-yt.md | 2 +- 4 files changed, 28 insertions(+), 18 deletions(-) diff --git a/doc/FAQs.md b/doc/FAQs.md index 98e5a97d..4d08ad57 100644 --- a/doc/FAQs.md +++ b/doc/FAQs.md @@ -2,7 +2,7 @@ ## When Installing -### Get Errors when Using CMake +### Get errors when using CMake Make sure the folder where CMake generates build files is empty or not exist yet by removing the folder: ```bash @@ -11,7 +11,18 @@ rm -rf cmake -S . -B ``` -### Unable to Link to Dependencies Fetched by libyt After Installation +### Get errors when linking shared library libyt.so + +If we compile Python from source, and we get errors like this when compiling `libyt`: +```text +ld: : relocation R_X86_64_32S against symbol `_Py_FalseStruct' can not be used when making a shared object; recompile with -fPIC +ld: failed to set dynamic section sizes: bad value +``` +We need to re-compile and install Python and add `--enable-shared` when configuring. + +The simplest way to avoid this is to install Python from package manager. + +### Unable to link to dependencies fetched by libyt after installation Keep `libyt` project repo after installation. `libyt` fetches and stores dependencies under `libyt/vendor` folder, so that the content can be reused in different builds. @@ -19,11 +30,11 @@ Keep `libyt` project repo after installation. `libyt` fetches and stores depende ## When Running Applications -### How Does libyt Run Python Script? +### How does libyt run Python script? `libyt` runs Python script synchronously, which means every MPI process runs the same piece of Python code. They do the job together under the process space of MPI tasks using [`mpi4py`](https://mpi4py.readthedocs.io/en/stable/index.html). -### Have Problems when Running libyt in Parallel Mode +### Have problems when running libyt in parallel mode If we happen to get errors related to one-sided MPI, for example: ```text @@ -42,7 +53,7 @@ Remember to set `OMPI_MCA_osc=sm,pt2pt` before running the application. It is fo OMPI_MCA_osc=sm,pt2pt mpirun -np 3 ./example ``` -### Why Does my Program Hang and How Do I Solve It? +### Why does my program hang and how do I solve it? Though `libyt` can execute any Python module, when it comes to reading simulation data, it requires every MPI process to participate. The program hanging problem is due to only some MPI processes are accessing the data, but not all of them. @@ -60,12 +71,12 @@ The program hanging problem is due to only some MPI processes are accessing the ### Have Problems when Using Interactive Prompt -#### Why Can't I Find the Prompt `>>>`? +#### Why can't I find the prompt `>>>`? `>>> ` is probably immersed inside the output. We can hit enter again, which is to provide an empty statement, and it will come out. We can make prompt more smoothly by setting [`YT_VERBOSE`](./libyt-api/yt_initialize.md#yt_param_libyt) to `YT_VERBOSE_INFO`. -#### Where Can I Use Interactive Mode? +#### Where can I use interactive mode? `libyt` interactive Python prompt only works on local machine or submit the job to HPC platforms using interactive jobs like `qsub -I` in PBS scheduler. The reason is that the user interface is exposed in the terminal. diff --git a/doc/how-to-install/how-to-install.md b/doc/how-to-install/how-to-install.md index ad80ba7e..ae05d042 100644 --- a/doc/how-to-install/how-to-install.md +++ b/doc/how-to-install/how-to-install.md @@ -22,13 +22,12 @@ To turn off some of the features, go to [Details](./details.md). - **Compiler** (>4.8): It should be able to support `c++14`. - `CXX`: Path to `g++` compiler. - `CC`: Path to `gcc` compiler. -- **[Python](https://www.python.org/)** (>=3.7): The Python environment we want to use when doing in situ analysis. - - `PYTHON_PATH`: Python installation prefix, the path contains folders `include`, `lib` etc. - - `NumPy`: Should have `NumPy` installed. - - Other [Python Dependencies](#python-dependencies) +- **[Python](https://www.python.org/)** (>=3.7): The Python environment we want to use when doing in situ analysis. (If we compile it from source, use `--enable-shared` when configuring.) + - `PYTHON_PATH`: Python installation prefix, the path contains folders `include`, `lib` etc. + - `numpy` and other [Python Dependencies](#python-dependencies) - **[OpenMPI](https://www.open-mpi.org/)** or **[MPICH](https://www.mpich.org/)**: MPI used for compiling simulations and `libyt` needs to be the same. - `MPI_PATH`: MPI installation prefix, the path contains folders `include`, `lib` etc. -- **[Readline](https://tiswww.case.edu/php/chet/readline/rltop.html)**: Readline library is already installed on Linux and macOS generally. If not, we can get through system package manager or compile from source ourselves. (Use `--with-curses` when configuring if we compile from source.) +- **[Readline](https://tiswww.case.edu/php/chet/readline/rltop.html)**: Readline library is already installed on Linux and macOS generally. If not, we can get through system package manager or compile from source ourselves. (If we compile it from source, use `--with-curses` when configuring.) - `READLINE_PATH`: `readline` installation prefix. Provide the path if it is not in system search path. **Follow the steps to install `libyt` that is fault-tolerant to Python code, and supports interactive Python prompt, reloading script, and Jupyter Notebook access:** @@ -67,7 +66,6 @@ To turn off some of the features, go to [Details](./details.md). ``` 5. `libyt` is now installed at `` folder. - ## Python Dependencies - **Option** indicates under what circumstances will we need this package. @@ -95,5 +93,6 @@ For example, when running `libyt` in HPC cluster, the Python environment is diff ## FAQs -- [Get Errors when Using CMake](../FAQs.md#get-errors-when-using-cmake) -- [Unable to Link to Dependencies Fetched by libyt After Installation](../FAQs.md#unable-to-link-to-dependencies-fetched-by-libyt-after-installation) +- [Get errors when using CMake](../FAQs.md#get-errors-when-using-cmake) +- [Get errors when linking shared library libyt.so](../FAQs.md#get-errors-when-linking-shared-library-libytso) +- [Unable to link to dependencies fetched by libyt after installation](../FAQs.md#unable-to-link-to-dependencies-fetched-by-libyt-after-installation) diff --git a/doc/in-situ-python-analysis/interactive-python-prompt.md b/doc/in-situ-python-analysis/interactive-python-prompt.md index 9ee94164..69094aa4 100644 --- a/doc/in-situ-python-analysis/interactive-python-prompt.md +++ b/doc/in-situ-python-analysis/interactive-python-prompt.md @@ -44,5 +44,5 @@ Then it will exit the API [`yt_run_InteractiveMode`](../libyt-api/yt_run_interac ## FAQs -- [Why Can't I Find the Prompt `>>>`?](../FAQs.md#why-cant-i-find-the-prompt-) -- [Where Can I Use Interactive Mode?](../FAQs.md#where-can-i-use-interactive-mode) \ No newline at end of file +- [Why can't I find the prompt `>>>`?](../FAQs.md#why-cant-i-find-the-prompt-) +- [Where can I use interactive mode?](../FAQs.md#where-can-i-use-interactive-mode) \ No newline at end of file diff --git a/doc/in-situ-python-analysis/using-yt.md b/doc/in-situ-python-analysis/using-yt.md index babb5e65..a7ff6549 100644 --- a/doc/in-situ-python-analysis/using-yt.md +++ b/doc/in-situ-python-analysis/using-yt.md @@ -97,7 +97,7 @@ libyt inherits field information (ex: units, name aliases) defined in yt fronten ## FAQs -- [Why Does my Program Hang and How Do I Solve It?](../FAQs.md#why-does-my-program-hang-and-how-do-i-solve-it) +- [Why does my program hang and how do I solve it?](../FAQs.md#why-does-my-program-hang-and-how-do-i-solve-it) [^1]: `annotate_quiver`, `annotate_cquiver`, `annotate_velocity`, `annotate_line_integral_convolution`, `annotate_magnetic_field`, and `annotate_particles` From a15ebc469bde16b11f2ef4788d3b804d856ba37d Mon Sep 17 00:00:00 2001 From: cindytsai Date: Thu, 9 May 2024 13:53:17 -0500 Subject: [PATCH 21/27] Add pip. --- doc/how-to-install/details.md | 18 +++++++++--------- doc/how-to-install/how-to-install.md | 18 +++++++++--------- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/doc/how-to-install/details.md b/doc/how-to-install/details.md index 4f1a3913..dc493de2 100644 --- a/doc/how-to-install/details.md +++ b/doc/how-to-install/details.md @@ -67,15 +67,15 @@ The options are mutually independent to each other. :::{table} :width: 100% -| Python package | Required version | Notes | Option | -|--------------------------------------------------------------------------------|------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-----------------------| -| [`NumPy`](https://numpy.org/) | <2.0 | The fundamental package for scientific computing with Python. | Always | -| [`yt`](https://yt-project.org/) | | The core analytic tool. | Always | -| [`yt_libyt`](https://github.com/data-exp-lab/yt_libyt) | | `yt` frontend for `libyt`. | Always | -| [`mpi4py`](https://mpi4py.readthedocs.io/en/stable/) | | Python bindings for the Message Passing Interface (MPI) standard.
{octicon}`alert;1em;sd-text-danger;` Make sure `mpi4py` used in Python and MPI used in simulation are matched. (Check how to install `mpi4py` [here](https://mpi4py.readthedocs.io/en/stable/install.html#installation).) | `-DSERIAL_MODE=OFF` | -| [`jedi`](https://jedi.readthedocs.io/en/latest/) | | Support auto-completion in Jupyter Notebook and JupyterLab. (We will have this if IPython is already installed.) | `-DJUPYTER_KERNEL=ON` | -| [`jupyter-client`](https://jupyter-client.readthedocs.io/en/latest/index.html) | >=8.0.0 | Jupyter Client. | `-DJUPYTER_KERNEL=ON` | -| [`jupyter_libyt`](https://github.com/yt-project/jupyter_libyt) | | Jupyter kernel provisioner for `libyt`. | `-DJUPYTER_KERNEL=ON` | +| Python package | pip | Required version | Notes | Option | +|--------------------------------------------------------------------------------|------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-----------------------| +| [`NumPy`](https://numpy.org/) | `numpy` | <2.0 | The fundamental package for scientific computing with Python. | Always | +| [`yt`](https://yt-project.org/) | `yt` | | The core analytic tool. | Always | +| [`yt_libyt`](https://github.com/data-exp-lab/yt_libyt) | `yt-libyt` | | `yt` frontend for `libyt`. | Always | +| [`mpi4py`](https://mpi4py.readthedocs.io/en/stable/) | `mpi4py` | | Python bindings for the Message Passing Interface (MPI) standard.
{octicon}`alert;1em;sd-text-danger;` Make sure `mpi4py` used in Python and MPI used in simulation are matched. (Check how to install `mpi4py` [here](https://mpi4py.readthedocs.io/en/stable/install.html#installation).) | `-DSERIAL_MODE=OFF` | +| [`jedi`](https://jedi.readthedocs.io/en/latest/) | `jedi` | | Support auto-completion in Jupyter Notebook and JupyterLab. (We will have this if IPython is already installed.) | `-DJUPYTER_KERNEL=ON` | +| [`jupyter-client`](https://jupyter-client.readthedocs.io/en/latest/index.html) | `jupyter-client` | >=8.0.0 | Jupyter Client. | `-DJUPYTER_KERNEL=ON` | +| [`jupyter_libyt`](https://github.com/yt-project/jupyter_libyt) | `jupyter-libyt` | | Jupyter kernel provisioner for `libyt`. | `-DJUPYTER_KERNEL=ON` | ::: ```{note} diff --git a/doc/how-to-install/how-to-install.md b/doc/how-to-install/how-to-install.md index ae05d042..ad3a1b7b 100644 --- a/doc/how-to-install/how-to-install.md +++ b/doc/how-to-install/how-to-install.md @@ -73,15 +73,15 @@ To turn off some of the features, go to [Details](./details.md). :::{table} :width: 100% -| Python package | Required version | Notes | Option | -|--------------------------------------------------------------------------------|------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-----------------------| -| [`NumPy`](https://numpy.org/) | <2.0 | The fundamental package for scientific computing with Python. | Always | -| [`yt`](https://yt-project.org/) | | The core analytic tool. | Always | -| [`yt_libyt`](https://github.com/data-exp-lab/yt_libyt) | | `yt` frontend for `libyt`. | Always | -| [`mpi4py`](https://mpi4py.readthedocs.io/en/stable/) | | Python bindings for the Message Passing Interface (MPI) standard.
{octicon}`alert;1em;sd-text-danger;` Make sure `mpi4py` used in Python and MPI used in simulation are matched. (Check how to install `mpi4py` [here](https://mpi4py.readthedocs.io/en/stable/install.html#installation).) | `-DSERIAL_MODE=OFF` | -| [`jedi`](https://jedi.readthedocs.io/en/latest/) | | Support auto-completion in Jupyter Notebook and JupyterLab. (We will have this if IPython is already installed.) | `-DJUPYTER_KERNEL=ON` | -| [`jupyter-client`](https://jupyter-client.readthedocs.io/en/latest/index.html) | >=8.0.0 | Jupyter Client. | `-DJUPYTER_KERNEL=ON` | -| [`jupyter_libyt`](https://github.com/yt-project/jupyter_libyt) | | Jupyter kernel provisioner for `libyt`. | `-DJUPYTER_KERNEL=ON` | +| Python package | pip | Required version | Notes | Option | +|--------------------------------------------------------------------------------|------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-----------------------| +| [`NumPy`](https://numpy.org/) | `numpy` | <2.0 | The fundamental package for scientific computing with Python. | Always | +| [`yt`](https://yt-project.org/) | `yt` | | The core analytic tool. | Always | +| [`yt_libyt`](https://github.com/data-exp-lab/yt_libyt) | `yt-libyt` | | `yt` frontend for `libyt`. | Always | +| [`mpi4py`](https://mpi4py.readthedocs.io/en/stable/) | `mpi4py` | | Python bindings for the Message Passing Interface (MPI) standard.
{octicon}`alert;1em;sd-text-danger;` Make sure `mpi4py` used in Python and MPI used in simulation are matched. (Check how to install `mpi4py` [here](https://mpi4py.readthedocs.io/en/stable/install.html#installation).) | `-DSERIAL_MODE=OFF` | +| [`jedi`](https://jedi.readthedocs.io/en/latest/) | `jedi` | | Support auto-completion in Jupyter Notebook and JupyterLab. (We will have this if IPython is already installed.) | `-DJUPYTER_KERNEL=ON` | +| [`jupyter-client`](https://jupyter-client.readthedocs.io/en/latest/index.html) | `jupyter-client` | >=8.0.0 | Jupyter Client. | `-DJUPYTER_KERNEL=ON` | +| [`jupyter_libyt`](https://github.com/yt-project/jupyter_libyt) | `jupyter-libyt` | | Jupyter kernel provisioner for `libyt`. | `-DJUPYTER_KERNEL=ON` | ::: ```{note} From a74453d436ee7760f1bc62d574e229d566b80d49 Mon Sep 17 00:00:00 2001 From: cindytsai Date: Fri, 10 May 2024 11:06:16 -0500 Subject: [PATCH 22/27] format requirements and remove label meanings at home page. --- doc/how-to-install/how-to-install.md | 24 +++++++++++++++++------- doc/index.md | 5 ----- 2 files changed, 17 insertions(+), 12 deletions(-) diff --git a/doc/how-to-install/how-to-install.md b/doc/how-to-install/how-to-install.md index ad3a1b7b..959cbcd7 100644 --- a/doc/how-to-install/how-to-install.md +++ b/doc/how-to-install/how-to-install.md @@ -17,17 +17,27 @@ To turn off some of the features, go to [Details](./details.md). **Requirements:** -- **[CMake](https://cmake.org/)** (>=3.15) -- **[pkg-config](https://www.freedesktop.org/wiki/Software/pkg-config/)**: Generally, Linux and macOS already have pkg-config installed. -- **Compiler** (>4.8): It should be able to support `c++14`. - - `CXX`: Path to `g++` compiler. +**[CMake](https://cmake.org/)** (>=3.15) +: A software build system. + +**[pkg-config](https://www.freedesktop.org/wiki/Software/pkg-config/)** +: Generally, Linux and macOS already have pkg-config installed. It is a helper tool for finding libraries and compiling applications. + +**Compiler** (It should support `c++14`) +: - `CXX`: Path to `g++` compiler. - `CC`: Path to `gcc` compiler. -- **[Python](https://www.python.org/)** (>=3.7): The Python environment we want to use when doing in situ analysis. (If we compile it from source, use `--enable-shared` when configuring.) + +**[Python](https://www.python.org/)** (>=3.7) +: The Python environment we want to use when doing in situ analysis. If we compile it from source, use `--enable-shared` when configuring. - `PYTHON_PATH`: Python installation prefix, the path contains folders `include`, `lib` etc. - `numpy` and other [Python Dependencies](#python-dependencies) -- **[OpenMPI](https://www.open-mpi.org/)** or **[MPICH](https://www.mpich.org/)**: MPI used for compiling simulations and `libyt` needs to be the same. + +**[OpenMPI](https://www.open-mpi.org/)** or **[MPICH](https://www.mpich.org/)** +: MPI used for compiling simulations and for compiling `libyt` should be the same. - `MPI_PATH`: MPI installation prefix, the path contains folders `include`, `lib` etc. -- **[Readline](https://tiswww.case.edu/php/chet/readline/rltop.html)**: Readline library is already installed on Linux and macOS generally. If not, we can get through system package manager or compile from source ourselves. (If we compile it from source, use `--with-curses` when configuring.) + +**[Readline](https://tiswww.case.edu/php/chet/readline/rltop.html)** +: Readline library is already installed on Linux and macOS generally. If not, we can get through system package manager or compile from source ourselves. If we compile it from source, use `--with-curses` when configuring. - `READLINE_PATH`: `readline` installation prefix. Provide the path if it is not in system search path. **Follow the steps to install `libyt` that is fault-tolerant to Python code, and supports interactive Python prompt, reloading script, and Jupyter Notebook access:** diff --git a/doc/index.md b/doc/index.md index 3a388bc7..e313adc7 100644 --- a/doc/index.md +++ b/doc/index.md @@ -37,8 +37,3 @@ debug-and-profiling/index - [**In Situ Python Analysis**](./in-situ-python-analysis/index.md): this is for users that are using `libyt` to do in situ analysis. - [**Debug and Time Profiling**](./debug-and-profiling/index.md): how to check inputs and run time profiling in `libyt`. -#### Label Meanings -- {octicon}`pencil;1em;sd-text-warning;` means `libyt` only borrows the variable and does not make a copy. -- {octicon}`info;1em;sd-text-info;` means more information on this topic. -- {octicon}`alert;1em;sd-text-danger;` means things we should be careful. -- {octicon}`calendar;1em;sd-text-secondary;` means things we are trying to improve. From 16b8012b5c6a34a687130da4fa9ffc43c2f6bd70 Mon Sep 17 00:00:00 2001 From: cindytsai Date: Fri, 10 May 2024 11:32:17 -0500 Subject: [PATCH 23/27] Update content. --- doc/index.md | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/doc/index.md b/doc/index.md index e313adc7..6b1ada2c 100644 --- a/doc/index.md +++ b/doc/index.md @@ -29,11 +29,16 @@ debug-and-profiling/index ``` #### Contents + +**For users:** - [**Quick Start**](./quick-start.md) +- [**Example**](./example.md): this demonstrates how to implement `libyt` in an amr simulation step by step. - [**How to Install**](./how-to-install/how-to-install.md#how-to-install): how to get `libyt`? - [**How it Works**](./how-it-works.md): what is happening behind the scene? -- [**Example**](./example.md): this demonstrates how to implement `libyt` in simulation step by step. -- [**libyt API**](./libyt-api/index.md): this is for simulation developers that wish to implement `libyt`. - [**In Situ Python Analysis**](./in-situ-python-analysis/index.md): this is for users that are using `libyt` to do in situ analysis. +- [**FAQs**](./FAQs.md) + +**For simulation developers:** +- [**libyt API**](./libyt-api/index.md): this is for simulation developers that wish to implement `libyt`. - [**Debug and Time Profiling**](./debug-and-profiling/index.md): how to check inputs and run time profiling in `libyt`. From 97dee95006dab4119a1d984931854d0cdf9d4abb Mon Sep 17 00:00:00 2001 From: cindytsai Date: Fri, 10 May 2024 11:54:19 -0500 Subject: [PATCH 24/27] use runner image macos-12, it supports python3.7~3.12. macos-latest which is now macos-14 only supports python 3.11 and 3.12 --- .github/workflows/cmake-build-test.yml | 2 +- .github/workflows/example-test-run.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/cmake-build-test.yml b/.github/workflows/cmake-build-test.yml index 9754a562..3f39679a 100644 --- a/.github/workflows/cmake-build-test.yml +++ b/.github/workflows/cmake-build-test.yml @@ -25,7 +25,7 @@ jobs: - os: ubuntu-latest mpi: 'openmpi' check_shared_lib: ldd lib/libyt.so - - os: macOS-latest + - os: macos-12 mpi: 'mpich' check_shared_lib: otool -l lib/libyt.dylib python-version: ['3.7', '3.8', '3.9', '3.10', '3.11', '3.12'] diff --git a/.github/workflows/example-test-run.yml b/.github/workflows/example-test-run.yml index effc4905..6302b836 100644 --- a/.github/workflows/example-test-run.yml +++ b/.github/workflows/example-test-run.yml @@ -24,7 +24,7 @@ jobs: platform: - os: ubuntu-latest mpi: 'openmpi' - - os: macOS-latest + - os: macos-12 mpi: 'mpich' python-version: ['3.7', '3.8', '3.9', '3.10', '3.11', '3.12'] From eb27c96ef38b6254677db76b6859ecff4d38c7a2 Mon Sep 17 00:00:00 2001 From: cindytsai Date: Sun, 19 May 2024 01:49:35 -0500 Subject: [PATCH 25/27] set velocity_unit in yt_param_yt. --- .../libyt-python-module.md | 1 + doc/libyt-api/yt_set_parameters.md | 27 ++++++++++--------- example/amr-example/example.cpp | 18 ++++++------- include/yt_type_param_yt.h | 3 +++ src/check_data.cpp | 1 + src/print_data.cpp | 1 + src/yt_set_Parameters.cpp | 1 + 7 files changed, 31 insertions(+), 21 deletions(-) diff --git a/doc/in-situ-python-analysis/libyt-python-module.md b/doc/in-situ-python-analysis/libyt-python-module.md index b6b17139..06497d51 100644 --- a/doc/in-situ-python-analysis/libyt-python-module.md +++ b/doc/in-situ-python-analysis/libyt-python-module.md @@ -34,6 +34,7 @@ import libyt | `param_yt["length_unit"]` | `length_unit` | `yt_set_Parameters` | | | `param_yt["mass_unit"]` | `mass_unit` | `yt_set_Parameters` | | | `param_yt["time_unit"]` | `time_unit` | `yt_set_Parameters` | | +| `param_yt["velocity_unit"]` | `velocity_unit` | `yt_set_Parameters` | | | `param_yt["magnetic_unit"]` | `magnetic_unit` | `yt_set_Parameters` | - Will be set to 1, if it's not set. | | `param_yt["cosmological_simulation"]` | `cosmological_simulation` | `yt_set_Parameters` | | | `param_yt["dimensionality"]` | `dimensionality` | `yt_set_Parameters` | | diff --git a/doc/libyt-api/yt_set_parameters.md b/doc/libyt-api/yt_set_parameters.md index 10291b3b..b59520f7 100644 --- a/doc/libyt-api/yt_set_parameters.md +++ b/doc/libyt-api/yt_set_parameters.md @@ -33,6 +33,8 @@ int yt_set_Parameters( yt_param_yt *param_yt ) - Usage: Simulation mass unit in g (CGS). - `double time_unit` (Default=`DBL_UNDEFINED`) - Usage: Simulation time unit in s (CGS). +- `double velocity_unit` (Default=`DBL_UNDEFINED`) + - Usage: Simulation velocity unit in cm / s (CGS). - `double magnetic_unit` (Default=`1.0`) - Usage: Simulation magnetic unit in gauss. - `int periodicity[3]` (Default=`INT_UNDEFINED`) @@ -72,18 +74,19 @@ int yt_set_Parameters( yt_param_yt *param_yt ) ## Example ```cpp yt_param_yt param_yt; -param_yt.frontend = "gamer"; // simulation frontend that libyt borrows field info from -param_yt.fig_basename = "FigName"; // figure base name (default=Fig) -param_yt.length_unit = 3.0857e21; // length unit (cm) -param_yt.mass_unit = 1.9885e33; // mass unit (g) -param_yt.time_unit = 3.1557e13; // time unit (sec) -param_yt.current_time = time; // simulation time in code units -param_yt.dimensionality = 3; // dimensionality, support 3 only -param_yt.refine_by = REFINE_BY; // refinement factor between a grid and its subgrid -param_yt.num_grids = num_grids; // number of grids -param_yt.num_grids_local = num_grids_local; // number of local grids -param_yt.num_fields = num_fields + 1; // number of fields, addition one for derived field demo -param_yt.num_par_types = num_par_types; // number of particle types +param_yt.frontend = "gamer"; // simulation frontend that libyt borrows field info from +param_yt.fig_basename = "FigName"; // figure base name (default=Fig) +param_yt.length_unit = 3.0857e21; // length unit (cm) +param_yt.mass_unit = 1.9885e33; // mass unit (g) +param_yt.time_unit = 3.1557e13; // time unit (sec) +param_yt.velocity_unit = param_yt.length_unit / param_yt.time_unit; // velocity unit (cm/s) +param_yt.current_time = time; // simulation time in code units +param_yt.dimensionality = 3; // dimensionality, support 3 only +param_yt.refine_by = REFINE_BY; // refinement factor between a grid and its subgrid +param_yt.num_grids = num_grids; // number of grids +param_yt.num_grids_local = num_grids_local; // number of local grids +param_yt.num_fields = num_fields + 1; // number of fields, addition one for derived field demo +param_yt.num_par_types = num_par_types; // number of particle types yt_par_type par_type_list[num_par_types]; par_type_list[0].par_type = "io"; diff --git a/example/amr-example/example.cpp b/example/amr-example/example.cpp index cea69639..8e0d1cd6 100644 --- a/example/amr-example/example.cpp +++ b/example/amr-example/example.cpp @@ -162,19 +162,19 @@ int main(int argc, char* argv[]) { // libyt: 2. provide YT-specific parameters // ========================================== yt_param_yt param_yt; - param_yt.frontend = "gamer"; // simulation frontend that libyt borrows field info from - param_yt.fig_basename = "FigName"; // figure base name (default=Fig) - param_yt.length_unit = 3.0857e21; // length unit (cm) - param_yt.mass_unit = 1.9885e33; // mass unit (g) - param_yt.time_unit = 3.1557e13; // time unit (sec) - param_yt.current_time = time; // simulation time in code units - param_yt.dimensionality = 3; // dimensionality, support 3 only + param_yt.frontend = "gamer"; // simulation frontend that libyt borrows field info from + param_yt.fig_basename = "FigName"; // figure base name (default=Fig) + param_yt.length_unit = 3.0857e21; // length unit (cm) + param_yt.mass_unit = 1.9885e33; // mass unit (g) + param_yt.time_unit = 3.1557e13; // time unit (sec) + param_yt.velocity_unit = param_yt.length_unit / param_yt.time_unit; // velocity unit (cm/s) + param_yt.current_time = time; // simulation time in code units + param_yt.dimensionality = 3; // dimensionality, support 3 only param_yt.refine_by = REFINE_BY; // refinement factor between a grid and its subgrid - param_yt.index_offset = 1; // grid id starts at 1. (default is 0-indexed) param_yt.num_grids = num_grids; // number of grids param_yt.num_grids_local = num_grids_local; // number of local grids param_yt.num_fields = num_fields + 1; // number of fields, addition one for derived field demo - param_yt.num_par_types = num_par_types; // number of particle types (or species) + param_yt.num_par_types = num_par_types; // number of particle types yt_par_type par_type_list[num_par_types]; par_type_list[0].par_type = "io"; diff --git a/include/yt_type_param_yt.h b/include/yt_type_param_yt.h index e513a197..c63fe889 100644 --- a/include/yt_type_param_yt.h +++ b/include/yt_type_param_yt.h @@ -27,6 +27,7 @@ // length_unit : Simulation length unit in cm (CGS) // mass_unit : Simulation mass unit in g (CGS) // time_unit : Simulation time unit in s (CGS) +// velocity_unit : Simulation velocity unit in cm / s (CGS) // magnetic_unit : Simulation magnetic unit in gauss // refine_by : Refinement factor between a grid and its subgrid // index_offset : Offset of the index. @@ -53,6 +54,7 @@ typedef struct yt_param_yt { double length_unit; double mass_unit; double time_unit; + double velocity_unit; double magnetic_unit; int periodicity[3]; int cosmological_simulation; @@ -90,6 +92,7 @@ typedef struct yt_param_yt { length_unit = DBL_UNDEFINED; mass_unit = DBL_UNDEFINED; time_unit = DBL_UNDEFINED; + velocity_unit = DBL_UNDEFINED; magnetic_unit = DBL_UNDEFINED; for (int d = 0; d < 3; d++) { diff --git a/src/check_data.cpp b/src/check_data.cpp index 6d7f103e..0775d213 100644 --- a/src/check_data.cpp +++ b/src/check_data.cpp @@ -345,6 +345,7 @@ int check_yt_param_yt(const yt_param_yt& param_yt) { if (param_yt.length_unit == DBL_UNDEFINED) YT_ABORT("\"%s\" has not been set!\n", "length_unit"); if (param_yt.mass_unit == DBL_UNDEFINED) YT_ABORT("\"%s\" has not been set!\n", "mass_unit"); if (param_yt.time_unit == DBL_UNDEFINED) YT_ABORT("\"%s\" has not been set!\n", "time_unit"); + if (param_yt.velocity_unit == DBL_UNDEFINED) YT_ABORT("\"%s\" has not been set!\n", "velocity_unit"); if (param_yt.magnetic_unit == DBL_UNDEFINED) log_warning("\"%s\" has not been set!\n", "magnetic_unit"); for (int d = 0; d < 3; d++) { diff --git a/src/print_data.cpp b/src/print_data.cpp index dba9c0da..b5123cc8 100644 --- a/src/print_data.cpp +++ b/src/print_data.cpp @@ -37,6 +37,7 @@ int print_yt_param_yt(const yt_param_yt& param_yt) { log_debug(" %-*s = %13.7e\n", width_scalar, "length_unit", param_yt.length_unit); log_debug(" %-*s = %13.7e\n", width_scalar, "mass_unit", param_yt.mass_unit); log_debug(" %-*s = %13.7e\n", width_scalar, "time_unit", param_yt.time_unit); + log_debug(" %-*s = %13.7e\n", width_scalar, "velocity_unit", param_yt.velocity_unit); if (param_yt.magnetic_unit == DBL_UNDEFINED) log_debug(" %-*s = %s\n", width_scalar, "magnetic_unit", "NOT SET, and will be set to 1."); else diff --git a/src/yt_set_Parameters.cpp b/src/yt_set_Parameters.cpp index 620d2330..cfad1995 100644 --- a/src/yt_set_Parameters.cpp +++ b/src/yt_set_Parameters.cpp @@ -83,6 +83,7 @@ int yt_set_Parameters(yt_param_yt* param_yt) { add_dict_scalar(g_py_param_yt, "length_unit", g_param_yt.length_unit); add_dict_scalar(g_py_param_yt, "mass_unit", g_param_yt.mass_unit); add_dict_scalar(g_py_param_yt, "time_unit", g_param_yt.time_unit); + add_dict_scalar(g_py_param_yt, "velocity_unit", g_param_yt.velocity_unit); if (g_param_yt.magnetic_unit == DBL_UNDEFINED) { add_dict_scalar(g_py_param_yt, "magnetic_unit", 1); From 848eeac89fa2ade181f6014a9b621fe16ceb5315 Mon Sep 17 00:00:00 2001 From: cindytsai Date: Wed, 29 May 2024 22:03:39 -0500 Subject: [PATCH 26/27] Remove uint, ulong typedef. --- doc/libyt-api/yt_set_userparameter.md | 16 +++++------ example/amr-example/example.cpp | 4 +-- include/libyt.h | 4 +-- include/yt_type.h | 4 --- src/add_dict.cpp | 39 +++++++++++++++------------ src/yt_set_UserParameter.cpp | 18 ++++++++----- 6 files changed, 45 insertions(+), 40 deletions(-) diff --git a/doc/libyt-api/yt_set_userparameter.md b/doc/libyt-api/yt_set_userparameter.md index 1029b823..a68023ef 100644 --- a/doc/libyt-api/yt_set_userparameter.md +++ b/doc/libyt-api/yt_set_userparameter.md @@ -2,14 +2,14 @@ ## `yt_set_UserParameter*` ```cpp -int yt_set_UserParameterInt ( const char *key, const int n, const int *input ); -int yt_set_UserParameterLong ( const char *key, const int n, const long *input ); -int yt_set_UserParameterLongLong ( const char *key, const int n, const long long *input ); -int yt_set_UserParameterUint ( const char *key, const int n, const uint *input ); -int yt_set_UserParameterUlong ( const char *key, const int n, const ulong *input ); -int yt_set_UserParameterFloat ( const char *key, const int n, const float *input ); -int yt_set_UserParameterDouble ( const char *key, const int n, const double *input ); -int yt_set_UserParameterString ( const char *key, const char *input ); +int yt_set_UserParameterInt ( const char *key, const int n, const int *input ); +int yt_set_UserParameterLong ( const char *key, const int n, const long *input ); +int yt_set_UserParameterLongLong ( const char *key, const int n, const long long *input ); +int yt_set_UserParameterUint ( const char *key, const int n, const unsigned int *input ); +int yt_set_UserParameterUlong ( const char *key, const int n, const unsigned long *input ); +int yt_set_UserParameterFloat ( const char *key, const int n, const float *input ); +int yt_set_UserParameterDouble ( const char *key, const int n, const double *input ); +int yt_set_UserParameterString ( const char *key, const char *input ); ``` - Usage: Add code or user specific parameters that is used in your input yt [`frontend`](./yt_set_parameters.md#yt_param_yt) `XXXDataset` class. `libyt` borrows field information (`class XXXFieldInfo`), so we need to set them properly. `libyt` will add them to `libytDataset` class as new attributes. You can also add whatever variables you like, it will be stored under [`libyt.param_user`](../in-situ-python-analysis/libyt-python-module.md#param_user). - Return: `YT_SUCCESS` or `YT_FAIL` diff --git a/example/amr-example/example.cpp b/example/amr-example/example.cpp index 8e0d1cd6..c176f265 100644 --- a/example/amr-example/example.cpp +++ b/example/amr-example/example.cpp @@ -211,8 +211,8 @@ int main(int argc, char* argv[]) { // demo of some other parameters we can add const int user_int = 1; const long user_long = 2; - const uint user_uint = 3; - const ulong user_ulong = 4; + const unsigned int user_uint = 3; + const unsigned long user_ulong = 4; const float user_float = 5.0; const double user_double = 6.0; const char* user_string = "my_string"; diff --git a/include/libyt.h b/include/libyt.h index 99fdf79b..9f42096e 100644 --- a/include/libyt.h +++ b/include/libyt.h @@ -33,8 +33,8 @@ int yt_get_GridsPtr(yt_grid** grids_local); int yt_set_UserParameterInt(const char* key, const int n, const int* input); int yt_set_UserParameterLong(const char* key, const int n, const long* input); int yt_set_UserParameterLongLong(const char* key, const int n, const long long* input); -int yt_set_UserParameterUint(const char* key, const int n, const uint* input); -int yt_set_UserParameterUlong(const char* key, const int n, const ulong* input); +int yt_set_UserParameterUint(const char* key, const int n, const unsigned int* input); +int yt_set_UserParameterUlong(const char* key, const int n, const unsigned long* input); int yt_set_UserParameterFloat(const char* key, const int n, const float* input); int yt_set_UserParameterDouble(const char* key, const int n, const double* input); int yt_set_UserParameterString(const char* key, const char* input); diff --git a/include/yt_type.h b/include/yt_type.h index 1eac796a..d37a058b 100644 --- a/include/yt_type.h +++ b/include/yt_type.h @@ -7,10 +7,6 @@ / ********************************************************************************/ -// short names for unsigned types -typedef unsigned int uint; -typedef unsigned long int ulong; - // enumerate types typedef enum yt_verbose { YT_VERBOSE_OFF = 0, YT_VERBOSE_INFO, YT_VERBOSE_WARNING, YT_VERBOSE_DEBUG } yt_verbose; diff --git a/src/add_dict.cpp b/src/add_dict.cpp index f6de16c3..d7258063 100644 --- a/src/add_dict.cpp +++ b/src/add_dict.cpp @@ -7,9 +7,10 @@ // Function : add_dict_scalar // Description : Auxiliary function for adding a scalar item to a Python dictionary // -// Note : 1. Overloaded with various data types: float, double, int, long, uint, ulong -// ==> (float,double) are converted to double internally -// (int,long,uint,ulong) are converted to long internally +// Note : 1. Overloaded with various data types: float, double, int, long, unsigned int, unsigned long +// ==> (float,double) are converted to double internally +// (int,long,unsigned int,unsigned long) are converted to long internally +// (long long) are converted to long long internally // // Parameter : dict : Target Python dictionary // key : Dictionary key @@ -31,15 +32,16 @@ int add_dict_scalar(PyObject* dict, const char* key, const T value) { if (typeid(T) == typeid(float) || typeid(T) == typeid(double)) py_obj = PyFloat_FromDouble((double)value); - else if (typeid(T) == typeid(int) || typeid(T) == typeid(long) || typeid(T) == typeid(uint) || - typeid(T) == typeid(ulong)) + else if (typeid(T) == typeid(int) || typeid(T) == typeid(long) || typeid(T) == typeid(unsigned int) || + typeid(T) == typeid(unsigned long)) py_obj = PyLong_FromLong((long)value); else if (typeid(T) == typeid(long long)) py_obj = PyLong_FromLongLong((long long)value); else - YT_ABORT("Unsupported data type (only support float, double, int, long, long long, uint, ulong)!\n"); + YT_ABORT( + "Unsupported data type (only support float, double, int, long, long long, unsigned int, unsigned long)!\n"); // insert "value" into "dict" with "key" if (PyDict_SetItemString(dict, key, py_obj) != 0) @@ -56,10 +58,10 @@ int add_dict_scalar(PyObject* dict, const char* key, const T value) { // Function : add_dict_vector_n // Description : Auxiliary function for adding an n-element vector item to a Python dictionary // -// Note : 1. Overloaded with various data types: float, double, int, long, uint, ulong -// ==> (float,double) are converted to double internally -// (int,long,uint,ulong) are converted to long internally -// (long long) are converted to long long internally +// Note : 1. Overloaded with various data types: float, double, int, long, unsigned int, unsigned long +// ==> (float,double) are converted to double internally +// (int,long,unsigned int,unsigned long) are converted to long internally +// (long long) are converted to long long internally // // Parameter : dict : Target Python dictionary // key : Dictionary key @@ -84,8 +86,8 @@ int add_dict_vector_n(PyObject* dict, const char* key, const int len, const T* v for (Py_ssize_t v = 0; v < VecSize; v++) { PyTuple_SET_ITEM(tuple, v, PyFloat_FromDouble((double)vector[v])); } - } else if (typeid(T) == typeid(int) || typeid(T) == typeid(long) || typeid(T) == typeid(uint) || - typeid(T) == typeid(ulong)) { + } else if (typeid(T) == typeid(int) || typeid(T) == typeid(long) || typeid(T) == typeid(unsigned int) || + typeid(T) == typeid(unsigned long)) { for (Py_ssize_t v = 0; v < VecSize; v++) { PyTuple_SET_ITEM(tuple, v, PyLong_FromLong((long)vector[v])); } @@ -94,7 +96,8 @@ int add_dict_vector_n(PyObject* dict, const char* key, const int len, const T* v PyTuple_SET_ITEM(tuple, v, PyLong_FromLongLong((long long)vector[v])); } } else { - YT_ABORT("Unsupported data type (only support float, double, int, long, long long, uint, ulong)!\n"); + YT_ABORT("Unsupported data type (only support float, double, int, long, long long, unsigned int, unsigned " + "long)!\n"); } } else { YT_ABORT("Creating a tuple object (key = \"%s\") ... failed!\n", key); @@ -148,16 +151,18 @@ template int add_dict_scalar(PyObject* dict, const char* key, const doub template int add_dict_scalar(PyObject* dict, const char* key, const int value); template int add_dict_scalar(PyObject* dict, const char* key, const long value); template int add_dict_scalar(PyObject* dict, const char* key, const long long value); -template int add_dict_scalar(PyObject* dict, const char* key, const uint value); -template int add_dict_scalar(PyObject* dict, const char* key, const ulong value); +template int add_dict_scalar(PyObject* dict, const char* key, const unsigned int value); +template int add_dict_scalar(PyObject* dict, const char* key, const unsigned long value); template int add_dict_vector_n(PyObject* dict, const char* key, const int len, const float* vector); template int add_dict_vector_n(PyObject* dict, const char* key, const int len, const double* vector); template int add_dict_vector_n(PyObject* dict, const char* key, const int len, const int* vector); template int add_dict_vector_n(PyObject* dict, const char* key, const int len, const long* vector); template int add_dict_vector_n(PyObject* dict, const char* key, const int len, const long long* vector); -template int add_dict_vector_n(PyObject* dict, const char* key, const int len, const uint* vector); -template int add_dict_vector_n(PyObject* dict, const char* key, const int len, const ulong* vector); +template int add_dict_vector_n(PyObject* dict, const char* key, const int len, + const unsigned int* vector); +template int add_dict_vector_n(PyObject* dict, const char* key, const int len, + const unsigned long* vector); //------------------------------------------------------------------------------------------------------- // Function : add_dict_field_list diff --git a/src/yt_set_UserParameter.cpp b/src/yt_set_UserParameter.cpp index a1ddca0b..9c8be4f8 100644 --- a/src/yt_set_UserParameter.cpp +++ b/src/yt_set_UserParameter.cpp @@ -16,8 +16,8 @@ static const int MaxParamNameWidth = 15; // Description : Add code-specific parameters // // Note : 1. All code-specific parameters are stored in "libyt.param_user" -// 2. Overloaded with various data types: float, double, int, long, long long, uint, ulong, -// char* +// 2. Overloaded with various data types: float, double, int, long, long long, unsigned int, +// unsigned long, char* // ==> But do not use c++ template since I don't know how to instantiating template // without function name mangling ... // @@ -33,8 +33,12 @@ int yt_set_UserParameterLong(const char* key, const int n, const long* input) { int yt_set_UserParameterLongLong(const char* key, const int n, const long long* input) { return add_nonstring(key, n, input); } -int yt_set_UserParameterUint(const char* key, const int n, const uint* input) { return add_nonstring(key, n, input); } -int yt_set_UserParameterUlong(const char* key, const int n, const ulong* input) { return add_nonstring(key, n, input); } +int yt_set_UserParameterUint(const char* key, const int n, const unsigned int* input) { + return add_nonstring(key, n, input); +} +int yt_set_UserParameterUlong(const char* key, const int n, const unsigned long* input) { + return add_nonstring(key, n, input); +} int yt_set_UserParameterFloat(const char* key, const int n, const float* input) { return add_nonstring(key, n, input); } int yt_set_UserParameterDouble(const char* key, const int n, const double* input) { return add_nonstring(key, n, input); @@ -54,7 +58,7 @@ static int add_nonstring(const char* key, const int n, const T* input) { // export data to libyt.param_user if (typeid(T) == typeid(float) || typeid(T) == typeid(double) || typeid(T) == typeid(int) || - typeid(T) == typeid(long) || typeid(T) == typeid(uint) || typeid(T) == typeid(ulong) || + typeid(T) == typeid(long) || typeid(T) == typeid(unsigned int) || typeid(T) == typeid(unsigned long) || typeid(T) == typeid(long long)) { // scalar and 3-element array if (n == 1) { @@ -63,8 +67,8 @@ static int add_nonstring(const char* key, const int n, const T* input) { if (add_dict_vector_n(g_py_param_user, key, n, input) == YT_FAIL) return YT_FAIL; } } else { - YT_ABORT( - "Unsupported data type (only support char*, float*, double*, int*, long*, long long*, uint*, ulong*)!\n"); + YT_ABORT("Unsupported data type (only support char*, float*, double*, int*, long*, long long*, unsigned int*, " + "unsigned long*)!\n"); } log_debug("Inserting code-specific parameter \"%-*s\" ... done\n", MaxParamNameWidth, key); From a8c45c73bc81b9328fa9826a7c4a3938684c9b45 Mon Sep 17 00:00:00 2001 From: cindytsai Date: Thu, 30 May 2024 20:43:53 -0500 Subject: [PATCH 27/27] typo I think I accidentally removed index_offset in the example. --- example/amr-example/example.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/example/amr-example/example.cpp b/example/amr-example/example.cpp index c176f265..cc48e7ea 100644 --- a/example/amr-example/example.cpp +++ b/example/amr-example/example.cpp @@ -171,6 +171,7 @@ int main(int argc, char* argv[]) { param_yt.current_time = time; // simulation time in code units param_yt.dimensionality = 3; // dimensionality, support 3 only param_yt.refine_by = REFINE_BY; // refinement factor between a grid and its subgrid + param_yt.index_offset = 1; // grid id starts at 1. (default is 0-indexed) param_yt.num_grids = num_grids; // number of grids param_yt.num_grids_local = num_grids_local; // number of local grids param_yt.num_fields = num_fields + 1; // number of fields, addition one for derived field demo