diff --git a/.cproject b/.cproject new file mode 100644 index 0000000..64218e3 --- /dev/null +++ b/.cproject @@ -0,0 +1,42 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..6867e71 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,11 @@ +# To get started with Dependabot version updates, you'll need to specify which +# package ecosystems to update and where the package manifests are located. +# Please see the documentation for all configuration options: +# https://docs.github.com/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file + +version: 2 +updates: + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "weekly" diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 0000000..58de1ec --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,53 @@ +name: Build + +on: + push: + paths: + - 'src/**' + - 'include/**' + - 'Makefile' + - '*.mk' + - '.github/workflows/build.yml' + +jobs: + + build: + name: Build library + + runs-on: ubuntu-latest + env: + CC: gcc + XCHANGE: ../xchange + REDISX: ../redisx + steps: + - name: Check out smax-clib + uses: actions/checkout@v4 + with: + path: smax-clib + + - name: Check out xchange dependency + uses: actions/checkout@v4 + with: + repository: Smithsonian/xchange + path: xchange + + - name: Check out RedisX dependency + uses: actions/checkout@v4 + with: + repository: Smithsonian/redisx + path: redisx + + - name: Build static library + run: make -C smax-clib static + + - name: Build xchange dependency + run: make -C xchange shared + + - name: Build xchange dependency + run: make -C redisx shared + + - name: Build shared library + run: make -C smax-clib shared + + + diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml new file mode 100644 index 0000000..ec99360 --- /dev/null +++ b/.github/workflows/check.yml @@ -0,0 +1,43 @@ +name: Static Analysis + +on: + push: + paths: + - 'src/**' + - 'include/**' + - 'Makefile' + - '*.mk' + - '.github/workflows/check.yml' + +jobs: + + cppcheck: + name: Check source code + + runs-on: ubuntu-latest + env: + CC: gcc + XCHANGE: xchange + REDISX: redisx + steps: + + - name: Check out smax-clib + uses: actions/checkout@v4 + + - name: Check out xchange dependency + uses: actions/checkout@v4 + with: + repository: Smithsonian/xchange + path: xchange + + - name: Check out RedisX dependency + uses: actions/checkout@v4 + with: + repository: Smithsonian/redisx + path: redisx + + - name: install package dependencies + run: sudo apt-get install -y cppcheck + + - name: Run cppcheck + run: make check diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml new file mode 100644 index 0000000..04e84e8 --- /dev/null +++ b/.github/workflows/codeql.yml @@ -0,0 +1,95 @@ +# For most projects, this workflow file will not need changing; you simply need +# to commit it to your repository. +# +# You may wish to alter this file to override the set of languages analyzed, +# or to provide custom queries or build logic. +# +# ******** NOTE ******** +# We have attempted to detect the languages in your repository. Please check +# the `language` matrix defined below to confirm you have the correct set of +# supported CodeQL languages. +# +name: "CodeQL Advanced" + +on: + push: + branches: [ "main" ] + pull_request: + branches: [ "main" ] + schedule: + - cron: '35 18 * * 5' + +jobs: + analyze: + name: Analyze (${{ matrix.language }}) + # Runner size impacts CodeQL analysis time. To learn more, please see: + # - https://gh.io/recommended-hardware-resources-for-running-codeql + # - https://gh.io/supported-runners-and-hardware-resources + # - https://gh.io/using-larger-runners (GitHub.com only) + # Consider using larger runners or machines with greater resources for possible analysis time improvements. + runs-on: ${{ (matrix.language == 'swift' && 'macos-latest') || 'ubuntu-latest' }} + permissions: + # required for all workflows + security-events: write + + # required to fetch internal or private CodeQL packs + packages: read + + # only required for workflows in private repositories + actions: read + contents: read + + strategy: + fail-fast: false + matrix: + include: + - language: c-cpp + build-mode: autobuild + # CodeQL supports the following values keywords for 'language': 'c-cpp', 'csharp', 'go', 'java-kotlin', 'javascript-typescript', 'python', 'ruby', 'swift' + # Use `c-cpp` to analyze code written in C, C++ or both + # Use 'java-kotlin' to analyze code written in Java, Kotlin or both + # Use 'javascript-typescript' to analyze code written in JavaScript, TypeScript or both + # To learn more about changing the languages that are analyzed or customizing the build mode for your analysis, + # see https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/customizing-your-advanced-setup-for-code-scanning. + # If you are analyzing a compiled language, you can modify the 'build-mode' for that language to customize how + # your codebase is analyzed, see https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/codeql-code-scanning-for-compiled-languages + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + path: smax-clib + + - name: Checkout xchange dependency + uses: actions/checkout@v4 + with: + repository: Smithsonian/xchange + path: xchange + + - name: Checkout RedisX dependency + uses: actions/checkout@v4 + with: + repository: Smithsonian/redisx + path: redisx + + # Initializes the CodeQL tools for scanning. + - name: Initialize CodeQL + uses: github/codeql-action/init@v3 + with: + languages: ${{ matrix.language }} + build-mode: manual + + - name: Manual build + shell: bash + env: + XCHANGE: ../xchange + REDISX: ../redisx + run: | + make -C xchange shared + make -C redisx shared + make -C smax-clib shared + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v3 + with: + working-directory: smax-clib + category: "/language:${{matrix.language}}" diff --git a/.github/workflows/dox.yml b/.github/workflows/dox.yml new file mode 100644 index 0000000..f282372 --- /dev/null +++ b/.github/workflows/dox.yml @@ -0,0 +1,150 @@ +name: API documentation + +on: + release: + types: [published] + + push: + paths: + - 'src/**' + - 'include/**' + - 'css/**' + - 'resources/**' + - 'Doxyfile' + - '*.md' + - '.github/workflows/dox.yml' + +jobs: + + apidocs: + name: Generate API documentation + + runs-on: ubuntu-latest + env: + CC: gcc + XCHANGE: xchange + REDISX: redisx + steps: + + - name: install doxygen + run: sudo apt-get install doxygen + + - name: Check out smax-clib + uses: actions/checkout@v4 + + - name: Generate docs + run: make local-dox + + site-update: + name: Update github pages + needs: apidocs + if: github.repository_owner == 'Smithsonian' && (github.event_name == 'release' || contains(github.event.head_commit.message, 'site update')) + + runs-on: ubuntu-latest + steps: + + - name: Checkout smax-clib + uses: actions/checkout@v4 + with: + repository: Smithsonian/smax-clib + path: smax-clib + + - name: Check out xchange dependency + uses: actions/checkout@v4 + with: + repository: Smithsonian/xchange + path: xchange + + - name: Check out RedisX dependency + uses: actions/checkout@v4 + with: + repository: Smithsonian/redisx + path: redisx + + - name: Generate headless README for xchange + run: make -C xchange README-orig.md + + - name: Generate headless README for RedisX + run: make -C redisx README-orig.md + + - name: Generate xchange apidocs + uses: mattnotmitt/doxygen-action@v1.9.8 + with: + additional-packages: font-roboto + working-directory: ./xchange + + - name: Generate RedisX apidocs + uses: mattnotmitt/doxygen-action@v1.9.8 + with: + additional-packages: font-roboto + working-directory: ./redisx + + - name: Generate headless README for smax-clib + run: make -C redisx README-orig.md + + - name: Generate smax-clib apidocs + uses: mattnotmitt/doxygen-action@v1.9.8 + with: + additional-packages: font-roboto + working-directory: ./smax-clib + + - name: Checkout gh-pages + uses: actions/checkout@v4 + with: + ref: 'gh-pages' + path: site + + - name: Assert site/doc/ + run: mkdir -p site/doc + + - name: Copy README + run: | + echo 'CfA logo' > site/doc/README.md + echo '
' >> site/doc/README.md + cat redisx/README-orig.md >> site/doc/README.md + + - name: Copy CHANGELOG + run: cp redisx/CHANGELOG.md site/doc/ + + - name: Copy API documentation + run: cp -a redisx/apidoc site/ + + - name: Push to pages + run: | + cd site + git config --global user.email "$GITHUB_JOB+github-actions[bot]@users.noreply.github.com" + git config --global user.name "github-actions" + git add -A + git commit -m "[automated site update]" && git push || true + + + changelog-update: + name: Update CHANGELOG on github pages + if: github.repository_owner == 'Smithsonian' && contains(github.event.head_commit.message, 'changelog update') + + runs-on: ubuntu-latest + steps: + + - name: Checkout source + uses: actions/checkout@v4 + + - name: Checkout gh-pages + uses: actions/checkout@v4 + with: + ref: 'gh-pages' + path: site + + - name: Assert site/doc/ + run: mkdir -p site/doc + + - name: Copy CHANGELOG + run: cp CHANGELOG.md site/doc/ + + - name: Push to pages + run: | + cd site + git config --global user.email "$GITHUB_JOB+github-actions[bot]@users.noreply.github.com" + git config --global user.name "github-actions" + git add -A + git commit -m "[automated site update]" && git push || true + diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..17a57a4 --- /dev/null +++ b/.gitignore @@ -0,0 +1,13 @@ +**/apidoc +**/obj +**/bin +**/dep +**/lib +*.d +*.o +*~ +*.swp +README-orig.md +TODO +xchange/** +redisx/** diff --git a/.project b/.project new file mode 100644 index 0000000..7ddd480 --- /dev/null +++ b/.project @@ -0,0 +1,28 @@ + + + smax-clib + + + redisx + xchange + + + + org.eclipse.cdt.managedbuilder.core.genmakebuilder + clean,full,incremental, + + + + + org.eclipse.cdt.managedbuilder.core.ScannerConfigBuilder + full,incremental, + + + + + + org.eclipse.cdt.core.cnature + org.eclipse.cdt.managedbuilder.core.managedBuildNature + org.eclipse.cdt.managedbuilder.core.ScannerConfigNature + + diff --git a/.settings/language.settings.xml b/.settings/language.settings.xml new file mode 100644 index 0000000..2b6f529 --- /dev/null +++ b/.settings/language.settings.xml @@ -0,0 +1,15 @@ + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..6f5756d --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,12 @@ +# Changelog + +All notable changes to the [Smithsonian/redisx](https://github.com/Smithsonian/smax-clib) library will be +documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to +[Semantic Versioning](https://semver.org/spec/v2.0.0.html). + + +## [Unreleased] + +Initial public release. diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 0000000..9c72ea1 --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,128 @@ +# Contributor Covenant Code of Conduct + +## Our Pledge + +We as members, contributors, and leaders pledge to make participation in our +community a harassment-free experience for everyone, regardless of age, body +size, visible or invisible disability, ethnicity, sex characteristics, gender +identity and expression, level of experience, education, socio-economic status, +nationality, personal appearance, race, religion, or sexual identity +and orientation. + +We pledge to act and interact in ways that contribute to an open, welcoming, +diverse, inclusive, and healthy community. + +## Our Standards + +Examples of behavior that contributes to a positive environment for our +community include: + +* Demonstrating empathy and kindness toward other people +* Being respectful of differing opinions, viewpoints, and experiences +* Giving and gracefully accepting constructive feedback +* Accepting responsibility and apologizing to those affected by our mistakes, + and learning from the experience +* Focusing on what is best not just for us as individuals, but for the + overall community + +Examples of unacceptable behavior include: + +* The use of sexualized language or imagery, and sexual attention or + advances of any kind +* Trolling, insulting or derogatory comments, and personal or political attacks +* Public or private harassment +* Publishing others' private information, such as a physical or email + address, without their explicit permission +* Other conduct which could reasonably be considered inappropriate in a + professional setting + +## Enforcement Responsibilities + +Community leaders are responsible for clarifying and enforcing our standards of +acceptable behavior and will take appropriate and fair corrective action in +response to any behavior that they deem inappropriate, threatening, offensive, +or harmful. + +Community leaders have the right and responsibility to remove, edit, or reject +comments, commits, code, wiki edits, issues, and other contributions that are +not aligned to this Code of Conduct, and will communicate reasons for moderation +decisions when appropriate. + +## Scope + +This Code of Conduct applies within all community spaces, and also applies when +an individual is officially representing the community in public spaces. +Examples of representing our community include using an official e-mail address, +posting via an official social media account, or acting as an appointed +representative at an online or offline event. + +## Enforcement + +Instances of abusive, harassing, or otherwise unacceptable behavior may be +reported to the community leaders responsible for enforcement at +the Center for Astrophysics | Harvard & Smithsonian. +All complaints will be reviewed and investigated promptly and fairly. + +All community leaders are obligated to respect the privacy and security of the +reporter of any incident. + +## Enforcement Guidelines + +Community leaders will follow these Community Impact Guidelines in determining +the consequences for any action they deem in violation of this Code of Conduct: + +### 1. Correction + +**Community Impact**: Use of inappropriate language or other behavior deemed +unprofessional or unwelcome in the community. + +**Consequence**: A private, written warning from community leaders, providing +clarity around the nature of the violation and an explanation of why the +behavior was inappropriate. A public apology may be requested. + +### 2. Warning + +**Community Impact**: A violation through a single incident or series +of actions. + +**Consequence**: A warning with consequences for continued behavior. No +interaction with the people involved, including unsolicited interaction with +those enforcing the Code of Conduct, for a specified period of time. This +includes avoiding interactions in community spaces as well as external channels +like social media. Violating these terms may lead to a temporary or +permanent ban. + +### 3. Temporary Ban + +**Community Impact**: A serious violation of community standards, including +sustained inappropriate behavior. + +**Consequence**: A temporary ban from any sort of interaction or public +communication with the community for a specified period of time. No public or +private interaction with the people involved, including unsolicited interaction +with those enforcing the Code of Conduct, is allowed during this period. +Violating these terms may lead to a permanent ban. + +### 4. Permanent Ban + +**Community Impact**: Demonstrating a pattern of violation of community +standards, including sustained inappropriate behavior, harassment of an +individual, or aggression toward or disparagement of classes of individuals. + +**Consequence**: A permanent ban from any sort of public interaction within +the community. + +## Attribution + +This Code of Conduct is adapted from the [Contributor Covenant][homepage], +version 2.0, available at +https://www.contributor-covenant.org/version/2/0/code_of_conduct.html. + +Community Impact Guidelines were inspired by [Mozilla's code of conduct +enforcement ladder](https://github.com/mozilla/diversity). + +[homepage]: https://www.contributor-covenant.org + +For answers to common questions about this code of conduct, see the FAQ at +https://www.contributor-covenant.org/faq. Translations are available at +https://www.contributor-covenant.org/translations. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..df71794 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,43 @@ +# Contributing to RedisX + + +The _SMA-X C/C++_ library is for everyone. And, it is developers like you who can make it better. Whether there is a +nagging issue you would like to fix, or a new feature you'd like to see, you can make a difference yourself. Make this +project a little bit your own, by submitting pull requests with fixes and enhancement. When you are ready, here are +the typical steps for contributing to the project: + +1. Old or new __Issue__? Whether you just found a bug, or you are missing a much needed feature, start by checking +open (and closed) [Issues](https://github.com/Smithsonian/smax/issues). If an existing issue seems like a +good match to yours, feel free to speak up, comment, or to offer help in resolving it. If you find no issues that +match, go ahead and create a new one. + +2. __Fork__. Is it something you'd like to help resolve? Great! You should start by creating your own fork of the +repository so you can work freely on your solution. I recommend that you place your work on a branch of your fork, +which is named either after the issue number, e.g. `issue-192`, or some other descriptive name, such as +`sentinel-support`. + +3. __Develop__. Experiment on your fork/branch freely. If you run into a dead-end, you can always abandon it (which is +why branches are great) and start anew. You can run `make all` to ensure that all components of the package and its +API documentation are also in order. Remember to synchronize your `main` branch by fetching changes from upstream +every once in a while, and merging them into your development branch. Don't forget to: + + - Add __doxygen__ markup your new code. You can keep it sweet and simple, but make sure it properly explains your + globally exposed functions, their arguments and return values. You should also cross-reference other functions / + constants that are similar, related, or relevant to what you just added. + +4. __Pull Request__. Once you feel your work can be integrated, create a pull request from your fork/branch. You can +do that easily from the github page of your fork/branch directly. In the pull request, provide a concise description +of what you added or changed. Your pull request will be reviewed. You may get some feedback at this point, and maybe +there will be discussions about possible improvements or regressions etc. It's a good thing too, and your changes will +likely end up with added polish as a result. You can be all the more proud of it in the end! + +5. If all goes well, your pull-request will get merged, and will be included in the upcoming release of _smax-clib_. +Congratulations for your excellent work, and many thanks for dedicating some of your time for making this library a +little bit better. There will be many who will appreciate it. :-) + + +If at any point you have questions, or need feedback, don't be afraid to ask. You can put your questions into the +issue you found or created, or your pull-request, or as a Q&A in +[Discussions](https://github.com/Smithsonian/smax_clib/discussions). + + diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..fdddb29 --- /dev/null +++ b/LICENSE @@ -0,0 +1,24 @@ +This is free and unencumbered software released into the public domain. + +Anyone is free to copy, modify, publish, use, compile, sell, or +distribute this software, either in source code form or as a compiled +binary, for any purpose, commercial or non-commercial, and by any +means. + +In jurisdictions that recognize copyright laws, the author or authors +of this software dedicate any and all copyright interest in the +software to the public domain. We make this dedication for the benefit +of the public at large and to the detriment of our heirs and +successors. We intend this dedication to be an overt act of +relinquishment in perpetuity of all present and future rights to this +software under copyright law. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +OTHER DEALINGS IN THE SOFTWARE. + +For more information, please refer to diff --git a/README.md b/README.md index 6b77f95..cde9b21 100644 --- a/README.md +++ b/README.md @@ -1,73 +1,704 @@ -# SMA information exchange (SMA-X) over Redis +![Build Status](https://github.com/Smithsonian/smax-clib/actions/workflows/build.yml/badge.svg) +![Static Analysis](https://github.com/Smithsonian/smax-clib/actions/workflows/check.yml/badge.svg) + + ![API documentation](https://github.com/Smithsonian/smax-clib/actions/workflows/dox.yml/badge.svg) + + + ![Project page](https://github.com/Smithsonian/smax-clib/actions/workflows/pages/pages-build-deployment/badge.svg) + + + + + + CfA logo + +
+ +# SMA-X: SMA information exchange Author: Attila Kovacs -Last Updated: 7 January 2021 +Last Updated: 14 September 2024 +## Table of Contents + - [Introduction](#introduction) + - [Prerequisites](#prerequisites) + - [Building the SMA-X C library](#building) + - [Initial configuration](#configuration) + - [Connecting to / disconnecting from SMA-X](#connecting) + - [Sharing and pulling data](#sharing-and-pulling) + - [Lazy pulling](#lazy-pulling) + - [Pipelined pulls](#pipelined-pulls) + - [Custom notification and update handling](#notifications) + - [Optional metadata](#optional-metadata) + - [Error handling](#error-handling) + - [Debug support](#debug-support) + - [Future plans](#future-plans) -## 1. Introduction +------------------------------------------------------------------------------ -### 1.1. Features at a glance + +## Introduction -#### Pipelined (queued) pulls +The SMA information eXchange (SMA-X) is a high-throughput and versatile data sharing platform for distributed software +systems. It is built around a central Redis database, and provides atomic access to structured data, including +specific branches and/or leaf nodes, with associated metadadata. -#### Lazy pulling +SMA-X consists of a set of server-side [LUA](https://lua.org/) scripts that run on [Redis](https://redis.io) (or one +of its forks / clones such as [Valkey](https://valkey.io) or [Dragonfly](https://dragonfly.io)); a set of libraries to +interface client applications; and a set of command-line tools build with them. Currently we provide client libraries +for C/C++ and Python 3. We may provide Java and/or Rust client libraries too in the future. +### Features at a glance +TODO -## 2. Building your program with SMA-X +------------------------------------------------------------------------------ + +## Prerequisites -## 3. API description and usage +The SMA-X C/C++ library has a build and runtime dependency on the __xchange__ and __RedisX__ libraries also available +at the Smithsonian Github repositories: + - [Smithsonian/xchange](https://github.com/Smithsonian/xchange) + - [Smithsonian/redisx](https://github.com/Smithsonian/redisx) -### 3.1. Basic usage +Additionally, to configure your Redis (or Valkey / Dragonfly) servers for SMA-X, you will need the +[Smithsonian/smax-server](https://github.com/Smithsonian/smax-server) repo also. -#### Connecting / Disconnecting -#### Sharing & Pulling +------------------------------------------------------------------------------ -#### Standard metadata + +## Building the SMA-X C library +The __smax-clib__ library can be built either as a shared (`libsmax.so[.1]`) and as a static (`libsmax.a`) library, +depending on what suits your needs best. +You can configure the build, either by editing `config.mk` or else by defining the relevant environment variables +prior to invoking `make`. The following build variables can be configured: -### 3.2. Configuration + - `XCHANGE`: the root of the location where the [Smithsonian/xchange](https://github.com/Smithsonian/xchange) library + is installed. It expects to find `xchange.h` under `$(XCHANGE)/include` and `libxchange.so` under `$(XCHANGE)/lib` + or else in the default `LD_LIBRARY_PATH`. + + - `REDISX`: the root of the location where the [Smithsonian/redisx](https://github.com/Smithsonian/redisx) library + is installed. It expects to find `redisx.h` under `$(REDISX)/include` and `libredisx.so` under `$(REDISX)/lib` + or else in the default `LD_LIBRARY_PATH`. + + - `CC`: The C compiler to use (default: `gcc`). -### 3.3. Simplified (easy) access + - `CPPFLAGS`: C pre-processor flags, such as externally defined compiler constants. + + - `CFLAGS`: Flags to pass onto the C compiler (default: `-Os -Wall`). Note, `-Iinclude` will be added automatically. + + - `LDFLAGS`: Linker flags (default is `-lm`). Note, `-lxchange -lredisx` will be added automatically. + - `BUILD_MODE`: You can set it to `debug` to enable debugging features: it will initialize the global `xDebug` + variable to `TRUE` and add `-g` to `CFLAGS`. + - `CHECKEXTRA`: Extra options to pass to `cppcheck` for the `make check` target + +After configuring, you can simply run `make`, which will build the `shared` (`lib/libsmax.so[.1]`) and `static` +(`lib/libsmax.a`) libraries, local HTML documentation (provided `doxygen` is available), and performs static +analysis via the `check` target. Or, you may build just the components you are interested in, by specifying the +desired `make` target(s). (You can use `make help` to get a summary of the available `make` targets). -### 3.4. Pipelined (queued) pulling +------------------------------------------------------------------------------ -#### Synchronization points + +## Initial configuration -#### Callbacks +Bu default, the library assumes that the Redis server used for SMA-X runs on a machine called `smax` (e.g. you may assign +`smax` to the IP address in `/etc/hosts`), and that the Redis is on the default port 6379/tcp. However, you can configure +to use a specific host and/or an alternative Redis port number to use instead. -### 3.5. Lazy pulling +```c + smaxSetServer("my-smax.example.com", 7033); +``` +Also, while SMA-X will normally run on database index 0, you can also specify a different database number to use. E.g.: +```c + smaxSetDB(3); +``` -### 3.6. Notifications & (remote) updates +(Note, you can switch the database later also, but beware that if you have an active subscription client open, you cannot +switch that client until the subscriptions are terminated.) -#### Monitoring updates +You can also set up the authentication credentials for using the SMA-X database on the Redis server: -#### Waiting for updates +```c + smaxSetUser("johndoe"); + smaxSetPassword("mySecretPassword); +``` +And finally, you can select the option to automatically try reconnect to the SMA-X server in case of lost connection or +network errors (and keep track of changes locally until then): -### 3.7. Optional extra metadata +```c + smaxSetResilient(TRUE); +``` -#### Descriptions +------------------------------------------------------------------------------ -#### Physical units + +## Connecting to / disconnecting from SMA-X + +Once you have configured the connection parameters, you can connec to the server by: + +```c + int status = smaxConnect(); + if(status < 0) { + // Oops, we could not connect to the server + ... + } +``` + +And, when you are done, you should disconnect with: + +```c + smaxDisconnect(); +``` + +------------------------------------------------------------------------------ + + +## Sharing and pulling data + + - [The basics](#basics) + - [Flexible types and sizes](#flexible-types-and-sizes) + - [Standard metadata](#metadata) + - [Scalar quantities](#scalars) + - [Arrays](#arrays) + - [Structures / substructures](#structures) + + +### The basics + +For SMA-X we use the terms sharing and pulling, instead of the more generic get/set terminology. The intention is to +make programmers conscious of the fact that the transactions are not some local access to memory, but that they +involve networking, and as such may be subject to unpredictable latencies, network outages, or other errors. + +At the lowest level, the library provides two functions accordingly: `smaxShare()` and `smaxPull()`, either to send +local data to store in SMA-X, or to retrieve data from SMA-X for local use, respectively. There are higher-level +functions too, which build on these, providing a simpler API for specific data types, or extra features, such as +lazy or pipelined pulls. These will be discussed in the sections below. But, before that, let's look into the basics +of how data is handled between you machine local data that you use in your C/C++ application and its +machine-independent representation in SMA-X. + +Here is an example for a generic sharing of a `double[]` array from C/C++: + +```c + double data[8] = ... // your local data + + // Share (send) this data to SMA-X as "system:subsystem:some_data" + int status = smaxShare("system:subsystem", "some_data", X_DOUBLE, 8); + if(status < 0) { + // Ooops, that did not work... + ... + } + +``` + + + +Pulling data back from SMA-X works similarly, e.g.: + +```c + double data[4][2] = ... // buffer in which we will receive data + XMeta meta; // (optional) metadata we can obtain + + // Retrieve 'data' from the SMA-X "system:subsystem:some_data", as 8 doubles + int status = smaxPull("system:subsystem", "some_data", X_DOUBLE, 8, data, &meta); + if(status < 0) { + // Oops, something went wrong... + return; + } + +``` + +The metadata argument is optional, and can be `NULL` if not required. + + +### Flexible types and sizes + +One nifty feature of the library is that as a consumer you need not be too concerned about what type or size of data +the producer provides. The program that produces the data may sometimes change, for example from writing 32-bit +floating point types to a 64-bit floating point types. Also while it produced data for say 10 units before (as an +array of 10), now it might report for just 9, or perhaps it now reports for 12. + +The point is that if your consumer application was written to expect ten 32-bit floating floating point values, it can +get that even if the producer changed the exact type or element count since you have written your client. The library +will simply apply the necessaty type conversion automatically, and then truncate, or else pad (with zeroes), the data +as necessary to get what you want. + +The type conversion can be both widening or narrowing. Strings and numerical values can be converted to one another +through the expected string representation of numbers and vice versa. Boolean `true` values are treated equivalently +to a numerical value of 1 and all non-zero numerical values will convert to boolean `true`. + +And, if you are concerned about the actual type or size (or shape) of the data stored, you have the option to inspect +the metadata, and make decisions based on it. Otherwise, the library will just take care of giving you the available +data in the format you expect. + + + +### Standard metadata + +For every variable (structure or leaf node) in SMA-X there is also a set of essential metadata that is stored in the +Redis database, which describe the data themselves, such as the native type (at the origin); the size as stored in +Redis; the array dimension and shape; the host (and program) which provided the data; the time it was last updated; +and a serial number. + +The user of the library has the option to retrieve metadata together with the actual data, and thus gain access to +this information. The header file `smax.h` defines the `Xmeta` type as: + + +```c + typedef struct XMeta { + int status; // Error code or X_SUCCESS. + XType storeType; // Type of variable as stored. + int storeDim; // Dimensionality of the data as stored. + int storeSizes[X_MAX_DIMS]; // Sizes along each dimension of the data as stored. + int storeBytes; // Total number of bytes stored. + char origin[SMAX_ORIGIN_LENGTH]; // Host name that last modified. + struct timespec timestamp; // Timestamp of the last modification. + int serial; // Number of times the variable was updated. + } XMeta; +``` + + +### Scalar quantities + +Often enough we deal with scalar quantities (not arrays), such as a single number, boolean value, or a +string (yes, we treat strings i.e., `char *`, as a scalar too!). + +Here are some examples of sharing scalars to SMA-X. Easy-peasy: + +```c + int status; + + // Put a boolean value into SMA-X + status = smaxShareBoolean("system:subsystem", "is_online", TRUE); + + // Put an integer value into SMA-X + status = smaxShareInt("system:subsystem", "some_int_value", 1012); + + // Put a floating-point value into SMA-X + status = smaxShareDouble("system:subsystem", "some_value", -3.4032e-11); + + // Put a string value into SMA-X + status = smaxShareString("system:subsystem", "name", "blah-blah"); +``` + +Or pulling them from SMA-X: + +```c + // Retrieve "system:subsystem:is_online" as a boolean, defaulting to FALSE + boolean isTrue = smaxPullBoolean("system:subsystem", "is_online", FALSE); + + // Retrieve "system:subsystem:some_int_value" as an int, with a default value of -1 + int c = smaxPullInt("system:subsystem", "some_int_value", -1); + + // Retrieve "system:subsystem:some_value" as a double (or else NAN if cannot). + double value = smaxPullDouble("system:subsystem", "some_value"); + if(!isnan(value)) { // check for NAN if need be + ... + } + + // Retrieve "system:subsystem:name" as a 0-terminated C string (or NULL if cannot). + char *str = smaxPullDouble("system:subsystem", "name"); + if(str != NULL) { // check for NULL + ... + } + + ... + + // Once the pulled string is no longer needed, destroy it. + free(str) + +``` + + +### Arrays + +The generic `smaxShare()` function readily handles 1D arrays, and the `smaxPull()` handles native (monolithic) arrays +of all types (e.g. `double[][]`, or `boolean[][][]`). However, you may want to share multidimensional arrays, noting +their specific shapes, or else pull array data for which you may not know in advance what size (or shape) of the +storage needed locally for the values stored in SMA-X for some variable. For these reasons, the library provides a set +of functions to make the handling of arrays a simpler too. + +Let's begin with sharing multi-dimensional array. Instead of `smaxShare()`, you can use `smaxShareArray()` which allows +you to define the multi-dimensional shape of the data beyond just the number of elements stored. E.g.: + +``` + float data[4][2] = ... // Your local 2D data array + int shape[] = { 4, 2 }; // The array shape as stored in SMA-X + + int status = smaxShareArray("system:subsystem", "my_2d_array", X_FLOAT, 2, shape); + if (status < 0) { + // Oops, did not work... + ... + } +``` + +Note, that the dimensions of how the data is stored in SMA-X is determined solely by the '2' dimensions specified +as the 4th argument, and the corresponding 2 elements in the `shape` array. The `data` could have been any pointer +to an array of floats containing at least the required number of element (8 in the example above). + +For 1D arrays, you have some convenience methods for shharing specific types. These can be convenient because they +eliminate one potential source of bugs, where the type argument to `smaxShare()` does not match the pointer type of +the data. Using, say, `smaxShareFloats()` instead to share a 1D floating-point array instead will allow the compiler +to check and warn if the data array is not the `float *` type. E.g.: + +```c + float *data = ... // pointer to a one or higher-dimensional C array of floats. + + // Send N elements from data to SMA-X as "system:subsystem:my_array" + int status = smaxShareFloats("system:subsystem", "my_array", data, N); + if (status < 0) { + // Oops, did not work... + ... + } +``` + +Similar functions are available for every built-in type (plus strings and booleans). For pulling arrays without +knowing a-priori the element count or shape, there are also convenience functions, such as: + +``` + XMeta meta; // (optional) we'll return the metadata in this + int status; // we'll return the status in this + + // Get whatever data is in "system:subsystem:my_array" as doubles. + double *data = smaxPullDoubles("system:subsystem", "my_array", &meta, &status); + if(status < 0) { + // Oops, we got an error + } + if(data == NULL) { + // Oops the data is NULL + } + + ... + + // When done using the data we obtained, destroy it. + free(data); +``` + +As illustrated, the above will return a dynamically allocated array with the required size to hold the data, and the +size and shape of the data is returned in the metadata that was also supplied with the call. After using the returned +data (and ensuring it is not `NULL`), you should always be sure to call `free()` on it to avoid memory leaks in your +application. + + +### Structures / substructures... + +You can share entire data structures, represented by an appropriate `XStructure` type (see the __xchange__ library for +a description and usage): + +```c + XStructure s = ... // The structured data you have prepared locally. + + int status = smaxShareStruct("syste:subsystem", &s); + if(status < 0) { + // Oops, something did not work + ... + } +``` + +Or, you can read a structure, including all embedded substructures in it, with `smaxPullStruct()`: + + +```c + XMeta meta; + int nElements; + + XStructure *s = smaxPullStruct("system", "subsystem", &meta, &n); + if(n < 0) { + // Oops there was an error... + return; + } + + double value = smaxGetDoubleField(s, "some_value", 0.0); + + ... + + // Once the pulled structure is no longer needed, destroy it... + xDestroyStruct(s); +``` + +Note, that the structure returned by `smaxPullStruct()` is in the serialized format of SMA-X. That is, all leaf nodes +are stored as strings, just as they appear in the Redis database. Hence, we used the `smaxGet...Field()` methods above +to deserialize the leaf nodes as needed on demand. If you want to use the methods of __xchange__ to access the +structure, you will need to convert to binary format first, using `smax2xStruct(XStructure *)`. + +Note also, that pulling large structures can be an expensive operation on the Redis server, and may block the server +for longer than usual periods, causing latencies for other programs that use SMA-X. It's best to use this method for +smallish structures only (with, say, a hundred or so or fewer leaf nodes). + + +------------------------------------------------------------------------------ + + +## Lazy pulling (high-frequency queries) + +What happens if you need the data frequently? Do you pound on the database at some high-frequency? No, you probably +no not want to do that, especially if the data you need is not necessaily changing fast. There is no point on wasting +network bandwidth only to return the same values again and again. This is where 'lazy' pulling excels. + +From the caller's perspective lazy pulling works just like regular SMA-X pulls, e.g.: + +```c + int data[10][4][2]; + int sizes[] = { 10, 4, 2 }; + XMeta meta; + + int status = smaxLazyPull("some_table", "some_data", X_INT, 3, sizes, data, &meta); +``` + +or + +```c + int status = smaxLazyPullDouble("some_table", "some_var"); +``` + +But, under the hood, it does something different. The first time a new variable is lazy pulled it is fetched from the +Redis database just like a regular pull. But, it also will cache the value, and watch for update notifications from +the SMA-X server. Thus, as long as no update notification is received, successive calls will simply return the locally +cached value. This can save big on network usage, and also provides orders of magnitude faster access so long as the +variable remains unchanged. + +When the vatiable is updated in SMA-X, our client library will be notified, and one of two things can happen: + + 1. it invalidates the cache, so that the next lazy pull will again work just like a regular pull, fetching the + updated value from SMA-X on demand. And again the library will cache that value and watch for notifications for + the next update. Or, + + 2. it will trigger a background process to update the cached value in the background with a pipelined + (high-throughput) pull. However, until the new value is actually fetched, it will return the previously cached + value promptly. + +The choice between the two is yours, and you can control which suits your need best. The default behavior for lazy pulls +is (1), but you may call `smaxLazyCache()` after the first pull of a variable, to indicate that you want to enable +background cache updates (2) for it. The advantage of (1) is that it will never serve you outdated data even if there +are significant network latencies -- but you may have to wait a little to fetch updates. On the other hand (2) will +always provide a recent value with effectively no latency, but this value may be outdated if there are delays on the +network updating the cache. The difference is typically at the micro-seconds level on a local LAN. However, (2) may +be preferable when you need to access SMA-X data from timing critical code blocks, where it is more important to ensure +that the value is returned quickly, rather than whether it is a millisecond too old or not. + +In either case, when you are done using lazy variables, you should let the library know that it no longer needs to watch +updates for these, by calling either `smaxLazyEnd()` on specific variables, or else `smaxLazyFlush()` to stop watching +updates for all lazy variables. (A successive lazy pull will automatically start watching for updates again, in case you +wish to re-enable). + +```c + // Lazy pull a bunch of data (typically in a loop). + for(...) { + smaxLazyPull("some_table", "some_var", ...); + smaxLaxyPull(...); + ... + } + + // Once we do not need "some_table:some_var" any more: + smaxLazyEnd("some_table", "some_var"); + + ... + + // And to stop lazy accessing all + smaxLazyFlush(); +``` + + +------------------------------------------------------------------------------ + + +## Pipelined pulling (high volume data retrievals) + +The regular pulling of data from SMA-X requires a separate round-trip for each and every request. That is, successive +pulls are sent only after the responses from the prior pull has been received. A lot of the time is spent on waiting +for responses to come back. With round trip times in the 100 μs range, this means that this method of fetching data +from SMA-X is suitable for obtaining at most a a few thousand values per second. + +However, sometimes you want to get access to a large number of values faster. This is what pipelined pulling is for. +In pipelined mode, a batch of pull requests are sent to the SMA-X Redis server in quick succession, without waiting +for responses. The values, when received are processed by a dedicated background thread. And, the user has an option +of either waiting until all data is collected, or ask for as callback when the data is ready. + +Again it works similarly to the basic pulling, except that you submit your pull request to a queue with +`smaxQueue()`. For example: + +```c + double d; // A value we will fill + XMeta meta; // (optional) metadata to fill (for the above value). + + int status = smaxQueue("some_table", "some_var", X_DOUBLE, 1, &d, &meta); +``` + +Pipelined (batched) pulls have dramatic effects on performance. Rather than being limited by round-trip times, you will +be limited by the performance of the Redis server itself (or the network bandwidth on some older infrastructure). As +such, instead of thousand of queries per second, you can pull 2-3 orders of magnitude more in a given time, with hudreds +of thousands to even millions of pull per second this way. + +### Synchronization points and waiting + +After you have submitted a batch of pull request to the queue, you can create a synchronization point as: + +```c + XSyncPoint *syncPoint = smaxCreateSyncPoint(); +``` + +A synchronization point is a marker in the queue that we can wait on. After the synchronization point is created, you +can sumbit more pull request to the same queue (e.g. for another processing block), or do some other things for a bit +(since it will take at least some microseconds before the data is ready). Then, when ready you can wait on the +specific synchronization point to ensure that data submitted prior to its creation is delivered from SMA-X: + +``` + // Wait for data submitted prior to syncPoint to be ready, or time out after 1000 ms. + int status = smaxSync(syncPoint, 1000); + + // Destroy the synchronization point if we no longer need it. + xDestroySyncPoint(syncPoint); + + // Check return status... + if(status == X_TIMEOUT) { + // We timed out + ... + } + else if(status < 0) { + // Some other error + ... + } +``` + +### Callbacks + +The alternative to synchronization points and waiting, is to provide a callback function, which will process your data +as soon as it is available, e.g.: + +```c + void my_pull_processor(void *arg) { + // Say, we expect a string tag passed along to identify what we need to process... + char *tag = (char *) arg; + + // Do what we need to do... + ... + } +``` + +Then submit this callback routine to the queue after the set of variables it requires with: + +```c + // We'll call my_pull_processor, with the argument "some_tag", when prior data has arrived. + smaxQueueCallback(my_pull_processor, "some_tag"); +``` + +### Finishing up + +If you might still have some pending pipelined pulls that have not received responses yet, you may want to wait until +all previously sumbitted requests have been collected. You can do that with: + +```c + // Wait for up to 3000 ms for all pipelined pulls to collect responses from SMA-X. + int status = smaxWaitQueueComplete(3000); + + // Check return status... + if(status == X_TIMEOUT) { + // We timed out + ... + } + else if(status < 0) { + // Some other error + ... + } +``` + + + + +------------------------------------------------------------------------------ + +------------------------------------------------------------------------------ + + +## Custom notifications and update handling + +### Monitoring updates + +### Waiting for updates + +### Status / error messages + + +------------------------------------------------------------------------------ + + +## Optional metadata + + +### Descriptions + +### Coordinate Systems + +### Physical units #### Coordinate systems -### 3.8. Program messages +----------------------------------------------------------------------------- + + +## Error handling + +The principal error handling of the library is an extension of that of __xchange__, with further error codes defined +in `smax.h` and `redisx.h`. The functions that return an error status (either directly, or into the integer designated +by a pointer argument), can be inspected by `smaxErrorDescription()`, e.g.: + +```c + int status = smaxShare(...); + if (status != X_SUCCESS) { + // Ooops, something went wrong... + fprintf(stderr, "WARNING! set value: %s", smaxErrorDescription(status)); + ... + } +``` + +----------------------------------------------------------------------------- + + +## Debug support + +You can enable verbose output of the library with `smaxSetVerbose(boolean)`. When enabled, it will produce status +messages to `stderr`so you can follow what's going on. In addition (or alternatively), you can enable debug messages +with `xSetDebug(boolean)`. When enabled, all errors encountered by the library (such as invalid arguments passed) will +be printed to `stderr`, including call traces, so you can walk back to see where the error may have originated from. +(You can also enable debug messages by default by defining the `DEBUG` constant for the compiler, e.g. by adding +`-DDEBUG` to `CFLAGS` prior to calling `make`). + +For helping to debug your application, the __xchange__ library provides two macros: `xvprintf()` and `xdprintf()`, +for printing verbose and debug messages to `stderr`. Both work just like `printf()`, but they are conditional on +verbosity being enabled via `xSetVerbose(boolean)` and `xSetDebug(boolean)`, respectively. Applications using this +library may use these macros to produce their own verbose and/or debugging outputs conditional on the same global +settings. + + + +----------------------------------------------------------------------------- + + +## Future plans + +Some obvious ways the library could evolve and grow in the not too distant future: + + - Automated regression testing and coverage tracking. -### 3.9. Miscellaneous utility functions +If you have an idea for a must have feature, please let me (Attila) know. Pull requests, for new features or fixes to +existing ones, are especially welcome! + +----------------------------------------------------------------------------- +Copyright (C) 2024 Attila Kovács diff --git a/resources/CfA-logo-dark.png b/resources/CfA-logo-dark.png new file mode 100644 index 0000000..2b5a13c Binary files /dev/null and b/resources/CfA-logo-dark.png differ diff --git a/resources/CfA-logo.png b/resources/CfA-logo.png new file mode 100644 index 0000000..1d3e67a Binary files /dev/null and b/resources/CfA-logo.png differ diff --git a/resources/doxygen_extra.css b/resources/doxygen_extra.css new file mode 100644 index 0000000..953f03b --- /dev/null +++ b/resources/doxygen_extra.css @@ -0,0 +1,181 @@ +* { + font-family: "Roboto", "Calibri", "Arial", sans-serif; +} + +.headertitle { + font-size: 125%; +} + +#nav-tree .label { + font-size: 110%; +} + +p, li { + font-size: 110%; +} + +h1, h2, h3, h4, h5, h6 { + margin-top: 24px; + margin-bottom: 16px; + font-weight: 600; + line-height: 1.25; + color: #8d0034; +} + +h1 { + font-size: 200%; + padding-top: 1em; + padding-bottom: 0.3em; + border-bottom: 2px solid; +} + +h2 { + font-size: 170%; + padding-top: 0.3em; + padding-bottom: 0.2em; + border-bottom: 1px solid; +} + +h3 { + font-size: 150%; +} + +h4 { + font-size: 125%; +} + +h5 { + font-size: 110%; +} + +h6 { + font-size: 87.5%; + color: #283266; +} + +div.fragment { + padding: 4px 16px 4px 16px; + border: 0px solid; +} + +.line, .keywordflow { + font-family: "Roboto Mono", "Calibri Mono", "SFMono-Regular", Consolas, "Liberation Mono", Menlo, Courier, monospace; + line-height: 1.8; +} + +span.comment { + color: #555D68; +} + + +div.line.glow { + background-color: auto; + box-shadow: none; + font-weight: bold; +} + +hr { + display: none; +} + +code { + font-family: "Roboto Mono", "Calibri Mono", "SFMono-Regular", Consolas, "Liberation Mono", Menlo, Courier, monospace; + margin: 0px 2px 0px 2px; + font-size: 87.5%; + font-weight: bold; +} + +a.hl_function, a.hl_struct, span.keywordtype, span.keywordflow { + font-weight: bold; +} + +span.keywordflow { + color: #C6809A; +} + +a { + color: var(--page-link-color); +} + +a:visited, a.dark-mode { + color: #894270; +} + +#main-nav a { + color: var(--menu-link-color); +} + +#main-menu a { + color: var(--menu-link-color); +} + +#nav-tree-contents a { + color: var(--menu-link-color); +} + +/* tint 15/85 80%: 894270 */ +/* tint 30/70 482562 -> 80%: 6D5182 */ +/* tint 50/50 5C1B55 */ +/* tint 70/30 701048 */ +/* 80% red A4335D */ +/* 100% blue 283266 */ +/* 80% blue 555DA3 */ +/* 70% blue 6B719E */ +/* 60% blue 8086AC */ +/* 50% blue 959ABA */ +/* 40% blue AAAEC8 */ +html { + --title-background-color: #D8D8D8; + --nav-background-color: #F0F0F0; + --memdef-title-background-color: #D8D8D8; + --memdef-proto-background-color: #D8D8D8; + --memdef-doc-background-color: #F0F0F0; + --fragment-foreground-color: #283266; + --menu-link-color: #303030; + --page-link-color: #8d0034; + --code-link-color: #8d0034; + --page-visited-link-color: #701048; + --code-type-keyword-color: #5C1B55; + --memdef-proto-text-color: #283266; +} + +html.dark-mode { + --title-background-color: #181818; + --page-background-color: black; + --nav-background-color: black; + --memdef-title-background-color: #181818; + --memdef-proto-background-color: #181818; + --memdef-doc-background-color: #303030; + --fragment-foreground-color: #959ABA; + --menu-link-color: #D0D0D0; + --page-link-color: #A4335D; + --code-link-color: #A4335D; + --page-visited-link-color: #894270; + --code-type-keyword-color: #6D5182; + --memdef-proto-text-color: #555DA3; +} + +.paramtype { + color: var(--code-type-keyword-color); +} + +.paramname { + color: #BB0085; +} + +.memtitle { + font-weight: bold; +} + +.memname { + font-weight: bold; + color: var(--fragment-foreground-color); +} + + +blockquote { + border-left: 2px solid #606060; + margin: 0 24px 0 4px; + padding: 0px 12px 0px 16px; +} + diff --git a/resources/header.html b/resources/header.html new file mode 100644 index 0000000..7de7c54 --- /dev/null +++ b/resources/header.html @@ -0,0 +1,93 @@ + + + + + + + + + + + + + + + + + + + + + +$projectname: $title +$title + + + + + + + + +$treeview +$search +$mathjax +$darkmode + +$extrastylesheet + + + + +
+ + + +
+ + +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+
$projectname $projectnumber +
+
$projectbrief
+
+
$projectbrief
+
$searchbox
$searchbox
+
+ + diff --git a/resources/smithsonian-logo-55x55.png b/resources/smithsonian-logo-55x55.png new file mode 100644 index 0000000..defb229 Binary files /dev/null and b/resources/smithsonian-logo-55x55.png differ