Merge branch 'dev' into ex-zd

.github/workflows/c-cpp.yml

@@ -33,6 +33,16 @@ jobs:
       run: make -j8
     - name: test
       run: make test
+  ubuntu_16_cmake:
+    name: Ubuntu 16 cmake
+    runs-on: ubuntu-20.04
+    container: ubuntu:16.04
+    steps:
+    - uses: actions/checkout@v2
+    - name: install packages
+      run: apt-get update && apt-get install -y zlib1g-dev gcc g++ autoconf automake make cmake
+    - name: build
+      run:  mkdir build && cd build && cmake -DSLOW5_LINK_STATIC=ON -DSLOW5_USE_ZSTD=OFF -DSLOW5_ENABLE_MT=OFF .. && make -j8
   ubuntu_18:
     name: Ubuntu 18
     runs-on: ubuntu-20.04
@@ -120,6 +130,19 @@ jobs:
       run: make test
     - name: make mt
       run: examples/mt/build.sh
+  ubuntu_24:
+    name: Ubuntu 24
+    runs-on: ubuntu-24.04
+    steps:
+    - uses: actions/checkout@v2
+    - name: install packages
+      run: sudo apt-get update && sudo apt-get install zlib1g-dev
+    - name: build-gcc
+      run: make -j8
+    - name: test-gcc
+      run: make test
+    - name: make mt
+      run: examples/mt/build.sh
   os_x_11:
     name: OSX 11
     runs-on: macos-11
@@ -153,6 +176,28 @@ jobs:
       run: make test
     - name: examples
       run: make examples
+  os_x_13:
+    name: OSX 13
+    runs-on: macos-13
+    steps:
+    - uses: actions/checkout@v2
+    - name: build
+      run: make -j8
+    - name: test
+      run: make test
+    - name: examples
+      run: make examples
+  os_x_14:
+    name: OSX 14
+    runs-on: macos-14
+    steps:
+    - uses: actions/checkout@v2
+    - name: build
+      run: make -j8
+    - name: test
+      run: make test
+    - name: examples
+      run: make examples
   arm64:
     name: ubuntu arm
     runs-on: ubuntu-latest

.github/workflows/python.yml

@@ -14,7 +14,7 @@ jobs:
     steps:
     - uses: actions/checkout@v2
     - name: install packages
-      run: sudo apt-get update && sudo apt-get install -y zlib1g-dev python3-pip && pip3 install setuptools cython "numpy>=1.13.3,<1.15.0" --user
+      run: sudo apt-get update && sudo apt-get install -y zlib1g-dev python3-pip wget && wget --no-check-certificate https://bootstrap.pypa.io/pip/3.4/get-pip.py && python3 get-pip.py && pip3 install setuptools cython "numpy>=1.13.3,<1.15.0" --user
     - name: env
       run: lsb_release -a &&  ldd --version && gcc --version && g++ --version && python3 --version
     - name: build
@@ -58,6 +58,28 @@ jobs:
       run: make pyslow5
     - name: test
       run: python3 -m unittest -v python/test.py
+  ubuntu_22:
+    name: pyslow5 Ubuntu 22
+    runs-on: ubuntu-22.04
+    steps:
+    - uses: actions/checkout@v2
+    - name: install packages
+      run: sudo apt-get update && sudo apt-get install -y zlib1g-dev python3-pip && pip3 install setuptools cython numpy
+    - name: build
+      run: make pyslow5
+    - name: test
+      run: python3 -m unittest -v python/test.py
+  ubuntu_24:
+    name: pyslow5 Ubuntu 24
+    runs-on: ubuntu-24.04
+    steps:
+    - uses: actions/checkout@v2
+    - name: install packages
+      run: sudo apt-get update && sudo apt-get install -y zlib1g-dev python3-pip && pip3 install --break-system-packages setuptools cython numpy
+    - name: build
+      run: make pyslow5
+    - name: test
+      run: python3 -m unittest -v python/test.py
   arm64:
     name: pyslow5 ubuntu arm
     runs-on: ubuntu-latest
@@ -77,7 +99,7 @@ jobs:
           make pyslow5 &&
           python3 -m unittest -v python/test.py"
   os_x_11:
-    name: pyslow5 mac os
+    name: pyslow5 mac os 11
     runs-on: macos-11
     steps:
     - uses: actions/checkout@v2
@@ -88,13 +110,35 @@ jobs:
     - name: test
       run: python3 -m unittest -v python/test.py
   os_x_12:
-    name: pyslow5 mac os
+    name: pyslow5 mac os 12
     runs-on: macos-12
     steps:
     - uses: actions/checkout@v2
     - name: install packages
       run: pip3 install numpy setuptools
     - name: build
       run: make pyslow5
+    - name: test
+      run: python3 -m unittest -v python/test.py
+  os_x_13:
+    name: pyslow5 mac os 13
+    runs-on: macos-13
+    steps:
+    - uses: actions/checkout@v2
+    - name: install packages
+      run: pip3 install numpy setuptools
+    - name: build
+      run: make pyslow5
+    - name: test
+      run: python3 -m unittest -v python/test.py
+  os_x_14:
+    name: pyslow5 mac os 14
+    runs-on: macos-14
+    steps:
+    - uses: actions/checkout@v2
+    - name: install packages
+      run: pip3 install numpy setuptools --break-system-packages
+    - name: build
+      run: make pyslow5
     - name: test
       run: python3 -m unittest -v python/test.py

.travis.yml

@@ -73,14 +73,29 @@ jobs:
           packages:
             - zlib1g-dev
       script: make && make test
-    # - name: "IBM Z CPUs"
-    #   arch: s390x
-    #   os: linux
-    #   addons:
-    #     apt:
-    #       packages:
-    #         - zlib1g-dev
-    #   script: make && make test
+    - name: "IBM Z CPUs"
+      arch: s390x
+      os: linux
+      addons:
+        apt:
+          packages:
+            - zlib1g-dev
+            - valgrind
+      script:
+        - make && make test
+        - ./test/test.sh mem
+    - name: "IBM Z CPUs with zstd"
+      arch: s390x
+      os: linux
+      addons:
+        apt:
+          packages:
+            - zlib1g-dev
+            - libzstd1-dev
+            - valgrind
+      script:
+        - make zstd=1 && make test zstd=1
+        - ./test/test.sh mem
     # - stage: deploy
     #   name: "GitHub Release"
     #   dist: trusty

CMakeLists.txt

@@ -37,8 +37,8 @@ unset(SLOW5_ENABLE_MT CACHE)
 
 # include_directories(${PROJECT_SOURCE_DIR}/src)
 # include_directories(${PROJECT_SOURCE_DIR}/test)
-include_directories(${PROJECT_SOURCE_DIR}/include)
-include_directories(${PROJECT_SOURCE_DIR}/thirdparty/streamvbyte/include)
+# include_directories(${PROJECT_SOURCE_DIR}/include)
+# include_directories(${PROJECT_SOURCE_DIR}/thirdparty/streamvbyte/include)
 
 set(slow5_ src/slow5.c)
 set(slow5_idx src/slow5_idx.c)
@@ -66,6 +66,7 @@ else()
 endif(SLOW5_LINK_STATIC)
 unset(SLOW5_LINK_STATIC CACHE)
 
+target_include_directories(slow5 PUBLIC ${PROJECT_SOURCE_DIR}/include)
 target_link_libraries(slow5 streamvbyte_slow5)
 
 # Build a static lib

Makefile

@@ -55,10 +55,10 @@ $(SHAREDLIB): $(OBJ) $(SVBLIB)
 $(SVBLIB):
 	make -C $(SVB) no_simd=$(no_simd) libstreamvbyte.a
 
-$(BUILD_DIR)/slow5.o: src/slow5.c src/slow5_extra.h src/slow5_idx.h src/slow5_misc.h src/klib/ksort.h $(SLOW5_H)
+$(BUILD_DIR)/slow5.o: src/slow5.c src/slow5_extra.h src/slow5_idx.h src/slow5_misc.h src/klib/ksort.h src/slow5_byte.h $(SLOW5_H)
 	$(CC) $(CFLAGS) $(CPPFLAGS) $< -c -fpic -o $@
 
-$(BUILD_DIR)/slow5_idx.o: src/slow5_idx.c src/slow5_idx.h src/slow5_extra.h src/slow5_misc.h $(SLOW5_H)
+$(BUILD_DIR)/slow5_idx.o: src/slow5_idx.c src/slow5_idx.h src/slow5_extra.h src/slow5_misc.h src/slow5_byte.h $(SLOW5_H)
 	$(CC) $(CFLAGS) $(CPPFLAGS) $< -c -fpic -o $@
 
 $(BUILD_DIR)/slow5_misc.o: src/slow5_misc.c src/slow5_misc.h include/slow5/slow5_error.h

README.md

@@ -15,8 +15,8 @@ SLOW5 ecosystem: https://hasindu2008.github.io/slow5<br/>
 [![BioConda Install](https://img.shields.io/conda/dn/bioconda/pyslow5.svg?style=flag&label=BioConda%20install)](https://anaconda.org/bioconda/pyslow5)
 [![PyPI](https://img.shields.io/pypi/v/pyslow5.svg?style=flat)](https://pypi.python.org/pypi/pyslow5)
 ![PyPI - Downloads](https://img.shields.io/pypi/dm/pyslow5?label=pyslow5%20PyPi)
-[![C/C++ CI](https://github.com/hasindu2008/slow5lib/actions/workflows/c-cpp.yml/badge.svg)](https://github.com/hasindu2008/slow5lib/actions/workflows/c-cpp.yml)
-[![Python CI](https://github.com/hasindu2008/slow5lib/actions/workflows/python.yml/badge.svg)](https://github.com/hasindu2008/slow5lib/actions/workflows/python.yml)
+[![C CI](https://github.com/hasindu2008/slow5lib/actions/workflows/c-cpp.yml/badge.svg)](https://github.com/hasindu2008/slow5lib/actions/workflows/c-cpp.yml)
+[![Py CI](https://github.com/hasindu2008/slow5lib/actions/workflows/python.yml/badge.svg)](https://github.com/hasindu2008/slow5lib/actions/workflows/python.yml)
 
 Please cite the following in your publications when using *slow5lib/pyslow5*:
 
@@ -33,6 +33,20 @@ Please cite the following in your publications when using *slow5lib/pyslow5*:
 }
 ```
 
+## Table of Contents
+
+- [Building](#building)
+  - [Optional zstd compression](#optional-zstd-compression)
+  - [Without SIMD](#without-simd)
+  - [Advanced building options](#advanced-building-options)
+- [Usage](#usage)
+  - [Examples](#examples)
+  - [pyslow5](#pyslow5)
+  - [Other languages](#other-languages)
+  - [Current limitations & future work](#current-limitations--future-work)
+  - [Notes](#notes)
+- [Acknowledgement](#acknowledgement)
+
 ## Building
 
 Building slow5lib requires a compiler that supports C99 standard (with X/Open 7 POSIX 2008 extensions), which is widely available.
@@ -54,7 +68,6 @@ On Fedora/CentOS : sudo dnf/yum install zlib-devel
 On OS X : brew install zlib
 ```
 
-
 #### Optional zstd compression
 
 You can optionally enable [*zstd* compression](https://facebook.github.io/zstd) support when building *slow5lib* by invoking `make zstd=1`. This requires __zstd 1.3 or higher development libraries__ installed on your system:
@@ -71,6 +84,9 @@ SLOW5 files compressed with *zstd* offer smaller file size and better performanc
 
 *slow5lib* from version 0.3.0 onwards uses code from [StreamVByte](https://github.com/lemire/streamvbyte) and by default requires vector instructions (SSSE3 or higher for Intel/AMD and neon for ARM). If your processor is an ancient processor with no such vector instructions, invoke make as `make no_simd=1`.
 
+#### Advanced building options
+
+- To support large files on 32-bit systems use: `CFLAGS="-D_FILE_OFFSET_BITS=64"  make`.
 
 ## Usage
 
@@ -112,6 +128,8 @@ Following examples will be added upon request. If you are interested, open a Git
 
 You can invoke `examples/build.sh` to compile the example programmes. Have a look at the script to see the commands used for compiling and linking. If you compiled *slow5lib* with *zstd* support enabled, make sure you append `-lzstd` to the compilation commands.
 
+Some examples demonstrating t the use of easy multi-thread API are available [here](https://github.com/hasindu2008/slow5lib/tree/master/examples/mt)
+
 
 ### pyslow5
 
@@ -124,11 +142,12 @@ A slow5 library for RUST programming language developed by [@bsaintjo](https://g
 
 ### Current limitations & future work
 
-slow5lib is a reference implementation for SLOW5 format. Depending on the interest from the community, the following limitations could be overcome and more performance optimisations can be performed. Open a GitHub issue if you are interested. Contributions are welcome.
+slow5lib is a reference implementation for SLOW5 format. Depending on the interest from the community, the following limitations could be overcome and more performance optimis  ations can be performed. Open a GitHub issue if you are interested. Contributions are welcome.
 
 - No native windows support: slow5lib works well on Windows through WSL, in fact, this is my primary development environment. I am not aware of anyone using native Windows for nanopore bioinformatics. However, if needed, [methods used for minimap2](https://github.com/lh3/minimap2/issues/19) can be adopted.
-- Does not support big-endian systems: Big-endian systems are rare nowadays and I do not have access to one to test. If necessary, it is a matter of writing a layer that swaps the bytes before/after writing to disk. Note: Not to be confused with big.LITTLE architecture which is something else on which slow5lib already works.
-- When running with >64 threads, malloc() calls could reduce the thread efficiency. If that is the case, frequent mallocs could be replaced with kalloc in [klib](https://github.com/attractivechaos/klib).
+- svb-zd compression does not big-endian systems: As of version 1.2.0, slow5lib supports big-endian systems (e.g., IBM Z), except for svb-zd compression that uses [StreamVByte](https://github.com/lemire/streamvbyte) that does not support big-endian.
+Note: Not to be confused with big.LITTLE architecture which is something else on which all features of slow5lib already works.
+- When running with >64 threads, malloc() calls could reduce the thread efficiency. If that is the case, frequent mallocs could be replaced with kalloc in [klib](https://github.com/attractivechaos/klib). Alternatively, preloading tcmalloc or jemalloc would do.
 - Aggressive compiler optimisations (e.g.,  -O3) and architecture-specific compiler optimisations (e.g., -march=native) are not used in the makefile. These flags will improve performance at the cost of limited portability. These could be provided in a separate make target.