mmap (#86)

* cleaner make static library

* first mmap implementation

* adding slow5_madvise api

* slow5_get_next_mem mmap support and bug fixes

* improving streamvbyte16 compilation options

* selective merge upstream/bench

* ignore static libraries

* removing redundant -mssse3 flag

* simplifying logic

* replacing getline

* offset bug fix

* mmap slow5_get

* extra mmap api and mmap slow5_get_next

* zstd_local for svb16

* removing relative zstd_local

---------

Co-authored-by: Sasha Jenner <[email protected]>

Makefile

@@ -3,6 +3,10 @@
 # or uncomment the following line
 #zstd=1
 
+# to compile with local zstd headers
+# run `make zstd_local=<dir>`
+# zstd_local must be an absolute path
+
 CC		= cc
 AR		= ar
 SVB		= thirdparty/streamvbyte
@@ -51,8 +55,7 @@ slow5lib: $(SHAREDLIB) $(STATICLIB)
 $(STATICLIB): $(OBJ) $(SVBLIB) $(SVB16LIB)
 	cp $(SVBLIB) $@
 	mkdir -p $(BUILD_DIR)/tmp
-	cp $(SVB16LIB) $(BUILD_DIR)/tmp/tmp.a
-	cd $(BUILD_DIR)/tmp && $(AR) x tmp.a && cd ../..
+	cd $(BUILD_DIR)/tmp && $(AR) x ../../$(SVB16LIB) && cd ../..
 	$(AR) rcs $@ $(OBJ) $(BUILD_DIR)/tmp/*.o
 	rm -rf $(BUILD_DIR)/tmp
 
@@ -63,7 +66,7 @@ $(SVBLIB):
 	make -C $(SVB) no_simd=$(no_simd) libstreamvbyte.a
 
 $(SVB16LIB):
-	make -C $(SVB16) no_simd=$(no_simd) zstd=$(zstd) libstreamvbyte16.a
+	make -C $(SVB16) no_simd=$(no_simd) zstd=$(zstd) zstd_local=$(zstd_local) libstreamvbyte16.a
 
 $(BUILD_DIR)/slow5.o: src/slow5.c src/slow5_extra.h src/slow5_idx.h src/slow5_misc.h src/klib/ksort.h $(SLOW5_H)
 	$(CC) $(CFLAGS) $(CPPFLAGS) $< -c -fpic -o $@

include/slow5/slow5.h

@@ -298,6 +298,10 @@ struct slow5_file_meta {
     uint64_t start_rec_offset;  ///< offset (in bytes) of the first SLOW5 record (skipping the SLOW5 header; used for indexing)
     char *fread_buffer;         ///< buffer for fread
     const char *mode;           ///< file mode
+
+    void *mp;                   ///< memory map pointer
+    off_t mlen;                 ///< slow5 file size
+    uint64_t moffset;           ///< current byte offset from the memory map pointer
 };
 typedef struct slow5_file_meta slow5_file_meta_t;
 
@@ -328,6 +332,19 @@ existing function prototypes must NOT be changed as such changes affects the bac
 newer functions can be added while keeping the existing ones intact
 */
 
+/*
+ * advise the usage of the memory mapped SLOW5 file
+ * s5p must be a valid pointer
+ * advice can be any valid advice option for posix_madvise
+ * include <sys/mman.h> to reference any of these option
+ * common options are:
+ * - POSIX_MADV_NORMAL: default
+ * - POSIX_MADV_SEQUENTIAL: expect read retrievals in sequential order
+ * - POSIX_MADV_RANDOM: expect read retrievals in random order
+ * return 0 on success, positive on error (see posix_madvise)
+ */
+int slow5_madvise(struct slow5_file *s5p, int advice);
+
 /**
  * Open a slow5 file with a specific mode given it's pathname.
  *