updated becnhmarks

test/.gitignore

@@ -35,8 +35,8 @@ data/test/same_diff_ids.slow5.idx
 data/exp/aux_array/exp_lossless_end_reason.slow5.idx
 test/data/exp/aux_array/exp_lossless_end_reason.slow5.idx
 
-bench/get_all_read_ids
-bench/get_all_samples
-bench/get_selected_read_ids_read_number
-bench/get_selected_read_ids_sample_count
-bench/get_selected_read_ids_samples
+bench/get_read_ids
+bench/get_samples
+bench/get_read_number_rand
+bench/get_sample_count_rand
+bench/get_samples_rand

test/bench/build.sh

@@ -3,8 +3,13 @@
 set -x
 set -e
 
-gcc -Wall -O2 -g -I include/ -o test/bench/get_all_read_ids test/bench/get_all_read_ids.c lib/libslow5.a -lm -lz -lzstd
-gcc -Wall -O2 -g -I include/ -o test/bench/get_all_samples test/bench/get_all_samples.c lib/libslow5.a -lm -lz -lzstd -lpthread  -fopenmp
-gcc -Wall -O2 -g -I include/ -o test/bench/get_selected_read_ids_samples test/bench/get_selected_read_ids_samples.c lib/libslow5.a -lm -lz -lzstd -lpthread  -fopenmp
-gcc -Wall -O2 -g -I include/ -o test/bench/get_selected_read_ids_sample_count test/bench/get_selected_read_ids_sample_count.c lib/libslow5.a -lm -lz -lzstd -lpthread
-gcc -Wall -O2 -g -I include/ -o test/bench/get_selected_read_ids_read_number test/bench/get_selected_read_ids_read_number.c lib/libslow5.a -lm -lz -lzstd -lpthread
+make clean && make -j zstd=1 slow5_mt=1
+gcc -Wall -O2 -g -I include/ -o test/bench/convert_to_pa test/bench/convert_to_pa.c lib/libslow5.a -lm -lz -lzstd -fopenmp
+gcc -Wall -O2 -g -I include/ -o test/bench/convert_to_pa_rand test/bench/convert_to_pa_rand.c lib/libslow5.a -lm -lz -lzstd -fopenmp
+gcc -Wall -O2 -g -I include/ -o test/bench/get_all test/bench/get_all.c lib/libslow5.a -lm -lz -lzstd -lpthread
+gcc -Wall -O2 -g -I include/ -o test/bench/get_read_ids test/bench/get_read_ids.c lib/libslow5.a -lm -lz -lzstd
+gcc -Wall -O2 -g -I include/ -o test/bench/get_samples test/bench/get_samples.c lib/libslow5.a -lm -lz -lzstd -lpthread  -fopenmp
+gcc -Wall -O2 -g -I include/ -o test/bench/get_samples_no_pipelining test/bench/get_samples_no_pipelining.c lib/libslow5.a -lm -lz -lzstd -lpthread  -fopenmp
+gcc -Wall -O2 -g -I include/ -o test/bench/get_samples_rand test/bench/get_samples_rand.c lib/libslow5.a -lm -lz -lzstd -lpthread  -fopenmp
+gcc -Wall -O2 -g -I include/ -o test/bench/get_sample_count_rand test/bench/get_sample_count_rand.c lib/libslow5.a -lm -lz -lzstd -lpthread
+gcc -Wall -O2 -g -I include/ -o test/bench/get_read_number_rand test/bench/get_read_number_rand.c lib/libslow5.a -lm -lz -lzstd -lpthread

test/bench/convert_to_pa.c

@@ -64,7 +64,7 @@ void free_raw_batch(char **mem_records, size_t *mem_bytes, int batch_size) {
 int main(int argc, char *argv[]) {
 
     if(argc != 4) {
-        fprintf(stderr, "Usage: %s in_file.blow5 num_thread batch_size\n", argv[0]);
+        fprintf(stderr, "Usage: %s reads.blow5 num_thread batch_size\n", argv[0]);
         return EXIT_FAILURE;
     }
 

test/bench/convert_to_pa_rand.c

@@ -23,7 +23,7 @@ static inline double realtime(void) {
 int main(int argc, char *argv[]) {
 
     if(argc != 5) {
-        fprintf(stderr, "Usage: %s in_file.blow5 rid_list.txt num_thread batch_size\n", argv[0]);
+        fprintf(stderr, "Usage: %s reads.blow5 rid_list.txt num_thread batch_size\n", argv[0]);
         return EXIT_FAILURE;
     }
 

test/bench/get_all.c

@@ -7,7 +7,7 @@
 #include <slow5/slow5.h>
 #include <omp.h>
 #include <sys/time.h>
-#include "../../python/slow5threads.h"
+#include <slow5/slow5_mt.h>
 
 static inline double realtime(void) {
     struct timeval tp;
@@ -20,7 +20,7 @@ static inline double realtime(void) {
 int main(int argc, char *argv[]) {
 
     if(argc != 5) {
-        fprintf(stderr, "Usage: %s in_file.blow5 out_file.csv num_thread batch_size\n", argv[0]);
+        fprintf(stderr, "Usage: %s reads.blow5 num_thread batch_size\n", argv[0]);
         return EXIT_FAILURE;
     }
 
@@ -29,13 +29,8 @@ int main(int argc, char *argv[]) {
     int batch_size = atoi(argv[4]);
     int num_thread = atoi(argv[3]);
 
-    FILE *fp = fopen(argv[2],"w");
-    if(fp==NULL){
-        fprintf(stderr,"Error in opening file %s for writing\n",argv[2]);
-        perror("perr: ");
-        exit(EXIT_FAILURE);
-    }
-    fputs("read_id,samples\n", fp);
+    FILE *fp = stdout;
+    fputs("#read_id\tsample_sum\n", fp);
 
     int64_t sum_samples = 0;
 
@@ -55,7 +50,7 @@ int main(int argc, char *argv[]) {
     while(ret > 0){
 
         t0 = realtime();
-        ret = slow5_get_next_batch(&rec,sp,batch_size,num_thread);
+        ret = slow5_get_next_batch_lazy(&rec,sp,batch_size,num_thread);
         tot_time += realtime() - t0;
         fprintf(stderr,"batch loaded with %d reads\n",ret);
 
@@ -64,7 +59,7 @@ int main(int argc, char *argv[]) {
         }
 
         t0 = realtime();
-        slow5_free_batch(&rec,ret);
+        slow5_free_batch_lazy(&rec,ret);
         tot_time += realtime() - t0;
 
         if(ret<batch_size){ //this indicates nothing left to read //need to handle errors
@@ -76,8 +71,6 @@ int main(int argc, char *argv[]) {
     slow5_close(sp);
     tot_time += realtime() - t0;
 
-    fclose(fp);
-
     fprintf(stderr,"Time for getting all records %f\n", tot_time);
     fprintf(stderr,"%f MSamples/s\n", (double)sum_samples/(tot_time*1000*1000));
 

test/bench/get_all_read_ids.c → test/bench/get_read_ids.c

@@ -1,6 +1,6 @@
 //get all the read ids and print them to stdout
 //make zstd=1
-//gcc -Wall -O2 -I include/ -o get_all_read_ids test/bench/get_all_read_ids.c lib/libslow5.a -lm -lz -lzstd
+//gcc -Wall -O2 -I include/ -o get_read_ids test/bench/get_read_ids.c lib/libslow5.a -lm -lz -lzstd
 
 #include <stdio.h>
 #include <stdlib.h>
@@ -17,8 +17,8 @@ static inline double realtime(void) {
 
 int main(int argc, char *argv[]) {
 
-    if(argc != 3) {
-        fprintf(stderr, "Usage: %s in_file.blow5 out_file.csv\n", argv[0]);
+    if(argc != 2) {
+        fprintf(stderr, "Usage: %s reads.blow5 > out_file.tsv\n", argv[0]);
         return EXIT_FAILURE;
     }
 
@@ -44,20 +44,10 @@ int main(int argc, char *argv[]) {
 
     tot_time += realtime() - t0;
 
-
-    FILE *fp = fopen(argv[2],"w");
-    if(fp==NULL){
-        fprintf(stderr,"Error in opening file %s for writing\n",argv[2]);
-        perror("perr: ");
-        exit(EXIT_FAILURE);
-    }
-
-    fputs("read_id\n", fp);
     for(int i=0; i<num_reads; i++) {
-        fputs(read_ids[i],fp);
-        fputc('\n',fp);
+        fputs(read_ids[i],stdout);
+        fputc('\n',stdout);
     }
-    fclose(fp);
 
     t0 = realtime();
     slow5_idx_unload(sp);

test/bench/get_selected_read_ids_read_number.c → test/bench/get_read_number_rand.c

@@ -1,6 +1,6 @@
 //get all the samples and sum them to stdout
 //make zstd=1 slow5_mt=1
-//gcc -Wall -O2 -I include/ -o get_selected_read_ids_read_number test/bench/get_selected_read_ids_read_number.c lib/libslow5.a -lm -lz -lzstd -lpthread
+//gcc -Wall -O2 -I include/ -o get_read_number_rand test/bench/get_read_number_rand.c lib/libslow5.a -lm -lz -lzstd -lpthread
 
 #include <stdio.h>
 #include <stdlib.h>
@@ -19,14 +19,14 @@ static inline double realtime(void) {
 
 int main(int argc, char *argv[]) {
 
-    if(argc != 6) {
-        fprintf(stderr, "Usage: %s in_file.blow5 inread_id.csv out_file.csv num_thread batch_size\n", argv[0]);
+    if(argc != 5) {
+        fprintf(stderr, "Usage: %s reads.blow5 inread_id.tsv num_thread batch_size\n", argv[0]);
         return EXIT_FAILURE;
     }
 
     int ret=0;
-    int batch_size = atoi(argv[5]);
-    int num_thread = atoi(argv[4]);
+    int batch_size = atoi(argv[4]);
+    int num_thread = atoi(argv[3]);
 
     uint64_t *sums = malloc(sizeof(uint64_t)*batch_size);
 
@@ -37,19 +37,9 @@ int main(int argc, char *argv[]) {
         perror("perr: ");;
         exit(EXIT_FAILURE);
     }
-    if(fscanf(fpr,"%s",tmp)<1 || strcmp(tmp,",read_id")!=0){
-        fprintf(stderr,"Bad input CSV file. CSV header is expected to have read_id\n");
-        exit(EXIT_FAILURE);
-    }
-
-    FILE *fpw = fopen(argv[3],"w");
-    if(fpw==NULL){
-        fprintf(stderr,"Error in opening file %s for writing\n",argv[3]);
-        perror("perr: ");;
-        exit(EXIT_FAILURE);
-    }
-    fputs("read_id,read_number\n", fpw);
 
+    FILE *fpw = stdout;
+    fputs("#read_id\tread_number\n", fpw);
 
     double tot_time = 0;
     double t0 = realtime();
@@ -75,8 +65,7 @@ int main(int argc, char *argv[]) {
 
         int i=0;
         for(i=0; i<batch_size; i++){
-            int crap;
-            if (fscanf(fpr,"%d,%s",&crap,tmp) < 2) {
+            if (fscanf(fpr,"%s",tmp) < 1) {
                 break;
             }
             rid[i] = strdup(tmp);
@@ -102,7 +91,7 @@ int main(int argc, char *argv[]) {
                 exit(EXIT_FAILURE);
             }
             tot_time += realtime() - t0;
-            fprintf(fpw,"%s,%d\n",rec[i]->read_id, rn);
+            fprintf(fpw,"%s\t%d\n",rec[i]->read_id, rn);
         }
         fprintf(stderr,"batch printed with %d reads\n",ret);
 
@@ -127,7 +116,6 @@ int main(int argc, char *argv[]) {
 
     free(sums);
     fclose(fpr);
-    fclose(fpw);
 
     free(rid);
 

test/bench/get_selected_read_ids_sample_count.c → test/bench/get_sample_count_rand.c

@@ -1,6 +1,6 @@
 //get all the samples and sum them to stdout
 //make zstd=1 slow5_mt=1
-//gcc -Wall -O2 -I include/ -o get_selected_read_ids_sample_count test/bench/get_selected_read_ids_sample_count.c lib/libslow5.a -lm -lz -lzstd -lpthread
+//gcc -Wall -O2 -I include/ -o get_sample_count_rand test/bench/get_sample_count_rand.c lib/libslow5.a -lm -lz -lzstd -lpthread
 
 #include <stdio.h>
 #include <stdlib.h>
@@ -19,14 +19,14 @@ static inline double realtime(void) {
 
 int main(int argc, char *argv[]) {
 
-    if(argc != 6) {
-        fprintf(stderr, "Usage: %s in_file.blow5 inread_id.csv out_file.csv num_thread batch_size\n", argv[0]);
+    if(argc != 5) {
+        fprintf(stderr, "Usage: %s reads.blow5 inread_id.tsv num_thread batch_size\n", argv[0]);
         return EXIT_FAILURE;
     }
 
     int ret=0;
-    int batch_size = atoi(argv[5]);
-    int num_thread = atoi(argv[4]);
+    int batch_size = atoi(argv[4]);
+    int num_thread = atoi(argv[3]);
 
     uint64_t *sums = malloc(sizeof(uint64_t)*batch_size);
 
@@ -37,19 +37,9 @@ int main(int argc, char *argv[]) {
         perror("perr: ");;
         exit(EXIT_FAILURE);
     }
-    if(fscanf(fpr,"%s",tmp)<1 || strcmp(tmp,",read_id")!=0){
-        fprintf(stderr,"Bad input CSV file. CSV header is expected to have read_id\n");
-        exit(EXIT_FAILURE);
-    }
-
-    FILE *fpw = fopen(argv[3],"w");
-    if(fpw==NULL){
-        fprintf(stderr,"Error in opening file %s for writing\n",argv[3]);
-        perror("perr: ");;
-        exit(EXIT_FAILURE);
-    }
-    fputs("read_id,sample_count\n", fpw);
 
+    FILE *fpw = stdout;
+    fputs("#read_id\tsample_count\n", fpw);
 
     double tot_time = 0;
     double t0 = realtime();
@@ -75,8 +65,7 @@ int main(int argc, char *argv[]) {
 
         int i=0;
         for(i=0; i<batch_size; i++){
-            int crap;
-            if (fscanf(fpr,"%d,%s",&crap,tmp) < 2) {
+            if (fscanf(fpr,"%s",tmp) < 1) {
                 break;
             }
             rid[i] = strdup(tmp);
@@ -94,7 +83,7 @@ int main(int argc, char *argv[]) {
         fprintf(stderr,"batch loaded with %d reads\n",ret);
 
         for(int i=0;i<ret;i++){
-            fprintf(fpw,"%s,%ld\n",rec[i]->read_id,rec[i]->len_raw_signal);
+            fprintf(fpw,"%s\t%ld\n",rec[i]->read_id,rec[i]->len_raw_signal);
         }
         fprintf(stderr,"batch printed with %d reads\n",ret);
 
@@ -119,7 +108,6 @@ int main(int argc, char *argv[]) {
 
     free(sums);
     fclose(fpr);
-    fclose(fpw);
 
     free(rid);
 

test/bench/get_all_samples.c → test/bench/get_samples.c

@@ -1,6 +1,6 @@
 //get all the samples and sum them to stdout
 //make zstd=1
-//gcc -Wall -O2 -I include/ -o get_all_samples test/bench/get_all_samples.c lib/libslow5.a -lm -lz -lzstd -lpthread  -fopenmp
+//gcc -Wall -O2 -I include/ -o get_samples test/bench/get_samples.c lib/libslow5.a -lm -lz -lzstd -lpthread  -fopenmp
 
 #include <stdio.h>
 #include <stdlib.h>
@@ -141,7 +141,7 @@ void* output_free_batch(void* voidargs){
 
 
     for(int i=0;i<batch_size;i++){
-        fprintf(fp,"%s,%ld\n",slow5_rec[i]->read_id,sums[i]);
+        fprintf(fp,"%s\t%ld\n",slow5_rec[i]->read_id,sums[i]);
     }
     //fprintf(stderr,"batch printed with %d reads\n",batch_size);
     free_raw_batch(mem_records, mem_bytes, batch_size);
@@ -162,25 +162,20 @@ void* output_free_batch(void* voidargs){
 
 int main(int argc, char *argv[]) {
 
-    if(argc != 5) {
-        fprintf(stderr, "Usage: %s in_file.blow5 out_file.csv num_thread batch_size\n", argv[0]);
+    if(argc != 4) {
+        fprintf(stderr, "Usage: %s reads.blow5 num_thread batch_size\n", argv[0]);
         return EXIT_FAILURE;
     }
 
 
-    int batch_size = atoi(argv[4]);
-    int num_thread = atoi(argv[3]);
+    int batch_size = atoi(argv[3]);
+    int num_thread = atoi(argv[2]);
     int ret=batch_size;
     threads = num_thread;
 
 
-    FILE *fp = fopen(argv[2],"w");
-    if(fp==NULL){
-        fprintf(stderr,"Error in opening file %s for writing\n",argv[2]);
-        perror("perr: ");
-        exit(EXIT_FAILURE);
-    }
-    fputs("read_id,samples\n", fp);
+    FILE *fp = stdout;
+    fputs("#read_id\tsample_sum\n", fp);
 
 
     double tot_time = 0;
@@ -272,8 +267,6 @@ int main(int argc, char *argv[]) {
     slow5_close(sp);
     tot_time += realtime() - t0;
 
-    fclose(fp);
-
     fprintf(stderr,"Time for getting raw bytes (exclude depress & parse) %f\n", tot_time);
 
     return 0;