Skip to content

Commit

Permalink
Prefer namespacing instead of suffix/prefix naming.
Browse files Browse the repository at this point in the history
This prefers using Namespacing to define function names that can apply
to both array and group nodes. So functions like create_array are now
Array.create.
  • Loading branch information
zoj613 committed Sep 17, 2024
1 parent dfd8b7e commit 3c2c8e0
Show file tree
Hide file tree
Showing 14 changed files with 622 additions and 641 deletions.
50 changes: 24 additions & 26 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -52,8 +52,6 @@ A similar example using the `Lwt`-backed Asynchronous API can be found [here][7]
### setup
```ocaml
open Zarr
open Zarr.Metadata
open Zarr.Node
open Zarr.Codecs
open Zarr.Indexing
open Zarr_sync.Storage
Expand All @@ -64,14 +62,14 @@ let store = FilesystemStore.create "testdata.zarr";;
```
### create group
```ocaml
let group_node = GroupNode.of_path "/some/group";;
FilesystemStore.create_group store group_node;;
let group_node = Node.Group.of_path "/some/group";;
FilesystemStore.Group.create store group_node;;
```
### create an array
```ocaml
let array_node = ArrayNode.(group_node / "name");;
let array_node = Node.Array.(group_node / "name");;
(* creates an array with char data type and fill value '?' *)
FilesystemStore.create_array
FilesystemStore.Array.create
~codecs:[`Transpose [|2; 0; 1|]; `Bytes BE; `Gzip L2]
~shape:[|100; 100; 50|]
~chunks:[|10; 15; 20|]
Expand All @@ -83,11 +81,11 @@ FilesystemStore.create_array
### read/write from/to an array
```ocaml
let slice = [|R [|0; 20|]; I 10; R [||]|];;
let x = FilesystemStore.read_array store array_node slice Ndarray.Char;;
let x = FilesystemStore.Array.read store array_node slice Ndarray.Char;;
(* Do some computation on the array slice *)
let x' = Zarr.Ndarray.map (fun _ -> Random.int 256 |> Char.chr) x;;
FilesystemStore.write_array store array_node slice x';;
let y = FilesystemStore.read_array store array_node slice Ndarray.Char;;
FilesystemStore.Array.write store array_node slice x';;
let y = FilesystemStore.Array.read store array_node slice Ndarray.Char;;
assert (Ndarray.equal x' y);;
```
### create an array with sharding
Expand All @@ -98,9 +96,9 @@ let config =
;index_codecs = [`Bytes BE; `Crc32c]
;index_location = Start};;
let shard_node = ArrayNode.(group_node / "another");;
let shard_node = Node.Array.(group_node / "another");;
FilesystemStore.create_array
FilesystemStore.Array.create
~codecs:[`ShardingIndexed config]
~shape:[|100; 100; 50|]
~chunks:[|10; 15; 20|]
Expand All @@ -111,30 +109,30 @@ FilesystemStore.create_array
```
### exploratory functions
```ocaml
let a, g = FilesystemStore.find_all_nodes store;;
List.map ArrayNode.to_path a;;
let a, g = FilesystemStore.hierarchy store;;
List.map Node.Array.to_path a;;
(*- : string list = ["/some/group/name"; "/some/group/another"] *)
List.map GroupNode.to_path g;;
List.map Node.Group.to_path g;;
(*- : string list = ["/"; "/some"; "/some/group"] *)
FilesystemStore.reshape store array_node [|25; 32; 10|];;
FilesystemStore.Array.reshape store array_node [|25; 32; 10|];;
let meta = FilesystemStore.group_metadata store group_node;;
GroupMetadata.show meta;; (* pretty prints the contents of the metadata *)
let meta = FilesystemStore.Group.metadata store group_node;;
Metadata.Group.show meta;; (* pretty prints the contents of the metadata *)
FilesystemStore.array_exists store shard_node;;
FilesystemStore.group_exists store group_node;;
FilesystemStore.Array.exists store shard_node;;
FilesystemStore.Group.exists store group_node;;
let a, g = FilesystemStore.find_child_nodes store group_node;;
List.map ArrayNode.to_path a;;
let a, g = FilesystemStore.Group.children store group_node;;
List.map Node.Array.to_path a;;
(*- : string list = ["/some/group/name"; "/some/group/another"] *)
List.map GroupNode.to_path g;;
List.map Node.Group.to_path g;;
(*- : string list = [] *)
FilesystemStore.erase_group_node store group_node;;
FilesystemStore.erase_all_nodes store;; (* clears the store *)
FilesystemStore.rename_group store group_node;;
FilesystemStore.rename_array store anode;;
FilesystemStore.Group.delete store group_node;;
FilesystemStore.clear store;; (* clears the store *)
FilesystemStore.Group.rename store group_node;;
FilesystemStore.Array.rename store anode;;
```

[1]: https://codecov.io/gh/zoj613/zarr-ml/graph/badge.svg?token=KOOG2Y1SH5
Expand Down
34 changes: 16 additions & 18 deletions examples/inmemory_zipstore.ml
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@
dune exec -- examples/inmemory_zipstore.exe
in your shell at the root of this project. *)

module MemoryZipStore : sig
module ZipStore : sig
include Zarr.Storage.STORE with type 'a Deferred.t = 'a Lwt.t
val with_open : ?level:Zipc_deflate.level -> string -> (t -> 'a Deferred.t) -> 'a Deferred.t
end = struct
Expand Down Expand Up @@ -143,32 +143,30 @@ end = struct
| Error e -> failwith e
in
Lwt.finalize (fun () -> f t) @@ fun () ->
Lwt_io.with_file
~flags:Unix.[O_WRONLY; O_TRUNC; O_CREAT; O_NONBLOCK]
~mode:Lwt_io.Output
t.path
(fun oc ->
Result.fold ~error:failwith ~ok:(Lwt_io.write oc) @@ Zipc.to_binary_string t.ic)
let flags = Unix.[O_WRONLY; O_TRUNC; O_CREAT; O_NONBLOCK] in
Lwt_io.with_file ~flags ~mode:Lwt_io.Output t.path @@ fun oc ->
Result.fold ~error:failwith ~ok:(Lwt_io.write oc) @@ Zipc.to_binary_string t.ic
end

let _ =
Lwt_main.run @@ begin
let open Zarr.Node in
let open Zarr in
let open Zarr.Ndarray in
let open Zarr.Indexing in
let open MemoryZipStore.Deferred.Syntax in
let open ZipStore.Deferred.Syntax in

MemoryZipStore.with_open "examples/data/testdata.zip" @@ fun store ->
let* xs, _ = MemoryZipStore.find_all_nodes store in
ZipStore.with_open "examples/data/testdata.zip" @@ fun store ->
let* xs, _ = ZipStore.hierarchy store in
let anode = List.hd @@ List.filter
(fun node -> ArrayNode.to_path node = "/some/group/name") xs in
(fun node -> Node.Array.to_path node = "/some/group/name") xs in
let slice = [|R [|0; 20|]; I 10; R [||]|] in
let* x = MemoryZipStore.read_array store anode slice Zarr.Ndarray.Char in
let* x = ZipStore.Array.read store anode slice Char in
let x' = x |> Zarr.Ndarray.map @@ fun _ -> Random.int 256 |> Char.chr in
let* () = MemoryZipStore.write_array store anode slice x' in
let* y = MemoryZipStore.read_array store anode slice Zarr.Ndarray.Char in
let* () = ZipStore.Array.write store anode slice x' in
let* y = ZipStore.Array.read store anode slice Char in
assert (Zarr.Ndarray.equal x' y);
let* () = MemoryZipStore.rename_array store anode "name2" in
let+ exists = MemoryZipStore.array_exists store @@ ArrayNode.of_path "/some/group/name2" in
let* () = ZipStore.Array.rename store anode "name2" in
let+ exists = ZipStore.Array.exists store @@ Node.Array.of_path "/some/group/name2" in
assert exists
end;
print_endline "Zip store has been update."
print_endline "Zip store has been updated."
35 changes: 18 additions & 17 deletions examples/readonly_zipstore.ml
Original file line number Diff line number Diff line change
Expand Up @@ -5,22 +5,22 @@
raise a Not_implemented exception for the set_* and erase_* family of
functions. This effectively allows us to create a read-only store since
calling any of the following functions would result in error:
- ReadOnlyZipStore.create_group
- ReadOnlyZipStore.create_array
- ReadOnlyZipStore.erase_group_node
- ReadOnlyZipStore.erase_array_node
- ReadOnlyZipStore.erase_all_nodes
- ReadOnlyZipStore.write_array
- ReadOnlyZipStore.reshape
- ReadOnlyZipStore.rename_array
- ReadOnlyZipStore.rename_group
- ZipStore.create_group
- ZipStore.create_array
- ZipStore.erase_group_node
- ZipStore.erase_array_node
- ZipStore.clear
- ZipStore.write_array
- ZipStore.reshape
- ZipStore.rename_array
- ZipStore.rename_group
Below we show how to implement this custom Zarr Store.
To compile & run this example execute the command
dune exec -- examples/zipstore.exe
in your shell at the root of this project. *)

module ReadOnlyZipStore : sig
module ZipStore : sig
exception Not_implemented
include Zarr.Storage.STORE with type 'a Deferred.t = 'a
val with_open : string -> (t -> 'a) -> 'a
Expand Down Expand Up @@ -94,12 +94,13 @@ end

let _ =
Eio_main.run @@ fun _ ->
let open Zarr.Node in
let open Zarr in
let open Zarr.Ndarray in

ReadOnlyZipStore.with_open "examples/data/testdata.zip" @@ fun store ->
let xs, _ = ReadOnlyZipStore.find_all_nodes store in
ZipStore.with_open "examples/data/testdata.zip" @@ fun store ->
let xs, _ = ZipStore.hierarchy store in
let anode = List.hd @@ Eio.Fiber.List.filter
(fun node -> ArrayNode.to_path node = "/some/group/name") xs in
let arr = ReadOnlyZipStore.read_array store anode [||] Zarr.Ndarray.Char in
try ReadOnlyZipStore.write_array store anode [||] arr with
| ReadOnlyZipStore.Not_implemented -> print_endline "Store is read-only"
(fun node -> Node.Array.to_path node = "/some/group/name") xs in
let arr = ZipStore.Array.read store anode [||] Char in
try ZipStore.Array.write store anode [||] arr with
| ZipStore.Not_implemented -> print_endline "Store is read-only"
88 changes: 43 additions & 45 deletions zarr-eio/test/test_eio.ml
Original file line number Diff line number Diff line change
@@ -1,13 +1,11 @@
open OUnit2
open Zarr
open Zarr.Indexing
open Zarr.Metadata
open Zarr.Node
open Zarr.Codecs
open Zarr_eio.Storage

let string_of_list = [%show: string list]
let print_node_pair = [%show: ArrayNode.t list * GroupNode.t list]
let print_node_pair = [%show: Node.Array.t list * Node.Group.t list]
let print_int_array = [%show : int array]

module type EIO_STORE = sig
Expand All @@ -17,103 +15,103 @@ end
let test_storage
(type a) (module M : EIO_STORE with type t = a) (store : a) =
let open M in
let gnode = GroupNode.root in
let gnode = Node.Group.root in

let nodes = find_all_nodes store in
let nodes = hierarchy store in
assert_equal ~printer:print_node_pair ([], []) nodes;

create_group store gnode;
let exists = group_exists store gnode in
Group.create store gnode;
let exists = Group.exists store gnode in
assert_equal ~printer:string_of_bool true exists;

let meta = group_metadata store gnode in
assert_equal ~printer:GroupMetadata.show GroupMetadata.default meta;
let meta = Group.metadata store gnode in
assert_equal ~printer:Metadata.Group.show Metadata.Group.default meta;

erase_group_node store gnode;
let exists = group_exists store gnode in
Group.delete store gnode;
let exists = Group.exists store gnode in
assert_equal ~printer:string_of_bool false exists;
let nodes = find_all_nodes store in
let nodes = hierarchy store in
assert_equal ~printer:print_node_pair ([], []) nodes;

let attrs = `Assoc [("questions", `String "answer")] in
create_group ~attrs store gnode;
let meta = group_metadata store gnode in
assert_equal ~printer:Yojson.Safe.show attrs @@ GroupMetadata.attributes meta;
Group.create ~attrs store gnode;
let meta = Group.metadata store gnode in
assert_equal ~printer:Yojson.Safe.show attrs @@ Metadata.Group.attributes meta;

let exists = array_exists store @@ ArrayNode.(gnode / "non-member") in
let exists = Array.exists store @@ Node.Array.(gnode / "non-member") in
assert_equal ~printer:string_of_bool false exists;

let cfg =
{chunk_shape = [|2; 5; 5|]
;index_location = End
;index_codecs = [`Bytes BE]
;codecs = [`Bytes LE]} in
let anode = ArrayNode.(gnode / "arrnode") in
let anode = Node.Array.(gnode / "arrnode") in
let slice = [|R [|0; 20|]; I 10; R [|0; 29|]|] in
let exp = Ndarray.init Complex32 [|21; 1; 30|] (Fun.const Complex.one) in

List.iter
(fun codecs ->
create_array
Array.create
~codecs ~shape:[|100; 100; 50|] ~chunks:[|10; 15; 20|]
Complex32 Complex.one anode store;
write_array store anode slice exp;
let got = read_array store anode slice Complex32 in
Array.write store anode slice exp;
let got = Array.read store anode slice Complex32 in
assert_equal exp got;
Ndarray.fill exp Complex.{re=2.0; im=0.};
write_array store anode slice exp;
let got = read_array store anode slice Complex32 in
Array.write store anode slice exp;
let got = Array.read store anode slice Complex32 in
assert_equal exp got;
Ndarray.fill exp Complex.{re=0.; im=3.0};
write_array store anode slice exp;
let got = read_array store anode slice Complex32 in
Array.write store anode slice exp;
let got = Array.read store anode slice Complex32 in
assert_equal exp got)
[[`ShardingIndexed cfg]; [`Bytes BE]];

let child = GroupNode.of_path "/some/child/group" in
create_group store child;
let arrays, groups = find_child_nodes store gnode in
let child = Node.Group.of_path "/some/child/group" in
Group.create store child;
let arrays, groups = Group.children store gnode in
assert_equal
~printer:string_of_list ["/arrnode"] (List.map ArrayNode.to_path arrays);
~printer:string_of_list ["/arrnode"] (List.map Node.Array.to_path arrays);
assert_equal
~printer:string_of_list ["/some"] (List.map GroupNode.to_path groups);
~printer:string_of_list ["/some"] (List.map Node.Group.to_path groups);

let c = find_child_nodes store @@ GroupNode.(root / "fakegroup") in
let c = Group.children store @@ Node.Group.(root / "fakegroup") in
assert_equal ([], []) c;

let ac, gc = find_all_nodes store in
let ac, gc = hierarchy store in
let got =
List.fast_sort String.compare @@
List.map ArrayNode.show ac @ List.map GroupNode.show gc in
List.map Node.Array.show ac @ List.map Node.Group.show gc in
assert_equal
~printer:string_of_list
["/"; "/arrnode"; "/some"; "/some/child"; "/some/child/group"] got;

(* tests for renaming nodes *)
let some = GroupNode.of_path "/some/child" in
rename_array store anode "ARRAYNODE";
rename_group store some "CHILD";
let ac, gc = find_all_nodes store in
let some = Node.Group.of_path "/some/child" in
Array.rename store anode "ARRAYNODE";
Group.rename store some "CHILD";
let ac, gc = hierarchy store in
let got =
List.fast_sort String.compare @@
List.map ArrayNode.show ac @ List.map GroupNode.show gc in
List.map Node.Array.show ac @ List.map Node.Group.show gc in
assert_equal
~printer:string_of_list
["/"; "/ARRAYNODE"; "/some"; "/some/CHILD"; "/some/CHILD/group"] got;
(* restore old array node name. *)
rename_array store (ArrayNode.of_path "/ARRAYNODE") "arrnode";
Array.rename store (Node.Array.of_path "/ARRAYNODE") "arrnode";

let nshape = [|25; 32; 10|] in
reshape store anode nshape;
let meta = array_metadata store anode in
assert_equal ~printer:print_int_array nshape @@ ArrayMetadata.shape meta;
Array.reshape store anode nshape;
let meta = Array.metadata store anode in
assert_equal ~printer:print_int_array nshape @@ Metadata.Array.shape meta;
assert_raises
(Zarr.Storage.Key_not_found "fakegroup/zarr.json")
(fun () -> array_metadata store ArrayNode.(gnode / "fakegroup"));
(fun () -> Array.metadata store Node.Array.(gnode / "fakegroup"));

erase_array_node store anode;
erase_all_nodes store;
let got = find_all_nodes store in
Array.delete store anode;
clear store;
let got = hierarchy store in
assert_equal ~printer:print_node_pair ([], []) got

let _ =
Expand Down
Loading

0 comments on commit 3c2c8e0

Please sign in to comment.