feat: support running on a read-only filesystem

Docker containers can run on a read-only filesystem, which has some
security benefits. The current updater implementation writes the .tgz
file to the filesystem, so it does not work in such an environment.

This provides a `read_only_fs?` configuration which stores the data in
memory while loading, and does not write the ETS table to disk for
future loading.

README.md

@@ -72,6 +72,12 @@ containing the `20xxx.ets` file that ships with this library.
 For instance with this config: `config :tzdata, :data_dir, "/etc/elixir_tzdata_data"`
 an `.ets` file such as `/etc/elixir_tzdata_data/release_ets/2017b.ets` should be present.
 
+If you are running in an environment where there is no writeable directory (such as a read-only Docker filesystem), you can set the `read_only_fs?` configuration to true which will not write these update files. The automatic updater will still run if configured (see below).
+
+``` elixir
+config :tzdata, :read_only_fs?, true
+```
+
 ## Automatic data updates
 
 By default Tzdata will poll for timezone database updates every day.

config/config.exs

@@ -4,4 +4,5 @@ use Mix.Config
 
 config :logger, utc_log: true
 config :tzdata, :autoupdate, :enabled
+config :tzdata, :read_only_fs?, false
 # config :tzdata, :data_dir, "/etc/elixir_tzdata_storage"

lib/tzdata/basic_data_map.ex

@@ -1,40 +1,53 @@
 defmodule Tzdata.BasicDataMap do
   @moduledoc false
 
+  alias Tzdata.DataLoader
   alias Tzdata.Parser
   alias Tzdata.ParserOrganizer, as: Organizer
+
   @file_names ~w(africa antarctica asia australasia backward etcetera europe northamerica southamerica)s
   def from_files_in_dir(dir_name) do
-    Enum.map(@file_names, fn file_name -> {String.to_atom(file_name), Parser.read_file(file_name, dir_name)} end)
-    |> make_map
+    from_file_names(dir_name, @file_names)
   end
 
   def from_single_file_in_dir(dir_name, file_name) do
-    [{String.to_atom(file_name), Parser.read_file(file_name, dir_name)}]
+    from_file_names(dir_name, [file_name])
+  end
+
+  defp from_file_names(dir_name, file_names) do
+    file_names
+    |> DataLoader.file_lines(dir_name)
+    |> Enum.map(fn {file_name, lines} ->
+      {String.to_atom(file_name), Parser.process_file(lines)}
+    end)
     |> make_map
   end
 
   def make_map(all_files_read) do
-    all_files_flattened = all_files_read |> Enum.map(fn {_name, read_file} -> read_file end) |> List.flatten
+    all_files_flattened =
+      all_files_read |> Enum.map(fn {_name, read_file} -> read_file end) |> List.flatten()
+
     rules = Organizer.rules(all_files_flattened)
     zones = Organizer.zones(all_files_flattened)
     links = Organizer.links(all_files_flattened)
     zone_list = Organizer.zone_list(all_files_flattened)
     link_list = Organizer.link_list(all_files_flattened)
     zone_and_link_list = Organizer.zone_and_link_list(all_files_flattened)
 
-    by_group = all_files_read
-    |> Enum.map(fn {name, file_read} -> {name, Organizer.zone_and_link_list(file_read)} end)
-    |> Enum.into(Map.new)
+    by_group =
+      all_files_read
+      |> Enum.map(fn {name, file_read} -> {name, Organizer.zone_and_link_list(file_read)} end)
+      |> Enum.into(Map.new())
+
     {:ok,
-      %{rules: rules,
-      zones: zones,
-      links: links,
-      zone_list: zone_list,
-      link_list: link_list,
-      zone_and_link_list: zone_and_link_list,
-      by_group: by_group,
-      }
-    }
+     %{
+       rules: rules,
+       zones: zones,
+       links: links,
+       zone_list: zone_list,
+       link_list: link_list,
+       zone_and_link_list: zone_and_link_list,
+       by_group: by_group
+     }}
   end
 end

lib/tzdata/data_builder.ex

@@ -13,7 +13,7 @@ defmodule Tzdata.DataBuilder do
 
     if release_version == current_version do
       # remove temporary tzdata dir
-      File.rm_rf(tzdata_dir)
+      DataLoader.cleanup(tzdata_dir)
 
       Logger.info(
         "Downloaded tzdata release from IANA is the same version as the version currently in use (#{
@@ -45,27 +45,35 @@ defmodule Tzdata.DataBuilder do
 
     map.zone_list
     |> Enum.each(fn zone_name ->
-         insert_periods_for_zone(table, map, zone_name)
-       end)
-
-    # remove temporary tzdata dir
-    File.rm_rf(tzdata_dir)
-    ets_tmp_file_name = "#{release_dir()}/#{release_version}.tmp"
-    ets_file_name = ets_file_name_for_release_version(release_version)
-    File.mkdir_p(release_dir())
-    # Create file using a .tmp line ending to avoid it being
-    # recognized as a complete file before writing to it is complete.
-    :ets.tab2file(table, :erlang.binary_to_list(ets_tmp_file_name))
-    :ets.delete(table)
-    # Then rename it, which should be an atomic operation.
-    :file.rename(ets_tmp_file_name, ets_file_name)
+      insert_periods_for_zone(table, map, zone_name)
+    end)
+
+    # cleanup temporary tzdata dir
+    DataLoader.cleanup(tzdata_dir)
+
+    case Application.fetch_env(:tzdata, :read_only_fs?) do
+      {:ok, true} ->
+        ets_tmp_file_name = "#{release_dir()}/#{release_version}.tmp"
+        ets_file_name = ets_file_name_for_release_version(release_version)
+        File.mkdir_p(release_dir())
+        # Create file using a .tmp line ending to avoid it being
+        # recognized as a complete file before writing to it is complete.
+        :ets.tab2file(table, :erlang.binary_to_list(ets_tmp_file_name))
+        :ets.delete(table)
+        # Then rename it, which should be an atomic operation.
+        :file.rename(ets_tmp_file_name, ets_file_name)
+
+      _ ->
+        :ok
+    end
+
     {:ok, content_length, release_version}
   end
 
   defp leap_sec_data(tzdata_dir), do: LeapSecParser.read_file(tzdata_dir)
 
   def ets_file_name_for_release_version(release_version) do
-    "#{release_dir()}/#{release_version}.v#{Tzdata.EtsHolder.file_version}.ets"
+    "#{release_dir()}/#{release_version}.v#{Tzdata.EtsHolder.file_version()}.ets"
   end
 
   def ets_table_name_for_release_version(release_version) do
@@ -79,10 +87,11 @@ defmodule Tzdata.DataBuilder do
     tuple_periods =
       periods
       |> Enum.map(fn period ->
-           period_to_tuple(key, period)
-         end)
+        period_to_tuple(key, period)
+      end)
 
-    tuple_periods |> Enum.each(fn tuple_period ->
+    tuple_periods
+    |> Enum.each(fn tuple_period ->
       :ets.insert(table, tuple_period)
     end)
   end

lib/tzdata/data_loader.ex

@@ -13,27 +13,74 @@ defmodule Tzdata.DataLoader do
     {:ok, last_modified} = last_modified_from_headers(headers)
 
     new_dir_name =
-      "#{data_dir()}/tmp_downloads/#{content_length}_#{:rand.uniform(100_000_000)}/"
+      case Application.fetch_env(:tzdata, :read_only_fs?) do
+        {:ok, true} ->
+          {:binary, body}
+
+        {:ok, false} ->
+          new_dir_name =
+            "#{data_dir()}/tmp_downloads/#{content_length}_#{:rand.uniform(100_000_000)}/"
+
+          File.mkdir_p!(new_dir_name)
+          target_filename = "#{new_dir_name}latest.tar.gz"
+          File.write!(target_filename, body)
+          extract(target_filename, new_dir_name)
+          new_dir_name
+      end
 
-    File.mkdir_p!(new_dir_name)
-    target_filename = "#{new_dir_name}latest.tar.gz"
-    File.write!(target_filename, body)
-    extract(target_filename, new_dir_name)
     release_version = release_version_for_dir(new_dir_name)
     Logger.debug("Tzdata data downloaded. Release version #{release_version}.")
     {:ok, content_length, release_version, new_dir_name, last_modified}
   end
 
-  defp extract(filename, target_dir) do
+  def stream_file(filename, target_dir) do
+    [{^filename, lines}] = file_lines([filename], target_dir)
+    lines
+  end
+
+  def file_lines(filenames, target_dir)
+
+  def file_lines(filenames, target_dir) when is_binary(target_dir) do
+    for filename <- filenames do
+      {filename, File.stream!(Path.join(target_dir, filename))}
+    end
+  end
+
+  def file_lines(filenames, {:binary, _} = body) do
+    filenames = Enum.map(filenames, &:binary.bin_to_list/1)
+    {:ok, extracted} = :erl_tar.extract(body, [:compressed, :memory, {:files, filenames}])
+
+    for {charlist, contents} <- extracted do
+      # split into lines, keeping the delimiters
+      lines =
+        ~r/[^\n]*\n/
+        |> Regex.scan(contents)
+        |> List.flatten()
+
+      {:binary.list_to_bin(charlist), lines}
+    end
+  end
+
+  defp extract(filename, target_dir) when is_binary(target_dir) do
     :erl_tar.extract(filename, [:compressed, {:cwd, target_dir}])
     # remove tar.gz file after extraction
     File.rm!(filename)
   end
 
+  def cleanup(dir_name)
+
+  def cleanup(dir_name) when is_binary(dir_name) do
+    File.rm_rf(dir_name)
+  end
+
+  def cleanup({:binary, _}) do
+    :ok
+  end
+
   def release_version_for_dir(dir_name) do
     [only_line_in_file] =
-      "#{dir_name}/version"
-      |> File.stream!()
+      "version"
+      |> stream_file(dir_name)
       |> Enum.to_list()
 
     only_line_in_file |> String.replace(~r/\s/, "")

lib/tzdata/leap_sec_parser.ex

@@ -8,7 +8,8 @@ defmodule Tzdata.LeapSecParser do
 
   @file_name "leap-seconds.list"
   def read_file(dir_prepend \\ "source_data", file_name \\ @file_name) do
-    File.stream!("#{dir_prepend}#{file_name}")
+    file_name
+    |> Tzdata.DataLoader.stream_file(dir_prepend)
     |> process_file
   end
 

test/test_helper.exs

@@ -1 +1,7 @@
 ExUnit.start()
+
+# Allow testing the read-only filesystem code with
+# `mix test --include read_only_fs`
+if :read_only_fs in ExUnit.configuration()[:include] do
+  Application.put_env(:tzdata, :read_only_fs?, true)
+end