Calcul et sauvegarde de métadonnées pour les ressources GBFS

apps/shared/lib/validation/gbfs_validator.ex

@@ -8,6 +8,7 @@ defmodule Shared.Validation.GBFSValidator do
     A structure holding validation results for a GBFS feed
     """
     @enforce_keys [:has_errors, :errors_count, :version_detected, :version_validated]
+    @derive Jason.Encoder
     defstruct has_errors: false, errors_count: nil, version_detected: nil, version_validated: nil
 
     @type t :: %__MODULE__{

apps/shared/test/validation/gbfs_validator_test.exs

@@ -1,13 +1,19 @@
 defmodule GBFSValidatorTest do
-  use ExUnit.Case, async: true
+  use ExUnit.Case, async: false
   doctest Shared.Validation.GBFSValidator
 
   import Mox
 
   alias Shared.Validation.GBFSValidator.Summary
   alias Shared.Validation.GBFSValidator.Wrapper, as: GBFSValidator
 
-  setup :verify_on_exit!
+  setup do
+    :verify_on_exit!
+    # Do not use a mock for the GBFS Validator as we'll mock HTTP calls
+    old_value = Application.fetch_env!(:transport, :gbfs_validator_impl)
+    on_exit(fn -> Application.put_env(:transport, :gbfs_validator_impl, old_value) end)
+    Application.put_env(:transport, :gbfs_validator_impl, Shared.Validation.GBFSValidator.HTTPClient)
+  end
 
   test "validate GBFS feed" do
     Transport.HTTPoison.Mock
@@ -42,4 +48,16 @@ defmodule GBFSValidatorTest do
     {:error, error} = GBFSValidator.validate("https://example.com/gbfs.json")
     assert String.starts_with?(error, "impossible to query GBFS Validator")
   end
+
+  test "can encode summary" do
+    assert """
+    {"errors_count":0,"has_errors":false,"version_detected":"1.1","version_validated":"1.1"}\
+    """
+     == Jason.encode!(%Summary{
+      errors_count: 0,
+      has_errors: false,
+      version_detected: "1.1",
+      version_validated: "1.1"
+    })
+  end
 end

apps/transport/lib/transport/data_checker.ex

@@ -4,7 +4,9 @@ defmodule Transport.DataChecker do
   """
   alias Datagouvfr.Client.{Datasets, Discussions}
   alias Mailjet.Client
-  alias DB.{Dataset, Repo}
+  alias DB.{Dataset, Repo, Resource}
+  alias Shared.Validation.GBFSValidator.Summary, as: GBFSValidationSummary
+  alias Shared.Validation.GBFSValidator.Wrapper, as: GBFSValidator
   import TransportWeb.Router.Helpers
   import Ecto.Query
   require Logger
@@ -80,6 +82,166 @@ defmodule Transport.DataChecker do
     )
   end
 
+  def set_gbfs_feeds_metadata do
+    resources =
+      Resource
+      |> join(:inner, [r], d in Dataset, on: r.dataset_id == d.id)
+      |> where([_r, d], d.type == "bike-scooter-sharing" and d.is_active)
+      |> where([r, _d], like(r.url, "%gbfs.json") or r.format == "gbfs")
+      |> where([r, _d], not fragment("? ~ ?", r.url, "station|free_bike"))
+      |> Repo.all()
+
+    Logger.info("Fetching details about #{Enum.count(resources)} GBFS feeds")
+
+    resources
+    |> Stream.map(fn resource ->
+      Logger.info("Fetching GBFS metadata for #{resource.url} (##{resource.id})")
+      changeset = Resource.changeset(resource, %{format: "gbfs", metadata: compute_gbfs_feed_metadata(resource)})
+      Repo.update(changeset)
+    end)
+    |> Stream.run()
+  end
+
+  @spec compute_gbfs_feed_metadata(Resource.t()) :: map()
+  def compute_gbfs_feed_metadata(resource) do
+    with {:ok, %{status_code: 200, body: body}} <- http_client().get(resource.url),
+         {:ok, json} <- Jason.decode(body) do
+      %{
+        validation: gbfs_validation(resource),
+        feeds: gbfs_feeds(json),
+        versions: gbfs_versions(json),
+        languages: gbfs_languages(json),
+        system_details: gbfs_system_details(json),
+        types: gbfs_types(json),
+        ttl: gbfs_ttl(json)
+      }
+    else
+      e ->
+        Logger.error(inspect(e))
+        %{}
+    end
+  end
+
+  @spec gbfs_validation(Resource.t()) :: GBFSValidationSummary.t() | nil
+  defp gbfs_validation(resource) do
+    case GBFSValidator.validate(resource.url) do
+      {:ok, %GBFSValidationSummary{} = summary} -> summary
+      {:error, _} -> nil
+    end
+  end
+
+  defp gbfs_types(%{"data" => _data} = payload) do
+    has_bike_status = gbfs_has_feed?(payload, "free_bike_status")
+    has_station_information = gbfs_has_feed?(payload, "station_information")
+
+    cond do
+      has_bike_status and has_station_information ->
+        ["free_floating", "stations"]
+
+      has_bike_status ->
+        ["free_floating"]
+
+      has_station_information ->
+        ["stations"]
+
+      true ->
+        Logger.error("Cannot detect GBFS types for feed #{inspect(payload)}")
+        nil
+    end
+  end
+
+  defp gbfs_ttl(%{"data" => _data} = payload) do
+    feed = payload |> gbfs_first_feed()
+
+    value =
+      case gbfs_types(payload) do
+        ["free_floating", "stations"] -> feed |> gbfs_feed_url_by_name("free_bike_status")
+        ["free_floating"] -> feed |> gbfs_feed_url_by_name("free_bike_status")
+        ["stations"] -> feed |> gbfs_feed_url_by_name("station_information")
+        nil -> payload["ttl"]
+      end
+
+    gbfs_feed_ttl(value)
+  end
+
+  defp gbfs_feed_ttl(value) when is_integer(value) and value >= 0, do: value
+
+  defp gbfs_feed_ttl(feed_url) when is_binary(feed_url) do
+    with {:ok, %{status_code: 200, body: body}} <- http_client().get(feed_url),
+         {:ok, json} <- Jason.decode(body) do
+      json["ttl"]
+    else
+      e ->
+        Logger.error("Cannot get GBFS ttl details: #{inspect(e)}")
+        nil
+    end
+  end
+
+  defp gbfs_system_details(%{"data" => _data} = payload) do
+    feed_url = payload |> gbfs_first_feed() |> gbfs_feed_url_by_name("system_information")
+
+    if not is_nil(feed_url) do
+      with {:ok, %{status_code: 200, body: body}} <- http_client().get(feed_url),
+           {:ok, json} <- Jason.decode(body) do
+        %{
+          timezone: json["data"]["timezone"],
+          name: json["data"]["name"]
+        }
+      else
+        e ->
+          Logger.error("Cannot get GBFS system_information details: #{inspect(e)}")
+          nil
+      end
+    end
+  end
+
+  defp gbfs_first_feed(%{"data" => data} = payload) do
+    (data["en"] || data["fr"] || data[payload |> gbfs_languages() |> Enum.at(0)])["feeds"]
+  end
+
+  defp gbfs_languages(%{"data" => data}) do
+    Map.keys(data)
+  end
+
+  @spec gbfs_versions(map()) :: [binary()] | nil
+  defp gbfs_versions(%{"data" => _data} = payload) do
+    gbfs_versions_url = payload |> gbfs_first_feed() |> gbfs_feed_url_by_name("gbfs_versions")
+
+    if is_nil(gbfs_versions_url) do
+      [Map.get(payload, "version", "1.0")]
+    else
+      with {:ok, %{status_code: 200, body: body}} <- http_client().get(gbfs_versions_url),
+           {:ok, json} <- Jason.decode(body) do
+        json["data"]["versions"] |> Enum.map(fn json -> json["version"] end) |> Enum.sort(:desc)
+      else
+        _ -> nil
+      end
+    end
+  end
+
+  @spec gbfs_feed_url_by_name(list(), binary()) :: binary() | nil
+  defp gbfs_feed_url_by_name(feeds, name) do
+    Enum.find(feeds, fn map -> gbfs_feed_is_named?(map, name) end)["url"]
+  end
+
+  @spec gbfs_feed_is_named?(map(), binary()) :: boolean()
+  def gbfs_feed_is_named?(map, name) do
+    # Many people make the mistake of appending `.json` to feed names
+    # so try to match this as well
+    Enum.member?([name, "#{name}.json"], map["name"])
+  end
+
+  @spec gbfs_has_feed?(map(), binary()) :: boolean()
+  def gbfs_has_feed?(%{"data" => _data} = payload, name) do
+    Enum.member?(gbfs_feeds(payload), name)
+  end
+
+  def gbfs_feeds(%{"data" => _data} = payload) do
+    # Remove potential ".json" at the end of feed names as people
+    # often make this mistake
+    payload |> gbfs_first_feed() |> Enum.map(fn feed -> String.replace(feed["name"], ".json", "") end)
+  end
+
   defp make_str({delay, datasets}) do
     r_str =
       datasets
@@ -184,4 +346,6 @@ defmodule Transport.DataChecker do
       false
     )
   end
+
+  defp http_client, do: Transport.Shared.Wrapper.HTTPoison.impl()
 end

apps/transport/test/support/mocks.ex

@@ -1,3 +1,4 @@
 Mox.defmock(Transport.ExAWS.Mock, for: ExAws.Behaviour)
 Mox.defmock(Transport.HTTPoison.Mock, for: HTTPoison.Base)
 Mox.defmock(Validation.Validator.Mock, for: Shared.Validation.Validator)
+Mox.defmock(Shared.Validation.GBFSValidator.Mock, for: Shared.Validation.GBFSValidator.Wrapper)