Add Crawler.Store.all_urls/0

CHANGELOG.md

@@ -7,6 +7,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## master
 
+- [Added] `Crawler.Store.all_urls/0` to find all scraped URLs
 - [Improved] Memory usage optimisations
 
 ## v1.1.2 [2021-10-14]

README.md

@@ -126,7 +126,7 @@ end
 Crawler provides `pause/1`, `resume/1` and `stop/1`, see below.
 
 ```elixir
-{:ok, opts} = Crawler.crawl("http://elixir-lang.org")
+{:ok, opts} = Crawler.crawl("https://elixir-lang.org")
 
 Crawler.pause(opts)
 
@@ -137,6 +137,12 @@ Crawler.stop(opts)
 
 Please note that when pausing Crawler, you would need to set a large enough `:timeout` (or even set it to `:infinity`) otherwise parser would timeout due to unprocessed links.
 
+## Find All Scraped URLs
+
+```elixir
+Crawler.Store.all_urls() # => ["https://elixir-lang.org", "https://google.com", ...]
+```
+
 ## API Reference
 
 Please see https://hexdocs.pm/crawler.

lib/crawler/store.ex

@@ -60,4 +60,8 @@ defmodule Crawler.Store do
   def processed(url) do
     {_new, _old} = Registry.update_value(DB, url, &%{&1 | processed: true})
   end
+
+  def all_urls do
+    Registry.select(DB, [{{:"$1", :_, :_}, [], [:"$1"]}])
+  end
 end

test/lib/crawler_test.exs

@@ -60,6 +60,14 @@ defmodule CrawlerTest do
       assert Store.find_processed(linked_url2)
       assert Store.find_processed(linked_url3)
       refute Store.find(linked_url4)
+
+      urls = Crawler.Store.all_urls()
+
+      assert Enum.member?(urls, url)
+      assert Enum.member?(urls, linked_url1)
+      assert Enum.member?(urls, linked_url2)
+      assert Enum.member?(urls, linked_url3)
+      refute Enum.member?(urls, linked_url4)
     end)
 
     wait(fn ->