Merge pull request #268 from burnettekm/bnk-438-fed-automated-fedach-…

…file-updates

Enable fedach and fedwire download from proxy

README.md

@@ -70,6 +70,14 @@ FRB_ROUTING_NUMBER=123456780
 FRB_DOWNLOAD_CODE=86cfa5a9-1ab9-4af5-bd89-0f84d546de13
 ```
 
+#### Download files from proxy
+
+Fed can download the files from a proxy or other HTTP resources. The optional URL template is configured as an environment variable. If the URL template is not configured, Fed will download the files directly from FRB eServices by default. This value is considered a template because when preparing the request Fed replaces `%s` in the path with the requested list name(`fedach` or `fedwire`).
+
+```
+FRB_DOWNLOAD_URL_TEMPLATE=https://my.example.com/files/%s?format=json
+```
+
 ### Docker
 
 We publish a [public Docker image `moov/fed`](https://hub.docker.com/r/moov/fed/) from Docker Hub or use this repository. No configuration is required to serve on `:8086` and metrics at `:9096/metrics` in Prometheus format. We also have Docker images for [OpenShift](https://quay.io/repository/moov/fed?tab=tags) published as `quay.io/moov/fed`.
@@ -181,17 +189,18 @@ PONG
 
 ### Configuration settings
 
-| Environmental Variable | Description | Default |
-|-----|-----|-----|
-| `FEDACH_DATA_PATH` | Filepath to FedACH data file | `./data/FedACHdir.txt` |
-| `FEDWIRE_DATA_PATH` | Filepath to Fedwire data file | `./data/fpddir.txt` |
-| `FRB_ROUTING_NUMBER` | Federal Reserve Board eServices (ABA) routing number used to download FedACH and FedWire files | Empty |
-| `FRB_DOWNLOAD_CODE` | Federal Reserve Board eServices (ABA) download code used to download FedACH and FedWire files | Empty |
-| `LOG_FORMAT` | Format for logging lines to be written as. | Options: `json`, `plain` - Default: `plain` |
-| `HTTP_BIND_ADDRESS` | Address for Fed to bind its HTTP server on. This overrides the command-line flag `-http.addr`. | Default: `:8086` |
-| `HTTP_ADMIN_BIND_ADDRESS` | Address for Fed to bind its admin HTTP server on. This overrides the command-line flag `-admin.addr`. | Default: `:9096` |
+| Environmental Variable | Description                                                                                                                         | Default |
+|-----|-------------------------------------------------------------------------------------------------------------------------------------|-----|
+| `FEDACH_DATA_PATH` | Filepath to FedACH data file                                                                                                        | `./data/FedACHdir.txt` |
+| `FEDWIRE_DATA_PATH` | Filepath to Fedwire data file                                                                                                       | `./data/fpddir.txt` |
+| `FRB_ROUTING_NUMBER` | Federal Reserve Board eServices (ABA) routing number used to download FedACH and FedWire files                                      | Empty |
+| `FRB_DOWNLOAD_CODE` | Federal Reserve Board eServices (ABA) download code used to download FedACH and FedWire files                                       | Empty |
+| `FRB_DOWNLOAD_URL_TEMPLATE` | URL Template for downloading files from alternate source                                                                            | `https://frbservices.org/EPaymentsDirectory/directories/%s?format=json`|
+| `LOG_FORMAT` | Format for logging lines to be written as.                                                                                          | Options: `json`, `plain` - Default: `plain` |
+| `HTTP_BIND_ADDRESS` | Address for Fed to bind its HTTP server on. This overrides the command-line flag `-http.addr`.                                      | Default: `:8086` |
+| `HTTP_ADMIN_BIND_ADDRESS` | Address for Fed to bind its admin HTTP server on. This overrides the command-line flag `-admin.addr`.                               | Default: `:9096` |
 | `HTTPS_CERT_FILE` | Filepath containing a certificate (or intermediate chain) to be served by the HTTP server. Requires all traffic be over secure HTTP. | Empty |
-| `HTTPS_KEY_FILE`  | Filepath of a private key matching the leaf certificate from `HTTPS_CERT_FILE`. | Empty |
+| `HTTPS_KEY_FILE`  | Filepath of a private key matching the leaf certificate from `HTTPS_CERT_FILE`.                                                     | Empty |
 
 #### Logos
 

cmd/server/reader.go

@@ -5,43 +5,49 @@
 package main
 
 import (
+	"errors"
 	"fmt"
-	"io"
-	"os"
-
 	"github.com/moov-io/base/log"
 	"github.com/moov-io/fed"
 	"github.com/moov-io/fed/pkg/download"
+	"io"
+	"os"
 )
 
 func fedACHDataFile(logger log.Logger) (io.Reader, error) {
-	if file, err := attemptFileDownload(logger, "fedach"); file != nil {
-		return file, nil
-	} else if err != nil {
+	file, err := attemptFileDownload(logger, "fedach")
+	if err != nil && !errors.Is(err, download.ErrMissingConfigValue) {
 		return nil, fmt.Errorf("problem downloading fedach: %v", err)
 	}
 
+	if file != nil {
+		return file, nil
+	}
+
 	path := readDataFilepath("FEDACH_DATA_PATH", "./data/FedACHdir.txt")
 	logger.Logf("search: loading %s for ACH data", path)
 
-	file, err := os.Open(path)
+	file, err = os.Open(path)
 	if err != nil {
 		return nil, fmt.Errorf("problem opening %s: %v", path, err)
 	}
 	return file, nil
 }
 
 func fedWireDataFile(logger log.Logger) (io.Reader, error) {
-	if file, err := attemptFileDownload(logger, "fedwire"); file != nil {
+	file, err := attemptFileDownload(logger, "fedach")
+	if err != nil && !errors.Is(err, download.ErrMissingConfigValue) {
+		return nil, fmt.Errorf("problem downloading fedach: %v", err)
+	}
+
+	if file != nil {
 		return file, nil
-	} else if err != nil {
-		return nil, fmt.Errorf("problem downloading fedwire: %v", err)
 	}
 
 	path := readDataFilepath("FEDWIRE_DATA_PATH", "./data/fpddir.txt")
 	logger.Logf("search: loading %s for Wire data", path)
 
-	file, err := os.Open(path)
+	file, err = os.Open(path)
 	if err != nil {
 		return nil, fmt.Errorf("problem opening %s: %v", path, err)
 	}
@@ -51,20 +57,18 @@ func fedWireDataFile(logger log.Logger) (io.Reader, error) {
 func attemptFileDownload(logger log.Logger, listName string) (io.Reader, error) {
 	routingNumber := os.Getenv("FRB_ROUTING_NUMBER")
 	downloadCode := os.Getenv("FRB_DOWNLOAD_CODE")
-
-	if routingNumber != "" && downloadCode != "" {
-		logger.Logf("download: attempting %s", listName)
-		client, err := download.NewClient(&download.ClientOpts{
-			RoutingNumber: routingNumber,
-			DownloadCode:  downloadCode,
-		})
-		if err != nil {
-			return nil, fmt.Errorf("client setup: %v", err)
-		}
-		return client.GetList(listName)
+	downloadURL := os.Getenv("FRB_DOWNLOAD_URL_TEMPLATE")
+
+	logger.Logf("download: attempting %s", listName)
+	client, err := download.NewClient(&download.ClientOpts{
+		RoutingNumber: routingNumber,
+		DownloadCode:  downloadCode,
+		DownloadURL:   downloadURL,
+	})
+	if err != nil {
+		return nil, fmt.Errorf("client setup: %w", err)
 	}
-
-	return nil, nil
+	return client.GetList(listName)
 }
 
 func readDataFilepath(env, fallback string) string {

pkg/download/download.go

@@ -14,16 +14,26 @@ import (
 	"time"
 )
 
+const DefaultFRBDownloadURLTemplate = "https://frbservices.org/EPaymentsDirectory/directories/%s?format=json"
+
+var (
+	ErrMissingConfigValue   = errors.New("missing config value")
+	ErrMissingRoutingNumber = errors.New("missing routing number")
+	ErrMissingDownloadCD    = errors.New("missing download code")
+)
+
 type Client struct {
 	httpClient *http.Client
 
 	routingNumber string // X_FRB_EPAYMENTS_DIRECTORY_ORG_ID header
 	downloadCode  string // X_FRB_EPAYMENTS_DIRECTORY_DOWNLOAD_CD
+	downloadURL   string // defaults to "https://frbservices.org/EPaymentsDirectory/directories/%s?format=json" where %s is the list name
+
 }
 
 type ClientOpts struct {
-	HTTPClient                  *http.Client
-	RoutingNumber, DownloadCode string
+	HTTPClient                               *http.Client
+	RoutingNumber, DownloadCode, DownloadURL string
 }
 
 func NewClient(opts *ClientOpts) (*Client, error) {
@@ -39,23 +49,29 @@ func NewClient(opts *ClientOpts) (*Client, error) {
 	}
 
 	if opts.RoutingNumber == "" {
-		return nil, errors.New("missing routing number")
+		return nil, fmt.Errorf("%w: %w", ErrMissingConfigValue, ErrMissingRoutingNumber)
+	}
+
+	if opts.RoutingNumber == "" {
+		return nil, fmt.Errorf("%w: %w", ErrMissingConfigValue, ErrMissingDownloadCD)
 	}
-	if opts.DownloadCode == "" {
-		return nil, errors.New("missing download code")
+
+	if opts.DownloadURL == "" {
+		opts.DownloadURL = DefaultFRBDownloadURLTemplate
 	}
 
 	return &Client{
 		httpClient:    opts.HTTPClient,
 		routingNumber: opts.RoutingNumber,
 		downloadCode:  opts.DownloadCode,
+		downloadURL:   opts.DownloadURL,
 	}, nil
 }
 
 // GetList downloads an FRB list and saves it into an io.Reader.
 // Example listName values: fedach, fedwire
 func (c *Client) GetList(listName string) (io.Reader, error) {
-	where, err := url.Parse(fmt.Sprintf("https://frbservices.org/EPaymentsDirectory/directories/%s?format=json", listName))
+	where, err := url.Parse(fmt.Sprintf(c.downloadURL, listName))
 	if err != nil {
 		return nil, fmt.Errorf("url: %v", err)
 	}
@@ -64,8 +80,11 @@ func (c *Client) GetList(listName string) (io.Reader, error) {
 	if err != nil {
 		return nil, fmt.Errorf("building %s url: %v", listName, err)
 	}
-	req.Header.Set("X_FRB_EPAYMENTS_DIRECTORY_ORG_ID", c.routingNumber)
-	req.Header.Set("X_FRB_EPAYMENTS_DIRECTORY_DOWNLOAD_CD", c.downloadCode)
+
+	if c.downloadCode != "" && c.routingNumber != "" {
+		req.Header.Set("X_FRB_EPAYMENTS_DIRECTORY_ORG_ID", c.routingNumber)
+		req.Header.Set("X_FRB_EPAYMENTS_DIRECTORY_DOWNLOAD_CD", c.downloadCode)
+	}
 
 	// perform our request
 	resp, err := c.httpClient.Do(req)

pkg/download/download_test.go

@@ -6,8 +6,12 @@ package download
 
 import (
 	"bytes"
+	"fmt"
 	"io"
+	"net/http"
+	"net/http/httptest"
 	"os"
+	"path/filepath"
 	"testing"
 
 	"github.com/stretchr/testify/require"
@@ -34,6 +38,41 @@ func TestClient__fedach(t *testing.T) {
 	}
 }
 
+func TestClient__fedach_custom_url(t *testing.T) {
+	file, err := os.ReadFile(filepath.Join("..", "..", "data", "fedachdir.json"))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	mockHTTPServer := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
+		fmt.Fprint(writer, string(file))
+	}))
+	defer mockHTTPServer.Close()
+
+	t.Setenv("FRB_DOWNLOAD_URL_TEMPLATE", mockHTTPServer.URL+"/%s")
+	t.Setenv("FRB_ROUTING_NUMBER", "123456789")
+	t.Setenv("FRB_DOWNLOAD_CODE", "a1b2c3d4-123b-9876-1234-z1x2y3a1b2c3")
+
+	client := setupClient(t)
+
+	fedach, err := client.GetList("fedach")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	buf, ok := fedach.(*bytes.Buffer)
+	require.True(t, ok)
+
+	if n := buf.Len(); n < 1024 {
+		t.Errorf("unexpected size of %d bytes", n)
+	}
+
+	bs, _ := io.ReadAll(io.LimitReader(fedach, 10024))
+	if !bytes.Equal(bs, file) {
+		t.Errorf("unexpected output:\n%s", string(bs))
+	}
+}
+
 func TestClient__fedwire(t *testing.T) {
 	client := setupClient(t)
 
@@ -55,18 +94,54 @@ func TestClient__fedwire(t *testing.T) {
 	}
 }
 
+func TestClient__wire_custom_url(t *testing.T) {
+	file, err := os.ReadFile(filepath.Join("..", "..", "data", "fedachdir.json"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	mockHTTPServer := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
+		fmt.Fprint(writer, string(file))
+	}))
+	defer mockHTTPServer.Close()
+
+	t.Setenv("FRB_DOWNLOAD_URL_TEMPLATE", mockHTTPServer.URL+"/%s")
+	t.Setenv("FRB_ROUTING_NUMBER", "123456789")
+	t.Setenv("FRB_DOWNLOAD_CODE", "a1b2c3d4-123b-9876-1234-z1x2y3a1b2c3")
+
+	client := setupClient(t)
+
+	fedach, err := client.GetList("fedwire")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	buf, ok := fedach.(*bytes.Buffer)
+	require.True(t, ok)
+
+	if n := buf.Len(); n < 1024 {
+		t.Errorf("unexpected size of %d bytes", n)
+	}
+
+	bs, _ := io.ReadAll(io.LimitReader(fedach, 10024))
+	if !bytes.Equal(bs, file) {
+		t.Errorf("unexpected output:\n%s", string(bs))
+	}
+}
+
 func setupClient(t *testing.T) *Client {
 	t.Helper()
 
 	routingNumber := os.Getenv("FRB_ROUTING_NUMBER")
 	downloadCode := os.Getenv("FRB_DOWNLOAD_CODE")
+	downloadURL := os.Getenv("FRB_DOWNLOAD_URL_TEMPLATE")
 	if routingNumber == "" || downloadCode == "" {
 		t.Skip("missing FRB routing number or download code")
 	}
 
 	client, err := NewClient(&ClientOpts{
 		RoutingNumber: routingNumber,
 		DownloadCode:  downloadCode,
+		DownloadURL:   downloadURL,
 	})
 	if err != nil {
 		t.Fatal(err)