diff --git a/README.md b/README.md index 45bf5845..d868853a 100644 --- a/README.md +++ b/README.md @@ -70,6 +70,14 @@ FRB_ROUTING_NUMBER=123456780 FRB_DOWNLOAD_CODE=86cfa5a9-1ab9-4af5-bd89-0f84d546de13 ``` +#### Download files from proxy + +Fed can download the files from a proxy or other HTTP resources. The optional URL template is configured as an environment variable. If the URL template is not configured, Fed will download the files directly from FRB eServices by default. This value is considered a template because when preparing the request Fed replaces `%s` in the path with the requested list name(`fedach` or `fedwire`). + +``` +FRB_DOWNLOAD_URL_TEMPLATE=https://my.example.com/files/%s?format=json +``` + ### Docker We publish a [public Docker image `moov/fed`](https://hub.docker.com/r/moov/fed/) from Docker Hub or use this repository. No configuration is required to serve on `:8086` and metrics at `:9096/metrics` in Prometheus format. We also have Docker images for [OpenShift](https://quay.io/repository/moov/fed?tab=tags) published as `quay.io/moov/fed`. @@ -181,17 +189,18 @@ PONG ### Configuration settings -| Environmental Variable | Description | Default | -|-----|-----|-----| -| `FEDACH_DATA_PATH` | Filepath to FedACH data file | `./data/FedACHdir.txt` | -| `FEDWIRE_DATA_PATH` | Filepath to Fedwire data file | `./data/fpddir.txt` | -| `FRB_ROUTING_NUMBER` | Federal Reserve Board eServices (ABA) routing number used to download FedACH and FedWire files | Empty | -| `FRB_DOWNLOAD_CODE` | Federal Reserve Board eServices (ABA) download code used to download FedACH and FedWire files | Empty | -| `LOG_FORMAT` | Format for logging lines to be written as. | Options: `json`, `plain` - Default: `plain` | -| `HTTP_BIND_ADDRESS` | Address for Fed to bind its HTTP server on. This overrides the command-line flag `-http.addr`. | Default: `:8086` | -| `HTTP_ADMIN_BIND_ADDRESS` | Address for Fed to bind its admin HTTP server on. This overrides the command-line flag `-admin.addr`. | Default: `:9096` | +| Environmental Variable | Description | Default | +|-----|-------------------------------------------------------------------------------------------------------------------------------------|-----| +| `FEDACH_DATA_PATH` | Filepath to FedACH data file | `./data/FedACHdir.txt` | +| `FEDWIRE_DATA_PATH` | Filepath to Fedwire data file | `./data/fpddir.txt` | +| `FRB_ROUTING_NUMBER` | Federal Reserve Board eServices (ABA) routing number used to download FedACH and FedWire files | Empty | +| `FRB_DOWNLOAD_CODE` | Federal Reserve Board eServices (ABA) download code used to download FedACH and FedWire files | Empty | +| `FRB_DOWNLOAD_URL_TEMPLATE` | URL Template for downloading files from alternate source | `https://frbservices.org/EPaymentsDirectory/directories/%s?format=json`| +| `LOG_FORMAT` | Format for logging lines to be written as. | Options: `json`, `plain` - Default: `plain` | +| `HTTP_BIND_ADDRESS` | Address for Fed to bind its HTTP server on. This overrides the command-line flag `-http.addr`. | Default: `:8086` | +| `HTTP_ADMIN_BIND_ADDRESS` | Address for Fed to bind its admin HTTP server on. This overrides the command-line flag `-admin.addr`. | Default: `:9096` | | `HTTPS_CERT_FILE` | Filepath containing a certificate (or intermediate chain) to be served by the HTTP server. Requires all traffic be over secure HTTP. | Empty | -| `HTTPS_KEY_FILE` | Filepath of a private key matching the leaf certificate from `HTTPS_CERT_FILE`. | Empty | +| `HTTPS_KEY_FILE` | Filepath of a private key matching the leaf certificate from `HTTPS_CERT_FILE`. | Empty | #### Logos diff --git a/cmd/server/reader.go b/cmd/server/reader.go index b2997beb..7c9d7cc7 100644 --- a/cmd/server/reader.go +++ b/cmd/server/reader.go @@ -5,26 +5,29 @@ package main import ( + "errors" "fmt" - "io" - "os" - "github.com/moov-io/base/log" "github.com/moov-io/fed" "github.com/moov-io/fed/pkg/download" + "io" + "os" ) func fedACHDataFile(logger log.Logger) (io.Reader, error) { - if file, err := attemptFileDownload(logger, "fedach"); file != nil { - return file, nil - } else if err != nil { + file, err := attemptFileDownload(logger, "fedach") + if err != nil && !errors.Is(err, download.ErrMissingConfigValue) { return nil, fmt.Errorf("problem downloading fedach: %v", err) } + if file != nil { + return file, nil + } + path := readDataFilepath("FEDACH_DATA_PATH", "./data/FedACHdir.txt") logger.Logf("search: loading %s for ACH data", path) - file, err := os.Open(path) + file, err = os.Open(path) if err != nil { return nil, fmt.Errorf("problem opening %s: %v", path, err) } @@ -32,16 +35,19 @@ func fedACHDataFile(logger log.Logger) (io.Reader, error) { } func fedWireDataFile(logger log.Logger) (io.Reader, error) { - if file, err := attemptFileDownload(logger, "fedwire"); file != nil { + file, err := attemptFileDownload(logger, "fedach") + if err != nil && !errors.Is(err, download.ErrMissingConfigValue) { + return nil, fmt.Errorf("problem downloading fedach: %v", err) + } + + if file != nil { return file, nil - } else if err != nil { - return nil, fmt.Errorf("problem downloading fedwire: %v", err) } path := readDataFilepath("FEDWIRE_DATA_PATH", "./data/fpddir.txt") logger.Logf("search: loading %s for Wire data", path) - file, err := os.Open(path) + file, err = os.Open(path) if err != nil { return nil, fmt.Errorf("problem opening %s: %v", path, err) } @@ -51,20 +57,18 @@ func fedWireDataFile(logger log.Logger) (io.Reader, error) { func attemptFileDownload(logger log.Logger, listName string) (io.Reader, error) { routingNumber := os.Getenv("FRB_ROUTING_NUMBER") downloadCode := os.Getenv("FRB_DOWNLOAD_CODE") - - if routingNumber != "" && downloadCode != "" { - logger.Logf("download: attempting %s", listName) - client, err := download.NewClient(&download.ClientOpts{ - RoutingNumber: routingNumber, - DownloadCode: downloadCode, - }) - if err != nil { - return nil, fmt.Errorf("client setup: %v", err) - } - return client.GetList(listName) + downloadURL := os.Getenv("FRB_DOWNLOAD_URL_TEMPLATE") + + logger.Logf("download: attempting %s", listName) + client, err := download.NewClient(&download.ClientOpts{ + RoutingNumber: routingNumber, + DownloadCode: downloadCode, + DownloadURL: downloadURL, + }) + if err != nil { + return nil, fmt.Errorf("client setup: %w", err) } - - return nil, nil + return client.GetList(listName) } func readDataFilepath(env, fallback string) string { diff --git a/pkg/download/download.go b/pkg/download/download.go index f0876e11..eeab7fde 100644 --- a/pkg/download/download.go +++ b/pkg/download/download.go @@ -14,16 +14,26 @@ import ( "time" ) +const DefaultFRBDownloadURLTemplate = "https://frbservices.org/EPaymentsDirectory/directories/%s?format=json" + +var ( + ErrMissingConfigValue = errors.New("missing config value") + ErrMissingRoutingNumber = errors.New("missing routing number") + ErrMissingDownloadCD = errors.New("missing download code") +) + type Client struct { httpClient *http.Client routingNumber string // X_FRB_EPAYMENTS_DIRECTORY_ORG_ID header downloadCode string // X_FRB_EPAYMENTS_DIRECTORY_DOWNLOAD_CD + downloadURL string // defaults to "https://frbservices.org/EPaymentsDirectory/directories/%s?format=json" where %s is the list name + } type ClientOpts struct { - HTTPClient *http.Client - RoutingNumber, DownloadCode string + HTTPClient *http.Client + RoutingNumber, DownloadCode, DownloadURL string } func NewClient(opts *ClientOpts) (*Client, error) { @@ -39,23 +49,29 @@ func NewClient(opts *ClientOpts) (*Client, error) { } if opts.RoutingNumber == "" { - return nil, errors.New("missing routing number") + return nil, fmt.Errorf("%w: %w", ErrMissingConfigValue, ErrMissingRoutingNumber) + } + + if opts.RoutingNumber == "" { + return nil, fmt.Errorf("%w: %w", ErrMissingConfigValue, ErrMissingDownloadCD) } - if opts.DownloadCode == "" { - return nil, errors.New("missing download code") + + if opts.DownloadURL == "" { + opts.DownloadURL = DefaultFRBDownloadURLTemplate } return &Client{ httpClient: opts.HTTPClient, routingNumber: opts.RoutingNumber, downloadCode: opts.DownloadCode, + downloadURL: opts.DownloadURL, }, nil } // GetList downloads an FRB list and saves it into an io.Reader. // Example listName values: fedach, fedwire func (c *Client) GetList(listName string) (io.Reader, error) { - where, err := url.Parse(fmt.Sprintf("https://frbservices.org/EPaymentsDirectory/directories/%s?format=json", listName)) + where, err := url.Parse(fmt.Sprintf(c.downloadURL, listName)) if err != nil { return nil, fmt.Errorf("url: %v", err) } @@ -64,8 +80,11 @@ func (c *Client) GetList(listName string) (io.Reader, error) { if err != nil { return nil, fmt.Errorf("building %s url: %v", listName, err) } - req.Header.Set("X_FRB_EPAYMENTS_DIRECTORY_ORG_ID", c.routingNumber) - req.Header.Set("X_FRB_EPAYMENTS_DIRECTORY_DOWNLOAD_CD", c.downloadCode) + + if c.downloadCode != "" && c.routingNumber != "" { + req.Header.Set("X_FRB_EPAYMENTS_DIRECTORY_ORG_ID", c.routingNumber) + req.Header.Set("X_FRB_EPAYMENTS_DIRECTORY_DOWNLOAD_CD", c.downloadCode) + } // perform our request resp, err := c.httpClient.Do(req) diff --git a/pkg/download/download_test.go b/pkg/download/download_test.go index 1709591d..7d0c0fec 100644 --- a/pkg/download/download_test.go +++ b/pkg/download/download_test.go @@ -6,8 +6,12 @@ package download import ( "bytes" + "fmt" "io" + "net/http" + "net/http/httptest" "os" + "path/filepath" "testing" "github.com/stretchr/testify/require" @@ -34,6 +38,41 @@ func TestClient__fedach(t *testing.T) { } } +func TestClient__fedach_custom_url(t *testing.T) { + file, err := os.ReadFile(filepath.Join("..", "..", "data", "fedachdir.json")) + if err != nil { + t.Fatal(err) + } + + mockHTTPServer := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { + fmt.Fprint(writer, string(file)) + })) + defer mockHTTPServer.Close() + + t.Setenv("FRB_DOWNLOAD_URL_TEMPLATE", mockHTTPServer.URL+"/%s") + t.Setenv("FRB_ROUTING_NUMBER", "123456789") + t.Setenv("FRB_DOWNLOAD_CODE", "a1b2c3d4-123b-9876-1234-z1x2y3a1b2c3") + + client := setupClient(t) + + fedach, err := client.GetList("fedach") + if err != nil { + t.Fatal(err) + } + + buf, ok := fedach.(*bytes.Buffer) + require.True(t, ok) + + if n := buf.Len(); n < 1024 { + t.Errorf("unexpected size of %d bytes", n) + } + + bs, _ := io.ReadAll(io.LimitReader(fedach, 10024)) + if !bytes.Equal(bs, file) { + t.Errorf("unexpected output:\n%s", string(bs)) + } +} + func TestClient__fedwire(t *testing.T) { client := setupClient(t) @@ -55,11 +94,46 @@ func TestClient__fedwire(t *testing.T) { } } +func TestClient__wire_custom_url(t *testing.T) { + file, err := os.ReadFile(filepath.Join("..", "..", "data", "fedachdir.json")) + if err != nil { + t.Fatal(err) + } + mockHTTPServer := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { + fmt.Fprint(writer, string(file)) + })) + defer mockHTTPServer.Close() + + t.Setenv("FRB_DOWNLOAD_URL_TEMPLATE", mockHTTPServer.URL+"/%s") + t.Setenv("FRB_ROUTING_NUMBER", "123456789") + t.Setenv("FRB_DOWNLOAD_CODE", "a1b2c3d4-123b-9876-1234-z1x2y3a1b2c3") + + client := setupClient(t) + + fedach, err := client.GetList("fedwire") + if err != nil { + t.Fatal(err) + } + + buf, ok := fedach.(*bytes.Buffer) + require.True(t, ok) + + if n := buf.Len(); n < 1024 { + t.Errorf("unexpected size of %d bytes", n) + } + + bs, _ := io.ReadAll(io.LimitReader(fedach, 10024)) + if !bytes.Equal(bs, file) { + t.Errorf("unexpected output:\n%s", string(bs)) + } +} + func setupClient(t *testing.T) *Client { t.Helper() routingNumber := os.Getenv("FRB_ROUTING_NUMBER") downloadCode := os.Getenv("FRB_DOWNLOAD_CODE") + downloadURL := os.Getenv("FRB_DOWNLOAD_URL_TEMPLATE") if routingNumber == "" || downloadCode == "" { t.Skip("missing FRB routing number or download code") } @@ -67,6 +141,7 @@ func setupClient(t *testing.T) *Client { client, err := NewClient(&ClientOpts{ RoutingNumber: routingNumber, DownloadCode: downloadCode, + DownloadURL: downloadURL, }) if err != nil { t.Fatal(err)