From c957d836f0f7ba20ff754113c9cb764d76a0ab2f Mon Sep 17 00:00:00 2001 From: davicorreiajr Date: Tue, 17 Mar 2020 12:49:51 -0300 Subject: [PATCH] Add select_streams option (to be added in the config file) to generate catalog file with some stream(s) pre-selected --- tap_google_sheets/__init__.py | 7 ++++--- tap_google_sheets/discover.py | 4 +++- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/tap_google_sheets/__init__.py b/tap_google_sheets/__init__.py index f97d4b8..9b5fa3e 100644 --- a/tap_google_sheets/__init__.py +++ b/tap_google_sheets/__init__.py @@ -20,10 +20,10 @@ 'user_agent' ] -def do_discover(client, spreadsheet_id): +def do_discover(client, spreadsheet_id, select_streams): LOGGER.info('Starting discover') - catalog = discover(client, spreadsheet_id) + catalog = discover(client, spreadsheet_id, select_streams) json.dump(catalog.to_dict(), sys.stdout, indent=2) LOGGER.info('Finished discover') @@ -44,9 +44,10 @@ def main(): config = parsed_args.config spreadsheet_id = config.get('spreadsheet_id') + select_streams = config.get('select_streams') if parsed_args.discover: - do_discover(client, spreadsheet_id) + do_discover(client, spreadsheet_id, select_streams) elif parsed_args.catalog: sync(client=client, config=config, diff --git a/tap_google_sheets/discover.py b/tap_google_sheets/discover.py index 6cf0d09..005385a 100644 --- a/tap_google_sheets/discover.py +++ b/tap_google_sheets/discover.py @@ -2,11 +2,13 @@ from tap_google_sheets.schema import get_schemas, STREAMS -def discover(client, spreadsheet_id): +def discover(client, spreadsheet_id, select_streams): schemas, field_metadata = get_schemas(client, spreadsheet_id) catalog = Catalog([]) for stream_name, schema_dict in schemas.items(): + if select_streams and stream_name in select_streams: + schema_dict['selected'] = True schema = Schema.from_dict(schema_dict) mdata = field_metadata[stream_name] key_properties = None