support 202309 (#83)

* refactor of sync and fetch logic to simplify and modernize approach --fetch-private for sync cache url fixes linting Co-authored-by: Nate Sales <[email protected]> * remove merge cruft and add changelog update * fix blank/null validation in object update * fix debug message * consistent object validation throughout update process * linting * debug log for cache url when loading from remote cache debug log of cache file when using local cache * linting * remove atomic transactions - in case of sync failure due to 429 they prevent the user from continuing the sync on the next run * linting --------- Co-authored-by: 20C <[email protected]> Co-authored-by: Nate Sales <[email protected]>
peeringdb · Nov 22, 2023 · a597426 · a597426
1 parent c16718a
commit a597426
Show file tree

Hide file tree

Showing 31 changed files with 1,236 additions and 1,476 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -5,8 +5,10 @@
 ### Added
 - django 4.2 support
 - python 3.11 support
+- refactor of sync and fetch logic to simplify and modernize approach
 - support for fetching data from peeringdb cache servers
 - support for quickly setting a local snapshot of peeringdb server via the `peeringdb server` commands
+- env variable support for config (see docs/config.md)
 ### Removed
 - python 3.7 support
 - django 2.2 support

diff --git a/CHANGELOG.yaml b/CHANGELOG.yaml
@@ -2,12 +2,14 @@ Unreleased:
   added:
   - django 4.2 support
   - python 3.11 support
+  - refactor of sync and fetch logic to simplify and modernize approach
   - support for fetching data from peeringdb cache servers
   - support for quickly setting a local snapshot of peeringdb server via the `peeringdb server` commands
+  - env variable support for config (see docs/config.md)
   fixed: []
   changed: []
   deprecated: []
-  removed: 
+  removed:
   - python 3.7 support
   - django 2.2 support
   - django 3.0 support

diff --git a/docs/cli.md b/docs/cli.md
@@ -111,4 +111,4 @@ $ peeringdb server --stop
 
 ## Error Handling
 
-If you try to start or stop the server without running the setup first, you will see an error stating that the `peeringdb_server` directory was not found. If this happens, run the `--setup` command and then try again.
+If you try to start or stop the server without running the setup first, you will see an error stating that the `peeringdb_server` directory was not found. If this happens, run the `--setup` command and then try again.
diff --git a/docs/config.md b/docs/config.md
@@ -0,0 +1,31 @@
+
+# PeeringDB Client Configuration
+
+This document describes the environment variables used to configure the PeeringDB client.
+
+## Sync Configuration
+
+- **PDB_SYNC_URL**: The main URL for syncing with PeeringDB. Default is `https://www.peeringdb.com/api`.
+- **PDB_SYNC_CACHE_URL**: The cache URL for syncing with PeeringDB. Default is `https://public.peeringdb.com`.
+- **PDB_SYNC_CACHE_DIR**: The directory for caching PeeringDB data. Default is `~/.cache/peeringdb`.
+- **PDB_SYNC_API_KEY**: The API key for authentication. No default value.
+- **PDB_SYNC_USER**: The username for authentication. No default value.
+- **PDB_SYNC_PASSWORD**: The password for authentication. No default value.
+- **PDB_SYNC_STRIP_TZ**: Strip timezone information (1 for true, 0 for false). Default is `1`.
+- **PDB_SYNC_ONLY**: Comma-separated list of data to sync. Default is all data (empty list).
+- **PDB_SYNC_TIMEOUT**: The timeout for syncing operations in seconds. Default is `0` (no timeout).
+
+## ORM Configuration
+
+### Database Configuration
+
+- **PDB_ORM_DB_ENGINE**: The database engine to use. Default is `sqlite3`.
+- **PDB_ORM_DB_NAME**: The name of the database. Default is `peeringdb.sqlite3`.
+- **PDB_ORM_DB_HOST**: The host of the database. No default value.
+- **PDB_ORM_DB_PORT**: The port of the database. Default is `0`.
+- **PDB_ORM_DB_USER**: The username for database authentication. No default value.
+- **PDB_ORM_DB_PASSWORD**: The password for database authentication. No default value.
+
+### General ORM Configuration
+
+- **PDB_ORM_BACKEND**: The backend to use for the ORM. Default is `django_peeringdb`.
diff --git a/poetry.lock b/poetry.lock
diff --git a/pyproject.toml b/pyproject.toml
@@ -10,18 +10,18 @@ readme = "README.md"
 repository = "https://github.com/peeringdb/peeringdb-py"
 authors = [ "PeeringDB <[email protected]>",]
 license = "Apache-2.0"
-classifiers = [ 
-    "Development Status :: 5 - Production/Stable",  
-    "Framework :: Django :: 3.2", 
-    "Framework :: Django :: 4.2", 
-    "Intended Audience :: Developers", 
-    "Intended Audience :: System Administrators", 
-    "Intended Audience :: Telecommunications Industry", 
-    "License :: OSI Approved :: Apache Software License", 
-    "Programming Language :: Python :: 3.8", 
-    "Programming Language :: Python :: 3.9", 
-    "Programming Language :: Python :: 3.10", 
-    "Programming Language :: Python :: 3.11", 
+classifiers = [
+    "Development Status :: 5 - Production/Stable",
+    "Framework :: Django :: 3.2",
+    "Framework :: Django :: 4.2",
+    "Intended Audience :: Developers",
+    "Intended Audience :: System Administrators",
+    "Intended Audience :: Telecommunications Industry",
+    "License :: OSI Approved :: Apache Software License",
+    "Programming Language :: Python :: 3.8",
+    "Programming Language :: Python :: 3.9",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
     "Topic :: Software Development :: Libraries :: Python Modules",
     "Topic :: Internet"
 ]
@@ -41,12 +41,14 @@ python = "^3.8"
 confu = "^1"
 munge = { extras = ["tomlkit", "yaml"], version = "^1.2.0" }
 "twentyc.rpc" = "^1"
+pyyaml = "^6.0.1"
+httpx = ">=0.24.1"
 
 [tool.poetry.dev-dependencies]
 
 # testing
+django_peeringdb = "^3.1.0"
 django = "~4.2"
-django_peeringdb = "^3.0.0"
 
 pytest = "^6.0.1"
 pytest-cov = "*"

diff --git a/src/peeringdb/__init__.py b/src/peeringdb/__init__.py
@@ -19,7 +19,7 @@ def _config_logs(lvl=None, name=None):
     _config_logs(L) => change log level
     """
     # print('_config_log', 'from %s' %name if name else '')
-    FORMAT = "%(message)s"
+    logging_format = "%(message)s"
     # maybe better for log files
     # FORMAT='[%(levelname)s]:%(message)s',
 
@@ -31,7 +31,7 @@ def _config_logs(lvl=None, name=None):
     if lvl:
         _log_level = lvl
 
-    logging.basicConfig(level=_log_level, format=FORMAT, stream=sys.stdout)
+    logging.basicConfig(level=_log_level, format=logging_format, stream=sys.stdout)
     _log = logging.getLogger(__name__)
     _log.setLevel(_log_level)
 
@@ -84,10 +84,10 @@ def initialize_backend(name, **kwargs):
     # Load internal module associated with the ORM module
     supportmod = import_module(modname)
     # Backend is any object returned from load_backend
-    B = supportmod.load_backend(**kwargs)
+    backend = supportmod.load_backend(**kwargs)
 
-    B.Backend.setup()
-    __backend = (B.Backend(), (name, B.__version__))
+    backend.Backend.setup()
+    __backend = (backend.Backend(), (name, backend.__version__))
 
 
 # TODO

diff --git a/src/peeringdb/_fetch.py b/src/peeringdb/_fetch.py