diff --git a/README.md b/README.md index f15626f..1a97ff9 100644 --- a/README.md +++ b/README.md @@ -48,8 +48,8 @@ for files. In this case, you can use urlchecker check: ```bash $ urlchecker check --help usage: urlchecker check [-h] [-b BRANCH] [--subfolder SUBFOLDER] [--cleanup] - [--force-pass] [--no-print] [--file-types FILE_TYPES] - [--files FILES] + [--force-pass] [--file-types FILE_TYPES] + [--files FILES] [--print-level PRINT_LEVEL] [--white-listed-urls WHITE_LISTED_URLS] [--white-listed-patterns WHITE_LISTED_PATTERNS] [--white-listed-files WHITE_LISTED_FILES] @@ -72,13 +72,14 @@ optional arguments: no cleaup) --force-pass force successful pass (return code 0) regardless of result - --no-print Skip printing results to the screen (defaults to - printing to console). --file-types FILE_TYPES comma separated list of file extensions to check (defaults to .md,.py) --files FILES comma separated list of exact files or patterns to check. + --print-level PRINT_LEVEL + print levels: all, files-with-urls-only, fails-only, + success-only, none --white-listed-urls WHITE_LISTED_URLS comma separated list of white listed urls (no spaces) --white-listed-patterns WHITE_LISTED_PATTERNS @@ -111,7 +112,7 @@ $ urlchecker check . branch: master cleanup: False file types: ['.md', '.py'] - print all: True + print level: all url whitetlist: [] url patterns: [] file patterns: [] @@ -188,7 +189,7 @@ $ urlchecker check --save results.csv . branch: master cleanup: False file types: ['.md', '.py'] - print all: True + print level: all url whitetlist: [] url patterns: [] file patterns: [] @@ -197,18 +198,18 @@ $ urlchecker check --save results.csv . save: results.csv timeout: 5 - /home/vanessa/Desktop/Code/urlstechie/urlchecker-test-repo/README.md + /home/vanessa/Desktop/Code/urlstechie/urlchecker-test-repo/README.md -------------------------------------------------------------------- No urls found. - /home/vanessa/Desktop/Code/urlstechie/urlchecker-test-repo/test_files/sample_test_file.py + /home/vanessa/Desktop/Code/urlstechie/urlchecker-test-repo/test_files/sample_test_file.py ----------------------------------------------------------------------------------------- https://github.com/SuperKogito/URLs-checker/README.md https://github.com/SuperKogito/URLs-checker/README.md https://www.google.com/ https://github.com/SuperKogito - /home/vanessa/Desktop/Code/urlstechie/urlchecker-test-repo/test_files/sample_test_file.md + /home/vanessa/Desktop/Code/urlstechie/urlchecker-test-repo/test_files/sample_test_file.md ----------------------------------------------------------------------------------------- https://github.com/SuperKogito/URLs-checker/blob/master/README.md https://github.com/SuperKogito/Voice-based-gender-recognition/issues @@ -296,7 +297,7 @@ checker = UrlChecker( file_types=[".md", ".py", ".rst"], include_patterns=[], white_listed_files=["README.md", "LICENSE"], - print_all=True, + print_level="all", ) ``` I can then run the checker like this: @@ -349,32 +350,32 @@ You can look at `checker.checks`, which is a dictionary of result objects, organized by the filename: ```python -for file_name, result in checker.checks.items(): - print() - print(result) - print("Total Results: %s " % result.count) - print("Total Failed: %s" % len(result.failed)) - print("Total Passed: %s" % len(result.passed)) +for file_name, result in checker.checks.items(): + print() + print(result) + print("Total Results: %s " % result.count) + print("Total Failed: %s" % len(result.failed)) + print("Total Passed: %s" % len(result.passed)) ... UrlCheck:/home/vanessa/Desktop/Code/urlstechie/urlchecker-python/tests/test_files/sample_test_file.md -Total Results: 26 +Total Results: 26 Total Failed: 6 Total Passed: 20 UrlCheck:/home/vanessa/Desktop/Code/urlstechie/urlchecker-python/.pytest_cache/README.md -Total Results: 1 +Total Results: 1 Total Failed: 0 Total Passed: 1 UrlCheck:/home/vanessa/Desktop/Code/urlstechie/urlchecker-python/.eggs/pytest_runner-5.2-py3.7.egg/ptr.py -Total Results: 0 +Total Results: 0 Total Failed: 0 Total Passed: 0 UrlCheck:/home/vanessa/Desktop/Code/urlstechie/urlchecker-python/docs/source/conf.py -Total Results: 3 +Total Results: 3 Total Failed: 0 Total Passed: 3 ``` @@ -425,7 +426,7 @@ checker = UrlCheckResult( file_name=file_name, white_listed_patterns=white_listed_patterns, white_listed_urls=white_listed_urls, - print_all=self.print_all, + print_level=self.print_level, ) ``` @@ -435,7 +436,7 @@ or you can provide all the parameters without the filename: checker = UrlCheckResult( white_listed_patterns=white_listed_patterns, white_listed_urls=white_listed_urls, - print_all=self.print_all, + print_level=self.print_level, ) ``` @@ -522,4 +523,3 @@ be fairly straight forward to always find what you are looking for. If you need help, or want to suggest a project for the organization, please [open an issue](https://github.com/urlstechie/urlchecker-python) - diff --git a/tests/test_client_check.py b/tests/test_client_check.py index cf0e8c9..b37e8c6 100644 --- a/tests/test_client_check.py +++ b/tests/test_client_check.py @@ -7,11 +7,11 @@ @pytest.mark.parametrize("config_fname", ["./tests/_local_test_config.conf"]) @pytest.mark.parametrize("cleanup", [False, True]) -@pytest.mark.parametrize("print_all", [False, True]) +@pytest.mark.parametrize("print_level", ["all", "files-with-urls-only", "fails-only", "success-only", "none"]) @pytest.mark.parametrize("force_pass", [False, True]) @pytest.mark.parametrize("rcount", [1, 3]) @pytest.mark.parametrize("timeout", [3, 5]) -def test_client_general(config_fname, cleanup, print_all, force_pass, rcount, timeout): +def test_client_general(config_fname, cleanup, print_level, force_pass, rcount, timeout): # init config parser config = configparser.ConfigParser() @@ -33,6 +33,8 @@ def test_client_general(config_fname, cleanup, print_all, force_pass, rcount, ti file_types, "--white-listed-files", "conf.py", + "--print_level", + print_level, "--white-listed-urls", white_listed_urls, "--white-listed_patterns", @@ -46,8 +48,7 @@ def test_client_general(config_fname, cleanup, print_all, force_pass, rcount, ti # Add boolean arguments if cleanup: cmd.append("--cleanup") - if print_all: - cmd.append("--print-all") + if force_pass: cmd.append("--force-pass") diff --git a/tests/test_core_check.py b/tests/test_core_check.py index 301020d..f205b8e 100644 --- a/tests/test_core_check.py +++ b/tests/test_core_check.py @@ -16,7 +16,10 @@ ["tests/test_files/sample_test_file.rst"], ], ) -@pytest.mark.parametrize("print_all", [False, True]) +@pytest.mark.parametrize( + "print_level", ["all", "files-with-urls-only", + "fails-only", "success-only", "none"] +) @pytest.mark.parametrize( "white_listed_urls", [["https://github.com/SuperKogito/SuperKogito.github.io"]] ) @@ -24,11 +27,11 @@ "white_listed_patterns", [[], ["https://github.com/SuperKogito/SuperKogito.github.io"]], ) -def test_check_files(file_paths, print_all, white_listed_urls, white_listed_patterns): +def test_check_files(file_paths, print_level, white_listed_urls, white_listed_patterns): """ test check repo function. """ - checker = UrlChecker(print_all=print_all) + checker = UrlChecker(print_level=print_level) checker.run( file_paths, white_listed_urls=white_listed_urls, @@ -46,7 +49,6 @@ def test_locally(local_folder_path, config_fname): # read input variables git_path = local_folder_path file_types = config["DEFAULT"]["file_types_test_values"].split(",") - print_all = True white_listed_urls = config["DEFAULT"]["white_listed_test_urls"].split(",") white_listed_patterns = config["DEFAULT"]["white_listed__test_patterns"].split(",") @@ -54,7 +56,7 @@ def test_locally(local_folder_path, config_fname): file_paths = get_file_paths(git_path, file_types) # check repo urls - checker = UrlChecker(print_all=print_all) + checker = UrlChecker(print_level="all") checker.run( file_paths=file_paths, white_listed_urls=white_listed_urls, @@ -69,7 +71,6 @@ def test_check_run_save(tmp_path, retry_count): # init vars git_path = "https://github.com/urlstechie/urlchecker-test-repo" file_types = [".py", ".md"] - print_all = True white_listed_urls = [ "https://superkogito.github.io/figures/fig2.html", "https://superkogito.github.io/figures/fig4.html", @@ -86,7 +87,7 @@ def test_check_run_save(tmp_path, retry_count): file_paths = get_file_paths(base_path, file_types) # check repo urls - checker = UrlChecker(print_all=print_all) + checker = UrlChecker(print_level="all") check_results = checker.run( file_paths=file_paths, white_listed_urls=white_listed_urls, diff --git a/tests/test_core_urlproc.py b/tests/test_core_urlproc.py index 6f6b4d9..2e16ebf 100644 --- a/tests/test_core_urlproc.py +++ b/tests/test_core_urlproc.py @@ -34,8 +34,6 @@ def test_check_urls(file): # Ensure one not whitelisted is failed assert "https://github.com/SuperKogito/URLs-checker/issues/1" in checker.failed - assert checker.print_all - # Run again with whitelist of exact urls checker = UrlCheckResult( white_listed_urls=["https://github.com/SuperKogito/URLs-checker/issues/1"] @@ -61,7 +59,11 @@ def test_get_user_agent(): assert isinstance(user_agent, str) -def test_check_response_status_code(): +@pytest.mark.parametrize( + "print_level", ["all", "files-with-urls-only", + "fails-only", "success-only", "none"] +) +def test_check_response_status_code(print_level): class failedResponse: status_code = 500 @@ -70,7 +72,7 @@ class successResponse: # Any failure returns True (indicating a retry is needed) assert not check_response_status_code( - "https://this-should-succeed", successResponse + "https://this-should-succeed", successResponse, print_level ) - assert check_response_status_code("https://this-should-fail", failedResponse) - assert check_response_status_code("https://this-should-also-fail", None) + assert check_response_status_code("https://this-should-fail", failedResponse, print_level) + assert check_response_status_code("https://this-should-also-fail", None, print_level) diff --git a/urlchecker/client/__init__.py b/urlchecker/client/__init__.py index 06337df..105d745 100755 --- a/urlchecker/client/__init__.py +++ b/urlchecker/client/__init__.py @@ -85,13 +85,6 @@ def get_parser(): action="store_true", ) - check.add_argument( - "--no-print", - help="Skip printing results to the screen (defaults to printing to console).", - default=False, - action="store_true", - ) - check.add_argument( "--file-types", dest="file_types", @@ -106,6 +99,17 @@ def get_parser(): default="", ) + # Printing + + check.add_argument( + "--print-level", + dest="print_level", + help="print levels: all, files-with-urls-only, fails-only, success-only, none", + choices=["all", "files-with-urls-only", "fails-only", "success-only", "none"], + default="all", + ) + + # White listing check.add_argument( @@ -158,7 +162,7 @@ def main(): parser = get_parser() def help(return_code=0): - """print help, including the software version and active client + """print help, including the software version and active client and exit with return code. """ diff --git a/urlchecker/client/check.py b/urlchecker/client/check.py index 14b58c6..d140a97 100644 --- a/urlchecker/client/check.py +++ b/urlchecker/client/check.py @@ -62,7 +62,7 @@ def main(args, extra): print(" cleanup: %s" % args.cleanup) print(" file types: %s" % file_types) print(" files: %s" % files) - print(" print all: %s" % (not args.no_print)) + print(" print_level: %s" % args.print_level) print(" url whitetlist: %s" % white_listed_urls) print(" url patterns: %s" % white_listed_patterns) print(" file patterns: %s" % white_listed_files) @@ -77,7 +77,7 @@ def main(args, extra): file_types=file_types, include_patterns=files, white_listed_files=white_listed_files, - print_all=not args.no_print, + print_level=args.print_level, ) check_results = checker.run( white_listed_urls=white_listed_urls, diff --git a/urlchecker/core/check.py b/urlchecker/core/check.py index f833023..3efdf4f 100644 --- a/urlchecker/core/check.py +++ b/urlchecker/core/check.py @@ -24,7 +24,7 @@ def __init__( path=None, file_types=None, white_listed_files=None, - print_all=True, + print_level=None, include_patterns=None, ): """ @@ -34,7 +34,7 @@ def __init__( Args: - path (str) : full path to the root folder to check. If not defined, no file_paths are parsed - - print_all (str) : control var for whether to print all checked file names or only the ones with urls. + - print_level (str) : control the printing level. - white_listed_files (list) : list of white-listed files and patterns for flies. - include_patterns (list) : list of files and patterns to check. """ @@ -45,7 +45,7 @@ def __init__( # Save run parameters self.white_listed_files = white_listed_files or [] self.include_patterns = include_patterns or [] - self.print_all = print_all + self.print_level = print_level self.path = path self.file_types = file_types or [".py", ".md"] self.file_paths = [] @@ -155,7 +155,7 @@ def run( file_name=file_name, white_listed_patterns=white_listed_patterns, white_listed_urls=white_listed_urls, - print_all=self.print_all, + print_level=self.print_level, ) # Check the urls diff --git a/urlchecker/core/urlproc.py b/urlchecker/core/urlproc.py index 2f02093..a4a1881 100644 --- a/urlchecker/core/urlproc.py +++ b/urlchecker/core/urlproc.py @@ -2,7 +2,7 @@ Copyright (c) 2020 Ayoub Malek and Vanessa Sochat -This source code is licensed under the terms of the MIT license. +This source code is licensed under the terms of the MIT license. For a copy, see . """ @@ -16,7 +16,7 @@ from urlchecker.logger import print_success, print_failure -def check_response_status_code(url, response): +def check_response_status_code(url, response, print_level): """ Check response status of an input url. Returns a boolean to indicate if retry is needed. @@ -30,16 +30,19 @@ def check_response_status_code(url, response): """ # Case 1: response is None indicating triggered error if not response: - print_failure(url) + if print_level not in ["success-only", "none"] : + print_failure(url) return True # Case 2: success! Retry is not needed. if response.status_code == 200: - print_success(url) + if print_level not in ["fails-only", "none"] : + print_success(url) return False # Case 3: failure of some kind - print_failure(url) + if print_level not in ["success-only", "none"] : + print_failure(url) return True @@ -101,10 +104,10 @@ def __init__( file_name=None, white_listed_patterns=None, white_listed_urls=None, - print_all=True, + print_level=None, ): self.file_name = file_name - self.print_all = print_all + self.print_level = print_level self.passed = [] self.failed = [] self.white_listed = [] @@ -167,13 +170,13 @@ def check_urls(self, urls=None, retry_count=1, timeout=5): # if no urls are found, mention it if required if not urls: - if self.print_all: + if self.print_level == "all": if self.file_name: print("\n", self.file_name, "\n", "-" * len(self.file_name)) print("No urls found.") return - if self.file_name: + if self.file_name and self.print_level != "none": print("\n", self.file_name, "\n", "-" * len(self.file_name)) # init seen urls list @@ -210,19 +213,22 @@ def check_urls(self, urls=None, retry_count=1, timeout=5): response = requests.get(url, timeout=pause, headers=headers) except requests.exceptions.Timeout as e: - print(e) + if self.print_level not in ["success-only", "none"]: + print(e) except requests.exceptions.ConnectionError as e: - print(e) + if self.print_level not in ["success-only", "none"]: + print(e) except Exception as e: - print(e) + if self.print_level not in ["success-only", "none"]: + print(e) # decrement retrials count rcount -= 1 # Break from the loop if we have success, update user - do_retry = check_response_status_code(url, response) + do_retry = check_response_status_code(url, response, self.print_level) # If we try again, pause for retry seconds and update retry seconds if rcount > 0 and do_retry: