From 7bba84ae63a995f579363e5c27ee778f1c320644 Mon Sep 17 00:00:00 2001
From: tassadarius <michael@altenhuber.net>
Date: Tue, 8 Mar 2022 21:43:33 +0100
Subject: [PATCH 1/2] NOISSUE Fix/Update Link checker script and fix dead links
 (#255)

* NOISSUE [DevelopmentTools] Fix and extend link availability check

Add the following features/fix the following mistakes:

* Fake browser User-Agent in the header as some sites block scripts
* Add parameter to search html files by pattern and default it to
  'nl/' directories
* Add parameter to exclude certain project names and let it default to
  'org.jcryptool.core.nl' as this uses another link format.
* Add parameter for an ignore file. The ignore file can contain links
  which are queried nevertheless. But if they fail they will not be
  counted as errors. A specific reason can be specified so it will only
  count as ignored if that reason is matched.
  It defaults to ./ignore_links.txt
  Add the ignored information to the summary
* Add meaningful return value: 0 if all links reachable or ignored, 1 if
  any link was not reachable (failed).
* Fix timeout case not being counted as timeout
* Improve formatting

* NOISSUE [OnlineHelp] Fix/Update dead links
---
 .../help/users/algorithms/signature/toc.html  |   2 +-
 .../help/users/cryptology/crypto_methods.html |   2 +-
 .../nl/de/help/users/cryptology/toc.html      |   2 +-
 .../nl/de/help/users/games/toc.html           |   2 +-
 org.jcryptool.core.help/nl/de/single/RSA.html |   2 +-
 .../help/users/algorithms/signature/toc.html  |   2 +-
 .../help/users/cryptology/crypto_methods.html |   2 +-
 .../nl/en/help/users/cryptology/toc.html      |   2 +-
 .../nl/en/help/users/games/toc.html           |   2 +-
 .../nl/en/help/users/visualizations/toc.html  |   2 +-
 org.jcryptool.core.help/nl/en/single/RSA.html |   2 +-
 org.jcryptool.releng/helper/ignored_links.txt |   4 +
 .../helper/link_availability_checker.py       | 205 +++++++++++++++---
 13 files changed, 186 insertions(+), 45 deletions(-)
 create mode 100644 org.jcryptool.releng/helper/ignored_links.txt

diff --git a/org.jcryptool.core.help/nl/de/help/users/algorithms/signature/toc.html b/org.jcryptool.core.help/nl/de/help/users/algorithms/signature/toc.html
index 11cd170878..fb9563dec9 100644
--- a/org.jcryptool.core.help/nl/de/help/users/algorithms/signature/toc.html
+++ b/org.jcryptool.core.help/nl/de/help/users/algorithms/signature/toc.html
@@ -30,7 +30,7 @@ <h2>Über Signaturen</h2>
 	<p>Digitale Signaturen sind eine Standardkomponente der meisten kryptografischen Protokoll-Suiten. Sie werden häufig für Software-Verteilung, Finanztransaktionen, Vertragsverwaltungssoftware und in weiteren Fällen verwendet, in denen es wichtig ist, Fälschungen oder Manipulationen zu erkennen.
 	<b>Digitale</b> Signaturen werden oft verwendet, um <b>elektronische</b> Signaturen zu implementieren.</p>
 
-	</p>Das folgende Bild zeigt eine visuelle Darstellung des Signier- und Verifizierungsprozesses (<a href="https://commons.wikimedia.org/wiki/File:Digital_Signature_diagram_de.svg">Diagramm Digitale Signatur</a> von Acdx, lizenziert unter <a href="https://creativecommons.org/choose/zero/">CC0</a>).</p>
+	</p>Das folgende Bild zeigt eine visuelle Darstellung des Signier- und Verifizierungsprozesses (<a href="https://commons.wikimedia.org/wiki/File:Digital_Signature_diagram_de.svg">Diagramm Digitale Signatur</a> von Acdx, lizenziert unter <a href="https://creativecommons.org/licenses/by-sa/3.0/deed.de">CC BY-SA 3.0</a>).</p>
 
 	<img src="PLUGINS_ROOT/org.jcryptool.core.help/$nl$/single/img/1500px-DSA-diagram.png" width="85%" alt="Visuelle Darstellung des Signier- und Verifizierungsprozesses" />
 
diff --git a/org.jcryptool.core.help/nl/de/help/users/cryptology/crypto_methods.html b/org.jcryptool.core.help/nl/de/help/users/cryptology/crypto_methods.html
index ce65686b64..9b1f60bd82 100644
--- a/org.jcryptool.core.help/nl/de/help/users/cryptology/crypto_methods.html
+++ b/org.jcryptool.core.help/nl/de/help/users/cryptology/crypto_methods.html
@@ -9,7 +9,7 @@
 
 <h1>Kryptografische Verfahren</h1>
 
-<P>Britannica (<a target="_blank" href="http://www.britannica.com">http://www.britannica.com</a>) beginnt seine Definition folgendermaßen:</P>
+<P>Britannica (<a target="_blank" href="https://www.britannica.com">http://www.britannica.com</a>) beginnt seine Definition folgendermaßen:</P>
 
 <P><B>Kryptologie</B> (vom Griechischen <I>kryptós, </I>"versteckt," und <I>lógos, </I>"Wort") ist die Wissenschaft von sicherer (allgemein geheimer) Kommunikation. Diese Sicherheit bedingt, dass die berechtigten Teilnehmer in der Lage sind, eine Nachricht mit Hilfe eines Schlüssels in einen Geheimtext zu transferieren und zurück. Obwohl der Geheimtext für jemand ohne den geheimen Schlüssel unlesbar und unfälschbar ist, kann der berechtigte Empfänger entweder das Chiffrat entschlüsseln, um die den verborgenen Klartext wieder zu erhalten, oder verifizieren, dass die Nachricht aller Wahrscheinlichkeit nach von jemand geschickt wurde, der den richtigen Schlüssel besaß.</P>
 
diff --git a/org.jcryptool.core.help/nl/de/help/users/cryptology/toc.html b/org.jcryptool.core.help/nl/de/help/users/cryptology/toc.html
index 1adc620016..6528a07c54 100644
--- a/org.jcryptool.core.help/nl/de/help/users/cryptology/toc.html
+++ b/org.jcryptool.core.help/nl/de/help/users/cryptology/toc.html
@@ -16,7 +16,7 @@ <h1>Kryptologie</h1>
 	<p>Weitere Informationen zum Einstieg finden Sie unter:</p>
 	<ul>
 	<li><a target="_blank" href="https://de.wikipedia.org/wiki/Kryptografie">https://de.wikipedia.org/wiki/Kryptografie</a></li>
-	<li><a target="_blank" href="https://www.cryptool.org/images/ctp/documents/CT-Book-de.pdf">Das CrypTool-Buch</a></li>
+	<li><a target="_blank" href="https://www.cryptool.org/download/ctb/CT-Book-de.pdf">Das CrypTool-Buch</a></li>
 	</ul>
 
 <br/>
diff --git a/org.jcryptool.core.help/nl/de/help/users/games/toc.html b/org.jcryptool.core.help/nl/de/help/users/games/toc.html
index 4af76ebbc1..35ca1ce71c 100644
--- a/org.jcryptool.core.help/nl/de/help/users/games/toc.html
+++ b/org.jcryptool.core.help/nl/de/help/users/games/toc.html
@@ -10,7 +10,7 @@
 <h1>Spiele</h1>
 
 	<p>Das JCT-Hauptmenü <strong>Spiele</strong> (in der Standard-Perspektive) hat 3 Einträge. Alle 3 Spiele haben Krypto-Bezug. Beispielsweise das 
-    <a href="http://org.eclipse.ui.intro/showHelpTopic?id=/org.jcryptool.games.numbershark/$nl$/help/gettingstarted/NumberSharkHelp-Tutorial.html">Zahlenhai</a>-Spiel, das Schülern spielerisch helfen kann, Teiler und Primzahlen besser zu verstehen. Hier wurde Lernstoff didaktisch verpackt.</p>
+    <a href="PLUGINS_ROOT/org.jcryptool.games.numbershark/$nl$/help/toc.html">Zahlenhai</a>-Spiel, das Schülern spielerisch helfen kann, Teiler und Primzahlen besser zu verstehen. Hier wurde Lernstoff didaktisch verpackt.</p>
 
 </p></br></br>
 </body>
diff --git a/org.jcryptool.core.help/nl/de/single/RSA.html b/org.jcryptool.core.help/nl/de/single/RSA.html
index 39709c9c5f..e8c573d0e6 100644
--- a/org.jcryptool.core.help/nl/de/single/RSA.html
+++ b/org.jcryptool.core.help/nl/de/single/RSA.html
@@ -47,7 +47,7 @@ <h2>Quellen</h2>
 
 <ul>
 	<li><a target="_blank" href="https://de.wikipedia.org/wiki/RSA-Kryptosystem" >[1] Wikipedia: RSA-Kryptosystem</a></li>
-	<li><a target="_blank" href="https://www.cryptool.org/assets/img/ctp/documents/CT-Book-en.pdf" >[2] Das CrypTool-Buch</a></li>
+	<li><a target="_blank" href="https://www.cryptool.org/download/ctb/CT-Book-de.pdf" >[2] Das CrypTool-Buch</a></li>
 	<li><a target="_blank" href="https://eprint.iacr.org/2003/159.pdf" >[3] Paixao: "An efficient variant of the RSA cryptosystem", eprint paper, 2003</a> </li>
 	<li><a target="_blank" href="https://hovav.net/ucsd/dist/survey.pdf" >[4] Boneh and Shacham: "Fast Variants of RSA", 2002</a></li>
 	<li><a target="_blank" href="https://de.wikipedia.org/wiki/Optimal_Asymmetric_Encryption_Padding" >[5] Wikipedia: Optimal Asymmetric Encryption Padding</a></li>
diff --git a/org.jcryptool.core.help/nl/en/help/users/algorithms/signature/toc.html b/org.jcryptool.core.help/nl/en/help/users/algorithms/signature/toc.html
index 009a6c6330..4d7cb6591d 100644
--- a/org.jcryptool.core.help/nl/en/help/users/algorithms/signature/toc.html
+++ b/org.jcryptool.core.help/nl/en/help/users/algorithms/signature/toc.html
@@ -29,7 +29,7 @@ <h2>About signatures</h2>
 	<p>Digital signatures are a standard element of most cryptographic protocol suites, and are commonly used for software distribution, financial transactions, contract management software, and in other cases where it is important to detect forgery or tampering.
 	<b>Digital</b> signatures are often used to implement <b>electronic</b> signatures.</p>
 
-	</p>The following image gives a visual representation of the signing and verification processes (<a href="https://commons.wikimedia.org/wiki/File:Digital_Signature_diagram.svg">Digital Signature Diagram</a> by Acdx, licensed under <a href="https://creativecommons.org/choose/zero/">CC0</a>).	</p>
+	</p>The following image gives a visual representation of the signing and verification processes (<a href="https://commons.wikimedia.org/wiki/File:Digital_Signature_diagram.svg">Digital Signature Diagram</a> by Acdx, licensed under <a href="https://creativecommons.org/licenses/by-sa/3.0/deed.en">CC BY-SA 3.0</a>).	</p>
 
 	<img src="PLUGINS_ROOT/org.jcryptool.core.help/$nl$/single/img/1500px-DSA-diagram.png" width="85%" alt="Visual representation of the signing and verification processes" />
 
diff --git a/org.jcryptool.core.help/nl/en/help/users/cryptology/crypto_methods.html b/org.jcryptool.core.help/nl/en/help/users/cryptology/crypto_methods.html
index bbf9ad021e..ba96704514 100644
--- a/org.jcryptool.core.help/nl/en/help/users/cryptology/crypto_methods.html
+++ b/org.jcryptool.core.help/nl/en/help/users/cryptology/crypto_methods.html
@@ -9,7 +9,7 @@
 
 <h1>Cryptographic Mechanisms</h1>
 
-<P>Britannica (<a target="_blank" href="http://www.britannica.com">http://www.britannica.com</a>) starts its definition with:</P>
+<P>Britannica (<a target="_blank" href="https://www.britannica.com">http://www.britannica.com</a>) starts its definition with:</P>
 
 <P><B>Cryptology</B> (from the Greek <I>kryptós, </I>"hidden," and <I>lógos, </I>"word") is the science of secure (generally secret) communications. This security obtains from legitimate users, the transmitter and the receiver, being able to transform information into a cipher by virtue of a key &ndash; <I>i.e.,</I> a piece of information known only to them. Although the cipher is inscrutable and often unforgeable to anyone without this secret key, the authorized receiver can either decrypt the cipher to recover the hidden information or verify that it was sent in all likelihood by someone possessing the key.</P>
 
diff --git a/org.jcryptool.core.help/nl/en/help/users/cryptology/toc.html b/org.jcryptool.core.help/nl/en/help/users/cryptology/toc.html
index 877a01bbc0..7f1d3cb87a 100644
--- a/org.jcryptool.core.help/nl/en/help/users/cryptology/toc.html
+++ b/org.jcryptool.core.help/nl/en/help/users/cryptology/toc.html
@@ -16,7 +16,7 @@ <h1>Cryptology</h1>
 	<p>Further information to start with can be found at:</p>
 	<ul>
 	<li><a target="_blank" href="https://en.wikipedia.org/wiki/Cryptography">https://en.wikipedia.org/wiki/Cryptography</a></li>
-	<li><a target="_blank" href="https://www.cryptool.org/images/ctp/documents/CT-Book-en.pdf">The CrypTool Book</a></li>
+	<li><a target="_blank" href="https://www.cryptool.org/download/ctb/CT-Book-en.pdf">The CrypTool Book</a></li>
 	</ul>
 
 <br/>	
diff --git a/org.jcryptool.core.help/nl/en/help/users/games/toc.html b/org.jcryptool.core.help/nl/en/help/users/games/toc.html
index 785d82f482..8fe8bbaf5c 100644
--- a/org.jcryptool.core.help/nl/en/help/users/games/toc.html
+++ b/org.jcryptool.core.help/nl/en/help/users/games/toc.html
@@ -10,7 +10,7 @@
 	<h1>Games</h1>
 
 	<p>The JCT main menu <strong>Games</strong> (in the Default Perspective) has 3 entries. All 3 games have crypto reference. For example the 
-    <a href="http://org.eclipse.ui.intro/showHelpTopic?id=/org.jcryptool.games.numbershark/$nl$/help/gettingstarted/NumberSharkHelp-Tutorial.html">Number Shark</a> game that can help students to better understand divisors and prime numbers in a playful way. Here, learning material was didactically packed.</p>
+    <a href="PLUGINS_ROOT/org.jcryptool.games.numbershark/$nl$/help/toc.html">Number Shark</a> game that can help students to better understand divisors and prime numbers in a playful way. Here, learning material was didactically packed.</p>
 
 
 </body>
diff --git a/org.jcryptool.core.help/nl/en/help/users/visualizations/toc.html b/org.jcryptool.core.help/nl/en/help/users/visualizations/toc.html
index c267d6432c..f24a1ea5b1 100644
--- a/org.jcryptool.core.help/nl/en/help/users/visualizations/toc.html
+++ b/org.jcryptool.core.help/nl/en/help/users/visualizations/toc.html
@@ -35,7 +35,7 @@ <h1>Visualizations</h1>
     <p> </br> </p>
 	<p>More cryptographic algorithms in JCT can be found at</p>
 	<ul>
-        <li><a href="./../algorithms/toc.html">Algorithms main menu</a> &ndash;  Performs classic and modern algorithms without visualizing them (in the <a href="http://org.eclipse.ui.intro/showHelpTopic?id=/org.jcryptool.core.help/$nl$/help/users/general/perspective_default.html&amp;embed=false;">Default Perspective</a>)</li>
+        <li><a href="./../algorithms/toc.html">Algorithms main menu</a> &ndash;  Performs classic and modern algorithms without visualizing them (in the <a href="PLUGINS_ROOT/org.jcryptool.core.help/$nl$/help/users/general/perspective_default.html">Default Perspective</a>)</li>
         <li><a href="./../general/perspective_algorithm.html">Algorithm Perspective</a> &ndash;  Offers the algorithms from the crypto providers Bouncy Castle and FlexiProvider (only modern algorithms)</li>
     </ul>
 
diff --git a/org.jcryptool.core.help/nl/en/single/RSA.html b/org.jcryptool.core.help/nl/en/single/RSA.html
index 7574dc9404..c4556c31e1 100644
--- a/org.jcryptool.core.help/nl/en/single/RSA.html
+++ b/org.jcryptool.core.help/nl/en/single/RSA.html
@@ -47,7 +47,7 @@ <h2>References</h2>
 
 <ul>
 	<li><a target="_blank" href="https://en.wikipedia.org/wiki/RSA_(cryptosystem)" >[1] Wikipedia: RSA (cryptosystem)</a></li>
-	<li><a target="_blank" href="https://www.cryptool.org/assets/img/ctp/documents/CT-Book-de.pdf" >[2] The CrypTool Book</a></li>
+	<li><a target="_blank" href="https://www.cryptool.org/download/ctb/CT-Book-en.pdf" >[2] The CrypTool Book</a></li>
 	<li><a target="_blank" href="https://eprint.iacr.org/2003/159.pdf" >[3] Paixao: "An efficient variant of the RSA cryptosystem", eprint paper, 2003</a> </li>
 	<li><a target="_blank" href="https://hovav.net/ucsd/dist/survey.pdf" >[4] Boneh and Shacham: "Fast Variants of RSA", 2002</a></li>
 	<li><a target="_blank" href="https://en.wikipedia.org/wiki/Optimal_asymmetric_encryption_padding" >[5] Wikipedia: Optimal asymmetric encryption padding (OAEP)</a></li>
diff --git a/org.jcryptool.releng/helper/ignored_links.txt b/org.jcryptool.releng/helper/ignored_links.txt
new file mode 100644
index 0000000000..ee2438e0fd
--- /dev/null
+++ b/org.jcryptool.releng/helper/ignored_links.txt
@@ -0,0 +1,4 @@
+# Some weird error with the servers DH key
+https://secg.org/sec1-v2.pdf                         dh key too small
+# Not queryable by script.
+https://www.eetimes.com/document.asp?doc_id=1274550  timed out
diff --git a/org.jcryptool.releng/helper/link_availability_checker.py b/org.jcryptool.releng/helper/link_availability_checker.py
index 93195c7cfe..be7ca378ba 100755
--- a/org.jcryptool.releng/helper/link_availability_checker.py
+++ b/org.jcryptool.releng/helper/link_availability_checker.py
@@ -6,7 +6,6 @@
 It requires Python 3.7 or higher and has no external dependencies.
 """
 from argparse import ArgumentParser, RawDescriptionHelpFormatter
-import itertools
 import http.client
 import multiprocessing
 import os
@@ -25,6 +24,16 @@
 from urllib.request import Request, urlopen
 
 
+class DefaultSettings(Enum):
+    FILTER_PATTERN = "nl/**/"
+    EXCLUDE_PROJECTS = "org.jcryptool.core.nl, org.jcryptool.games.sudoku"
+    USER_AGENT = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_5) AppleWebKit/537.36 " \
+                 "(KHTML, like Gecko) Chrome/50.0.2661.102 Safari/537.36"
+    RETRY_COUNT = 3
+    TIMEOUT = 5
+    IGNORE_FILE = "ignored_links.txt"
+
+
 class Status(Enum):
     """Link target status"""
     PENDING = 0
@@ -32,6 +41,7 @@ class Status(Enum):
     OK = 2
     TIMEOUT = 3
     PROCESSING = 4
+    IGNORED = 5
 
 
 class HTMLLinkParser(HTMLParser, ABC):
@@ -65,6 +75,8 @@ class PingContext:
     error_reason: Optional[str] = field(default=None, hash=False)
     retry_count: int = field(default=0, hash=True)
     files: Set[Union[Path, str]] = field(default=None, hash=False)
+    is_ignored: bool = field(default=False, hash=False)
+    ignore_pattern: str = field(default=None, hash=False)
 
     def __str__(self):
         code_extension = ""
@@ -83,7 +95,7 @@ def search_links(html: str) -> List[str]:
 
 def send_request(context: PingContext) -> PingContext:
     """Send a request and save the return status code and possible errors."""
-    req = Request(context.url)
+    req = Request(context.url, headers={"User-Agent": DefaultSettings.USER_AGENT.value})
     try:
         _response = urlopen(req, timeout=context.timeout)
         context.status = Status.OK
@@ -95,7 +107,12 @@ def send_request(context: PingContext) -> PingContext:
             context.error_reason = e.reason
         elif hasattr(e, 'code'):
             context.http_code = e.code
-        context.status = Status.FAILED
+        if "urlopen error timed out" in str(e):
+            context.status = Status.TIMEOUT
+            context.retry_count += 1
+            context.timeout *= 2
+        else:
+            context.status = Status.FAILED
         return context
     except socket.timeout as e:
         context.status = Status.TIMEOUT
@@ -124,10 +141,12 @@ def is_legit_external_link(url: str) -> bool:
         return False
 
 
-def check_and_put(output: Dict[str, PingContext],
-                    links: List[str],
-                    current_file: Path,
-                    target_path: Path) -> None:
+def check_and_put(
+        output: Dict[str, PingContext],
+        links: List[str],
+        current_file: Path,
+        target_path: Path
+) -> None:
     for link in links:
         if is_legit_external_link(link):
             _rel_path = current_file.relative_to(target_path)
@@ -135,20 +154,36 @@ def check_and_put(output: Dict[str, PingContext],
             if link not in output:
                 output[link] = PingContext(link,
                                            Status.PENDING,
-                                           timeout=global_default_timeout,
+                                           timeout=DefaultSettings.TIMEOUT.value,
                                            files={str(_rel_path)}
                                            )
             else:
                 output[link].files.add(str(_rel_path))
 
 
-def collect_all_links(target_path: Union[Path, List[Path]]) -> List[PingContext]:
+def in_exclude(file: Path, exclude_list: List[str]) -> bool:
+    for exclude_pattern in exclude_list:
+        if exclude_pattern in str(file):
+            print_verbose(2, f"Excluding {file} because it matches the exclude "
+                             f"'{exclude_pattern}'")
+            return True
+    return False
+
+
+def collect_all_links(
+        target_path: Union[Path, List[Path]], filter_prefix: str, exclude: List[str]
+) -> List[PingContext]:
     """Search all html files in a given directory path and collect links."""
     result = {}
+    file_filter = f"{filter_prefix}*.html"
     if not isinstance(target_path, list):
         target_path = [target_path]
     for current_target_path in target_path:
-        for file in current_target_path.rglob('*.html'):
+        for file in current_target_path.rglob(file_filter):
+
+            if in_exclude(file, exclude):
+                continue
+
             with open(file, 'r') as file_descriptor:
                 data = file_descriptor.read()
             links = search_links(data)
@@ -164,9 +199,11 @@ def fill_waiting_queue(collected_links: List, mp_queue: Queue):
 
 def query(
         targets: Queue,
-        retry_count: int, successful_targets:
-        List[PingContext], failed_targets:
-        List[PingContext]) -> None:
+        retry_count: int,
+        successful_targets: List[PingContext],
+        failed_targets: List[PingContext],
+        ignored_targets: List[PingContext],
+) -> None:
     """Query a target link and check if it's reachable or not."""
     while not targets.empty():
         target = targets.get()
@@ -174,11 +211,24 @@ def query(
             print_verbose(2, f"[{os.getpid()}] Working on {target.url}")
             target.status = Status.PROCESSING
             target = send_request(target)
-            if target.status == Status.TIMEOUT and target.retry_count < retry_count:
-                print_verbose(2, f"Timeout on {target.url}, "
+            if target.is_ignored and target.status != Status.OK:
+                if target.ignore_pattern in str(target.error_reason):
+                    target.status = Status.IGNORED
+                    msg = f"[{os.getpid()}] Ignored '{target.url}' after it had an " \
+                          f"error {target.error_reason}."
+                    print_verbose(2, msg)
+                    ignored_targets.append(target)
+                    continue
+                else:
+                    msg = f"[{os.getpid()}] Link '{target.url}' on ignore list " \
+                          f"returned with an error but the pattern did not match: " \
+                          f"'{target.ignore_pattern}' not in '{target.error_reason}'"
+                    print_verbose(2, msg)
+
+            if target.status == Status.TIMEOUT and target.retry_count <= retry_count:
+                print_verbose(2, f"[{os.getpid()}] Timeout on {target.url}, "
                                  f"retry {target.retry_count}/{retry_count}")
                 target.status = Status.PENDING
-                target.retry_count += 1
                 targets.put(target)
             elif target.status == Status.OK:
                 successful_targets.append(target)
@@ -196,12 +246,43 @@ def print_verbose(verbosity_level: int, message: str, *args, **kwargs) -> None:
         print(message, *args, **kwargs)
 
 
-def set_properties(target_links: List[PingContext], timeout: int) -> List[PingContext]:
+def set_properties(
+        target_links: List[PingContext],
+        timeout: int, ignore_links: Optional[Dict[str, str]]
+) -> List[PingContext]:
     for target in target_links:
+        if ignore_links and target.url in ignore_links:
+            target.is_ignored = True
+            target.ignore_pattern = ignore_links[target.url]
+            print_verbose(2, f"Ignoring {target.url}")
         target.timeout = timeout
     return target_links
 
 
+def load_ignore_list(path: Path) -> Dict[str, str]:
+    output = dict()
+    with open(path, "r") as fp:
+        lines = fp.readlines()
+    for line in lines:
+        line = line.strip()
+        if line.startswith('#'):
+            continue
+        tokens = line.split(" ")
+        length = len(tokens)
+        if length == 1:
+            url = tokens[0]
+            reason_filter = ""
+        elif length >= 2:
+            url = tokens[0]
+            reason_filter = " ".join(tokens[1:]).strip()
+        else:
+            print_verbose(0, f"Could not parse ignore entry '{line}'", file=sys.stderr)
+            continue
+        output[url] = reason_filter
+    print_verbose(1, f"Loaded {len(output)} ignore entries from '{str(path)}'")
+    return output
+
+
 description = """HTML link availability checker.
 
 Recursively look up HTML files in a directory, extract all links (href) and
@@ -209,8 +290,15 @@ def set_properties(target_links: List[PingContext], timeout: int) -> List[PingCo
 
 epilog = f"""Examples:
 
-python3 {Path(__file__).name} --summary --results-dead git/core
-   Search in git/core, print the summary and output unreachable links to stdout
+Returns:
+0 (success) if all links were reachable or 1 (error) if at least one was unreachable
+
+python3 {Path(__file__).name} git/core git/crypto
+    Just check if any links are unavailable (check return code) in both core and crypto.
+    Recommended way to call for batch jobs.
+
+python3 {Path(__file__).name} --summary --results-dead git/core git/crypto
+   Search in git/core and crypto, print the summary and unreachable links to stdout
    
 python3 {Path(__file__).name} -v --results --result-paths -rf out.txt git/crypto
    Search in git/crypto, enable verbose output, print results with paths to file out.txt
@@ -235,10 +323,10 @@ def parse_arguments():
                         help="Print a (short) summary at the end", action="store_true")
     parser.add_argument("-c", "--retry-count",
                         help="Retry counter if a connection times out",
-                        type=int, default=3)
+                        type=int, default=DefaultSettings.RETRY_COUNT.value)
     parser.add_argument("-t", "--timeout",
                         help="Time to wait for an answer per request",
-                        type=int, default=global_default_timeout)
+                        type=int, default=DefaultSettings.TIMEOUT.value)
     parser.add_argument("-p", "--processes",
                         help="Processes to start (defaults to logical CPU cores or "
                              "at least 4 processes)",
@@ -261,6 +349,33 @@ def parse_arguments():
                         help="Also print corresponding file paths in the result output",
                         action="store_true")
     parser.add_argument("-rf", "--result-file", help="Redirect result output to file")
+    parser.add_argument(
+        "-f",
+        "--filter-pattern",
+        help="A filter pattern to apply to the html file search The term '*.html' is "
+             "automatically added. Defaults to "
+             f"'{DefaultSettings.FILTER_PATTERN.value}'",
+        default=DefaultSettings.FILTER_PATTERN.value,
+    )
+    parser.add_argument(
+        "-e",
+        "--exclude",
+        help="Projects (directories) to exclude from search as comma separated list. "
+             f"Defaults to '{DefaultSettings.EXCLUDE_PROJECTS.value}'.",
+        default=DefaultSettings.EXCLUDE_PROJECTS.value,
+    )
+    parser.add_argument(
+        "-i",
+        "--ignore",
+        help="Path to a file with URls to ignore. The format is 'URL reason(optional)' "
+             "with one entry per line. Note that the link will be queried everytime, "
+             "but if it fails and is on the ignore list, it will not count toward the "
+             "final result. The reason can be a textual representation of a "
+             "http error of urllib, e.g. 'forbidden', 'timeout' or "
+             "'certificate verify failed'. "
+             f"Defaults to '{DefaultSettings.IGNORE_FILE.value}'.",
+        default=DefaultSettings.IGNORE_FILE.value,
+    )
 
     verbose_group = parser.add_mutually_exclusive_group()
     verbose_group.add_argument("-v", help="Enable verbose output", action="store_true")
@@ -278,7 +393,6 @@ def print_files(files: List[str], file=None) -> None:
 
 global_verbosity_level = 0
 global_dot_pattern = re.compile(r'\.((?!html?).)')
-global_default_timeout = 5
 
 
 def main():
@@ -303,16 +417,26 @@ def main():
     global_verbosity_level = 0
     global_verbosity_level += 1 if args.v else 0
     global_verbosity_level += 2 if args.vv else 0
+    exclude_projects = [string.strip() for string in args.exclude.split(",")]
+    ignore_link_file = Path(args.ignore)
+
+    if not ignore_link_file.exists():
+        print_verbose(1, f"Specified --ignore file at '{str(ignore_link_file)}' could "
+                         f"not be found. Ignoring it...", file=sys.stderr)
+        target_ignore_links = None
+    else:
+        target_ignore_links = load_ignore_list(ignore_link_file)
 
     # Initialize multiprocessing data structures
     mp_manager = Manager()
     waiting_queue = mp_manager.Queue()
     successful_links = mp_manager.list()
     failed_links = mp_manager.list()
+    ignored_links = mp_manager.list()
 
     # Search for links in the target directory and set their timeout
-    target_links = collect_all_links(target_dir)
-    target_links = set_properties(target_links, timeout)
+    target_links = collect_all_links(target_dir, args.filter_pattern, exclude_projects)
+    target_links = set_properties(target_links, timeout, target_ignore_links)
 
     print_verbose(1, f"Collected {len(target_links)} links "
                      f"from *.html files in {', '.join([str(d) for d in target_dir])}")
@@ -322,10 +446,16 @@ def main():
     # Start the processes to query the links for availability.
     processes = []
     for _ in range(process_count):
-        _process = Process(target=query, args=(waiting_queue,
-                                              retry_count,
-                                              successful_links,
-                                              failed_links))
+        _process = Process(
+            target=query,
+            args=(
+                waiting_queue,
+                retry_count,
+                successful_links,
+                failed_links,
+                ignored_links
+            )
+        )
         processes.append(_process)
         _process.start()
 
@@ -335,7 +465,7 @@ def main():
 
     # From here on is output only. There are two major types of output,
     # the detailed results and the summary. The detailed results are
-    # redirectable to a file, the summary is not.
+    # redirect-able to a file, the summary is not.
     if print_result_use_file:
         print_result_handler = open(print_result_target, 'w')
     else:
@@ -364,21 +494,28 @@ def main():
     if print_summary:
         _len_successful = len(successful_links)
         _len_failed = len(failed_links)
-        _len_all = _len_successful + _len_failed
+        _len_ignored = len(ignored_links)
+        _len_all = _len_successful + _len_failed + _len_ignored
         if _len_all > 1:
-            success_rate = _len_successful / (_len_successful + _len_failed)
-            fail_rate = _len_failed / (_len_successful + _len_failed)
+            success_rate = _len_successful / _len_all
+            fail_rate = _len_failed / _len_all
+            ignore_rate = _len_ignored / _len_all
         else:
             success_rate = 0
             fail_rate = 0
+            ignore_rate = 0
         summary_msg = "Summary - Tested unique external links: "
         print(f"\n{summary_msg}{len(target_links)}")
         # Print as many '=' characters as the line above has.
         print(f"{'=' * (len(summary_msg) + len(str(len(target_links))))}")
-        print(f"\tReachable     {_len_successful:>5} ({success_rate:.2%})")
-        print(f"\tNot Reachable {_len_failed:>5} ({fail_rate:.2%})")
+        print(f"\tReachable     {_len_successful:>5} ({success_rate:>6.2%})")
+        print(f"\tNot Reachable {_len_failed:>5} ({fail_rate:>6.2%})")
+        if _len_ignored > 0:
+            print(f"\tIgnored       {_len_ignored:>5} ({ignore_rate:>6.2%})")
 
+    exit_code = 1 if failed_links else 0
     print_verbose(1, f"Done.")
+    sys.exit(exit_code)
 
 
 if __name__ == '__main__':

From 5ee9adfacd03908715e429e5c3d5f6c9a3041990 Mon Sep 17 00:00:00 2001
From: Simon Leischnig <simon-jena@gmx.net>
Date: Wed, 30 Mar 2022 15:10:52 +0200
Subject: [PATCH 2/2] update version in text file

---
 org.jcryptool.core.help/nl/de/developer_about.txt | 2 +-
 org.jcryptool.core.help/nl/en/developer_about.txt | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/org.jcryptool.core.help/nl/de/developer_about.txt b/org.jcryptool.core.help/nl/de/developer_about.txt
index b71989ecb4..5f2af331ba 100644
--- a/org.jcryptool.core.help/nl/de/developer_about.txt
+++ b/org.jcryptool.core.help/nl/de/developer_about.txt
@@ -1,4 +1,4 @@
-JCrypTool v1.0.6 wird Ihnen präsentiert von Kernentwicklungsteam:
+JCrypTool v1.0.8 wird Ihnen präsentiert von Kernentwicklungsteam:
 ------------------------------------------------------------------
 
 - Prof. Bernhard Esslinger
diff --git a/org.jcryptool.core.help/nl/en/developer_about.txt b/org.jcryptool.core.help/nl/en/developer_about.txt
index 5a98f2563e..79a906530a 100644
--- a/org.jcryptool.core.help/nl/en/developer_about.txt
+++ b/org.jcryptool.core.help/nl/en/developer_about.txt
@@ -1,4 +1,4 @@
-JCrypTool v1.0.6 is proudly presented by the core developer team:
+JCrypTool v1.0.8 is proudly presented by the core developer team:
 -----------------------------------------------------------------
 
 - Prof. Bernhard Esslinger