2captcha/amazon waf

Pipfile

@@ -33,6 +33,7 @@ types-beautifulsoup4 = "*"
 selenium = "*"
 gunicorn = "22.0.0"
 flask-api = {editable = true, ref = "develop", git = "git+https://github.com/flask-api/flask-api.git"}
+2captcha-python = "*"
 
 [dev-packages]
 

flathunter/abstract_crawler.py

@@ -4,6 +4,9 @@
 from time import sleep
 from typing import Optional, Any
 
+from io import BytesIO
+import base64
+
 import backoff
 import requests
 # pylint: disable=unused-import
@@ -12,10 +15,11 @@
 from bs4 import BeautifulSoup
 
 from selenium.common.exceptions import NoSuchElementException, TimeoutException
-from selenium.webdriver import Chrome
+from selenium.webdriver import Chrome, Keys
 from selenium.webdriver.common.by import By
 from selenium.webdriver.support import expected_conditions as EC
 from selenium.webdriver.support.wait import WebDriverWait
+from selenium.webdriver.common.action_chains import ActionChains
 
 from flathunter import proxies
 from flathunter.captcha.captcha_solver import CaptchaUnsolvableError
@@ -193,6 +197,57 @@ def resolve_geetest(self, driver):
             driver.refresh()
             raise
 
+    @backoff.on_exception(wait_gen=backoff.constant,
+                          exception=CaptchaUnsolvableError,
+                          max_tries=3)
+    def resolve_amazon(self, driver):
+        """Resolve Amazon Captcha"""
+        try:
+            driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")
+            sleep(3)
+            shadowelement = driver.execute_script("return document.querySelector('awswaf-captcha').shadowRoot")
+            my_img = shadowelement.find_element(By.ID, "root")
+            size = my_img.size
+            select_l = my_img.find_element(By.TAG_NAME, "select")
+            select_l.click()
+            sleep(1)
+            select_l.send_keys(Keys.DOWN)
+            sleep(3)
+            shadowelement = driver.execute_script("return document.querySelector('awswaf-captcha').shadowRoot")
+            my_img = shadowelement.find_element(By.ID, "root")
+            screenshot = my_img.screenshot_as_png
+            screenshot_bytes = BytesIO(screenshot)
+            base64_screenshot = base64.b64encode(screenshot_bytes.getvalue()).decode('utf-8')
+            result = self.captcha_solver.solve_amazon(base64_screenshot)  # Send image in 2captcha service
+            logger.info(result['code'])
+            l = result['code'].split(':')[1].split(';')
+            l = [[int(val.split('=')[1]) for val in coord.split(',')] for coord in l]
+            button_coord = [size['width'] - 30, size['height'] - 30]
+            l.append(button_coord)
+            actions = ActionChains(driver)
+            for i in l:
+                actions.move_to_element_with_offset(my_img, i[0] - 160, i[1] - 211).click()
+                actions.perform()
+                sleep(0.5)
+                actions.reset_actions()
+            sleep(1)
+            try:
+                confirm_button = my_img.find_element(By.ID, "amzn-btn-verify-internal")
+                actions.move_to_element_with_offset(confirm_button, 40, 15).click()
+                actions.perform()
+                sleep(4)
+            except:
+                pass
+            try:
+                driver.find_element(By.TAG_NAME, "awswaf-captcha")
+            except:
+                logger.info("Captcha solved")
+            else:
+                raise CaptchaUnsolvableError()
+        except Exception as ex:
+            driver.refresh()
+            raise CaptchaUnsolvableError()
+
     @backoff.on_exception(wait_gen=backoff.constant,
                           exception=CaptchaUnsolvableError,
                           max_tries=3)

flathunter/captcha/captcha_solver.py

@@ -17,7 +17,6 @@ class RecaptchaResponse:
     """Response from reCAPTCHA"""
     result: str
 
-
 class CaptchaSolver:
     """Interface for Captcha solvers"""
 
@@ -38,6 +37,9 @@ def solve_recaptcha(self, google_site_key: str, page_url: str) -> RecaptchaRespo
         """Should be implemented in subclass"""
         raise NotImplementedError()
 
+    def solve_amazon(self, image):
+        raise NotImplementedError()
+
 class CaptchaUnsolvableError(Exception):
     """Raised when Captcha was unsolveable"""
     def __init__(self):

flathunter/captcha/twocaptcha_solver.py

@@ -4,6 +4,7 @@
 from time import sleep
 import backoff
 import requests
+from twocaptcha import TwoCaptcha
 
 from flathunter.logging import logger
 from flathunter.captcha.captcha_solver import (
@@ -46,6 +47,11 @@ def solve_recaptcha(self, google_site_key: str, page_url: str) -> RecaptchaRespo
         captcha_id = self.__submit_2captcha_request(params)
         return RecaptchaResponse(self.__retrieve_2captcha_result(captcha_id))
 
+    def solve_amazon(self, image):
+        logger.info("Trying to solve amazon.")
+        solver = TwoCaptcha(self.api_key, defaultTimeout=60, pollingInterval=5)
+        result = solver.coordinates(image, lang='en')
+        return result
 
     @backoff.on_exception(**CaptchaSolver.backoff_options)
     def __submit_2captcha_request(self, params: Dict[str, str]) -> str:

flathunter/crawler/immobilienscout.py

@@ -34,7 +34,7 @@ class Immobilienscout(Crawler):
 
     URL_PATTERN = STATIC_URL_PATTERN
 
-    JSON_PATH_PARSER_ENTRIES = parse("$..['resultlist.realEstate']")
+    JSON_PATH_PARSER_ENTRIES = parse("$..['resultlistEntries']..['resultlist.realEstate']")
     JSON_PATH_PARSER_IMAGES = parse("$..galleryAttachments"
                                     "..attachment[?'@xsi.type'=='common:Picture']"
                                     "..['@href'].`sub(/(.*\\\\.jpe?g).*/, \\\\1)`")
@@ -116,6 +116,8 @@ def get_results(self, search_url, max_pages=None):
 
     def get_entries_from_javascript(self):
         """Get entries from JavaScript"""
+        if "Warum haben wir deine Anfrage blockiert?" in self.get_driver_force().page_source:
+            self.resolve_amazon(self.get_driver_force())
         try:
             result_json = self.get_driver_force().execute_script('return window.IS24.resultList;')
         except JavascriptException:

flathunter/gmaps_duration_processor.py

@@ -24,6 +24,8 @@ def process_expose(self, expose):
 
     def get_formatted_durations(self, address):
         """Return a formatted list of GoogleMaps durations"""
+        if address is None:
+            return ""
         out = ""
         for duration in self.config.get('durations', []):
             if 'destination' in duration and 'name' in duration: