use all of response.text in get_base_url

Flared · Aug 26, 2024 · 9e77ba6 · 9e77ba6
1 parent 0f59b5e
commit 9e77ba6
Show file tree

Hide file tree

Showing 2 changed files with 12 additions and 2 deletions.
diff --git a/scrapy/utils/response.py b/scrapy/utils/response.py
@@ -26,9 +26,8 @@
 def get_base_url(response: TextResponse) -> str:
     """Return the base url of the given response, joined with the response url"""
     if response not in _baseurl_cache:
-        text = response.text[0:4096]
         _baseurl_cache[response] = html.get_base_url(
-            text, response.url, response.encoding
+            response.text, response.url, response.encoding
         )
     return _baseurl_cache[response]
 

diff --git a/tests/test_utils_response.py b/tests/test_utils_response.py
@@ -91,6 +91,17 @@ def test_get_base_url(self):
         )
         self.assertEqual(get_base_url(resp2), "http://www.example.com")
 
+    def test_get_base_url_with_long_head(self):
+        resp = HtmlResponse(
+            "http://www.example.com",
+            body=f"""
+        <html>
+        <head>{'A' * 5000}<base href="http://www.example.com/img/" target="_blank"></head>
+        <body>blahablsdfsal&amp;</body>
+        </html>""".encode(),
+        )
+        self.assertEqual(get_base_url(resp), "http://www.example.com/img/")
+
     def test_response_status_message(self):
         self.assertEqual(response_status_message(200), "200 OK")
         self.assertEqual(response_status_message(404), "404 Not Found")