From 191ff4a6ab3ae4b2580d02559f2542084db6c98e Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 6 Mar 2024 13:12:38 +0500 Subject: [PATCH] Make the savefixture command work without a Scrapy project. --- scrapy_poet/commands.py | 5 +++++ tests/test_commands.py | 43 +++++++++++++++++++++++++++++++++++++++-- 2 files changed, 46 insertions(+), 2 deletions(-) diff --git a/scrapy_poet/commands.py b/scrapy_poet/commands.py index 07f103fc..d27eb1cf 100644 --- a/scrapy_poet/commands.py +++ b/scrapy_poet/commands.py @@ -1,5 +1,6 @@ import datetime import logging +import sys from pathlib import Path from typing import Optional, Type @@ -109,6 +110,10 @@ def run(self, args, opts): type_name = args[0] url = args[1] + if "" not in sys.path: + # when running without a Scrapy project the current dir may not be in sys.path, + # but the user may expect modules in the current dir to be available + sys.path.insert(0, "") cls = load_object(type_name) if not issubclass(cls, ItemPage): raise UsageError(f"Error: {type_name} is not a descendant of ItemPage") diff --git a/tests/test_commands.py b/tests/test_commands.py index 91455047..116fe11f 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -22,9 +22,12 @@ pytest_plugins = ["pytester"] -def call_scrapy_command(cwd: str, *args: str) -> None: +def call_scrapy_command(cwd: str, *args: str, run_module: bool = True) -> None: with tempfile.TemporaryFile() as out: - args = (sys.executable, "-m", "scrapy.cmdline") + args + if run_module: + args = (sys.executable, "-m", "scrapy.cmdline") + args + else: + args = ("scrapy",) + args status = subprocess.call(args, stdout=out, stderr=out, cwd=cwd) out.seek(0) assert status == 0, out.read().decode() @@ -328,3 +331,39 @@ async def to_item(self): os.chdir(cwd) result = pytester.runpytest_subprocess() result.assert_outcomes(passed=4) + + +def test_savefixture_without_project(pytester) -> None: + cwd = Path(pytester.path) + type_name = "po.BTSBookPage" + (cwd / "po.py").write_text( + """ +from web_poet import WebPage + + +class BTSBookPage(WebPage): + + async def to_item(self): + return { + 'url': self.url, + 'name': self.css("h1.name::text").get(), + } +""" + ) + with MockServer(CustomResource, pythonpath=_get_pythonpath()) as server: + call_scrapy_command( + str(cwd), + "savefixture", + type_name, + f"{server.root_url}", + run_module=False, # python -m adds '' to sys.path, making the test always pass + ) + fixtures_dir = cwd / "fixtures" + fixture_dir = fixtures_dir / type_name / "test-1" + fixture = Fixture(fixture_dir) + assert fixture.is_valid() + assert fixture.meta_path.exists() + item = json.loads(fixture.output_path.read_bytes()) + assert item["name"] == "Chocolate" + result = pytester.runpytest_subprocess() + result.assert_outcomes(passed=4)