Skip to content

Commit

Permalink
meta: Added simple reporting to reftest.
Browse files Browse the repository at this point in the history
  • Loading branch information
@sleepy-monax
sleepy-monax committed Sep 30, 2024
1 parent 39e1567 commit 1ca1949
Show file tree
Hide file tree
Showing 11 changed files with 895 additions and 450 deletions.
122 changes: 95 additions & 27 deletions meta/plugins/reftest.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -49,29 +49,47 @@ class RefTestArgs(model.TargetArgs):


@cli.command(None, "reftests", "Manage the reftests")
def _(): ...


TESTS_DIR: Path = Path(__file__).parent.parent.parent / "tests"
TEST_REPORT = TESTS_DIR / "report"


@cli.command(None, "reftests/clean", "Manage the reftests")
def _(args: RefTestArgs):
paperMuncher = buildPaperMuncher(args)
for f in TEST_REPORT.glob("*.*"):
f.unlink()
TEST_REPORT.rmdir()
print(f"Cleaned {TEST_REPORT}")

test_folder = Path(__file__).parent.parent.parent / "tests"
test_tmp_folder = test_folder / "_local"
test_tmp_folder.mkdir(parents=True, exist_ok=True)
# for temp in test_tmp_folder.glob('*.*'):
# temp.unlink()

temp_file = test_tmp_folder / "reftest.xhtml"
@cli.command(None, "reftests/run", "Manage the reftests")
def _(args: RefTestArgs):
paperMuncher = buildPaperMuncher(args)

def update_temp_file(container, rendering):
TEST_REPORT.mkdir(parents=True, exist_ok=True)
report = """
<!DOCTYPE html>
<html>
<head>
<title>Reftest</title>
</head>
<body>
"""

def update_temp_file(path, container, rendering):
# write xhtml into the temporary file
xhtml = re.sub(r"<slot\s*/>", rendering, container) if container else rendering
with temp_file.open("w") as f:
with path.open("w") as f:
f.write(f"<!DOCTYPE html>\n{textwrap.dedent(xhtml)}")

REG_INFO = re.compile(r"""(\w+)=['"]([^'"]+)['"]""")

def getInfo(txt):
return {prop: value for prop, value in REG_INFO.findall(txt)}

for file in test_folder.glob(args.glob or "*/*.xhtml"):
for file in TESTS_DIR.glob(args.glob or "*/*.xhtml"):
if file.suffix != ".xhtml":
continue
print(f"Running comparison test {file}...")
Expand All @@ -94,7 +112,7 @@ def getInfo(txt):

expected_xhtml = None
expected_image: bytes | None = None
expected_image_url = test_tmp_folder / f"{temp_file_name}.expected.bmp"
expected_image_url = TEST_REPORT / f"{temp_file_name}.expected.bmp"
if props.get("id"):
ref_image = file.parent / f"{props.get('id')}.bmp"
if ref_image.exists():
Expand All @@ -116,13 +134,15 @@ def getInfo(txt):
print(f"{vt100.YELLOW}Skip test{vt100.RESET}")
continue

update_temp_file(container, rendering)
input_path = TEST_REPORT / f"{temp_file_name}-{num}.xhtml"

update_temp_file(input_path, container, rendering)

# generate temporary bmp
img_path = test_tmp_folder / f"{temp_file_name}-{num}.bmp"
img_path = TEST_REPORT / f"{temp_file_name}-{num}.bmp"

if props.get("size") == "full":
paperMuncher.popen("render", "-sdlpo", img_path, temp_file)
paperMuncher.popen("render", "-sdlpo", img_path, input_path)
else:
size = props.get("size", "200")
paperMuncher.popen(
Expand All @@ -133,7 +153,7 @@ def getInfo(txt):
size,
"-sdlpo",
img_path,
temp_file,
input_path,
)

with img_path.open("rb") as imageFile:
Expand All @@ -144,7 +164,7 @@ def getInfo(txt):
expected_xhtml = rendering
if not expected_image:
expected_image = output_image
with (test_tmp_folder / f"{temp_file_name}.expected.bmp").open(
with (TEST_REPORT / f"{temp_file_name}.expected.bmp").open(
"wb"
) as imageWriter:
imageWriter.write(expected_image)
Expand All @@ -153,16 +173,21 @@ def getInfo(txt):
# check if the rendering is different
assert expected_image is not None
assert output_image is not None
if compareImages(expected_image, output_image) == (tag == "rendering"):
img_path.unlink()

passed = compareImages(expected_image, output_image) == (
tag == "rendering"
)

if passed:
# img_path.unlink()
print(f"{vt100.GREEN}Passed{vt100.RESET}")
else:
# generate temporary file for debugging
paperMuncher.popen(
"print",
"-sdlpo",
test_tmp_folder / f"{temp_file_name}-{num}.pdf",
temp_file,
TEST_REPORT / f"{temp_file_name}-{num}.pdf",
input_path,
)

help = renderingProps.get("help")
Expand All @@ -176,10 +201,10 @@ def getInfo(txt):
print(f"{vt100.WHITE}{expected_image_url}{vt100.RESET}")
print(f"{vt100.BLUE}{rendering[1:].rstrip()}{vt100.RESET}")
print(
f"{vt100.BLUE}{test_tmp_folder / f'{temp_file_name}-{num}.pdf'}{vt100.RESET}"
f"{vt100.BLUE}{TEST_REPORT / f'{temp_file_name}-{num}.pdf'}{vt100.RESET}"
)
print(
f"{vt100.BLUE}{test_tmp_folder / f'{temp_file_name}-{num}.bmp'}{vt100.RESET}"
f"{vt100.BLUE}{TEST_REPORT / f'{temp_file_name}-{num}.bmp'}{vt100.RESET}"
)
if help:
print(f"{vt100.BLUE}{help}{vt100.RESET}")
Expand All @@ -192,15 +217,58 @@ def getInfo(txt):
print(f"{vt100.WHITE}{expected_image_url}{vt100.RESET}")
print(f"{vt100.BLUE}{rendering[1:].rstrip()}{vt100.RESET}")
print(
f"{vt100.BLUE}{test_tmp_folder / f'{temp_file_name}-{num}.pdf'}{vt100.RESET}"
f"{vt100.BLUE}{TEST_REPORT / f'{temp_file_name}-{num}.pdf'}{vt100.RESET}"
)
print(
f"{vt100.BLUE}{test_tmp_folder / f'{temp_file_name}-{num}.bmp'}{vt100.RESET}"
f"{vt100.BLUE}{TEST_REPORT / f'{temp_file_name}-{num}.bmp'}{vt100.RESET}"
)
if help:
print(f"{vt100.BLUE}{help}{vt100.RESET}")

if args.fast:
break
report += f"""
<div class="test-case {passed and 'passed' or 'failed'}">
<h2>{props.get('name')}</h2>
<div class="outputs">
<img class="expected" src="{expected_image_url}" />
<img class="actual" src="{TEST_REPORT / f'{temp_file_name}-{num}.bmp'}" />
<iframe src="{input_path}" style="background-color: white; width: 200px; height: 200px;"></iframe>
</div>
<a href="{TEST_REPORT / f'{temp_file_name}-{num}.pdf'}">PDF</a>
<a href="{expected_image_url}">Expected</a>
<a href="{input_path}">Source</a>
</div>
<hr />
"""

if args.fast:
break

report += """
</body>
<style>
.test-case {
padding: 8px;
border-radius: 4px;
}
.passed {
background-color: lightgreen;
}
.failed {
background-color: lightcoral;
}
.outputs {
display: flex;
gap: 8px;
}
</style>
</html>
"""

with (TEST_REPORT / "report.html").open("w") as f:
f.write(report)

print(f"Report: {TEST_REPORT / 'report.html'}")

temp_file.unlink()
1 change: 1 addition & 0 deletions tests/.gitignore
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
report
4 changes: 2 additions & 2 deletions tests/css/box-model/padding.xhtml
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
<test id="padding-0"></test>
<container>
<html lang="en">
<html lang="en" xmlns="http://www.w3.org/1999/xhtml">

<head>
<style>
Expand Down Expand Up @@ -52,7 +52,7 @@

<test id="padding-1"></test>
<container>
<html lang="en">
<html lang="en" xmlns="http://www.w3.org/1999/xhtml">

<head>
<style>
Expand Down
Binary file modified tests/css/display-block-basic.bmp
View file
Binary file not shown.
Loading

0 comments on commit 1ca1949

Please sign in to comment.