[rqd] Python tests for rqd only v2

.github/workflows/testing-pipeline.yml

@@ -124,3 +124,73 @@ jobs:
         uses: tj-actions/changed-files@v41
       - name: Check for Version Change
         run: ci/check_version_bump.py ${{ steps.get_changed_files.outputs.all_changed_and_modified_files }}
+
+  test_rqd_python_3_7:
+    name: Test RQD with python 3.7
+    runs-on: ubuntu-latest
+    container:
+      image: python:3.7
+    steps:
+      - uses: actions/checkout@v4
+      - name: Run Python Tests
+        run: ci/run_rqd_python_tests.sh
+
+  test_rqd_python_3_8:
+    name: Test RQD with python 3.8
+    runs-on: ubuntu-latest
+    container:
+      image: python:3.8
+    steps:
+      - uses: actions/checkout@v4
+      - name: Run Python Tests
+        run: ci/run_rqd_python_tests.sh
+
+  test_rqd_python_3_9:
+    name: Test RQD with python 3.9
+    runs-on: ubuntu-latest
+    container:
+      image: python:3.9
+    steps:
+      - uses: actions/checkout@v4
+      - name: Run Python Tests
+        run: ci/run_rqd_python_tests.sh
+
+  test_rqd_python_3_10:
+    name: Test RQD with python 3.10
+    runs-on: ubuntu-latest
+    container:
+      image: python:3.10
+    steps:
+      - uses: actions/checkout@v4
+      - name: Run Python Tests
+        run: ci/run_rqd_python_tests.sh
+
+  test_rqd_python_3_11:
+    name: Test RQD with python 3.11
+    runs-on: ubuntu-latest
+    container:
+      image: python:3.11
+    steps:
+      - uses: actions/checkout@v4
+      - name: Run Python Tests
+        run: ci/run_rqd_python_tests.sh
+
+  test_rqd_python_3_12:
+    name: Test RQD with python 3.12
+    runs-on: ubuntu-latest
+    container:
+      image: python:3.12
+    steps:
+      - uses: actions/checkout@v4
+      - name: Run Python Tests
+        run: ci/run_rqd_python_tests.sh
+
+  test_rqd_python_3_13:
+    name: Test RQD with python 3.13
+    runs-on: ubuntu-latest
+    container:
+      image: python:3.13
+    steps:
+      - uses: actions/checkout@v4
+      - name: Run Python Tests
+        run: ci/run_rqd_python_tests.sh

ci/build_sphinx_docs.sh

@@ -10,7 +10,7 @@ python -m grpc_tools.protoc -I=proto/ --python_out=pycue/opencue/compiled_proto
 
 # Fix imports to work in both Python 2 and 3. See
 # <https://github.com/protocolbuffers/protobuf/issues/1491> for more info.
-2to3 -wn -f import pycue/opencue/compiled_proto/*_pb2*.py
+python ci/fix_compiled_proto.py pycue/opencue/compiled_proto
 
 # Build the docs and treat warnings as errors
 ~/.local/bin/sphinx-build -W -b html -d docs/_build/doctrees docs docs/_build/html

ci/fix_compiled_proto.py

@@ -0,0 +1,26 @@
+#!/usr/bin/env python
+
+"""Script that makes the imports in the generated compiled_proto python files relative.
+
+"""
+import os
+import re
+import sys
+import glob
+
+PYTHON_SCRIPT_PATH = sys.argv[1]
+
+if os.path.isdir(PYTHON_SCRIPT_PATH):
+    pattern = re.compile(r"^import \w+ as \w+_pb2")
+    for filepath in glob.glob(os.path.join(PYTHON_SCRIPT_PATH, "*_pb2*.py")):
+        filedata = []
+        with open(filepath) as f:
+            for line in f.readlines():
+                match = pattern.match(line)
+                if match is not None:
+                    line = f"from . {line}"
+                filedata.append(line.strip("\n"))
+        with open(filepath, "w") as f:
+            f.write("\n".join(filedata))
+else:
+    print("Argument is not a directory")

ci/python_coverage_report.sh

@@ -9,8 +9,8 @@ python -m pip install coverage pytest-xvfb
 # Protos need to have their Python code generated in order for tests to pass.
 python -m grpc_tools.protoc -I=proto/ --python_out=pycue/opencue/compiled_proto --grpc_python_out=pycue/opencue/compiled_proto proto/*.proto
 python -m grpc_tools.protoc -I=proto/ --python_out=rqd/rqd/compiled_proto --grpc_python_out=rqd/rqd/compiled_proto proto/*.proto
-2to3 -wn -f import pycue/opencue/compiled_proto/*_pb2*.py
-2to3 -wn -f import rqd/rqd/compiled_proto/*_pb2*.py
+python ci/fix_compiled_proto.py pycue/opencue/compiled_proto
+python ci/fix_compiled_proto.py rqd/rqd/compiled_proto
 
 # Run coverage for each component individually, but append it all into the same report.
 python -m coverage run --source=pycue/opencue/,pycue/FileSequence/ --omit=pycue/opencue/compiled_proto/* pycue/tests/test_suite.py

ci/run_python_lint.sh

@@ -13,8 +13,8 @@ python -m grpc_tools.protoc -I=proto/ --python_out=rqd/rqd/compiled_proto --grpc
 
 # Fix imports to work in both Python 2 and 3. See
 # <https://github.com/protocolbuffers/protobuf/issues/1491> for more info.
-2to3 -wn -f import pycue/opencue/compiled_proto/*_pb2*.py
-2to3 -wn -f import rqd/rqd/compiled_proto/*_pb2*.py
+python ci/fix_compiled_proto.py pycue/opencue/compiled_proto
+python ci/fix_compiled_proto.py rqd/rqd/compiled_proto
 
 echo "Running lint for pycue/..."
 cd pycue

ci/run_python_tests.sh

@@ -19,14 +19,14 @@ python -m grpc_tools.protoc -I=proto/ --python_out=rqd/rqd/compiled_proto --grpc
 
 # Fix imports to work in both Python 2 and 3. See
 # <https://github.com/protocolbuffers/protobuf/issues/1491> for more info.
-2to3 -wn -f import pycue/opencue/compiled_proto/*_pb2*.py
-2to3 -wn -f import rqd/rqd/compiled_proto/*_pb2*.py
-
-python3 -m unittest discover -s pycue/tests -t pycue -p "*.py"
-PYTHONPATH=pycue python3 -m unittest discover -s pyoutline/tests -t pyoutline -p "*.py"
-PYTHONPATH=pycue python3 -m unittest discover -s cueadmin/tests -t cueadmin -p "*.py"
-PYTHONPATH=pycue:pyoutline python3 -m unittest discover -s cuesubmit/tests -t cuesubmit -p "*.py"
-python3 -m unittest discover -s rqd/tests -t rqd -p "*.py"
+python ci/fix_compiled_proto.py pycue/opencue/compiled_proto
+python ci/fix_compiled_proto.py rqd/rqd/compiled_proto
+
+python -m unittest discover -s pycue/tests -t pycue -p "*.py"
+PYTHONPATH=pycue python -m unittest discover -s pyoutline/tests -t pyoutline -p "*.py"
+PYTHONPATH=pycue python -m unittest discover -s cueadmin/tests -t cueadmin -p "*.py"
+PYTHONPATH=pycue:pyoutline python -m unittest discover -s cuesubmit/tests -t cuesubmit -p "*.py"
+python -m unittest discover -s rqd/tests -t rqd -p "*.py"
 
 
 # Xvfb no longer supports Python 2.

ci/run_rqd_python_tests.sh

@@ -0,0 +1,27 @@
+#!/bin/bash
+
+# Script for running OpenCue RQD unit tests with vanilla python
+#
+# This script is written to be run within the OpenCue GitHub Actions environment.
+# See `.github/workflows/testing-pipeline.yml`.
+
+set -e
+
+args=("$@")
+python_version=$(python -V 2>&1)
+echo "Will run tests using ${python_version}"
+
+pip install --user -r requirements/rqd.txt -r requirements/tests.txt
+
+# Protos need to have their Python code generated in order for tests to pass.
+python -m grpc_tools.protoc -I=proto/ --python_out=pycue/opencue/compiled_proto --grpc_python_out=pycue/opencue/compiled_proto proto/*.proto
+python -m grpc_tools.protoc -I=proto/ --python_out=rqd/rqd/compiled_proto --grpc_python_out=rqd/rqd/compiled_proto proto/*.proto
+
+# Fix imports to work in both Python 2 and 3. See
+# <https://github.com/protocolbuffers/protobuf/issues/1491> for more info.
+python ci/fix_compiled_proto.py pycue/opencue/compiled_proto
+python ci/fix_compiled_proto.py rqd/rqd/compiled_proto
+
+python -m unittest discover -s pycue/tests -t pycue -p "*.py"
+python -m unittest discover -s rqd/tests -t rqd -p "*.py"
+

connectors/prometheus_metrics/Dockerfile

@@ -48,7 +48,7 @@ RUN python -m grpc_tools.protoc \
 
 # Fix imports to work in both Python 2 and 3. See
 # <https://github.com/protocolbuffers/protobuf/issues/1491> for more info.
-RUN 2to3 -wn -f import pycue/opencue/compiled_proto/*_pb2*.py
+RUN python3 ci/fix_compiled_proto.py pycue/opencue/compiled_proto
 
 RUN cd pycue && python setup.py install
 

cueadmin/Dockerfile

@@ -24,7 +24,7 @@ RUN python3 -m grpc_tools.protoc \
 
 # Fix imports to work in both Python 2 and 3. See
 # <https://github.com/protocolbuffers/protobuf/issues/1491> for more info.
-RUN 2to3 -wn -f import pycue/opencue/compiled_proto/*_pb2*.py
+RUN python3 ci/fix_compiled_proto.py pycue/opencue/compiled_proto
 
 COPY cueadmin/README.md ./cueadmin/
 COPY cueadmin/setup.py ./cueadmin/

cuegui/Dockerfile

@@ -54,7 +54,7 @@ RUN python3.6 -m grpc_tools.protoc \
 
 # Fix imports to work in both Python 2 and 3. See
 # <https://github.com/protocolbuffers/protobuf/issues/1491> for more info.
-RUN 2to3 -wn -f import pycue/opencue/compiled_proto/*_pb2*.py
+RUN python3 ci/fix_compiled_proto.py pycue/opencue/compiled_proto
 
 COPY cuegui/README.md ./cuegui/
 COPY cuegui/setup.py ./cuegui/

cuesubmit/Dockerfile

@@ -41,7 +41,7 @@ RUN python3.6 -m grpc_tools.protoc \
 
 # Fix imports to work in both Python 2 and 3. See
 # <https://github.com/protocolbuffers/protobuf/issues/1491> for more info.
-RUN 2to3 -wn -f import pycue/opencue/compiled_proto/*_pb2*.py
+RUN python3 ci/fix_compiled_proto.py pycue/opencue/compiled_proto
 
 COPY pyoutline/README.md ./pyoutline/
 COPY pyoutline/setup.py ./pyoutline/

proto/README.md

@@ -14,15 +14,14 @@ To generate:
 
 ```sh
 python -m grpc_tools.protoc -I=. --python_out=../rqd/rqd/compiled_proto --grpc_python_out=../rqd/rqd/compiled_proto ./*.proto
-2to3 -wn -f import ../rqd/rqd/compiled_proto/*_pb2*.py
+python ../ci/fix_compiled_proto.py ../rqd/rqd/compiled_proto
 ```
 
 For Windows (Powershell):
 
 ```powershell
 python -m grpc_tools.protoc --proto_path=. --python_out=../rqd/rqd/compiled_proto --grpc_python_out=../rqd/rqd/compiled_proto (ls *.proto).Name
-cd ..\rqd\rqd\compiled_proto\
-2to3 -wn -f import (ls *_pb2*.py).Name
+python ../ci/fix_compiled_proto.py ../rqd/rqd/compiled_proto
 ```
 
 
@@ -32,15 +31,14 @@ To generate:
 
 ```sh
 python -m grpc_tools.protoc -I=. --python_out=../pycue/opencue/compiled_proto --grpc_python_out=../pycue/opencue/compiled_proto ./*.proto
-2to3 -wn -f import ../pycue/opencue/compiled_proto/*_pb2*.py
+python ../ci/fix_compiled_proto.py ../pycue/opencue/compiled_proto
 ```
 
 For Windows (Powershell):
 
 ```powershell
 python -m grpc_tools.protoc --proto_path=. --python_out=../pycue/opencue/compiled_proto --grpc_python_out=../pycue/opencue/compiled_proto (ls *.proto).Name
-cd ..\pycue\opencue\compiled_proto\
-2to3 -wn -f import (ls *_pb2*.py).Name
+python ../ci/fix_compiled_proto.py ../pycue/opencue/compiled_proto
 ```
 
 

pycue/Dockerfile

@@ -25,7 +25,7 @@ RUN python3 -m grpc_tools.protoc \
 
 # Fix imports to work in both Python 2 and 3. See
 # <https://github.com/protocolbuffers/protobuf/issues/1491> for more info.
-RUN 2to3 -wn -f import pycue/opencue/compiled_proto/*_pb2*.py
+RUN python ci/fix_compiled_proto.py pycue/opencue/compiled_proto
 
 COPY VERSION.in VERSIO[N] ./
 RUN test -e VERSION || echo "$(cat VERSION.in)" | tee VERSION

pyoutline/Dockerfile

@@ -24,7 +24,7 @@ RUN python3 -m grpc_tools.protoc \
 
 # Fix imports to work in both Python 2 and 3. See
 # <https://github.com/protocolbuffers/protobuf/issues/1491> for more info.
-RUN 2to3 -wn -f import pycue/opencue/compiled_proto/*_pb2*.py
+RUN python ci/fix_compiled_proto.py pycue/opencue/compiled_proto
 
 COPY pyoutline/README.md ./pyoutline/
 COPY pyoutline/setup.py ./pyoutline/

requirements.txt

@@ -1,4 +1,3 @@
-2to3==1.0
 enum34==1.1.6
 future==1.0.0
 grpcio==1.48.2;python_version<"3.7"

requirements/rqd.txt

@@ -0,0 +1,9 @@
+future==1.0.0
+grpcio==1.48.2;python_version<"3.7"
+grpcio-tools==1.48.2;python_version<"3.7"
+grpcio==1.53.2;python_version>="3.7" and python_version<"3.12"
+grpcio-tools==1.53.0;python_version>="3.7" and python_version<"3.12"
+grpcio==1.66.2;python_version>="3.12"
+grpcio-tools==1.66.2;python_version>="3.12"
+PyYAML==5.1
+psutil==5.9.8

requirements/tests.txt

@@ -0,0 +1,4 @@
+mock==2.0.0
+pyfakefs==3.6;python_version<"3.7"
+pyfakefs==5.2.3;python_version>="3.7"
+pynput==1.7.6

rqd/Dockerfile

@@ -35,7 +35,7 @@ RUN python3.9 -m grpc_tools.protoc \
 
 # Fix imports to work in both Python 2 and 3. See
 # <https://github.com/protocolbuffers/protobuf/issues/1491> for more info.
-RUN 2to3 -wn -f import rqd/rqd/compiled_proto/*_pb2*.py
+RUN python ci/fix_compiled_proto.py rqd/rqd/compiled_proto
 
 COPY VERSION.in VERSIO[N] ./
 RUN test -e VERSION || echo "$(cat VERSION.in)" | tee VERSION

sandbox/install-client-sources.sh

@@ -10,7 +10,7 @@ python -m grpc_tools.protoc -I=. \
   --python_out=../pycue/opencue/compiled_proto \
   --grpc_python_out=../pycue/opencue/compiled_proto ./*.proto
 cd ..
-2to3 -wn -f import pycue/opencue/compiled_proto/*_pb2*.py
+python ../ci/fix_compiled_proto.py pycue/opencue/compiled_proto
 
 # Install all client packages.
 pip install pycue/ pyoutline/ cueadmin/ cuesubmit/ cuegui/