Misc lint, and go over CHANGES.rst

CHANGES.rst

@@ -4,11 +4,13 @@ CHANGES
 1.3.1
 ------
 
-Python 3.8 is now the minimum Python supported. Various dependecies elsehwere force this.
+Python 3.8 is now the minimum Python supported. Python 3.12 suported.
+Various dependecies elsehwere force 3.8 or newer.
 
 
-Packaging was redone to be able to support Python 3.12.
-
+* Packaging was redone to be able to support Python 3.12.
+* Files now follow current Python black formatting and isort import ordering
+* Some Python code linting
 
 1.3.0
 ------

admin-tools/pyenv-versions

@@ -5,4 +5,4 @@ if [[ $0 == ${BASH_SOURCE[0]} ]] ; then
     echo "This script should be *sourced* rather than run directly through bash"
     exit 1
 fi
-export PYVERSIONS='3.8.18 3.9.18 3.10.13 pyston-2.3.5 3.11.8 3.12.2'
+export PYVERSIONS='3.8 3.9 3.10 3.11 3.12'

docs/source/conf.py

@@ -17,8 +17,8 @@
 # -- Project information -----------------------------------------------------
 
 project = "mathics-scanner"
-copyright = "2021, The Mathics Team"
-author = "The Mathics Team"
+copyright = "2021, 2024 The Mathics Team"
+author = "The Mathics3 Team"
 
 # The full version, including alpha/beta/rc tags
 release = "1.0.1"

mathics_scanner/__init__.py

@@ -6,45 +6,44 @@
 named characters, their Unicode/ASCII equivalents and code-points.
 """
 
-from mathics_scanner.version import __version__
-
 from mathics_scanner.characters import (
     aliased_characters,
     named_characters,
     replace_unicode_with_wl,
     replace_wl_with_plain_text,
 )
-
-# TODO: Move is_symbol_name to the characters module
-from mathics_scanner.tokeniser import is_symbol_name, Tokeniser, Token
 from mathics_scanner.errors import (
-    InvalidSyntaxError,
     IncompleteSyntaxError,
+    InvalidSyntaxError,
     ScanError,
     TranslateError,
 )
 from mathics_scanner.feed import (
-    LineFeeder,
-    SingleLineFeeder,
     FileLineFeeder,
+    LineFeeder,
     MultiLineFeeder,
+    SingleLineFeeder,
 )
 
+# TODO: Move is_symbol_name to the characters module
+from mathics_scanner.tokeniser import Token, Tokeniser, is_symbol_name
+from mathics_scanner.version import __version__
+
 __all__ = [
-    FileLineFeeder,
-    IncompleteSyntaxError,
-    InvalidSyntaxError,
-    LineFeeder,
-    MultiLineFeeder,
-    ScanError,
-    SingleLineFeeder,
-    Token,
-    Tokeniser,
-    TranslateError,
-    __version__,
-    aliased_characters,
-    is_symbol_name,
-    named_characters,
-    replace_unicode_with_wl,
-    replace_wl_with_plain_text,
+    "FileLineFeeder",
+    "IncompleteSyntaxError",
+    "InvalidSyntaxError",
+    "LineFeeder",
+    "MultiLineFeeder",
+    "ScanError",
+    "SingleLineFeeder",
+    "Token",
+    "Tokeniser",
+    "TranslateError",
+    "__version__",
+    "aliased_characters",
+    "is_symbol_name",
+    "named_characters",
+    "replace_unicode_with_wl",
+    "replace_wl_with_plain_text",
 ]

mathics_scanner/feed.py

@@ -5,6 +5,7 @@
 """
 
 from abc import ABCMeta, abstractmethod
+from typing import List
 
 
 class LineFeeder(metaclass=ABCMeta):
@@ -19,7 +20,7 @@ def __init__(self, filename: str):
         :param filename: A string that describes the source of the feeder, i.e.
                          the filename that is being feed.
         """
-        self.messages = []
+        self.messages: List[str] = []
         self.lineno = 0
         self.filename = filename
 

mathics_scanner/tokeniser.py

@@ -19,8 +19,8 @@
 (\*\^(\+|-)?\d+)?                           (?# Exponent)
 """
 base_symbol_pattern = r"((?![0-9])([0-9${0}{1}])+)".format(_letters, _letterlikes)
-full_symbol_pattern = r"(`?{0}(`{0})*)".format(base_symbol_pattern)
-pattern_pattern = r"{0}?_(\.|(__?)?{0}?)?".format(full_symbol_pattern)
+full_symbol_pattern_str = r"(`?{0}(`{0})*)".format(base_symbol_pattern)
+pattern_pattern = r"{0}?_(\.|(__?)?{0}?)?".format(full_symbol_pattern_str)
 slot_pattern = r"\#(\d+|{0})?".format(base_symbol_pattern)
 FILENAME_PATTERN = r"""
 (?P<quote>\"?)                              (?# Opening quotation mark)
@@ -39,7 +39,7 @@
     ("Number", NUMBER_PATTERN),
     ("String", r'"'),
     ("Pattern", pattern_pattern),
-    ("Symbol", full_symbol_pattern),
+    ("Symbol", full_symbol_pattern_str),
     ("SlotSequence", r"\#\#\d*"),
     ("Slot", slot_pattern),
     ("Out", r"\%(\%+|\d+)?"),
@@ -312,7 +312,7 @@ def compile_tokens(token_list):
 token_indices = find_indices(literal_tokens)
 tokens = compile_tokens(tokens)
 filename_tokens = compile_tokens(filename_tokens)
-full_symbol_pattern = compile_pattern(full_symbol_pattern)
+full_symbol_pattern_re: re.Pattern = compile_pattern(full_symbol_pattern_str)
 
 
 def is_symbol_name(text: str) -> bool:
@@ -321,7 +321,7 @@ def is_symbol_name(text: str) -> bool:
     ``False``.
     """
     # Can't we just call match here?
-    return full_symbol_pattern.sub("", text) == ""
+    return full_symbol_pattern_re.sub("", text) == ""
 
 
 class Token:

setup.py

@@ -35,6 +35,7 @@
 
 
 def get_srcdir():
+    """Return the directory of the location if this code"""
     filename = osp.normcase(osp.dirname(osp.abspath(__file__)))
     return osp.realpath(filename)
 
@@ -43,23 +44,23 @@ def get_srcdir():
 for kind in ("dev", "full"):
     extras_require = []
     requirements_file = f"requirements-{kind}.txt"
-    for line in open(requirements_file).read().split("\n"):
+    for line in open(requirements_file, encoding="utf-8").read().split("\n"):
         if line and not line.startswith("#"):
             requires = re.sub(r"([^#]+)(\s*#.*$)?", r"\1", line)
             extras_require.append(requires)
     EXTRAS_REQUIRE[kind] = extras_require
 
 
 class table_building_egg_info(egg_info):
-    # This runs as part of building an sdist
+    """This runs as part of building an sdist"""
 
     def finalize_options(self):
         """Run program to create JSON tables"""
         build_tables_program = osp.join(
             get_srcdir(), "mathics_scanner", "generate", "build_tables.py"
         )
         print(f"Building JSON tables via {build_tables_program}")
-        result = subprocess.run([sys.executable, build_tables_program])
+        result = subprocess.run([sys.executable, build_tables_program], check=False)
         if result.returncode:
             raise RuntimeError(
                 f"Running {build_tables_program} exited with code {result.returncode}"