Merge branch 'master' into resume-download

.github/dependabot.yml

@@ -0,0 +1,13 @@
+# Keep GitHub Actions up to date with GitHub's Dependabot...
+# https://docs.github.com/en/code-security/dependabot/working-with-dependabot/keeping-your-actions-up-to-date-with-dependabot
+# https://docs.github.com/en/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file#package-ecosystem
+version: 2
+updates:
+  - package-ecosystem: github-actions
+    directory: /
+    groups:
+      github-actions:
+        patterns:
+          - "*"  # Group all Actions updates into a single larger pull request
+    schedule:
+      interval: weekly

.github/workflows/lint_python.yml

@@ -4,8 +4,8 @@ jobs:
   lint_python:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
-      - uses: actions/setup-python@v4
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
         with:
           cache: pip
           python-version: 3.x

.github/workflows/pre-commit.yml

@@ -9,8 +9,8 @@ jobs:
   pre-commit:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
-      - uses: actions/setup-python@v4
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
         with:
           python-version: 3.x
           cache: pip

.github/workflows/test_install.yml

@@ -11,6 +11,6 @@ jobs:
       matrix:
         setuptools-version: ["45.2.0", "58.1.0", "62.4.0"]
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - run: pip install setuptools=="${{ matrix.setuptools-version }}"
       - run: pip install .

.github/workflows/tox.yml

@@ -7,10 +7,10 @@ jobs:
       fail-fast: false
       max-parallel: 1  # Avoid timeout errors
       matrix:
-        python: ['3.7', '3.8', '3.9', '3.10', '3.11', 'pypy-3.7', 'pypy-3.8', 'pypy-3.9']
+        python: ['3.8', '3.9', '3.10', '3.11', '3.12', 'pypy-3.9', 'pypy-3.10']
     steps:
-      - uses: actions/checkout@v3
-      - uses: actions/setup-python@v4
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python }}
           cache: pip

.pre-commit-config.yaml

@@ -21,7 +21,7 @@ repos:
       - id: trailing-whitespace
 
   - repo: https://github.com/charliermarsh/ruff-pre-commit
-    rev: v0.0.261
+    rev: v0.0.269
 
     hooks:
       - id: ruff
@@ -42,7 +42,7 @@ repos:
       - id: codespell  # See setup.cfg for args
 
   - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v1.1.1
+    rev: v1.3.0
     hooks:
       - id: mypy
         additional_dependencies:

HISTORY.rst

@@ -3,13 +3,31 @@
 Release History
 ---------------
 
-3.6.0 (?)
-+++++++++
+3.7.0 (2024-03-19)
+++++++++++++++++++
+
+**Features and Improvements**
+
+- Added support for JSON Patch test operations, via the ``expect`` parameter.
+- Added support for moving values via --append-list
+  (Now, rather than ignoring any requests where the value is already present,
+  --append-list will move the value to the end of the list).
+- Switched to importlib-metadata to drop deprecated pkg_resources.
+
+**Bugfixes**
+
+- Fixed automatic size hint on uploads.
+- Fixed bug where auth wasn't being sent for searches with user_aggs params.
+
+3.6.0 (2023-12-27)
+++++++++++++++++++
 
 **Features and Improvements**
 
 - Added ``set_scanner`` and ``--no-scanner`` options to upload to stop ia's default behavior
   of setting the scanner field in meta.xml on initial upload.
+- ``0`` is now returned instead of an exception when search fails to retrieve the total number
+  of hits for a query.
 
 3.5.0 (2023-05-09)
 ++++++++++++++++++

README.rst

@@ -30,7 +30,7 @@ You can install this module via pip:
 
 .. code:: bash
 
-    $ pip install internetarchive
+    $ python3 -m pip install internetarchive
 
 Binaries of the command-line tool are also available:
 

docs/source/installation.rst

@@ -5,37 +5,46 @@ Installation
 
 
 System-Wide Installation
-------------------------
+-------------------------
 
 Installing the ``internetarchive`` library globally on your system can be done with `pip <http://www.pip-installer.org/>`_.
-This is the recommended method for installing ``internetarchive`` (`see below <installation.html#installing-pip>`_ for details on installing pip)::
+This is the recommended method for installing ``internetarchive`` (`see below <installation.html#installing-pip>`_ for details on installing pip).
+If you are on Mac OS X, refer to the `Mac OS X section <installation.html#mac-os-x>`_ below before proceeding.
+Once you're ready to install, run the following command::
 
-    $ sudo pip install internetarchive
+    $ sudo python3 -m pip install internetarchive
 
-or, with `easy_install <http://pypi.python.org/pypi/setuptools>`_::
+Updating Your $PATH
+~~~~~~~~~~~~~~~~~~~
 
-    $ sudo easy_install internetarchive
+Once you have successfully installed ``internetarchive``, you may need to update your ``$PATH`` (e.g. if running ``ia`` in your terminal returns an error).
+If you receive a command not found error, run the following command to update your ``$PATH``::
 
-Either of these commands will install the ``internetarchive`` Python library and ``ia`` command-line tool on your system.
+    $ echo "$(python3 -m site --user-base)/bin" | sudo tee -a /etc/paths
 
-**Note**: Some versions of Mac OS X come with Python libraries that are required by ``internetarchive`` (e.g. the Python package ``six``).
-This can cause installation issues. If your installation is failing with a message that looks something like::
+Updating ia
+~~~~~~~~~~~
 
-    OSError: [Errno 1] Operation not permitted: '/var/folders/bk/3wx7qs8d0x79tqbmcdmsk1040000gp/T/pip-TGyjVo-uninstall/System/Library/Frameworks/Python.framework/Versions/2.7/Extras/lib/python/six-1.4.1-py2.7.egg-info'
+To update, you can run the following command::
 
-You can use the ``--ignore-installed`` parameter in ``pip`` to ignore the libraries that are already installed, and continue with the rest of the installation::
+    $  sudo python3 -m pip install --upgrade internetarchive
 
-    $ sudo pip install --ignore-installed internetarchive
+Mac OS X
+~~~~~~~~
 
-More details on this issue can be found here: https://github.com/pypa/pip/issues/3165
+While newer versions Mac OS X ship with Python 3 installed, it is recommended to install an updated version of Python 3.
+You can do so with `Homebrew <https://brew.sh/#install>`_::
+
+    $ brew install python3
 
 Installing Pip
 ~~~~~~~~~~~~~~
 
-Pip can be `installed with the get-pip.py script <https://pip.pypa.io/en/stable/installing/>`_::
+If you are running Python 3.4+, you should already have ``pip`` installed.
+If it is not already installed, it can be `installed with the get-pip.py script <https://pip.pypa.io/en/stable/installing/>`_::
 
     $ curl -LOs https://bootstrap.pypa.io/get-pip.py
-    $ python get-pip.py
+    $ python3 get-pip.py
 
 
 virtualenv
@@ -45,11 +54,7 @@ If you don't want to, or can't, install the package system-wide you can use ``vi
 
 First, make sure ``virtualenv`` is installed on your system. If it's not, you can do so with pip::
 
-    $ sudo pip install virtualenv
-
-With ``easy_install``::
-
-    $ sudo easy_install virtualenv
+    $ sudo python3 -m pip install virtualenv
 
 Or your systems package manager, ``apt-get`` for example::
 
@@ -92,23 +97,14 @@ If you are on an older operating system that only has Python 2 installed, it's h
 
 You can install and use version v2.3.0 with pip::
 
-    $ pip install internetarchive==2.3.0
+    $ sudo python2 -m pip install internetarchive==2.3.0
 
 You can also download a binary of v2.3.0::
 
     $ curl -LOs https://archive.org/download/ia-pex/ia-py2
     $ chmod +x ia-py2
 
 
-Snap
-----
-
-You can install the latest ``ia`` `snap <https://snapcraft.io>`_, and help testing the most recent changes of the master branch in `all the supported Linux distros <https://snapcraft.io/docs/core/install>`_ with::
-
-    $ sudo snap install ia --edge
-
-Every time a new version of ``ia`` is pushed to the store, you will get it updated automatically.
-
 Get the Code
 ------------
 

internetarchive/__version__.py

@@ -1 +1 @@
-__version__ = '3.6.0.dev2'
+__version__ = '3.7.0'

internetarchive/cli/ia.py

@@ -64,7 +64,11 @@
 import sys
 
 from docopt import docopt, printable_usage
-from pkg_resources import DistributionNotFound, iter_entry_points
+
+if sys.version_info < (3, 10):
+    from importlib_metadata import entry_points  # type: ignore[import]
+else:
+    from importlib.metadata import entry_points
 from schema import Or, Schema, SchemaError  # type: ignore[import]
 
 from internetarchive import __version__
@@ -97,11 +101,11 @@ def load_ia_module(cmd: str):
             return __import__(_module, fromlist=['internetarchive.cli'])
         else:
             _module = f'ia_{cmd}'
-            for ep in iter_entry_points('internetarchive.cli.plugins'):
+            for ep in entry_points(group='internetarchive.cli.plugins'):
                 if ep.name == _module:
                     return ep.load()
             raise ImportError
-    except (ImportError, DistributionNotFound):
+    except (ImportError):
         print(f"error: '{cmd}' is not an ia command! See 'ia help'",
               file=sys.stderr)
         matches = '\t'.join(difflib.get_close_matches(cmd, cmd_aliases.values()))

internetarchive/cli/ia_metadata.py

@@ -22,17 +22,20 @@
     ia metadata <identifier>... [--exists | --formats] [--header=<key:value>...]
     ia metadata <identifier>... --modify=<key:value>... [--target=<target>]
                                 [--priority=<priority>] [--header=<key:value>...]
-                                [--timeout=<value>]
+                                [--timeout=<value>] [--expect=<key:value>...]
     ia metadata <identifier>... --remove=<key:value>... [--priority=<priority>]
                                 [--header=<key:value>...] [--timeout=<value>]
+                                [--expect=<key:value>...]
     ia metadata <identifier>... [--append=<key:value>... | --append-list=<key:value>...]
                                 [--priority=<priority>] [--target=<target>]
                                 [--header=<key:value>...] [--timeout=<value>]
+                                [--expect=<key:value>...]
     ia metadata <identifier>... --insert=<key:value>... [--priority=<priority>]
                                 [--target=<target>] [--header=<key:value>...]
-                                [--timeout=<value>]
+                                [--timeout=<value>] [--expect=<key:value>...]
     ia metadata --spreadsheet=<metadata.csv> [--priority=<priority>]
                 [--modify=<key:value>...] [--header=<key:value>...] [--timeout=<value>]
+                [--expect=<key:value>...]
     ia metadata --help
 
 options:
@@ -42,8 +45,10 @@
     -t, --target=<target>               The metadata target to modify.
     -a, --append=<key:value>...         Append a string to a metadata element.
     -A, --append-list=<key:value>...    Append a field to a metadata element.
-    -i, --insert=<key:value>...      Insert a value into a multi-value field given
+    -i, --insert=<key:value>...         Insert a value into a multi-value field given
                                         an index (e.g. `--insert=collection[0]:foo`).
+    -E, --expect=<key:value>...         Test an expectation server-side before applying
+                                        patch to item metadata.
     -s, --spreadsheet=<metadata.csv>    Modify metadata in bulk using a spreadsheet as
                                         input.
     -e, --exists                        Check if an item exists
@@ -79,13 +84,14 @@
 
 def modify_metadata(item: item.Item, metadata: Mapping, args: Mapping) -> Response:
     append = bool(args['--append'])
+    expect = get_args_dict(args['--expect'])
     append_list = bool(args['--append-list'])
     insert = bool(args['--insert'])
     try:
         r = item.modify_metadata(metadata, target=args['--target'], append=append,
-                                 priority=args['--priority'], append_list=append_list,
-                                 headers=args['--header'], insert=insert,
-                                 timeout=args['--timeout'])
+                                 expect=expect, priority=args['--priority'],
+                                 append_list=append_list, headers=args['--header'],
+                                 insert=insert, timeout=args['--timeout'])
         assert isinstance(r, Response)  # mypy: modify_metadata() -> Request | Response
     except ItemLocateError as exc:
         print(f'{item.identifier} - error: {exc}', file=sys.stderr)
@@ -178,6 +184,7 @@ def main(argv: dict, session: session.ArchiveSession) -> None:
         str: bool,
         '<identifier>': list,
         '--modify': list,
+        '--expect': list,
         '--header': Or(None, And(Use(get_args_header_dict), dict),
                error='--header must be formatted as --header="key:value"'),
         '--append': list,

internetarchive/cli/ia_reviews.py

@@ -68,7 +68,7 @@ def main(argv, session: ArchiveSession) -> None:
             print(r.text)
             sys.exit(0)
         except HTTPError as exc:
-            if exc.response.status_code == 404:
+            if exc.response.status_code == 404:  # type: ignore
                 sys.exit(0)
             else:
                 raise exc