Support tool calling.

.github/workflows/publish.yml

@@ -12,7 +12,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
+        python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"]
     steps:
     - uses: actions/checkout@v4
     - name: Set up Python ${{ matrix.python-version }}

.github/workflows/test.yml

@@ -11,7 +11,7 @@ jobs:
     strategy:
       matrix:
         os: [ubuntu-latest, macos-latest, windows-latest]
-        python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
+        python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"]
         pydantic: ["==1.10.2", ">=2.0.0"]
     steps:
     - uses: actions/checkout@v4

Justfile

@@ -17,7 +17,7 @@
   echo "  cog"
   pipenv run cog --check \
     -p "import sys, os; sys._called_from_test=True; os.environ['LLM_USER_PATH'] = '/tmp'" \
-    README.md docs/*.md
+    README.md docs/**/*.md docs/*.md
   echo "  mypy"
   pipenv run mypy llm
   echo "  ruff"

docs/help.md

@@ -74,6 +74,7 @@ Commands:
   plugins       List installed plugins
   similar       Return top N similar IDs from a collection
   templates     Manage stored prompt templates
+  tools         Manage available tools
   uninstall     Uninstall Python packages from the LLM environment
 ```
 
@@ -92,6 +93,7 @@ Options:
   -o, --option <TEXT TEXT>...  key/value options for the model
   -t, --template TEXT          Template to use
   -p, --param <TEXT TEXT>...   Parameters for template
+  --enable-tools               Enable tool usage for supported models
   --no-stream                  Do not stream output
   -n, --no-log                 Don't log to database
   --log                        Log prompt and response to the database
@@ -117,6 +119,7 @@ Options:
   -t, --template TEXT          Template to use
   -p, --param <TEXT TEXT>...   Parameters for template
   -o, --option <TEXT TEXT>...  key/value options for the model
+  --enable-tools               Enable tool usage for supported models
   --no-stream                  Do not stream output
   --key TEXT                   API key to use
   --help                       Show this message and exit.
@@ -298,6 +301,32 @@ Options:
   --help  Show this message and exit.
 ```
 
+(help-tools)=
+### llm tools --help
+```
+Usage: llm tools [OPTIONS] COMMAND [ARGS]...
+
+  Manage available tools
+
+Options:
+  --help  Show this message and exit.
+
+Commands:
+  list*  List available tools
+```
+
+(help-tools-list)=
+#### llm tools list --help
+```
+Usage: llm tools list [OPTIONS]
+
+  List available tools
+
+Options:
+  --schema  Show JSON schema for each tool
+  --help    Show this message and exit.
+```
+
 (help-templates)=
 ### llm templates --help
 ```

docs/openai-models.md

@@ -31,18 +31,18 @@ models = [line for line in result.output.split("\n") if line.startswith("OpenAI
 cog.out("```\n{}\n```".format("\n".join(models)))
 ]]] -->
 ```
-OpenAI Chat: gpt-3.5-turbo (aliases: 3.5, chatgpt)
-OpenAI Chat: gpt-3.5-turbo-16k (aliases: chatgpt-16k, 3.5-16k)
-OpenAI Chat: gpt-4 (aliases: 4, gpt4)
-OpenAI Chat: gpt-4-32k (aliases: 4-32k)
-OpenAI Chat: gpt-4-1106-preview
-OpenAI Chat: gpt-4-0125-preview
-OpenAI Chat: gpt-4-turbo-2024-04-09
-OpenAI Chat: gpt-4-turbo (aliases: gpt-4-turbo-preview, 4-turbo, 4t)
-OpenAI Chat: gpt-4o (aliases: 4o)
-OpenAI Chat: gpt-4o-mini (aliases: 4o-mini)
-OpenAI Chat: o1-preview
-OpenAI Chat: o1-mini
+OpenAI Chat: gpt-3.5-turbo (aliases: 3.5, chatgpt) (supports tool calling)
+OpenAI Chat: gpt-3.5-turbo-16k (aliases: chatgpt-16k, 3.5-16k) (supports tool calling)
+OpenAI Chat: gpt-4 (aliases: 4, gpt4) (supports tool calling)
+OpenAI Chat: gpt-4-32k (aliases: 4-32k) (supports tool calling)
+OpenAI Chat: gpt-4-1106-preview (supports tool calling)
+OpenAI Chat: gpt-4-0125-preview (supports tool calling)
+OpenAI Chat: gpt-4-turbo-2024-04-09 (supports tool calling)
+OpenAI Chat: gpt-4-turbo (aliases: gpt-4-turbo-preview, 4-turbo, 4t) (supports tool calling)
+OpenAI Chat: gpt-4o (aliases: 4o) (supports tool calling)
+OpenAI Chat: gpt-4o-mini (aliases: 4o-mini) (supports tool calling)
+OpenAI Chat: o1-preview (supports tool calling)
+OpenAI Chat: o1-mini (supports tool calling)
 OpenAI Completion: gpt-3.5-turbo-instruct (aliases: 3.5-instruct, chatgpt-instruct)
 ```
 <!-- [[[end]]] -->

docs/plugins/index.md

@@ -5,6 +5,8 @@ LLM plugins can enhance LLM by making alternative Large Language Models availabl
 
 Plugins can also add new commands to the `llm` CLI tool.
 
+Plugins can also add new Python functions that some LLM models can invoke - LLM tool calling.
+
 The {ref}`plugin directory <plugin-directory>` lists available plugins that you can install and use.
 
 {ref}`tutorial-model-plugin` describes how to build a new plugin in detail.
@@ -17,5 +19,6 @@ installing-plugins
 directory
 plugin-hooks
 tutorial-model-plugin
+tool-calling
 plugin-utilities
 ```

docs/plugins/llm-sampletools/llm_sampletools.py

@@ -0,0 +1,108 @@
+import random
+import sys
+from typing import Annotated
+import enum
+import datetime
+
+import pydantic
+import llm
+
+
+@llm.hookimpl
+def register_tools(register):
+    # Annotated function, will be introspected
+    register(llm.Tool(random_number))
+    register(llm.Tool(best_restaurant_in))
+
+    # Generate parameter schema from pydantic model
+    register(llm.Tool(current_temperature, WeatherInfo.model_json_schema()))
+
+    # No parameters, no parameter schema needed - no doc comment so provide description
+    register(
+        llm.Tool(best_restaurant, description="Find the best restaurant in the world.")
+    )
+
+    # Manually specify parameter schema
+    register(
+        llm.Tool(
+            current_time,
+            {
+                "type": "object",
+                "properties": {
+                    "time_format": {
+                        "description": "The format to use for the returned datetime, either ISO 8601 or unix ctime format.",
+                        "type": "string",
+                        "enum": ["iso", "ctime"],
+                    },
+                },
+                "required": ["time_format"],
+                "additionalProperties": False,
+            },
+        )
+    )
+
+
+##########
+
+
+def random_number(
+    minimum: Annotated[int, "The minimum value of the random number, default is 0"] = 0,
+    maximum: Annotated[
+        int, f"The maximum value of the random number, default is {sys.maxsize}."
+    ] = sys.maxsize,
+) -> str:
+    """Generate a random number."""
+    return str(random.randrange(minimum, maximum))  # noqa: S311
+
+
+##########
+
+
+def best_restaurant():
+    return "WorldsBestRestaurant"
+
+
+##########
+
+
+def best_restaurant_in(
+    location: Annotated[str, "The city the restaurant is located in."]
+) -> str:
+    """Find the best restaurant in the given location."""
+    return "CitiesBestRestaurant"
+
+
+##########
+
+
+class Degrees(enum.Enum):
+    CELSIUS = "celsius"
+    FAHRENHEIT = "fahrenheit"
+
+
+class WeatherInfo(pydantic.BaseModel):
+    location: str = pydantic.Field(
+        description="The location to return the current temperature for."
+    )
+    degrees: Degrees = pydantic.Field(
+        description="The degree scale to return temperature in."
+    )
+
+
+def current_temperature(**weather_info) -> str:
+    """Return the current temperature in the provided location."""
+    info = WeatherInfo(**weather_info)
+    return f"The current temperature in {info.location} is 42° {info.degrees.value}."
+
+
+##########
+
+
+def current_time(time_format):
+    """Return the current date and time in UTC using the specified format."""
+    time = datetime.datetime.now(datetime.timezone.utc)
+    if time_format == "iso":
+        return time.isoformat()
+    elif time_format == "ctime":
+        return time.ctime()
+    raise ValueError(f"Unsupported time format: {time_format}")

docs/plugins/llm-sampletools/pyproject.toml

@@ -0,0 +1,10 @@
+[project]
+name = "llm-sampletools"
+version = "0.1"
+dependencies = [
+    "llm",
+    "pydantic>=2.0",
+]
+
+[project.entry-points.llm]
+markov = "llm_sampletools"

docs/plugins/plugin-hooks.md

@@ -44,3 +44,10 @@ class HelloWorld(llm.Model):
 ```
 
 {ref}`tutorial-model-plugin` describes how to use this hook in detail.
+
+## register_tools(register)
+
+This hook can be used to register one or more Python callables as `llm.Tool`s.
+Models that support tool calling will be able to use these tools
+
+{ref}`tool-calling` describes this hook in more detail.

docs/plugins/tool-calling.md

@@ -0,0 +1,91 @@
+(tool-calling)=
+# Tool calling
+
+A plugin can expose additional tools to any supporting model via the `register_tools` plugin hook.
+A plugin that implements a new LLM model can consume installed tools if the model supports tool calling.
+
+## Registering new tools
+
+Tools are `llm.Tool` instances holding a Python callable and a JSON schema describing the function parameters.
+The callable must have a docstring that describes what it does.
+If a parameter JSON schema is not provided, `llm.Tool` will introspect the callable and attempt to generate one.
+For this to work, each paramater needs a `typing.Annotation` that contains the parameter type and a text description.
+The function must return a string. It can raise a descriptive exception to be returned to the LLM if the tool fails.
+If it raises `llm.ModelError`, that exception will be forwarded to the user.
+
+```python
+from typing import Annotated
+import llm
+
+@llm.hookimpl
+def register_tools(register):
+    register(llm.Tool(best_restaurant_in))
+
+def best_restaurant_in(
+    location: Annotated[str, "The city the restaurant is located in."]
+) -> str:
+    """Find the best restaurant in the given location."""
+    return "CitiesBestRestaurant"
+```
+
+Now when the user enables tool calling, if the model supports tool calling
+(the default OpenAI chat models do), then the model can invoke the tool.
+```shell-session
+$ llm --enable-tools -m 4o-mini 'What is the best restaurant in Asbury Park, NJ?'
+The best restaurant in Asbury Park, NJ, is called "Cities Best Restaurant."
+```
+
+You can generate a parameters JSON schema using [pydantic.ModelBase.model_json_schema()](https://docs.pydantic.dev/latest/api/base_model/#pydantic.BaseModel.model_json_schema), or write one by hand and pass it in to the `llm.Tool` initializer.
+Here are some examples of both:
+
+```{literalinclude} llm-sampletools/llm_sampletools.py
+:language: python
+```
+
+## Using tools in models
+
+If your plugin is implementing a new `llm.Model` class that can support tool calling,
+then you can set `supports_tool_calling = True` in your model class.
+
+You can then use the `Model.tools` property to access tools registered by your or other plugins.
+The `tools` property contains a `dict` of tool names mapped to `llm.Tool` instances.
+The `Tool.schema` property contains a Python dict representing the JSON schema for that tool function.
+`Tool` is callable - it should be passed a JSON string representing the callables parameters.
+The Tool handles any exceptions raised other than `llm.ModelError`.
+
+Here is a skeleton implementation for a hypothetical LLM API that supports tool calling.
+```python
+import llmapi  # hypothetical API
+
+class MyToolCallingModel(llm.Model):
+    model_id = "toolcaller"
+    supports_tool_calling = True
+
+    def execute(self, prompt, stream, response, conversation):
+        messages = [{"role": "user", "content": prompt.prompt}]
+        # Invoke our hypothetical LLM API, passing in all registered tool schemas.
+        completion = llmapi.chat.completion(
+            messages=messages,
+            tools=[tool.schema for tool in self.tools.values()]
+        )
+        if completion.tool_calls:
+            messages.append({"role": "assistant", "tool_calls": completion.tool_calls})
+            for tool_call in completion.tool_calls:
+                # Find the named tool and invoke it, adding the result to messages
+                tool = self.tools.get(tool_call.function.name)
+                if tool:
+                    # Invoke the tool with the JSON string arguments.
+                    tool_response = tool(tool_call.function.arguments)
+                    messages.append({"role": "tool", "content": tool_response, "tool_call_id": tool_call.id})
+            # Send the tool results back to the LLM
+            completion = llmapi.chat.completion(messages=messages)
+            yield completion.content
+        else:
+            yield completion.content
+```
+
+A number of LLM APIs support tool function calling using JSON schemas to define the tools.
+For example [OpenAI](https://platform.openai.com/docs/guides/function-calling),
+[Anthropic](https://docs.anthropic.com/en/docs/build-with-claude/tool-use),
+[Google Gemini](https://ai.google.dev/gemini-api/docs/function-calling#function_declarations),
+[Ollama](https://github.com/ollama/ollama/blob/main/docs/api.md#chat-request-with-tools).