From 301c73a6f7e9fa9cd2198fecf4bbcb70abee2a56 Mon Sep 17 00:00:00 2001 From: Alex O'Connell Date: Fri, 7 Jun 2024 00:09:46 -0400 Subject: [PATCH 1/8] remove llama-cpp-python builds for python3.11 --- .github/workflows/create-release.yml | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/.github/workflows/create-release.yml b/.github/workflows/create-release.yml index ffdc252..18904b0 100644 --- a/.github/workflows/create-release.yml +++ b/.github/workflows/create-release.yml @@ -20,12 +20,8 @@ jobs: matrix: include: # ARM variants - - home_assistant_version: "2023.12.4" - arch: "aarch64" - home_assistant_version: "2024.2.1" arch: "aarch64" - - home_assistant_version: "2023.12.4" - arch: "armhf" - home_assistant_version: "2024.2.1" arch: "armhf" @@ -34,18 +30,10 @@ jobs: suffix: "-noavx" arch: "amd64" extra_defines: "-DLLAMA_AVX=OFF -DLLAMA_AVX2=OFF -DLLAMA_FMA=OFF -DLLAMA_F16C=OFF" - - home_assistant_version: "2023.12.4" - arch: "amd64" - suffix: "-noavx" - extra_defines: "-DLLAMA_AVX=OFF -DLLAMA_AVX2=OFF -DLLAMA_FMA=OFF -DLLAMA_F16C=OFF" - home_assistant_version: "2024.2.1" arch: "i386" suffix: "-noavx" extra_defines: "-DLLAMA_AVX=OFF -DLLAMA_AVX2=OFF -DLLAMA_FMA=OFF -DLLAMA_F16C=OFF" - - home_assistant_version: "2023.12.4" - arch: "i386" - suffix: "-noavx" - extra_defines: "-DLLAMA_AVX=OFF -DLLAMA_AVX2=OFF -DLLAMA_FMA=OFF -DLLAMA_F16C=OFF" # AVX2 and AVX512 - home_assistant_version: "2024.2.1" From 5ddf0d09d572ce1cdc33e98391948e48eeea93d9 Mon Sep 17 00:00:00 2001 From: Alex O'Connell Date: Fri, 7 Jun 2024 08:48:50 -0400 Subject: [PATCH 2/8] restrict services that can be called and add format url function to make behavior standard --- .../llama_conversation/__init__.py | 24 ++++++++++++++++--- custom_components/llama_conversation/agent.py | 21 ++++++++++++---- .../llama_conversation/config_flow.py | 16 ++++++++++--- custom_components/llama_conversation/const.py | 1 - custom_components/llama_conversation/utils.py | 3 +++ 5 files changed, 53 insertions(+), 12 deletions(-) diff --git a/custom_components/llama_conversation/__init__.py b/custom_components/llama_conversation/__init__.py index 362700b..3738f5a 100644 --- a/custom_components/llama_conversation/__init__.py +++ b/custom_components/llama_conversation/__init__.py @@ -2,6 +2,7 @@ from __future__ import annotations import logging +from typing import Final import homeassistant.components.conversation as ha_conversation from homeassistant.config_entries import ConfigEntry @@ -107,8 +108,8 @@ async def async_migrate_entry(hass, config_entry: ConfigEntry): class HassServiceTool(llm.Tool): """Tool to get the current time.""" - name = SERVICE_TOOL_NAME - description = "Executes a Home Assistant service" + name: Final[str] = SERVICE_TOOL_NAME + description: Final[str] = "Executes a Home Assistant service" # Optional. A voluptuous schema of the input parameters. parameters = vol.Schema({ @@ -125,6 +126,17 @@ class HassServiceTool(llm.Tool): vol.Optional('item'): str, }) + ALLOWED_SERVICES: Final[list[str]] = [ + "turn_on", "turn_off", "toggle", "press", "increase_speed", "decrease_speed", "open_cover", "close_cover", "stop_cover", + "lock", "unlock", "start", "stop", "return_to_base", "pause", "cancel", "add_item" + ] + ALLOWED_DOMAINS: Final[list[str]] = [ + "light", "switch", "button", "fan", "cover", "lock", "media_player", "climate", "vacuum", "todo", "timer", "script", + ] + ALLOWED_SERVICE_CALL_ARGUMENTS: Final[list[str]] = [ + "rgb_color", "brightness", "temperature", "humidity", "fan_mode", "hvac_mode", "preset_mode", "item", "duration", + ] + async def async_call( self, hass: HomeAssistant, tool_input: llm.ToolInput, llm_context: llm.LLMContext ) -> JsonObjectType: @@ -132,8 +144,14 @@ async def async_call( domain, service = tuple(tool_input.tool_args["service"].split(".")) target_device = tool_input.tool_args["target_device"] + if domain not in self.ALLOWED_DOMAINS or service not in self.ALLOWED_SERVICES: + return { "result": "unknown service" } + + if domain == "script" and service not in ["reload", "turn_on", "turn_off", "toggle"]: + return { "result": "unknown service" } + service_data = {ATTR_ENTITY_ID: target_device} - for attr in ALLOWED_LEGACY_SERVICE_CALL_ARGUMENTS: + for attr in self.ALLOWED_SERVICE_CALL_ARGUMENTS: if attr in tool_input.tool_args.keys(): service_data[attr] = tool_input.tool_args[attr] try: diff --git a/custom_components/llama_conversation/agent.py b/custom_components/llama_conversation/agent.py index f9a4f9f..5707be8 100644 --- a/custom_components/llama_conversation/agent.py +++ b/custom_components/llama_conversation/agent.py @@ -29,8 +29,9 @@ import voluptuous_serialize +from . import HassServiceTool from .utils import closest_color, flatten_vol_schema, custom_custom_serializer, install_llama_cpp_python, \ - validate_llama_cpp_python_installation + validate_llama_cpp_python_installation, format_url from .const import ( CONF_CHAT_MODEL, CONF_MAX_TOKENS, @@ -106,7 +107,6 @@ TEXT_GEN_WEBUI_CHAT_MODE_CHAT, TEXT_GEN_WEBUI_CHAT_MODE_INSTRUCT, TEXT_GEN_WEBUI_CHAT_MODE_CHAT_INSTRUCT, - ALLOWED_LEGACY_SERVICE_CALL_ARGUMENTS, DOMAIN, HOME_LLM_API_ID, SERVICE_TOOL_NAME, @@ -670,7 +670,7 @@ def expose_attributes(attributes): for name, service in service_dict.get(domain, {}).items(): args = flatten_vol_schema(service.schema) - args_to_expose = set(args).intersection(ALLOWED_LEGACY_SERVICE_CALL_ARGUMENTS) + args_to_expose = set(args).intersection(HassServiceTool.ALLOWED_SERVICE_CALL_ARGUMENTS) service_schema = vol.Schema({ vol.Optional(arg): str for arg in args_to_expose }) @@ -1042,7 +1042,13 @@ class GenericOpenAIAPIAgent(LocalLLMAgent): model_name: str def _load_model(self, entry: ConfigEntry) -> None: - self.api_host = f"{'https' if entry.data[CONF_SSL] else 'http'}://{entry.data[CONF_HOST]}:{entry.data[CONF_PORT]}" + self.api_host = format_url( + hostname=entry.data[CONF_HOST], + port=entry.data[CONF_PORT], + ssl=entry.data[CONF_SSL], + path="" + ) + self.api_key = entry.data.get(CONF_OPENAI_API_KEY) self.model_name = entry.data.get(CONF_CHAT_MODEL) @@ -1249,7 +1255,12 @@ class OllamaAPIAgent(LocalLLMAgent): model_name: str def _load_model(self, entry: ConfigEntry) -> None: - self.api_host = f"{'https' if entry.data[CONF_SSL] else 'http'}://{entry.data[CONF_HOST]}:{entry.data[CONF_PORT]}" + self.api_host = format_url( + hostname=entry.data[CONF_HOST], + port=entry.data[CONF_PORT], + ssl=entry.data[CONF_SSL], + path="" + ) self.api_key = entry.data.get(CONF_OPENAI_API_KEY) self.model_name = entry.data.get(CONF_CHAT_MODEL) diff --git a/custom_components/llama_conversation/config_flow.py b/custom_components/llama_conversation/config_flow.py index f07424e..50d986d 100644 --- a/custom_components/llama_conversation/config_flow.py +++ b/custom_components/llama_conversation/config_flow.py @@ -38,7 +38,7 @@ from homeassistant.util.package import is_installed from importlib.metadata import version -from .utils import download_model_from_hf, install_llama_cpp_python, MissingQuantizationException +from .utils import download_model_from_hf, install_llama_cpp_python, format_url, MissingQuantizationException from .const import ( CONF_CHAT_MODEL, CONF_MAX_TOKENS, @@ -503,7 +503,12 @@ def _validate_text_generation_webui(self, user_input: dict) -> tuple: headers["Authorization"] = f"Bearer {api_key}" models_result = requests.get( - f"{'https' if self.model_config[CONF_SSL] else 'http'}://{self.model_config[CONF_HOST]}:{self.model_config[CONF_PORT]}/v1/internal/model/list", + format_url( + hostname=self.model_config[CONF_HOST], + port=self.model_config[CONF_PORT], + ssl=self.model_config[CONF_SSL], + path="/v1/internal/model/list" + ), timeout=5, # quick timeout headers=headers ) @@ -535,7 +540,12 @@ def _validate_ollama(self, user_input: dict) -> tuple: headers["Authorization"] = f"Bearer {api_key}" models_result = requests.get( - f"{'https' if self.model_config[CONF_SSL] else 'http'}://{self.model_config[CONF_HOST]}:{self.model_config[CONF_PORT]}/api/tags", + format_url( + hostname=self.model_config[CONF_HOST], + port=self.model_config[CONF_PORT], + ssl=self.model_config[CONF_SSL], + path="/api/tags" + ), timeout=5, # quick timeout headers=headers ) diff --git a/custom_components/llama_conversation/const.py b/custom_components/llama_conversation/const.py index 75159f5..27e4c03 100644 --- a/custom_components/llama_conversation/const.py +++ b/custom_components/llama_conversation/const.py @@ -76,7 +76,6 @@ DEFAULT_SSL = False CONF_EXTRA_ATTRIBUTES_TO_EXPOSE = "extra_attributes_to_expose" DEFAULT_EXTRA_ATTRIBUTES_TO_EXPOSE = ["rgb_color", "brightness", "temperature", "humidity", "fan_mode", "media_title", "volume_level", "item", "wind_speed"] -ALLOWED_LEGACY_SERVICE_CALL_ARGUMENTS = ["rgb_color", "brightness", "temperature", "humidity", "fan_mode", "hvac_mode", "preset_mode", "item", "duration"] CONF_PROMPT_TEMPLATE = "prompt_template" PROMPT_TEMPLATE_CHATML = "chatml" PROMPT_TEMPLATE_COMMAND_R = "command-r" diff --git a/custom_components/llama_conversation/utils.py b/custom_components/llama_conversation/utils.py index 3ff1695..f3a0a45 100644 --- a/custom_components/llama_conversation/utils.py +++ b/custom_components/llama_conversation/utils.py @@ -203,3 +203,6 @@ def install_llama_cpp_python(config_dir: str): time.sleep(0.5) # I still don't know why this is required return True + +def format_url(*, hostname: str, port: str, ssl: bool, path: str): + return f"{'https' if ssl else 'http'}://{hostname}{ ':' + port if port else ''}{path}" \ No newline at end of file From 249298bb99a780bc44691b4e24ab8f7a3e1a839c Mon Sep 17 00:00:00 2001 From: Alex O'Connell Date: Fri, 7 Jun 2024 17:42:03 -0400 Subject: [PATCH 3/8] better device prompting with area support + fix circular import --- .../llama_conversation/__init__.py | 7 +-- custom_components/llama_conversation/agent.py | 61 ++++++++++++++----- custom_components/llama_conversation/const.py | 20 +++++- .../llama_conversation/manifest.json | 2 +- 4 files changed, 66 insertions(+), 24 deletions(-) diff --git a/custom_components/llama_conversation/__init__.py b/custom_components/llama_conversation/__init__.py index 3738f5a..ca64a1e 100644 --- a/custom_components/llama_conversation/__init__.py +++ b/custom_components/llama_conversation/__init__.py @@ -32,7 +32,7 @@ BACKEND_TYPE_GENERIC_OPENAI, BACKEND_TYPE_LLAMA_CPP_PYTHON_SERVER, BACKEND_TYPE_OLLAMA, - ALLOWED_LEGACY_SERVICE_CALL_ARGUMENTS, + ALLOWED_SERVICE_CALL_ARGUMENTS, DOMAIN, HOME_LLM_API_ID, SERVICE_TOOL_NAME, @@ -133,9 +133,6 @@ class HassServiceTool(llm.Tool): ALLOWED_DOMAINS: Final[list[str]] = [ "light", "switch", "button", "fan", "cover", "lock", "media_player", "climate", "vacuum", "todo", "timer", "script", ] - ALLOWED_SERVICE_CALL_ARGUMENTS: Final[list[str]] = [ - "rgb_color", "brightness", "temperature", "humidity", "fan_mode", "hvac_mode", "preset_mode", "item", "duration", - ] async def async_call( self, hass: HomeAssistant, tool_input: llm.ToolInput, llm_context: llm.LLMContext @@ -151,7 +148,7 @@ async def async_call( return { "result": "unknown service" } service_data = {ATTR_ENTITY_ID: target_device} - for attr in self.ALLOWED_SERVICE_CALL_ARGUMENTS: + for attr in ALLOWED_SERVICE_CALL_ARGUMENTS: if attr in tool_input.tool_args.keys(): service_data[attr] = tool_input.tool_args[attr] try: diff --git a/custom_components/llama_conversation/agent.py b/custom_components/llama_conversation/agent.py index 5707be8..5af80fb 100644 --- a/custom_components/llama_conversation/agent.py +++ b/custom_components/llama_conversation/agent.py @@ -23,13 +23,12 @@ from homeassistant.const import ATTR_ENTITY_ID, CONF_HOST, CONF_PORT, CONF_SSL, MATCH_ALL, CONF_LLM_HASS_API from homeassistant.core import HomeAssistant, callback from homeassistant.exceptions import ConfigEntryNotReady, ConfigEntryError, TemplateError, HomeAssistantError -from homeassistant.helpers import config_validation as cv, intent, template, entity_registry as er, llm +from homeassistant.helpers import config_validation as cv, intent, template, entity_registry as er, llm, area_registry as ar from homeassistant.helpers.event import async_track_state_change, async_call_later from homeassistant.util import ulid, color import voluptuous_serialize -from . import HassServiceTool from .utils import closest_color, flatten_vol_schema, custom_custom_serializer, install_llama_cpp_python, \ validate_llama_cpp_python_installation, format_url from .const import ( @@ -114,6 +113,7 @@ TOOL_FORMAT_FULL, TOOL_FORMAT_REDUCED, TOOL_FORMAT_MINIMAL, + ALLOWED_SERVICE_CALL_ARGUMENTS, ) # make type checking work for llama-cpp-python without importing it directly at runtime @@ -445,6 +445,7 @@ def _async_get_exposed_entities(self) -> tuple[dict[str, str], list[str]]: entity_states = {} domains = set() entity_registry = er.async_get(self.hass) + area_registry = ar.async_get(self.hass) for state in self.hass.states.async_all(): if not async_should_expose(self.hass, CONVERSATION_DOMAIN, state.entity_id): @@ -456,11 +457,15 @@ def _async_get_exposed_entities(self) -> tuple[dict[str, str], list[str]]: attributes["state"] = state.state if entity and entity.aliases: attributes["aliases"] = entity.aliases + + if entity and entity.area_id: + area = area_registry.async_get_area(entity.area_id) + attributes["area_id"] = area.id + attributes["area_name"] = area.name + entity_states[state.entity_id] = attributes domains.add(state.domain) - # _LOGGER.debug(f"Exposed entities: {entity_states}") - return entity_states, list(domains) def _format_prompt( @@ -556,6 +561,9 @@ def _generate_icl_examples(self, num_examples, entity_names): entity_names = entity_names[:] entity_domains = set([x.split(".")[0] for x in entity_names]) + area_registry = ar.async_get(self.hass) + all_areas = list(area_registry.async_list_areas()) + in_context_examples = [ x for x in self.in_context_examples if x["type"] in entity_domains @@ -575,7 +583,7 @@ def _generate_icl_examples(self, num_examples, entity_names): response = chosen_example["response"] random_device = [ x for x in entity_names if x.split(".")[0] == chosen_example["type"] ][0] - random_area = "bedroom" # todo, pick a random area + random_area = random.choice(all_areas).name random_brightness = round(random.random(), 2) random_color = random.choice(list(color.COLORS.keys())) @@ -619,8 +627,8 @@ def _generate_system_prompt(self, prompt_template: str, llm_api: llm.APIInstance extra_attributes_to_expose = self.entry.options \ .get(CONF_EXTRA_ATTRIBUTES_TO_EXPOSE, DEFAULT_EXTRA_ATTRIBUTES_TO_EXPOSE) - def expose_attributes(attributes): - result = attributes["state"] + def expose_attributes(attributes) -> list[str]: + result = [] for attribute_name in extra_attributes_to_expose: if attribute_name not in attributes: continue @@ -644,19 +652,38 @@ def expose_attributes(attributes): elif attribute_name == "humidity": value = f"{value}%" - result = result + ";" + str(value) + result.append(str(value)) return result - device_states = [] + devices = [] + formatted_devices = "" # expose devices and their alias as well for name, attributes in entities_to_expose.items(): - device_states.append(f"{name} '{attributes.get('friendly_name')}' = {expose_attributes(attributes)}") + state = attributes["state"] + exposed_attributes = expose_attributes(attributes) + str_attributes = ";".join([state] + exposed_attributes) + + formatted_devices = formatted_devices + f"{name} '{attributes.get('friendly_name')}' = {str_attributes}\n" + devices.append({ + "entity_id": name, + "name": attributes.get('friendly_name'), + "state": state, + "attributes": exposed_attributes, + "area_name": attributes.get("area_name"), + "area_id": attributes.get("area_id") + }) if "aliases" in attributes: for alias in attributes["aliases"]: - device_states.append(f"{name} '{alias}' = {expose_attributes(attributes)}") - - formatted_states = "\n".join(device_states) + "\n" + formatted_devices = formatted_devices + f"{name} '{alias}' = {str_attributes}\n" + devices.append({ + "entity_id": name, + "name": alias, + "state": state, + "attributes": exposed_attributes, + "area_name": attributes.get("area_name"), + "area_id": attributes.get("area_id") + }) if llm_api: if llm_api.api.id == HOME_LLM_API_ID: @@ -670,7 +697,7 @@ def expose_attributes(attributes): for name, service in service_dict.get(domain, {}).items(): args = flatten_vol_schema(service.schema) - args_to_expose = set(args).intersection(HassServiceTool.ALLOWED_SERVICE_CALL_ARGUMENTS) + args_to_expose = set(args).intersection(ALLOWED_SERVICE_CALL_ARGUMENTS) service_schema = vol.Schema({ vol.Optional(arg): str for arg in args_to_expose }) @@ -681,17 +708,21 @@ def expose_attributes(attributes): self._format_tool(*tool) for tool in all_services ] + formatted_tools = ", ".join(tools) else: tools = [ self._format_tool(tool.name, tool.parameters, tool.description) for tool in llm_api.tools ] + formatted_tools = json.dumps(tools) else: tools = "No tools were provided. If the user requests you interact with a device, tell them you are unable to do so." render_variables = { - "devices": formatted_states, + "devices": devices, + "formatted_devices": formatted_devices, "tools": tools, + "formatted_tools": formatted_tools, "response_examples": [] } diff --git a/custom_components/llama_conversation/const.py b/custom_components/llama_conversation/const.py index 27e4c03..27da763 100644 --- a/custom_components/llama_conversation/const.py +++ b/custom_components/llama_conversation/const.py @@ -15,12 +15,25 @@ The current time and date is {{ (as_timestamp(now()) | timestamp_custom("%I:%M %p on %A %B %d, %Y", "")) }} Tools: {{ tools | to_json }} Devices: -{{ devices }}""" +{% for device in devices | selectattr('area_id', 'none'): %} +{{ device.entity_id }} '{{ device.name }}' = {{ device.state }}{{ ([""] + device.attributes) | join(";") }} +{% endfor %} +{% for area in devices | rejectattr('area_id', 'none') | groupby('area_name') %} +## Area: {{ area.grouper }} +{% for device in area.list %} +{{ device.entity_id }} '{{ device.name }}' = {{ device.state }};{{ device.attributes | join(";") }} +{% endfor %} +{% endfor %} +{% for item in response_examples %} +{{ item.request }} +{{ item.response }} + {{ item.tool | to_json }} +{% endfor %}""" DEFAULT_PROMPT_BASE_LEGACY = """ The current time and date is {{ (as_timestamp(now()) | timestamp_custom("%I:%M %p on %A %B %d, %Y", "")) }} -Services: {{ tools | join(", ") }} +Services: {{ formatted_tools }} Devices: -{{ devices }}""" +{{ formatted_devices }}""" ICL_EXTRAS = """ {% for item in response_examples %} {{ item.request }} @@ -76,6 +89,7 @@ DEFAULT_SSL = False CONF_EXTRA_ATTRIBUTES_TO_EXPOSE = "extra_attributes_to_expose" DEFAULT_EXTRA_ATTRIBUTES_TO_EXPOSE = ["rgb_color", "brightness", "temperature", "humidity", "fan_mode", "media_title", "volume_level", "item", "wind_speed"] +ALLOWED_SERVICE_CALL_ARGUMENTS = ["rgb_color", "brightness", "temperature", "humidity", "fan_mode", "hvac_mode", "preset_mode", "item", "duration" ] CONF_PROMPT_TEMPLATE = "prompt_template" PROMPT_TEMPLATE_CHATML = "chatml" PROMPT_TEMPLATE_COMMAND_R = "command-r" diff --git a/custom_components/llama_conversation/manifest.json b/custom_components/llama_conversation/manifest.json index e90a943..9844034 100644 --- a/custom_components/llama_conversation/manifest.json +++ b/custom_components/llama_conversation/manifest.json @@ -1,7 +1,7 @@ { "domain": "llama_conversation", "name": "Local LLM Conversation", - "version": "0.2.17", + "version": "0.3", "codeowners": ["@acon96"], "config_flow": true, "dependencies": ["conversation"], From 3e4a3510546f333e3e5d3709a897f229d057d5d2 Mon Sep 17 00:00:00 2001 From: Alex O'Connell Date: Fri, 7 Jun 2024 17:44:25 -0400 Subject: [PATCH 4/8] tweak requests version to be compatible with new HA --- custom_components/llama_conversation/manifest.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/custom_components/llama_conversation/manifest.json b/custom_components/llama_conversation/manifest.json index 9844034..748bca3 100644 --- a/custom_components/llama_conversation/manifest.json +++ b/custom_components/llama_conversation/manifest.json @@ -9,8 +9,8 @@ "integration_type": "service", "iot_class": "local_polling", "requirements": [ - "requests==2.31.0", + "requests>=2.31.0", "huggingface-hub==0.23.0", - "webcolors==1.13" + "webcolors>=1.13" ] } From e42981414a7a4b5c1f10c01729e0402b46f8403e Mon Sep 17 00:00:00 2001 From: Alex O'Connell Date: Fri, 7 Jun 2024 23:22:02 -0400 Subject: [PATCH 5/8] documentation updates --- docs/Backend Configuration.md | 7 +- docs/Model Prompting.md | 137 +++++++++++++++++++++++----------- docs/Setup.md | 2 +- 3 files changed, 97 insertions(+), 49 deletions(-) diff --git a/docs/Backend Configuration.md b/docs/Backend Configuration.md index 2f4dcc1..e89ac6d 100644 --- a/docs/Backend Configuration.md +++ b/docs/Backend Configuration.md @@ -44,15 +44,12 @@ After the wheel file has been copied to the correct folder, attempt the wheel in Pre-built wheel files (`*.whl`) are provided as part of the [GitHub release](https://github.com/acon96/home-llm/releases/latest) for the integration. To ensure compatibility with your Home Assistant and Python versions, select the correct `.whl` file for your hardware's architecture: -- For Home Assistant `2024.1.4` and older, use the Python 3.11 wheels (`cp311`) - For Home Assistant `2024.2.0` and newer, use the Python 3.12 wheels (`cp312`) - **ARM devices** (e.g., Raspberry Pi 4/5): - - Example filenames: - - `llama_cpp_python-{version}-cp311-cp311-musllinux_1_2_aarch64.whl` + - Example filename: - `llama_cpp_python-{version}-cp312-cp312-musllinux_1_2_aarch64.whl` - **x86_64 devices** (e.g., Intel/AMD desktops): - - Example filenames: - - `llama_cpp_python-{version}-cp311-cp311-musllinux_1_2_x86_64.whl` + - Example filename: - `llama_cpp_python-{version}-cp312-cp312-musllinux_1_2_x86_64.whl` ## Build your own diff --git a/docs/Model Prompting.md b/docs/Model Prompting.md index 4ddbc9b..5df7cbc 100644 --- a/docs/Model Prompting.md +++ b/docs/Model Prompting.md @@ -1,19 +1,101 @@ # Model Prompting -This integration allows for full customization of the system prompt using Home Assistant's [built in templating engine](https://www.home-assistant.io/docs/configuration/templating/). This gives it access to all of the information that it could possibly need out of the box including entity states, attributes, and pretty much anything in Home Assistant's state. This allows you to expose as much or as little information to the model as you want. +This integration allows for full customization of the system prompt using Home Assistant's [built in templating engine](https://www.home-assistant.io/docs/configuration/templating/). This gives it access to all of the information that it could possibly need out of the box including entity states, attributes, which allows you to expose as much or as little information to the model as you want. In addition to having access to all of this information, extra variables have been added to make it easier to build a useful prompt. ## System Prompt Template -The default system prompt is: +The default system prompt for non-fine tuned models is: ``` You are 'Al', a helpful AI Assistant that controls the devices in a house. Complete the following task as instructed with the information provided only. -Services: {{ services }} +The current time and date is {{ (as_timestamp(now()) | timestamp_custom("%I:%M %p on %A %B %d, %Y", "")) }} +Tools: {{ tools | to_json }} Devices: -{{ devices }} +{% for device in devices | selectattr('area_id', 'none'): %} +{{ device.entity_id }} '{{ device.name }}' = {{ device.state }}{{ ([""] + device.attributes) | join(";") }} +{% endfor %} +{% for area in devices | rejectattr('area_id', 'none') | groupby('area_name') %} +## Area: {{ area.grouper }} +{% for device in area.list %} +{{ device.entity_id }} '{{ device.name }}' = {{ device.state }};{{ device.attributes | join(";") }} +{% endfor %} +{% endfor %} +{% for item in response_examples %} +{{ item.request }} +{{ item.response }} + {{ item.tool | to_json }} +{% endfor %} +``` + +This prompt provides the following pieces of information to the model: +1. it gives the model a quick personality +2. provides the time and date +3. provides the available tools. + - Most models understand JSON so you can simply convert the provided variable to JSON and insert it into the prompt +4. provides the exposed devices. + - uses the `selectattr` filter to gather all the devices that do not have an area and puts them at the top + - uses the `rejectattr` filter to gather the opposite set of devices (the ones that do have areas) and then uses `groupby` make groups for the devices that are in an area. +5. provides "in-context-learning" examples to the model, so that it better understands the format that it should produce + +This all results in something that looks like this: ``` - -The `services` and `devices` variables are special variables that are provided by the integration and NOT Home Assistant. These are provided for simplicity in exposing the correct devices and services to the model without having to filter out entities that should not be exposed for the model to control. -- `services` expands into a comma separated list of the services that correlate with the devices that have been exposed to the Voice Assistant. -- `devices` expands into a multi-line block where each line is the format ` ' = ;` +You are 'Al', a helpful AI Assistant that controls the devices in a house. Complete the following task as instructed with the information provided only. +The current time and date is 09:40 PM on Friday June 07, 2024 +Tools: [{"name":"HassTurnOn","description":"Turns on/opens a device or entity","parameters":{"properties":{"name":"string","area":"string","floor":"string","domain":"string","device_class":"string"},"required":[]}},{"name":"HassTurnOff","description":"Turns off/closes a device or entity","parameters":{"properties":{"name":"string","area":"string","floor":"string","domain":"string","device_class":"string"},"required":[]}},{"name":"HassSetPosition","description":"Sets the position of a device or entity","parameters":{"properties":{"name":"string","area":"string","floor":"string","domain":"string","device_class":"string","position":"integer"},"required":["position"]}},{"name":"HassListAddItem","description":"Add item to a todo list","parameters":{"properties":{"item":"string","name":"string"},"required":[]}},{"name":"HassHumidifierSetpoint","description":"Set desired humidity level","parameters":{"properties":{"name":"string","humidity":"integer"},"required":["name","humidity"]}},{"name":"HassHumidifierMode","description":"Set humidifier mode","parameters":{"properties":{"name":"string","mode":"string"},"required":["name","mode"]}},{"name":"HassLightSet","description":"Sets the brightness or color of a light","parameters":{"properties": {"name":"string","area":"string","floor":"string","domain":"string","device_class":"string","color":"string","temperature":"integer","brightness":"integer"},"required":[]}},{"name":"HassMediaUnpause","description":"Resumes a media player","parameters":{"properties":{"name":"string","area":"string","floor":"string","domain":"string","device_class":"string"},"required":[]}},{"name":"HassMediaPause","description":"Pauses a media player","parameters":{"properties":{"name":"string","area":"string","floor":"string","domain":"string","device_class":"string"},"required":[]}},{"name":"HassVacuumStart","description":"Starts a vacuum","parameters":{"properties":{"name":"string","area":"string","floor":"string","domain":"string","device_class":"string"},"required":[]}},{"name":"HassVacuumReturnToBase","description":"Returns a vacuum to base","parameters":{"properties":{"name":"string","area":"string","floor":"string","domain":"string","device_class":"string"},"required":[]}}] +Devices: +button.push 'Push' = unknown +climate.heatpump 'HeatPump' = heat;68F +climate.ecobee_thermostat 'Ecobee Thermostat' = cool;70F;67.4%;on_high +cover.kitchen_window 'Kitchen Window' = closed +cover.hall_window 'Hall Window' = open +cover.living_room_window 'Living Room Window' = open +cover.garage_door 'Garage Door' = closed +fan.ceiling_fan 'Ceiling Fan' = off +fan.percentage_full_fan 'Percentage Full Fan' = off +humidifier.humidifier 'Humidifier' = on;68% +light.bed_light 'Bed Light' = off +light.ceiling_lights 'Ceiling Lights' = on;sandybrown (255, 164, 81);70% +light.ceiling_lights 'Dan's Lights' = on;sandybrown (255, 164, 81);70% +light.kitchen_lights 'Kitchen Lights' = on;tomato (255, 63, 111);70% +light.office_rgbw_lights 'Office RGBW Lights' = on;salmon (255, 128, 128);70% +light.living_room_rgbww_lights 'Living Room RGBWW Lights' = on;salmon (255, 127, 125);70% +lock.front_door 'Front Door' = locked +lock.kitchen_door 'Kitchen Door' = unlocked +lock.poorly_installed_door 'Poorly Installed Door' = unlocked +lock.openable_lock 'Openable Lock' = locked +sensor.carbon_dioxide 'Carbon dioxide' = 54 +switch.decorative_lights 'Decorative Lights' = on +vacuum.1_first_floor '1_First_floor' = docked +todo.shopping_list 'Shopping_list' = 2 +## Area: Living Room: +fan.living_room_fan 'Living Room Fan' = on; +Make the lights in Living Room greenyellow +The color should be changed now. + {"name":"HassLightSet","arguments":{"area":"Living Room","color":"greenyellow"}} +Set the brightness for light.bed_light to 0.47 +Setting the brightness now. + {"name":"HassLightSet","arguments":{"name":"light.bed_light","brightness":0.47}} +Can you open the cover.hall_window? +Opening the garage door for you. + {"name":"HassOpenCover","arguments":{"name":"cover.hall_window"}} +Stop the vacuum.1_first_floor vacuum +Sending the vacuum back to its base. + {"name":"HassVacuumReturnToBase","arguments":{"name":"vacuum.1_first_floor"}} +``` + +There are a few variables that are exposed to the template to allow the prompt to expose the devices in your home as well as the various tools that the model can call. + +Prompt Variables: +- `devices`: each item in the provided array contains the `entity_id`, `name`, `state`, and `attributes` properties +- `tools`: can be one of 3 formats as selected by the user + - Minimal: Tools are passed as an array of strings in a Python inspired function definition format. Uses the fewest tokens. ex: `climate.set_hvac_mode(hvac_mode)` + - Reduced: Tools are passed as an array of dictionaries where each tool contains the following fields: `name`, `description`, `parameters`. + - Full: Tools are passed as an array of dictionaries where the structure of each tool matches the tool format used by the OpenAI APIs. Uses the most tokens. +- `formatted_devices` expands into a multi-line block where each line is the format ` ' = ;` +- `formatted_tools`: when using Reduced, or Full tool format is selected, the entire array is converted to JSON. When using Minimal, each tool is returned separated by a comma. +- `response_examples`: an array of randomly generated in-context-learning examples containing the `request`, `response`, and `tool` properties that can be used to build a properly formatted ICL example + +`formatted_devices` and `formatted_tools` are provided for simplicity in exposing the correct devices and tools to the model if you are using the Home-LLM model or do not want to customize the formatting of the devices and tools. + +The examples used for the `response_examples` variable are loaded from the `in_context_examples.csv` file in the `/custom_components/llama_conversation/` folder. ### Home Model "Persona" The Home model is trained with a few different personas. They can be activated by using their system prompt found below: @@ -62,39 +144,8 @@ Currently supported prompt formats are: 4. Mistral 5. Zephyr w/ eos token `<|endoftext|>` 6. Zephyr w/ eos token `` -7. Llama 3 -8. None (useful for foundation models) - -## Prompting other models with In Context Learning -It is possible to use models that are not fine-tuned with the dataset via the usage of In Context Learning (ICL) examples. These examples condition the model to output the correct JSON schema without any fine-tuning of the model. - -Here is an example configuration of using Mixtral-7B-Instruct-v0.2. -First, download and set up the model on the desired backend. - -Then, navigate to the conversation agent's configuration page and set the following options: - -System Prompt: -``` -You are 'Al', a helpful AI Assistant that controls the devices in a house. Complete the following task as instructed with the information provided only. -Services: {{ services }} -Devices: -{{ devices }} - -Respond to the following user instruction by responding in the same format as the following examples: -{{ response_examples }} - -User instruction: -``` -Prompt Format: `Mistral` -Service Call Regex: `({[\S \t]*?})` -Enable in context learning (ICL) examples: Checked - -### Explanation -Enabling in context learning examples exposes the additional `{{ response_examples }}` variable for the system prompt. This variable is expanded to include various examples in the following format: -``` -{"to_say": "Switching off the fan as requested.", "service": "fan.turn_off", "target_device": "fan.ceiling_fan"} -{"to_say": "the todo has been added to your todo list.", "service": "todo.add_item", "target_device": "todo.shopping_list"} -{"to_say": "Starting media playback.", "service": "media_player.media_play", "target_device": "media_player.bedroom"} -``` +7. Zephyr w/ eos token `<|end|>` +8. Llama 3 +9. Command-R +10. None (useful for foundation models) -These examples are loaded from the `in_context_examples.csv` file in the `/custom_components/llama_conversation/` folder. \ No newline at end of file diff --git a/docs/Setup.md b/docs/Setup.md index 19d195c..6752d44 100644 --- a/docs/Setup.md +++ b/docs/Setup.md @@ -57,7 +57,7 @@ The next step is to specify which model will be used by the integration. You may **Model Name**: Use either `acon96/Home-3B-v3-GGUF` or `acon96/Home-1B-v3-GGUF` **Quantization Level**: The model will be downloaded in the selected quantization level from the HuggingFace repository. If unsure which level to choose, select `Q4_K_M`. -Pressing `Submit` will download the model from HuggingFace. +Pressing `Submit` will download the model from HuggingFace. The downloaded files will be stored by default in `/media/models/`. **Note for Docker/sanboxed HA install users:** The model download may fail if it does not have the permissions to create the ```media``` folder in your Home Assistant install. To fix this, you will need to manually create the folder beside your existing ```config``` folder called ```media``` and set the permissions accordingly so that the addon can access it. If you're using Docker or similar, you may need to map the folder in your Compose file too and ```Update the Stack```. Once created and updated, you can open the model download screen again and it should now download as normal. From 0a6e558fdaef59cfa5731aa40e8cac270907ac11 Mon Sep 17 00:00:00 2001 From: Alex O'Connell Date: Sat, 8 Jun 2024 12:14:24 -0400 Subject: [PATCH 6/8] Release v0.3.1 --- README.md | 1 + custom_components/llama_conversation/__init__.py | 4 ++++ custom_components/llama_conversation/agent.py | 9 +++++++-- custom_components/llama_conversation/config_flow.py | 2 ++ custom_components/llama_conversation/const.py | 7 +------ custom_components/llama_conversation/manifest.json | 2 +- 6 files changed, 16 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 1e55a0d..c78c59d 100644 --- a/README.md +++ b/README.md @@ -132,6 +132,7 @@ In order to facilitate running the project entirely on the system where Home Ass ## Version History | Version | Description | |---------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| v0.3.1 | Fix for broken requirements, fix issue with formatted tools, fix default prompt, fix custom API not registering on startup properly | | v0.3 | Adds support for Home Assistant LLM APIs, improved model prompting and tool formatting options, and automatic detection of GGUF quantization levels on HuggingFace | | v0.2.17 | Disable native llama.cpp wheel optimizations, add Command R prompt format | | v0.2.16 | Fix for missing huggingface_hub package preventing startup | diff --git a/custom_components/llama_conversation/__init__.py b/custom_components/llama_conversation/__init__.py index ca64a1e..fee13e5 100644 --- a/custom_components/llama_conversation/__init__.py +++ b/custom_components/llama_conversation/__init__.py @@ -55,6 +55,10 @@ async def update_listener(hass: HomeAssistant, entry: ConfigEntry): async def async_setup_entry(hass: HomeAssistant, entry: ConfigEntry) -> bool: """Set up Local LLM Conversation from a config entry.""" + # make sure the API is registered + if not any([x.id == HOME_LLM_API_ID for x in llm.async_get_apis(hass)]): + llm.async_register_api(hass, HomeLLMAPI(hass)) + def create_agent(backend_type): agent_cls = None diff --git a/custom_components/llama_conversation/agent.py b/custom_components/llama_conversation/agent.py index 5af80fb..57c3673 100644 --- a/custom_components/llama_conversation/agent.py +++ b/custom_components/llama_conversation/agent.py @@ -708,15 +708,20 @@ def expose_attributes(attributes) -> list[str]: self._format_tool(*tool) for tool in all_services ] - formatted_tools = ", ".join(tools) + else: tools = [ self._format_tool(tool.name, tool.parameters, tool.description) for tool in llm_api.tools ] + + if self.entry.options.get(CONF_TOOL_FORMAT, DEFAULT_TOOL_FORMAT) == TOOL_FORMAT_MINIMAL: + formatted_tools = ", ".join(tools) + else: formatted_tools = json.dumps(tools) else: - tools = "No tools were provided. If the user requests you interact with a device, tell them you are unable to do so." + tools = ["No tools were provided. If the user requests you interact with a device, tell them you are unable to do so."] + formatted_tools = tools[0] render_variables = { "devices": devices, diff --git a/custom_components/llama_conversation/config_flow.py b/custom_components/llama_conversation/config_flow.py index 50d986d..ea00f4f 100644 --- a/custom_components/llama_conversation/config_flow.py +++ b/custom_components/llama_conversation/config_flow.py @@ -489,6 +489,8 @@ async def async_step_download( self.download_task = None return self.async_show_progress_done(next_step_id=next_step) + # TODO: add validate for generic openAI API and hit the `/v1/models endpoint to check + def _validate_text_generation_webui(self, user_input: dict) -> tuple: """ Validates a connection to text-generation-webui and that the model exists on the remote server diff --git a/custom_components/llama_conversation/const.py b/custom_components/llama_conversation/const.py index 27da763..80b3309 100644 --- a/custom_components/llama_conversation/const.py +++ b/custom_components/llama_conversation/const.py @@ -23,11 +23,6 @@ {% for device in area.list %} {{ device.entity_id }} '{{ device.name }}' = {{ device.state }};{{ device.attributes | join(";") }} {% endfor %} -{% endfor %} -{% for item in response_examples %} -{{ item.request }} -{{ item.response }} - {{ item.tool | to_json }} {% endfor %}""" DEFAULT_PROMPT_BASE_LEGACY = """ The current time and date is {{ (as_timestamp(now()) | timestamp_custom("%I:%M %p on %A %B %d, %Y", "")) }} @@ -328,5 +323,5 @@ } } -INTEGRATION_VERSION = "0.3" +INTEGRATION_VERSION = "0.3.1" EMBEDDED_LLAMA_CPP_PYTHON_VERSION = "0.2.77" \ No newline at end of file diff --git a/custom_components/llama_conversation/manifest.json b/custom_components/llama_conversation/manifest.json index 748bca3..8f515d7 100644 --- a/custom_components/llama_conversation/manifest.json +++ b/custom_components/llama_conversation/manifest.json @@ -1,7 +1,7 @@ { "domain": "llama_conversation", "name": "Local LLM Conversation", - "version": "0.3", + "version": "0.3.1", "codeowners": ["@acon96"], "config_flow": true, "dependencies": ["conversation"], From ce4508b9edecb9da1ac259c643471409af196038 Mon Sep 17 00:00:00 2001 From: Alex O'Connell Date: Sat, 8 Jun 2024 12:20:02 -0400 Subject: [PATCH 7/8] fix release notes --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index c78c59d..957a925 100644 --- a/README.md +++ b/README.md @@ -132,7 +132,7 @@ In order to facilitate running the project entirely on the system where Home Ass ## Version History | Version | Description | |---------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| v0.3.1 | Fix for broken requirements, fix issue with formatted tools, fix default prompt, fix custom API not registering on startup properly | +| v0.3.1 | Adds basic area support in prompting, Fix for broken requirements, fix for issue with formatted tools, fix custom API not registering on startup properly | | v0.3 | Adds support for Home Assistant LLM APIs, improved model prompting and tool formatting options, and automatic detection of GGUF quantization levels on HuggingFace | | v0.2.17 | Disable native llama.cpp wheel optimizations, add Command R prompt format | | v0.2.16 | Fix for missing huggingface_hub package preventing startup | From 50bcd2e7ea1d5df0c4dbae0e95ed9d314295fa2e Mon Sep 17 00:00:00 2001 From: Alex O'Connell Date: Sat, 8 Jun 2024 13:05:57 -0400 Subject: [PATCH 8/8] update requirements + fix error handler for get api --- custom_components/llama_conversation/agent.py | 3 ++- requirements.txt | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/custom_components/llama_conversation/agent.py b/custom_components/llama_conversation/agent.py index 57c3673..83f20d1 100644 --- a/custom_components/llama_conversation/agent.py +++ b/custom_components/llama_conversation/agent.py @@ -254,11 +254,12 @@ async def async_process( ) except HomeAssistantError as err: _LOGGER.error("Error getting LLM API: %s", err) + intent_response = intent.IntentResponse(language=user_input.language) intent_response.async_set_error( intent.IntentResponseErrorCode.UNKNOWN, f"Error preparing LLM API: {err}", ) - return conversation.ConversationResult( + return ConversationResult( response=intent_response, conversation_id=user_input.conversation_id ) diff --git a/requirements.txt b/requirements.txt index da3bcd9..5450ae1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -18,7 +18,7 @@ huggingface-hub==0.23.0 webcolors==1.13 # types from Home Assistant -homeassistant +homeassistant>=2024.6.1 hassil home-assistant-intents