Add external indexers for RAG (#4457)

cozy.example.yaml

@@ -146,6 +146,8 @@ jobs:
   #   - "service":           launching services
   #   - "migrations":        transforming a VFS with Swift to layout v3
   #   - "notes-save":        saving notes to the VFS
+  #   - "rag-index":         send data to the RAG server for being indexed
+  #   - "rag-query":         send a query to the RAG server
   #   - "push":              sending push notifications
   #   - "sms":               sending SMS notifications
   #   - "sendmail":          sending mails
@@ -200,6 +202,17 @@ konnectors:
   # cmd: ./scripts/konnector-rkt-run.sh # run connectors with rkt
   # cmd: ./scripts/konnector-nsjail-node8-run.sh # run connectors with nsjail
 
+# rag are the URL of the RAG server(s) for AI.
+rag:
+  # A cozy will use the rag server for its context, or if the context is not
+  # declared, for default.
+  default:
+    url: http://localhost:8000
+    api_key: $3cr3t
+  beta:
+    url: http://localhost:8001
+    api_key: $3cr3t
+
 # mail service parameters for sending email via SMTP
 mail:
   # mail noreply address - flags: --mail-noreply-address

docs/README.md

@@ -72,6 +72,7 @@ designing new services.
 
 ### List of services
 
+-   `/ai` - [AI](ai.md)
 -   `/auth` - [Authentication & OAuth](auth.md)
     -   [Delegated authentication](delegated-auth.md)
 -   `/apps` - [Applications Management](apps.md)

docs/ai.md

@@ -0,0 +1,121 @@
+[Table of contents](README.md#table-of-contents)
+
+# AI for personal data
+
+## Introduction
+
+AI can be used for interacting with the personal data of a Cozy. This is
+currently an experimental feature. Retrieval-Augmented Generation (RAG) is
+a classical pattern in the AI world. Here, it is specific to each Cozy.
+
+![Architecture with a RAG server](diagrams/ai.svg)
+
+## Indexation
+
+First of all, the RAG server must be installed with its dependencies. It is
+not mandatory to install them on the same servers as the cozy-stack. And the
+URL of RAG must be filled in cozy-stack configuration file (in `rag`).
+
+For the moment, the feature is experimental, and a trigger must be created
+manually on the Cozy:
+
+```sh
+$ COZY=cozy.localhost:8080
+$ TOKEN=$(cozy-stack instances token-cli $COZY io.cozy.triggers)
+$ curl "http://${COZY}/jobs/triggers" -H "Authorization: Bearer $TOKEN" -d '{ "data": { "attributes": { "type": "@event", "arguments": "io.cozy.files", "debounce": "1m", "worker": "rag-index", "message": {"doctype": "io.cozy.files"} } } }'
+```
+
+It can also be a good idea to start a first indexation with:
+
+```sh
+$ cozy-stack triggers launch --domain $COZY $TRIGGER_ID
+```
+
+In practice, when files are uploaded/modified/deleted, the trigger will create
+a job for the index worker (with debounce). The index worker will look at the
+changed feed, and will call the RAG for each entry in the changes feed.
+
+## Chat
+
+When a user starts a chat, their prompts are sent to the RAG that can use the
+vector database to find relevant documents (technically, only some parts of
+the documents called chunks). Those documents are added to the prompt, so
+that the LLM can use them as a context when answering.
+
+### POST /ai/chat/conversations/:id
+
+This route can be used to ask AI for a chat completion. The id in the path
+must be the identifier of a chat conversation. The client can generate a random
+identifier for a new chat conversation.
+
+The stack will respond after pushing a job for this task, but without the
+response. The client must use the real-time websocket and subscribe to
+`io.cozy.ai.chat.events`.
+
+#### Request
+
+```http
+POST /ai/chat/conversations/e21dce8058b9013d800a18c04daba326 HTTP/1.1
+Content-Type: application/json
+```
+
+```json
+{
+  "q": "Why the sky is blue?"
+}
+```
+
+#### Response
+
+```http
+HTTP/1.1 202 Accepted
+Content-Type: application/vnd.api+json
+```
+
+```json
+{
+  "data": {
+    "type": "io.cozy.ai.chat.conversations",
+    "id": "e21dce8058b9013d800a18c04daba326",
+    "rev": "1-23456",
+    "attributes": {
+      "messages": [
+        {
+          "id": "eb17c3205bf1013ddea018c04daba326",
+          "role": "user",
+          "content": "Why the sky is blue?",
+          "createdAt": "2024-09-24T13:24:07.576Z"
+        }
+      ]
+    }
+  },
+  "cozyMetadata": {
+    "createdAt": "2024-09-24T13:24:07.576Z",
+    "createdOn": "http://cozy.localhost:8080/",
+    "doctypeVersion": "1",
+    "metadataVersion": 1,
+    "updatedAt": "2024-09-24T13:24:07.576Z"
+  }
+}
+```
+
+### Real-time via websockets
+
+```
+client > {"method": "AUTH", "payload": "token"}
+client > {"method": "SUBSCRIBE",
+          "payload": {"type": "io.cozy.ai.chat.events"}}
+server > {"event": "CREATED",
+          "payload": {"id": "eb17c3205bf1013ddea018c04daba326",
+                      "type": "io.cozy.ai.chat.events",
+                      "doc": {"object": "delta", "content": "The ", "position": 0}}}
+server > {"event": "CREATED",
+          "payload": {"id": "eb17c3205bf1013ddea018c04daba326",
+                      "type": "io.cozy.ai.chat.events",
+                      "doc": {"object": "delta", "content": "sky ", "position": 1}}}
+[...]
+server > {"event": "CREATED",
+          "payload": {"id": "eb17c3205bf1013ddea018c04daba326",
+                      "type": "io.cozy.ai.chat.events",
+                      "doc": {"object": "done"}}}
+```

docs/diagrams/ai.d2

@@ -0,0 +1,17 @@
+# https://d2lang.com/
+
+stack: {label: "Cozy-Stack"}
+rag: {label: "RAG"}
+llm: {label: "LLM"; shape: diamond}
+embed: {label: "Embeddings model"; shape: diamond}
+vector: {label: "Vector DB"; shape: cylinder}
+couchdb: {label: "CouchDB"; shape: cylinder}
+swift: {label: "Swift"; shape: cylinder}
+
+stack -> rag
+stack -> couchdb
+stack -> swift
+
+rag -> embed
+rag -> vector
+rag -> llm

docs/files.md

@@ -958,8 +958,8 @@ Get an image that shows the first page of a PDF (at most 1080x1920).
 Get a thumbnail of a file (for an image & pdf only). `:format` can be `tiny` (96x96)
 `small` (640x480), `medium` (1280x720), or `large` (1920x1080).
 
-This API does not require authentication because the secret acts as a token. 
-This secret is valid for 10 minutes, after which the link will return an error. 
+This API does not require authentication because the secret acts as a token.
+This secret is valid for 10 minutes, after which the link will return an error.
 To retrieve a new functional link, you must query the files API again to obtain
 a new secret.
 
@@ -1400,6 +1400,79 @@ Content-Type: application/vnd.api+json
 
 The same status codes can be encountered as the `PATCH /files/:file-id` route.
 
+### POST /files/:id/description
+
+This endpoint fills the `metadata.description` field of a file with a
+description generated by the IA from the content of this file.
+
+#### Request
+
+```http
+POST /files/9152d568-7e7c-11e6-a377-37cbfb190b4b/description HTTP/1.1
+```
+
+#### Response
+
+```http
+HTTP/1.1 200 OK
+Content-Type: application/json.vnd+api
+```
+
+```json
+{
+  "data": {
+    "type": "io.cozy.files",
+    "id": "9152d568-7e7c-11e6-a377-37cbfb190b4b",
+    "meta": {
+      "rev": "2-20900ae0"
+    },
+    "attributes": {
+      "type": "file",
+      "name": "hi.txt",
+      "trashed": false,
+      "md5sum": "ODZmYjI2OWQxOTBkMmM4NQo=",
+      "created_at": "2016-09-19T12:38:04Z",
+      "updated_at": "2016-09-19T12:38:04Z",
+      "tags": ["poem"],
+      "size": 12,
+      "executable": false,
+      "class": "document",
+      "mime": "text/plain",
+      "metadata": {
+        "description": "Explores love's complexities through vivid imagery and heartfelt emotions"
+      },
+      "cozyMetadata": {
+        "doctypeVersion": "1",
+        "metadataVersion": 1,
+        "createdAt": "2016-09-20T18:32:49Z",
+        "createdByApp": "drive",
+        "createdOn": "https://cozy.example.com/",
+        "updatedAt": "2016-09-22T13:32:51Z",
+        "uploadedAt": "2016-09-21T04:27:50Z",
+        "uploadedOn": "https://cozy.example.com/",
+        "uploadedBy": {
+          "slug": "drive"
+        }
+      }
+    },
+    "relationships": {
+      "parent": {
+        "links": {
+          "related": "/files/f2f36fec-8018-11e6-abd8-8b3814d9a465"
+        },
+        "data": {
+          "type": "io.cozy.files",
+          "id": "f2f36fec-8018-11e6-abd8-8b3814d9a465"
+        }
+      }
+    },
+    "links": {
+      "self": "/files/9152d568-7e7c-11e6-a377-37cbfb190b4b"
+    }
+  }
+}
+```
+
 ### POST /files/archive
 
 Create an archive. The body of the request lists the files and directories that

docs/toc.yml

@@ -23,6 +23,7 @@
   - "Sharing design": ./sharing-design.md
   - "Workflow of the konnectors": ./konnectors-workflow.md
 - List of services:
+  - "/ai - AI": ./ai.md
   - "/auth - Authentication & OAuth": ./auth.md
   - " /oidc - Delegated authentication": ./delegated-auth.md
   - "/apps - Applications Management": ./apps.md

docs/workers.md

@@ -412,3 +412,9 @@ It can be launched from command-line with:
 ```sh
 $ cozy-stack jobs run migrations --domain example.mycozy.cloud --json '{"type": "to-swift-v3"}'
 ```
+
+## index
+
+This worker is used for sending data to a RAG. It looks at the changes feed for
+the given doctype, send the changes to an external indexer that will generate
+embeddings for the data and put them in a vector database.

model/instance/instance.go

@@ -439,6 +439,17 @@ func (i *Instance) Registries() []*url.URL {
 	return context
 }
 
+// RAGServer returns the RAG server for the instance (AI features).
+func (i *Instance) RAGServer() config.RAGServer {
+	contexts := config.GetConfig().RAGServers
+	if i.ContextName != "" {
+		if server, ok := contexts[i.ContextName]; ok {
+			return server
+		}
+	}
+	return contexts[config.DefaultInstanceContext]
+}
+
 // HasForcedOIDC returns true only if the instance is in a context where the
 // config says that the stack shouldn't allow to authenticate with the
 // password.