From 3434edb4f5bcdb5806cd639823ce50bf1dee885e Mon Sep 17 00:00:00 2001 From: ArthurTemporim Date: Sat, 10 Aug 2019 18:20:35 -0300 Subject: [PATCH 01/10] Update Rasa version Signed-off-by: ArthurTemporim --- docker-compose.yml | 39 ++++++++++++++++++++------------------- requirements.txt | 4 +--- 2 files changed, 21 insertions(+), 22 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index b0842fe8..80a4e41f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,25 +1,6 @@ version: '2' services: - # ============================= Rocketchat ================================== - rocketchat: - container_name: rocketchat - image: rocket.chat:0.69.2 - restart: unless-stopped - env_file: - - docker/rocketchat.env - ports: - - 3000:3000 - depends_on: - - mongo - - mongo: - image: mongo:3.2 - restart: unless-stopped - volumes: - - ./data/db:/data/db - command: mongod --smallfiles --oplogSize 128 - # =============================== Coach ================================= coach: container_name: coach @@ -77,6 +58,26 @@ services: - ./bot/Makefile:/bot/Makefile command: sh -c "make run-actions" + # ============================= Rocketchat ================================== + rocketchat: + container_name: rocketchat + image: rocket.chat:0.69.2 + restart: unless-stopped + env_file: + - docker/rocketchat.env + ports: + - 3000:3000 + depends_on: + - mongo + + mongo: + image: mongo:3.2 + restart: unless-stopped + volumes: + - ./data/db:/data/db + command: mongod --smallfiles --oplogSize 128 + + # =============================== Notebooks ================================= notebooks: container_name: notebooks diff --git a/requirements.txt b/requirements.txt index 6dd715bc..6cda4e3e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,4 @@ -rasa-nlu==0.15.0 -rasa-core==0.14.1 -rasa-core-sdk==0.14.0 +rasa==1.2.2 sklearn-crfsuite==0.3.6 nltk==3.3 rocketchat-py-sdk==0.0.8 From 99513d847af9ebefd6cceaf0544a66082e07b133 Mon Sep 17 00:00:00 2001 From: ArthurTemporim Date: Sat, 10 Aug 2019 18:21:35 -0300 Subject: [PATCH 02/10] Update Rasa usage to version 1.2.2 Signed-off-by: ArthurTemporim --- .gitignore | 10 +++++----- coach/Makefile | 21 ++++++++++----------- coach/core_config.yml | 24 ++++++++++++++++++++++++ coach/policy_config.yml | 18 ------------------ coach/train.py | 23 ----------------------- 5 files changed, 39 insertions(+), 57 deletions(-) create mode 100644 coach/core_config.yml delete mode 100644 coach/policy_config.yml delete mode 100644 coach/train.py diff --git a/.gitignore b/.gitignore index c3d0128a..dcc2b4db 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,8 @@ bot/models rasa_core.log errors.json +models/dialogue/ +models/nlu/current/ ### Rasa evaluation ### .ipynb_checkpoints/ @@ -34,10 +36,11 @@ notebooks/stories/matrix.pdf /data uploads - -### database +### database ### web/postegres-data +### PyCharm ### +.idea/ # Created by https://www.gitignore.io/api/vim,linux,macos,python @@ -197,6 +200,3 @@ tags # End of https://www.gitignore.io/api/vim,linux,macos,python -models/dialogue/ - -models/nlu/current/ diff --git a/coach/Makefile b/coach/Makefile index 3e82a743..bda43c0d 100644 --- a/coach/Makefile +++ b/coach/Makefile @@ -1,16 +1,15 @@ train-nlu: - python3 -m rasa_nlu.train -c nlu_config.yml --fixed_model_name current \ - --data data/intents/ -o /src_models --project nlu --verbose + rasa train nlu -vv \ + --config nlu_config.yml \ + --fixed-model-name current \ + --nlu data/intents/ \ + --out /src_models train-core: - python3 train.py + rasa train core -vv \ + --config core_config.yml \ + -d domain.yml \ + -s data/stories/ \ + --out /src_models/dialogue/ train: train-nlu train-core - -train-online: - python -m rasa_core.train \ - interactive -o models/dialogue \ - -d domain.yml -c policy_config.yml \ - -s data/stories \ - --nlu models/nlu/current/ \ - --endpoints endpoints.yml diff --git a/coach/core_config.yml b/coach/core_config.yml new file mode 100644 index 00000000..fadcda73 --- /dev/null +++ b/coach/core_config.yml @@ -0,0 +1,24 @@ +policies: + - name: MemoizationPolicy + - name: KerasPolicy + - name: MappingPolicy + +# TODO: Update this config file with previous configurations +#policies: +# - name: KerasPolicy +# priority: 5 +# epochs: 20 +# batch_size: 10 +# featurizer: +# - name: FullDialogueTrackerFeaturizer +# state_featurizer: +# - name: LabelTokenizerSingleStateFeaturizer +# - name: FallbackPolicy +# nlu_threshold: 0.6 +# core_threshold: 0.6 +# priority: 1 +# - name: MemoizationPolicy +# priority: 2 +# max_history: 2 +# +# diff --git a/coach/policy_config.yml b/coach/policy_config.yml deleted file mode 100644 index 93a26eb6..00000000 --- a/coach/policy_config.yml +++ /dev/null @@ -1,18 +0,0 @@ -policies: - - name: KerasPolicy - priority: 5 - epochs: 20 - batch_size: 10 - featurizer: - - name: FullDialogueTrackerFeaturizer - state_featurizer: - - name: LabelTokenizerSingleStateFeaturizer - - name: FallbackPolicy - nlu_threshold: 0.6 - core_threshold: 0.6 - priority: 1 - - name: MemoizationPolicy - priority: 2 - max_history: 2 - - diff --git a/coach/train.py b/coach/train.py deleted file mode 100644 index 6a02f1a8..00000000 --- a/coach/train.py +++ /dev/null @@ -1,23 +0,0 @@ -import logging - -from rasa_core import utils, train - -logger = logging.getLogger(__name__) - -utils.configure_colored_logging(loglevel='DEBUG') - - -def train_dialogue(domain_file, - model_path, - training_folder, - policy_config): - return train(domain_file=domain_file, - stories_file=training_folder, - output_path=model_path, - policy_config=policy_config, - kwargs={'augmentation_factor': 50, - 'validation_split': 0.2,} - ) - -if __name__ == "__main__": - train_dialogue('domain.yml', '/src_models/dialogue', 'data/stories/', 'policy_config.yml') From 54ce567b2761b014a56a5ebc88d5e4cd6a5e1d49 Mon Sep 17 00:00:00 2001 From: ArthurTemporim Date: Tue, 20 Aug 2019 09:47:14 -0300 Subject: [PATCH 03/10] Reorganize docker-compose file Signed-off-by: ArthurTemporim --- docker-compose.yml | 59 +++++++++++++++++++++++++++++++++++----------- 1 file changed, 45 insertions(+), 14 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 9e467d0d..ea3ad9c0 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -2,6 +2,7 @@ version: '2' services: # =============================== Coach ================================= + # All the models are trained by this coach. coach: container_name: coach image: botcoach:latest @@ -10,16 +11,14 @@ services: command: sh -c "cp -r /src_models/* /notebook_models" # ================================= Bot ===================================== + # Generic Rasa bot, used to run console for example. bot: image: lappis/bot:boilerplate - container_name: bot_rocketchat + container_name: bot build: context: . dockerfile: ./docker/bot.Dockerfile restart: unless-stopped - command: make run-rocketchat - env_file: - - docker/bot-rocketchat.env ports: - 5002:5002 depends_on: @@ -27,8 +26,24 @@ services: volumes: - ./bot:/bot - ./scripts:/scripts + command: sh -c "rasa run" + + # ================================= Actions ================================= + # Rasa middleware used to connect with external APIs. + actions: + container_name: actions + build: + context: . + dockerfile: ./docker/actions.Dockerfile + ports: + - 5055:5055 + volumes: + - ./bot/actions:/bot/actions + - ./bot/Makefile:/bot/Makefile + command: sh -c "make run-actions" # =============================== Analytics ================================= + # Analitics ElasticSearch Stack. elasticsearch: build: context: . @@ -44,6 +59,7 @@ services: - ./elasticsearch/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml - esdata:/usr/share/elasticsearch/data + # Visualization layer of Analitics Stack. kibana: image: docker.elastic.co/kibana/kibana:6.4.2 restart: unless-stopped @@ -55,8 +71,8 @@ services: depends_on: - elasticsearch - # =============================== Broker ==================================== + # Custom broker to store Rasa tracker data. rabbitmq: image: rabbitmq:3-management restart: unless-stopped @@ -68,6 +84,7 @@ services: - RABBITMQ_DEFAULT_USER=admin - RABBITMQ_DEFAULT_PASS=admin + # Custom broker consumer responsible to store data into ElasticSearch. rabbitmq-consumer: build: context: . @@ -81,6 +98,7 @@ services: - RABBITMQ_DEFAULT_USER=admin - RABBITMQ_DEFAULT_PASS=admin + # TODO: move this env vars to a envfile # Analytics config - ELASTICSEARCH_URL=elasticsearch:9200 - ENVIRONMENT_NAME=localhost @@ -92,6 +110,7 @@ services: command: python3 /opt/scripts/consume_bot_messages.py # ============================ Telegram Bot ================================= + # Specific Rasa bot integrated with Telegram. bot_telegram: container_name: bot_telegram build: @@ -108,22 +127,31 @@ services: - ./scripts:/scripts command: sh -c "make run-telegram" - # ================================= Actions ================================= - actions: - container_name: actions + # ================================= Bot ===================================== + # Specific Rasa bot integrated with Rocket.Chat. + bot_rocketchat: + image: lappis/bot:boilerplate + container_name: bot_rocketchat build: context: . - dockerfile: ./docker/actions.Dockerfile + dockerfile: ./docker/bot.Dockerfile + restart: unless-stopped + env_file: + - docker/bot-rocketchat.env ports: - - 5055:5055 + - 5002:5002 + depends_on: + - actions volumes: - - ./bot/actions:/bot/actions - - ./bot/Makefile:/bot/Makefile - command: sh -c "make run-actions" + - ./bot:/bot + - ./scripts:/scripts + command: sh -c "rasa run" # ============================= Rocketchat ================================== + # Rocketchat container instance avaible on localhost:3000. rocketchat: container_name: rocketchat + # TODO: Update to 1.x.x version. image: rocket.chat:0.69.2 restart: unless-stopped env_file: @@ -133,6 +161,7 @@ services: depends_on: - mongo + # Rocketchat local database. mongo: image: mongo:3.2 restart: unless-stopped @@ -142,6 +171,7 @@ services: # =============================== Notebooks ================================= + # Rasa lab to enhance hyperparameters. notebooks: container_name: notebooks build: @@ -158,8 +188,9 @@ services: depends_on: - coach +# Shared volumes between containers (coach and bot). volumes: notebook_models: esbackup: esdata: - driver: local \ No newline at end of file + driver: local From a22a5e6d251ac0a71c88041149c3c92326a47dd3 Mon Sep 17 00:00:00 2001 From: ArthurTemporim Date: Tue, 20 Aug 2019 10:09:08 -0300 Subject: [PATCH 04/10] Boilerplate general improvements Signed-off-by: ArthurTemporim --- bot/credentials.yml | 14 ++++++++++---- docker-compose.yml | 2 +- docker/bot-rocketchat.env | 1 + requirements.txt | 3 --- 4 files changed, 12 insertions(+), 8 deletions(-) diff --git a/bot/credentials.yml b/bot/credentials.yml index 766a4230..fc537763 100644 --- a/bot/credentials.yml +++ b/bot/credentials.yml @@ -1,4 +1,10 @@ -telegram: - access_token: ${TELEGRAM_TOKEN} - verify: ${TELEGRAM_BOT_USERNAME} - webhook_url: ${TELEGRAM_WEBHOOK} +#telegram: +# access_token: ${TELEGRAM_TOKEN} +# verify: ${TELEGRAM_BOT_USERNAME} +# webhook_url: ${TELEGRAM_WEBHOOK} + +rocketchat: + user: ${ROCKETCHAT_BOT_USERNAME} + password: ${ROCKETCHAT_BOT_PASSWORD} + server_url: ${ROCKETCHAT_URL} + diff --git a/docker-compose.yml b/docker-compose.yml index ea3ad9c0..91c5a543 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -145,7 +145,7 @@ services: volumes: - ./bot:/bot - ./scripts:/scripts - command: sh -c "rasa run" + command: sh -c "rasa run --credentials credentials.yml --model /models" # ============================= Rocketchat ================================== # Rocketchat container instance avaible on localhost:3000. diff --git a/docker/bot-rocketchat.env b/docker/bot-rocketchat.env index 74f8bab9..80f7ce6a 100644 --- a/docker/bot-rocketchat.env +++ b/docker/bot-rocketchat.env @@ -15,6 +15,7 @@ ENVIRONMENT_NAME=localhost # Last commit hash using this command: git rev-parse HEAD BOT_VERSION=last-commit-hash +# TODO: These configs should be in a analytics.env not here. # Analytics config ENABLE_ANALYTICS=False diff --git a/requirements.txt b/requirements.txt index e393dab4..f253f0a4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,8 +1,5 @@ rasa==1.2.2 -sklearn-crfsuite==0.3.6 nltk==3.3 rocketchat-py-sdk==0.0.8 elasticsearch==6.3.1 -flake8==3.7.8 - From d7158ce071a0774cc5b2c1dac015a12b374d4c59 Mon Sep 17 00:00:00 2001 From: ArthurTemporim Date: Tue, 20 Aug 2019 18:36:27 -0300 Subject: [PATCH 05/10] Remove old rasa method calls Signed-off-by: ArthurTemporim --- bot/Makefile | 2 +- bot/connector.py | 2 +- bot/{run-rocketchat.py => run_rocketchat.py} | 0 bot/test_stories.py | 22 +++++++++----------- bot/tracker_store.py | 4 ++-- 5 files changed, 14 insertions(+), 16 deletions(-) rename bot/{run-rocketchat.py => run_rocketchat.py} (100%) diff --git a/bot/Makefile b/bot/Makefile index e7d87c9d..e540bc8c 100644 --- a/bot/Makefile +++ b/bot/Makefile @@ -1,6 +1,6 @@ run-rocketchat: make config-bot - python3 run-rocketchat.py + python3 run_rocketchat.py run-telegram: python -m rasa_core.run -d /models/dialogue -u /models/nlu/current --port 5001 --credentials credentials.yml --endpoints endpoints.yml diff --git a/bot/connector.py b/bot/connector.py index f00f5eaf..3d231b16 100644 --- a/bot/connector.py +++ b/bot/connector.py @@ -6,7 +6,7 @@ from flask import Blueprint, request, jsonify, make_response -from rasa_core.channels.channel import UserMessage, OutputChannel, InputChannel +from rasa.core.channels.channel import UserMessage, OutputChannel, InputChannel logger = logging.getLogger(__name__) diff --git a/bot/run-rocketchat.py b/bot/run_rocketchat.py similarity index 100% rename from bot/run-rocketchat.py rename to bot/run_rocketchat.py diff --git a/bot/test_stories.py b/bot/test_stories.py index 9620eb2f..62231fa9 100644 --- a/bot/test_stories.py +++ b/bot/test_stories.py @@ -1,18 +1,16 @@ -from rasa_core.test import _generate_trackers, collect_story_predictions - -import logging -from rasa_core import utils -from rasa_core.utils import AvailableEndpoints -from rasa_core.run import load_agent -from rasa_core.interpreter import NaturalLanguageInterpreter -from collections import namedtuple +import os import re import sys -import os +import logging +import argparse from os import listdir from os.path import isfile, join -import argparse +from collections import namedtuple +from rasa.core import utils +from rasa.core import run +from rasa.core.interpreter import NaturalLanguageInterpreter +from rasa.core.test import _generate_trackers, collect_story_predictions logger = logging.getLogger(__name__) @@ -90,9 +88,9 @@ def run_evaluation(file_to_evaluate, max_stories=None, use_e2e=False): - _endpoints = AvailableEndpoints.read_endpoints(None) + _endpoints = utils.AvailableEndpoints.read_endpoints(None) _interpreter = NaturalLanguageInterpreter.create(NLU_DIR) - _agent = load_agent(CORE_DIR, + _agent = run.load_agent(CORE_DIR, interpreter=_interpreter, endpoints=_endpoints) diff --git a/bot/tracker_store.py b/bot/tracker_store.py index 1df22c7f..859223f6 100644 --- a/bot/tracker_store.py +++ b/bot/tracker_store.py @@ -5,8 +5,8 @@ import hashlib import json -from rasa_core.tracker_store import InMemoryTrackerStore -from rasa_core.events import ActionExecuted, BotUttered, UserUttered +from rasa.core.tracker_store import InMemoryTrackerStore +from rasa.core.events import ActionExecuted, BotUttered, UserUttered from elasticsearch import Elasticsearch From c5b7f98102f3aeb75e37487ff8e31b6cf8f8b05f Mon Sep 17 00:00:00 2001 From: ArthurTemporim Date: Thu, 29 Aug 2019 19:10:35 -0300 Subject: [PATCH 06/10] Update nltk version to 3.4.5 Signed-off-by: ArthurTemporim --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index f253f0a4..438f83ad 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ rasa==1.2.2 -nltk==3.3 +nltk==3.4.5 rocketchat-py-sdk==0.0.8 elasticsearch==6.3.1 From 3b0d82451e7fe9d965701cbbc57530e54a8a027c Mon Sep 17 00:00:00 2001 From: ArthurTemporim Date: Thu, 29 Aug 2019 20:11:57 -0300 Subject: [PATCH 07/10] Update run-console script Signed-off-by: ArthurTemporim --- bot/Makefile | 2 +- bot/credentials.yml | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/bot/Makefile b/bot/Makefile index e540bc8c..8b24fd93 100644 --- a/bot/Makefile +++ b/bot/Makefile @@ -9,7 +9,7 @@ config-rocket: python /scripts/bot_config.py run-console: - python3 -m rasa_core.run -d /models/dialogue -u /models/nlu/current --debug + rasa shell -m /models/dialogue -vv run-console-broker: python3 -m rasa_core.run -d /models/dialogue -u /models/nlu/current --debug --endpoints endpoints.yml diff --git a/bot/credentials.yml b/bot/credentials.yml index fc537763..9a204236 100644 --- a/bot/credentials.yml +++ b/bot/credentials.yml @@ -3,8 +3,8 @@ # verify: ${TELEGRAM_BOT_USERNAME} # webhook_url: ${TELEGRAM_WEBHOOK} -rocketchat: - user: ${ROCKETCHAT_BOT_USERNAME} - password: ${ROCKETCHAT_BOT_PASSWORD} - server_url: ${ROCKETCHAT_URL} +#rocketchat: +# user: ${ROCKETCHAT_BOT_USERNAME} +# password: ${ROCKETCHAT_BOT_PASSWORD} +# server_url: ${ROCKETCHAT_URL} From 7dc3ccfcd3082f4a0408fd3e9b2dcf914252e70d Mon Sep 17 00:00:00 2001 From: ArthurTemporim Date: Thu, 29 Aug 2019 20:15:35 -0300 Subject: [PATCH 08/10] Comment e2e configuration Signed-off-by: ArthurTemporim --- Makefile | 7 ++----- bot/Makefile | 37 +++++++++++++++++-------------------- 2 files changed, 19 insertions(+), 25 deletions(-) diff --git a/Makefile b/Makefile index ab0a1098..03a3ad29 100644 --- a/Makefile +++ b/Makefile @@ -6,11 +6,8 @@ train: docker build . -f docker/coach.Dockerfile -t lappis/coach:boilerplate docker-compose build bot -run-telegram: - docker-compose up telegram_bot - run-console: docker-compose run bot make run-console -test-dialogue: - docker-compose run --rm bot make e2e \ No newline at end of file +#test-dialogue: +# docker-compose run --rm bot make e2e diff --git a/bot/Makefile b/bot/Makefile index 8b24fd93..74b53dcd 100644 --- a/bot/Makefile +++ b/bot/Makefile @@ -3,7 +3,8 @@ run-rocketchat: python3 run_rocketchat.py run-telegram: - python -m rasa_core.run -d /models/dialogue -u /models/nlu/current --port 5001 --credentials credentials.yml --endpoints endpoints.yml + rasa run -m /models/dialogue --port 5001 --credentials credentials.yml \ + --endpoints endpoints.yml config-rocket: python /scripts/bot_config.py @@ -12,28 +13,24 @@ run-console: rasa shell -m /models/dialogue -vv run-console-broker: - python3 -m rasa_core.run -d /models/dialogue -u /models/nlu/current --debug --endpoints endpoints.yml + rasa shell -m /models/dialogue -vv --endpoints endpoints.yml run-api: - python3 -m rasa_core.run -d /models/dialogue -u /models/nlu/current \ - --endpoints endpoints.yml --debug --enable_api - -evaluate: - python3 -m rasa_core.evaluate -d /models/dialogue -s data/core/ + rasa run -m /models/dialogue -vv --endpoints endpoints.yml --enable_api run-actions: - python -m rasa_core_sdk.endpoint --actions actions + rasa run actions --actions actions # Force Makefile to execute e2e command every time -.PHONY: all e2e clean -test-dialogue: - make e2e - -e2e: - python3 -m rasa_core.evaluate default --core /models/dialogue --nlu /models/nlu/current --stories e2e/e2e_stories.md --e2e - -test-stories: - python3 test_stories.py --stories e2e/e2e_stories.md --e2e - -config-bot: - python /scripts/bot_config.py -r ${ROCKETCHAT_URL} -an ${ROCKETCHAT_ADMIN_USERNAME} -ap ${ROCKETCHAT_ADMIN_PASSWORD} -bu ${ROCKETCHAT_BOT_USERNAME} -bp ${ROCKETCHAT_BOT_PASSWORD} +#.PHONY: all e2e clean +#test-dialogue: +# make e2e +# +#e2e: +# python3 -m rasa_core.evaluate default --core /models/dialogue --nlu /models/nlu/current --stories e2e/e2e_stories.md --e2e +# +#test-stories: +# python3 test_stories.py --stories e2e/e2e_stories.md --e2e +# +#config-bot: +# python /scripts/bot_config.py -r ${ROCKETCHAT_URL} -an ${ROCKETCHAT_ADMIN_USERNAME} -ap ${ROCKETCHAT_ADMIN_PASSWORD} -bu ${ROCKETCHAT_BOT_USERNAME} -bp ${ROCKETCHAT_BOT_PASSWORD} From f66a1819e0ea6b167c2bee5bad2ec33ae470fd21 Mon Sep 17 00:00:00 2001 From: ArthurTemporim Date: Sun, 15 Sep 2019 10:07:43 -0300 Subject: [PATCH 09/10] Update gitlab-ci file Signed-off-by: ArthurTemporim --- .gitlab-ci.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 2fe7df52..1e318965 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,5 +1,9 @@ image: python:3.6-slim +before_script: + - apt-get update --yes + - apt-get install gcc + variables: BOT_IMAGE: $DOCKERHUB_USER/bot COACH_IMAGE: $DOCKERHUB_USER/coach From 42227209d4b64ac44a2e0d7cbf7e0cca49ac0ca5 Mon Sep 17 00:00:00 2001 From: ArthurTemporim Date: Thu, 19 Sep 2019 00:01:19 -0300 Subject: [PATCH 10/10] Improve README and Makefile Signed-off-by: ArthurTemporim --- Makefile | 10 ++-- README.md | 150 ++++++++++++++---------------------------------------- 2 files changed, 45 insertions(+), 115 deletions(-) diff --git a/Makefile b/Makefile index 03a3ad29..7596656e 100644 --- a/Makefile +++ b/Makefile @@ -2,12 +2,16 @@ build-bot: ./docker/build-base.sh make train +run-analytics: + docker-compose up -d rabbitmq + docker-compose up -d rabbitmq-consumer + docker-compose up -d elasticsearch + docker-compose run --rm -v $PWD/analytics:/analytics bot python /analytics/setup_elastic.py + docker-compose up -d kibana + train: docker build . -f docker/coach.Dockerfile -t lappis/coach:boilerplate docker-compose build bot run-console: docker-compose run bot make run-console - -#test-dialogue: -# docker-compose run --rm bot make e2e diff --git a/README.md b/README.md index f0dbe797..d269a393 100644 --- a/README.md +++ b/README.md @@ -3,13 +3,33 @@ +## Tutorial para configurar todo o projeto + +```sh +``` + +```sh +sudo docker-compose up -d rocketchat + +sudo docker-compose up -d rabbitmq +sudo docker-compose up -d rabbitmq-consumer + +sudo docker-compose up -d elasticsearch +sudo docker-compose run --rm -v $PWD/analytics:/analytics bot python /analytics/setup_elastic.py +sudo docker-compose up -d kibana + +sudo docker-compose up -d bot +``` + +## Introdução + ### For English version, see [README-en](docs/README-en.md) Um projeto feito em Rasa com configurações necessárias para a construção de um projeto grande de chatbot. -Este projeto teve como base a [Tais](http://github.com/lappis-unb/tais). +Este projeto teve como base o projeto [Tais](http://github.com/lappis-unb/tais). -# Entenda a Arquitetura +### Entenda a Arquitetura É utilizado no boilerplate diversas tecnologias que interagem entre si para obter um melhor resultado. Veja a arquitetura implementada: @@ -23,120 +43,45 @@ Os notebooks avaliam o funcionamento de acordo com o formato das *intents* e *st O elasticsearch coleta os dados da conversa e armazena para a análise feita pelo kibana, que gera gráficos para avaliação das conversas dos usuários e do boilerplate. -## Bot - +### Bot Este script foi configurado para construir as imagens genéricas necessárias para execução deste ambiente. Caso seu projeto utilize este boilerplate e vá realizar uma integração contínua ou similar, é interessante -criar um repositório para as imagens e substitua os nomes das imagens "lappis/bot", "lappis/coach" e "lappis/botrequirements" pelas -suas respectivas novas imagens, por exemplo "/bot" em repositório público. +criar um repositório para as imagens e substitua os nomes das imagens "lappis/bot", e "lappis/botrequirements" pelas suas respectivas novas imagens, por exemplo "/bot" em repositório público. -### RocketChat -Para testar o assistente virtual utilizando da plataforma do RocketChat, siga os seguintes comandos para subir os containers em seu computador: +### Treinamento +**Atenção**: o comando de treinamento é usado para criar os modelos necessários na conversação do bot para treinar o seu chatbot execute o comando: ```sh -sudo docker-compose up -d rocketchat - -# Caso não vá subir a stack do Analytics -# aguarde o container do rocketchat subir -sudo docker-compose up -d bot -``` - -Caso queira subir toda a stack e vá utilizar do analytics é necessário substituir a seguinte variável de ambiente no `docker/bot-rocketchat.env` de **False** para **True**. -``` -# Analytics config -ENABLE_ANALYTICS=True - -# E logo após trocar, subir o container do bot -sudo docker-compose up -d bot -``` - - -Após esses comandos o RocketChat deve estar disponível na porta 3000 do seu computador. Entre em http://localhost:3000 para acessar. Será pedido que faça login. Por padrão é gerado um usuário admin: username: admin senha: admin. Nas próximas telas apenas clique na opção `Continue` e `Go to your workspace`. - -Para configurar o bot no rocketchat e conseguir conversar com ele pelo próprio rocket, acesse o [link](http://github.com/lappis-unb/rasa-ptbr-boilerplate/tree/master/docs/add_bot_rocketchat.md). - - -Opcionalmente, é possível fazer uma configuração para que o assistente virtual inicie a conversa, para isso você deve criar um `trigger`. -Para criar um `trigger` entre no rocketchat como `admin`, e vá no painel do Livechat na -seção de Triggers, clique em `New Trigger`. Preencha o Trigger da seguinte forma: - - -```yaml -Enabled: Yes -Name: Start Talk -Description: Start Talk -Condition: Visitor time on site - Value: 3 -Action: Send Message - Value: Impersonate next agent from queue - Value: Olá! +sudo make train ``` +### Console -#### Instalação - -Para executar o bot em um site você precisa inserir o seguinte Javascript na sua página - -```js - - - +```sh +sudo make run-console ``` -**Atenção**: Você precisa alterar a variavel `host` dentro do código acima para a url do site onde estará o seu Rocket.Chat. - ### Telegram Após realizar o [tutorial](/docs/setup_telegram.md) de exportação de todas variávies de ambiente necessárias, é possível realizar a execução do bot no telegram corretamente. -Antes de seguir adiante. Importante: As variáveis de ambiente são necessárias para o correto funcionamento do bot, por isso não esqueça de exportá-las. +**Antes de seguir adiante. Importante:** As variáveis de ambiente são necessárias para o correto funcionamento do bot, por isso não esqueça de exportá-las. Se ainda não tiver treinado seu bot execute antes: -```sh -make train -``` -**Atenção**: o comando "make train" executa um container docker, caso precise de sudo em seu computador -para execução docker, utilize "sudo make train". - Depois execute o bot no telegram: ```sh sudo docker-compose up bot_telegram ``` - -### Console - -```sh -make train -sudo docker-compose run --rm bot make run-console -``` - -### Train Online - -``` -make train -sudo docker-compose run --rm coach make train-online -``` - -## Analytics +### Analytics Para a visualização dos dados da interação entre o usuário e o chatbot nós utilizamos uma parte da Stack do Elastic, composta pelo ElasticSearch e o Kibana. Com isso, utilizamos um broker para fazer a gerência de mensagens. Então conseguimos adicionar mensagens ao ElasticSearch independente do tipo de mensageiro que estamos utilizando. -#### Setup RabbitMQ +### Configuração do RabbitMQ Em primeiro lugar para fazer o setup do analytics é necessário subir o RabiitMQ e suas configurações. @@ -163,9 +108,7 @@ RABBITMQ_DEFAULT_PASS=admin Sendo que as configurações de `RABBITMQ_DEFAULT_USER` e `RABBITMQ_DEFAULT_PASS` devem ser as mesmas definidas no serviço do `rabbitmq`. - - -### Execução +#### Integração com Rasa Existem duas formas para executar a Tais com o *broker*. A primeira delas é via linha de comando. Para utilizar esta forma é preciso definir Dentro do arquivo `endpoints.yml` as configurações do broker: @@ -201,7 +144,7 @@ Ao final é necessário buildar novamente o container do bot. sudo docker-compose up --build -d bot ``` -### Setup ElasticSearch +### Configuração ElasticSearch O ElasticSearch é o serviço responsável por armazenar os dados provenientes da interação entre o usuário e o chatbot. @@ -221,12 +164,13 @@ ENVIRONMENT_NAME=localhost BOT_VERSION=last-commit-hash ``` -### Setup Kibana (Visualização) +#### Setup Kibana (Visualização) + Para a análise dos dados das conversas com o usuário, utilize o kibana, e veja como os usuários estão interagindo com o bot, os principais assuntos, média de usuários e outras informações da análise de dados. O Kibana nos auxilia com uma interface para criação de visualização para os dados armazenados nos índices do ElasticSearch. -``` +```sh sudo docker-compose up -d kibana ``` @@ -282,24 +226,6 @@ docker-compose up -d notebooks Acesse o notebook em `localhost:8888` - - -## Tutorial para levantar toda a stack - -```sh -sudo docker-compose up -d rocketchat - -sudo docker-compose up -d rabbitmq -sudo docker-compose up -d rabbitmq-consumer - -sudo docker-compose up -d elasticsearch -sudo docker-compose run --rm -v $PWD/analytics:/analytics bot python /analytics/setup_elastic.py -sudo docker-compose up -d kibana - -sudo docker-compose up -d bot -``` - - # Como conseguir ajuda Parte da documentação técnica do framework da Tais está disponível na