diff --git a/src/wandbot/evaluation/eval/compare_query_enhancer.py b/src/wandbot/evaluation/eval/compare_query_enhancer.py deleted file mode 100644 index eb727ec..0000000 --- a/src/wandbot/evaluation/eval/compare_query_enhancer.py +++ /dev/null @@ -1,102 +0,0 @@ -import wandb -import json -import pandas as pd -from langchain_openai import ChatOpenAI -from tqdm import tqdm - -from wandbot.chat.query_enhancer import QueryHandler -from wandbot.query_handler.query_enhancer import load_query_enhancement_chain -from wandbot.utils import get_logger -from wandbot.chat.config import ChatConfig -from wandbot.chat.schemas import ChatRequest - -logger = get_logger(__name__) - -df = pd.read_json( - "data/eval/wandbot_cleaned_annotated_dataset_11-12-2023.jsonl", - lines=True, - orient="records", -) -correct_df = df[ - (df["is_wandb_query"] == "YES") & (df["correctness"] == "correct") -] - -query_enhancer_v1 = QueryHandler() - -config = ChatConfig() -llm = ChatOpenAI(model=config.chat_model_name, temperature=0) -lang_detect_path = "data/cache/models/lid.176.bin" -query_enhancer_v2 = load_query_enhancement_chain(llm, lang_detect_path) - -df = pd.DataFrame( - columns=[ - "question", - "qe1_condensed_query", - "qe2_question", - "qe1_intent", - "qe2_intent", - "qe1_keywords", - "qe2_keywords", - "qe1_subqueries", - "qe2_vector_search", - "qe2_web_answer", - ] -) - -project = "wandbot-eval" -entity = "wandbot" - -run = wandb.init(project=project, entity=entity) - -for idx, row in tqdm(correct_df.iterrows()): - query = row["question"] - chat_request = ChatRequest( - question=query, - chat_history=[], - language="en", - application="slack", - ) - complete_query = query_enhancer_v1(chat_request) - condensed_query = complete_query.condensed_query - intent = complete_query.intent_hints - keywords = ", ".join(complete_query.keywords) - subqueries = "; ".join(complete_query.sub_queries) - - #### - - result = query_enhancer_v2.invoke({"query": query, "chat_history": []}) - - try: - enhanced_question = result.get('standalone_question') - enhanced_intent = result.get('intents').get('intent_hints') + "\n" + ", ".join(result.get('intents').get('intent_labels')) - enhanced_keywords = ", ".join(result['keywords']) # Join list of keywords into a single string - vector_search = result['vector_search'] - web_answer = result['web_results']['web_answer'] - except: - enhanced_question = None - enhanced_intent = None - enhanced_keywords = None - vector_search = None - web_answer = None - - data_dict = { - "question": query, - "qe1_condensed_query": condensed_query, - "qe2_question": enhanced_question, - "qe1_intent": intent, - "qe2_intent": enhanced_intent, - "qe1_keywords": keywords, - "qe2_keywords": enhanced_keywords, - "qe1_subqueries": subqueries, - "qe2_vector_search": vector_search, - "qe2_web_answer": web_answer, - } - - with open("data/eval/compare_query_enhancer.jsonl", "w+") as outfile: - _data_dict = json.dumps(data_dict) - outfile.write(_data_dict + "\n") - - new_row = pd.DataFrame([data_dict]) - df = pd.concat([df, new_row], ignore_index=True) - -run.log({"Compare Query Enhancers Results": df}) diff --git a/src/wandbot/evaluation/evaluate_retrieval.ipynb b/src/wandbot/evaluation/evaluate_retrieval.ipynb deleted file mode 100644 index 44f9cec..0000000 --- a/src/wandbot/evaluation/evaluate_retrieval.ipynb +++ /dev/null @@ -1,814 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 1, - "id": "91bef3bf-c9a8-4b4d-a538-3db9a49b21d5", - "metadata": {}, - "outputs": [], - "source": [ - "%load_ext autoreload\n", - "%autoreload 2" - ] - }, - { - "cell_type": "code", - "execution_count": 39, - "id": "3ec9dd30-6287-48f7-9af7-d6a0136681da", - "metadata": {}, - "outputs": [], - "source": [ - "import re\n", - "import json\n", - "import wandb\n", - "import pandas as pd\n", - "import numpy as np\n", - "from tqdm import tqdm" - ] - }, - { - "cell_type": "markdown", - "id": "13986df9-9f62-411c-a158-711455b44e9d", - "metadata": {}, - "source": [ - "## Get Ground Truth (Ref) Contexts" - ] - }, - { - "cell_type": "code", - "execution_count": 34, - "id": "0f8907c5-b366-4910-b315-52c5acef87d2", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "98" - ] - }, - "execution_count": 34, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "df = pd.read_json(\n", - " \"/Users/ayushthakur/integrations/wandbot-eval/wbeval/wandbot/data/eval/wandbot_cleaned_annotated_dataset_11-12-2023.jsonl\",\n", - " lines=True,\n", - " orient=\"records\",\n", - ")\n", - "df = df[\n", - " (df[\"is_wandb_query\"] == \"YES\") & (df[\"correctness\"] == \"correct\")\n", - "]\n", - "len(df)" - ] - }, - { - "cell_type": "code", - "execution_count": 35, - "id": "b4d7a835-3773-4ef6-a169-bcf76cb42177", - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
questionanswercontextcorrectnessis_wandb_querynotes
0Hey I have a question about using wandb with f...When integrating `wandb` (Weights & Biases) wi...Source:\\thttps://docs.wandb.ai/guides/track/tr...correctYESThe answer clearly explains the recommended pr...
1Hey with wandb is it possible to link from the...Yes, with `wandb`, you can link to the best ru...Source:\\thttps://docs.wandb.ai/guides/track/pu...correctYESThis answer correctly explains how to use the ...
2Explain how I can version datasets with Weight...Versioning datasets with Weights & Biases (W&B...Source:\\thttps://github.com/wandb/examples/tre...correctYESThe answer correctly summarizes the informatio...
4Hi, can anybody help me with this issue? wandb...The `wandb.sdk.service.service.ServiceStartTim...Source:\\thttps://docs.wandb.ai/guides/track/lo...correctYESThis requires more data to debug and probably ...
5what is the difference between artifact.add_fi...`artifact.add_file` and `wandb.save` are both ...Source:\\thttps://docs.wandb.ai/guides/artifact...correctYESThe answer correctly identifies the distinctio...
\n", - "
" - ], - "text/plain": [ - " question \\\n", - "0 Hey I have a question about using wandb with f... \n", - "1 Hey with wandb is it possible to link from the... \n", - "2 Explain how I can version datasets with Weight... \n", - "4 Hi, can anybody help me with this issue? wandb... \n", - "5 what is the difference between artifact.add_fi... \n", - "\n", - " answer \\\n", - "0 When integrating `wandb` (Weights & Biases) wi... \n", - "1 Yes, with `wandb`, you can link to the best ru... \n", - "2 Versioning datasets with Weights & Biases (W&B... \n", - "4 The `wandb.sdk.service.service.ServiceStartTim... \n", - "5 `artifact.add_file` and `wandb.save` are both ... \n", - "\n", - " context correctness \\\n", - "0 Source:\\thttps://docs.wandb.ai/guides/track/tr... correct \n", - "1 Source:\\thttps://docs.wandb.ai/guides/track/pu... correct \n", - "2 Source:\\thttps://github.com/wandb/examples/tre... correct \n", - "4 Source:\\thttps://docs.wandb.ai/guides/track/lo... correct \n", - "5 Source:\\thttps://docs.wandb.ai/guides/artifact... correct \n", - "\n", - " is_wandb_query notes \n", - "0 YES The answer clearly explains the recommended pr... \n", - "1 YES This answer correctly explains how to use the ... \n", - "2 YES The answer correctly summarizes the informatio... \n", - "4 YES This requires more data to debug and probably ... \n", - "5 YES The answer correctly identifies the distinctio... " - ] - }, - "execution_count": 35, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "df.head()" - ] - }, - { - "cell_type": "code", - "execution_count": 36, - "id": "6cdad9a6-f20f-465b-9c2b-8a6bc7da8196", - "metadata": {}, - "outputs": [], - "source": [ - "import re\n", - "\n", - "ref_query_contexts = dict()\n", - "\n", - "def split_contexts(text):\n", - " # This pattern looks for 'Source:' followed by any characters (non-greedy), a URL, and ends with '\\n---\\n'\n", - " pattern = r\"(Source:\\s*https?://[^\\s]+\\s*.*?)(?=\\n---\\n|$)\"\n", - " contexts = [match.group().strip() for match in re.finditer(pattern, text, re.DOTALL)]\n", - " return contexts\n", - "\n", - "# Assuming df['context'] is a column containing the text to be split\n", - "for idx, row in df.iterrows():\n", - " contexts = split_contexts(row['context'])\n", - " assert len(contexts) == 5 # Ensure there are exactly 5 contexts\n", - " ref_query_contexts[row['question']] = [\n", - " re.sub(r\"Source:\\s*https?://[^\\s]+\\s*\", \"\", context).strip() for context in contexts\n", - " ]" - ] - }, - { - "cell_type": "markdown", - "id": "e9f594d9-b763-4788-8f21-5e050d7f3d26", - "metadata": {}, - "source": [ - "## Get Contexts for Best System" - ] - }, - { - "cell_type": "code", - "execution_count": 71, - "id": "23dfdee3-0027-4442-82a6-8f79141ce3f0", - "metadata": {}, - "outputs": [], - "source": [ - "with open(\n", - " \"/Users/ayushthakur/integrations/wandbot-eval/wbeval/wandbot/artifacts/run-3b3vex63-EvaluationResults:v0/Evaluation Results.table.json\"\n", - ") as f:\n", - " data = json.load(f)\n", - " columns = data[\"columns\"]\n", - " data = data[\"data\"]\n", - " df = pd.DataFrame(columns=columns, data=data)" - ] - }, - { - "cell_type": "code", - "execution_count": 79, - "id": "b4d9f2a9-fb0b-4472-990c-1b27aced87cd", - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
idxsystem_promptquestionanswermodelsourcessource_documentstotal_tokensprompt_tokenscompletion_tokens...answer_relevancy_resultanswer_relevancy_reasonanswer_relevancy_score_(ragas)answer_faithfulness_scoreanswer_faithfulness_resultanswer_faithfulness_reasonanswer_faithfulness_score_(ragas)answer_similarity_score_(ragas)context_precision_scorecontext_recall_score
00system: You are wandbot, an expert support ass...Hey I have a question about using wandb with f...When integrating `wandb` with FastAPI or any o...gpt-4-1106-previewhttps://docs.wandb.ai/guides/integrations/fast...[{\"source\": \"https://docs.wandb.ai/guides/inte...55085026482...TrueThe generated answer is relevant and addresses...0.8606861FalseThe generated answer provides a recommendation...NaN0.6936240.01.0
\n", - "

1 rows × 31 columns

\n", - "
" - ], - "text/plain": [ - " idx system_prompt \\\n", - "0 0 system: You are wandbot, an expert support ass... \n", - "\n", - " question \\\n", - "0 Hey I have a question about using wandb with f... \n", - "\n", - " answer model \\\n", - "0 When integrating `wandb` with FastAPI or any o... gpt-4-1106-preview \n", - "\n", - " sources \\\n", - "0 https://docs.wandb.ai/guides/integrations/fast... \n", - "\n", - " source_documents total_tokens \\\n", - "0 [{\"source\": \"https://docs.wandb.ai/guides/inte... 5508 \n", - "\n", - " prompt_tokens completion_tokens ... answer_relevancy_result \\\n", - "0 5026 482 ... True \n", - "\n", - " answer_relevancy_reason \\\n", - "0 The generated answer is relevant and addresses... \n", - "\n", - " answer_relevancy_score_(ragas) answer_faithfulness_score \\\n", - "0 0.860686 1 \n", - "\n", - " answer_faithfulness_result \\\n", - "0 False \n", - "\n", - " answer_faithfulness_reason \\\n", - "0 The generated answer provides a recommendation... \n", - "\n", - " answer_faithfulness_score_(ragas) answer_similarity_score_(ragas) \\\n", - "0 NaN 0.693624 \n", - "\n", - " context_precision_score context_recall_score \n", - "0 0.0 1.0 \n", - "\n", - "[1 rows x 31 columns]" - ] - }, - "execution_count": 79, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "df.head(1)" - ] - }, - { - "cell_type": "code", - "execution_count": 80, - "id": "e07be0d9-70cc-4e70-8058-cba6ed2c779e", - "metadata": { - "scrolled": true - }, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "98it [00:00, 5600.02it/s]\n" - ] - } - ], - "source": [ - "context_sys_best = dict()\n", - "\n", - "for idx, row in tqdm(df.iterrows()):\n", - " context_sys_best[row.question] = [\n", - " context_dict[\"text\"] for context_dict in eval(row.source_documents)\n", - " ]" - ] - }, - { - "cell_type": "markdown", - "id": "a774a08e-8d75-4b25-b222-d1ca154d5218", - "metadata": {}, - "source": [ - "## Get Contexts for Current Commit" - ] - }, - { - "cell_type": "code", - "execution_count": 91, - "id": "5811d13d-8abd-4e8b-a9fe-fb240e855bae", - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\u001b[34m\u001b[1mwandb\u001b[0m: 1 of 1 files downloaded. \n", - "wandb: WARNING Source type is set to 'repo' but some required information is missing from the environment. A job will not be created from this run. See https://docs.wandb.ai/guides/launch/create-job\n", - "\u001b[34m\u001b[1mwandb\u001b[0m: 1 of 1 files downloaded. \n", - "wandb: WARNING Source type is set to 'repo' but some required information is missing from the environment. A job will not be created from this run. See https://docs.wandb.ai/guides/launch/create-job\n" - ] - } - ], - "source": [ - "api = wandb.Api()\n", - "run = api.run(\"wandbot/wandbot-eval/8kfxwv2c\")\n", - "\n", - "for artifact in run.logged_artifacts():\n", - " artifact.download()" - ] - }, - { - "cell_type": "code", - "execution_count": 92, - "id": "38a5762f-440a-4cc1-a8ee-18ffae48d9e0", - "metadata": {}, - "outputs": [], - "source": [ - "with open(\n", - " \"/Users/ayushthakur/integrations/wandbot-eval/wbeval/wandbot/artifacts/run-8kfxwv2c-EvaluationResults:v0/Evaluation Results.table.json\"\n", - ") as f:\n", - " data = json.load(f)\n", - " columns = data[\"columns\"]\n", - " data = data[\"data\"]\n", - " df = pd.DataFrame(columns=columns, data=data)" - ] - }, - { - "cell_type": "code", - "execution_count": 96, - "id": "e7f5eb9a-3c44-4a14-8f23-51cc1ed59992", - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
idxsystem_promptquestionanswermodelsourcessource_documentstotal_tokensprompt_tokenscompletion_tokens...contextsanswer_correctness_scoreanswer_correctness_resultanswer_correctness_reasonanswer_relevancy_scoreanswer_relevancy_resultanswer_relevancy_reasonanswer_faithfulness_scoreanswer_faithfulness_resultanswer_faithfulness_reason
00System: As Wandbot - a support expert in Weigh...Hey I have a question about using wandb with f...When using `wandb` with FastAPI or any other w...gpt-4-1106-previewhttps://docs.wandb.ai/quickstart\\nhttps://gith...source: https://docs.wandb.ai/quickstart\\nsour...50314492539...source: https://docs.wandb.ai/quickstart\\nsour...3TrueThe generated answer provides a correct and co...3TrueThe generated answer is relevant and provides ...1FalseThe generated answer provides a recommendation...
\n", - "

1 rows × 25 columns

\n", - "
" - ], - "text/plain": [ - " idx system_prompt \\\n", - "0 0 System: As Wandbot - a support expert in Weigh... \n", - "\n", - " question \\\n", - "0 Hey I have a question about using wandb with f... \n", - "\n", - " answer model \\\n", - "0 When using `wandb` with FastAPI or any other w... gpt-4-1106-preview \n", - "\n", - " sources \\\n", - "0 https://docs.wandb.ai/quickstart\\nhttps://gith... \n", - "\n", - " source_documents total_tokens \\\n", - "0 source: https://docs.wandb.ai/quickstart\\nsour... 5031 \n", - "\n", - " prompt_tokens completion_tokens ... \\\n", - "0 4492 539 ... \n", - "\n", - " contexts \\\n", - "0 source: https://docs.wandb.ai/quickstart\\nsour... \n", - "\n", - " answer_correctness_score answer_correctness_result \\\n", - "0 3 True \n", - "\n", - " answer_correctness_reason answer_relevancy_score \\\n", - "0 The generated answer provides a correct and co... 3 \n", - "\n", - " answer_relevancy_result answer_relevancy_reason \\\n", - "0 True The generated answer is relevant and provides ... \n", - "\n", - " answer_faithfulness_score answer_faithfulness_result \\\n", - "0 1 False \n", - "\n", - " answer_faithfulness_reason \n", - "0 The generated answer provides a recommendation... \n", - "\n", - "[1 rows x 25 columns]" - ] - }, - "execution_count": 96, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "df.head(1)" - ] - }, - { - "cell_type": "code", - "execution_count": 123, - "id": "76985b90-d9cb-4b40-97ca-e2cf15676ace", - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "98it [00:00, 1971.16it/s]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "5\n", - "21\n", - "22\n", - "28\n", - "50\n", - "58\n", - "61\n", - "76\n", - "79\n", - "80\n", - "97\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\n" - ] - } - ], - "source": [ - "import re\n", - "\n", - "context_current = dict()\n", - "\n", - "def split_contexts(text):\n", - " # Regex pattern to capture blocks starting correctly after 'source:'\n", - " # Ensuring we skip unnecessary '---' lines and capture till the next 'source:' or end of text\n", - " pattern = r\"(?:---\\s*\\n)*?(source:\\s*https?://[^\\s]+\\nsource_type:\\s*.*?\\nhas_code:\\s*.*?\\n)((?:.*?)(?=(?:\\n---\\s*\\nsource:|\\n---\\s*$)))\"\n", - " \n", - " # Use re.DOTALL to match across multiple lines including newlines\n", - " contexts = [match.group(1) + match.group(2) for match in re.finditer(pattern, text, re.DOTALL)]\n", - " \n", - " return contexts\n", - "\n", - "# Assuming `df['context']` is a column containing the text blocks to be split\n", - "for idx, row in tqdm(df.iterrows()):\n", - " contexts = split_contexts(row['source_documents'] + \"\\n---\\n\")\n", - " if len(contexts) != 10:\n", - " print(idx)\n", - " # Store cleaned contexts, where we remove the 'source:', 'source_type:', and 'has_code:' lines if necessary\n", - " context_current[row['question']] = [\n", - " re.sub(r\"(source:.*\\n)|(source_type:.*\\n)|(has_code:.*\\n)\", \"\", context).strip() for context in contexts\n", - " ]" - ] - }, - { - "cell_type": "code", - "execution_count": 82, - "id": "38a74caa-c964-4ee0-8652-7abb0c5b6539", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "{'precision': 0.6, 'recall': 0.6, 'f1_score': 0.6, 'jaccard_index': 0.42857142857142855}\n" - ] - } - ], - "source": [ - "def calculate_metrics(a, b):\n", - " # Convert lists to sets\n", - " set_a = set(a)\n", - " set_b = set(b)\n", - " \n", - " # Calculate intersection and union\n", - " intersection = set_a.intersection(set_b)\n", - " union = set_a.union(set_b)\n", - " \n", - " # Calculate precision, recall, and F1 score\n", - " precision = len(intersection) / len(set_b) if set_b else 0\n", - " recall = len(intersection) / len(set_a) if set_a else 0\n", - " f1_score = 2 * (precision * recall) / (precision + recall) if (precision + recall) else 0\n", - " \n", - " # Calculate Jaccard Index\n", - " jaccard_index = len(intersection) / len(union) if union else 0\n", - " \n", - " return {\n", - " \"precision\": precision,\n", - " \"recall\": recall,\n", - " \"f1_score\": f1_score,\n", - " \"jaccard_index\": jaccard_index\n", - " }\n", - "\n", - "# Given lists\n", - "a = [\"text 1\", \"text 2\", \"text 3\", \"text 4\", \"text 5\"]\n", - "b = [\"text 3\", \"text 5\", \"text 7\", \"text 1\", \"text 10\"]\n", - "\n", - "# Calculate metrics\n", - "metrics = calculate_metrics(a, b)\n", - "print(metrics)" - ] - }, - { - "cell_type": "code", - "execution_count": 86, - "id": "0431f281-4932-4e28-b900-db643c83a8e2", - "metadata": {}, - "outputs": [], - "source": [ - "k = df.question.values[37]" - ] - }, - { - "cell_type": "code", - "execution_count": 87, - "id": "f775b2b5-92cb-4d67-a832-5ca54141753c", - "metadata": {}, - "outputs": [], - "source": [ - "a = ref_query_contexts[k]\n", - "b = context_sys_best[k]" - ] - }, - { - "cell_type": "code", - "execution_count": 88, - "id": "72e8cc5c-3909-412b-a8c9-2848aa693d78", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "{'precision': 0.0, 'recall': 0.0, 'f1_score': 0, 'jaccard_index': 0.0}" - ] - }, - "execution_count": 88, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "calculate_metrics(a, b)" - ] - }, - { - "cell_type": "code", - "execution_count": 89, - "id": "25677584-776d-477a-a30e-d0a7d982707d", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "['Project defaults\\n\\n\\nChange the default behavior for your account within the **Project** **Defaults** section. You can manage the proceeding:\\n\\n\\n* **Default location to create new projects** - Select the dropdown menu and choose the entity to set as the new default. Specify either your account or a team you are a member of.\\n* **Default projects privacy in your personal account** - Set a project to public (anyone can view), private (only you can view and contribute) or open (anyone can submit runs or write the reports) automatically when you create a project. You can optionally create a team to collaborate on private projects.\\n* **Enable code savings in your personal account** - Permit Weights and Biases to save the latest git commit hash by default. To enable code saving, toggle the Enable code savings in your personal account option. For more information about saving and comparing code, see Code Saving.',\n", - " 'Settings Page\\n\\n\\nWithin your individual user account you can edit: your profile picture, display name, geography location, biography information, emails associated to your account, and manage alerts for runs. You can also use the settings page to link your GitHub repository and delete your account. For more information, see User settings.\\n\\n\\nUse the team settings page to invite or remove new members to a team, manage alerts for team runs, change privacy settings, and view and manage storage usage. For more information about team settings, see Team settings.',\n", - " 'How do I kill a job with wandb?\\n\\n\\nPress `Ctrl+D` on your keyboard to stop a script that is instrumented with wandb.',\n", - " 'Team settings\\n\\n\\nNavigate to your team’s profile page and select the **Team settings** icon to manage team settings. Not all members in a team can modify team settings. The account type (Administrator, Member, or Service) of a member determines what settings that member can modify. For example, only Administration account types can change team privacy settings and remove a member from a team.',\n", - " 'Privacy\\n\\n\\nNavigate to the **Privacy** section to change privacy settings. Only members with Administrative roles can modify privacy settings. Administrator roles can:\\n\\n\\n* Force projects in the team to be private.\\n* Enable code saving by default.']" - ] - }, - "execution_count": 89, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "a" - ] - }, - { - "cell_type": "code", - "execution_count": 90, - "id": "e2f5e709-f1d4-42f4-afc4-b53092cf1c30", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "['Versions tab\\nThe versions tab shows all versions of the artifact as well as columns for each of the numeric values of the Run History at the time of logging the version. This allows you to compare performance and quickly identify versions of interest.\\nProject Defaults\\nYou can change your project default settings manually in your User Settings at\\n/settings.\\n- Default location to create new projects: This is set to your own personal entity by default. By clicking on the dropdown, you can switch between your personal entity and the teams you\\'re part of.\\n- Default project privacy in your personal account: This is set to \\'Private\\' by default. In other words, your projects will be private and can only be accessed by you.\\n- Enable code saving in your personal account: This is turned off by default. You can turn this on to save the main script or notebook to W&B.\\nThese settings can also be specified by passing arguments to\\nwandb.init.\\nFrequently Asked Questions\\nHow can I delete projects?\\nResize panels\\n- Standard layout: All panels maintain the same size, and there are pages of panels. You can resize the panels by clicking and dragging the lower right corner. Resize the section by clicking and dragging the lower right corner of the section.\\n- Custom layout: All panels are sized individually, and there are no pages.\\nSearch for metrics\\nUse the search box in the workspace to filter down the panels. This search matches the panel titles, which are by default the name of the metrics visualized.\\nTable Tab\\nUse the table to filter, group, and sort your results.\\nTable operations\\nUse the W&B App to sort, filter, and group your W&B Tables.\\n- Sort\\n- Filter\\n- Group\\nSort all rows in a Table by the value in a given column.\\n- Hover your mouse over the column title. A kebob menu will appear (three vertical docs).\\n- Select on the kebob menu (three vertical dots).\\n- Choose Sort Asc or Sort Desc to sort the rows in ascending or descending order, respectively.\\nProject Page\\nThe project Workspace gives you a personal sandbox to compare experiments. Use projects to organize models that can be compared, working on the same problem with different architectures, hyperparameters, datasets, preprocessing etc.\\nProject page tabs:\\n- Overview: snapshot of your project\\n- Workspace: personal visualization sandbox\\n- Table: bird\\'s eye view of all runs\\n- Reports: saved snapshots of notes, runs, and graphs\\n- Sweeps: automated exploration and optimization\\nOverview Tab\\n- Project name: click to edit the project name\\n- Project description: click to edit the project description and add notes\\n- Delete project: click the dot menu in the right corner to delete a project\\n- Project privacy: edit who can view runs and reports— click the lock icon\\n- Last active: see when the most recent data was logged to this project\\n- Total compute: we add up all the run times in your project to get this total\\nBy default, this turns other numeric columns into histograms showing the distribution of values for that column across the group. Grouping is helpful for understanding higher-level patterns in your data.\\nReports Tab\\nSee all the snapshots of results in one place, and share findings with your team.\\nSweeps Tab\\nStart a new sweep from your project.\\nArtifacts Tab\\nView all the artifacts associated with a project, from training datasets and fine-tuned models to tables of metrics and media.\\nOverview panel\\nOn the overview panel, you\\'ll find a variety of high-level information about the artifact, including its name and version, the hash digest used to detect changes and prevent duplication, the creation date, and any aliases. You can add or remove aliases here, take notes on both the version as well as the artifact as a whole.\\nMetadata panel\\nMetadata panel\\nThe metadata panel provides access to the artifact\\'s metadata, which is provided when the artifact is constructed. This metadata might include configuration arguments required to reconstruct the artifact, URLs where more information can be found, or metrics produced during the run which logged the artifact. Additionally, you can see the configuration for the run which produced the artifact as well as the history metrics at the time of logging the artifact.\\nUsage panel\\nThe Usage panel provides a code snippet for downloading the artifact for use outside of the web app, for example on a local machine. This section also indicates and links to the run which output the artifact and any runs which use the artifact as an input.\\nFiles panel\\nThe files panel lists the files and folders associated with the artifact. You can navigate through this file tree and view the contents directly in the W&B web app.\\nHow can I delete projects?\\nYou can delete your project by clicking the three dots on the right of the overview tab.\\nIf the project is empty (i.e. it has no runs), you can delete it by clicking the dropdown menu in the top-right and selecting \"Delete project\".\\nWhere are the privacy settings for projects? How can I make a project public or private?\\nClick the lock in the navigation bar at the top of the page to change project privacy settings. You can edit who can view or submit runs to your project. These settings include all runs and reports in the project. If you\\'d like to share your results with just a few people, you can create a private team.\\nHow do I reset my workspace?\\nIf you see an error like the one below on your project page, here\\'s how to reset your workspace.\\n\"objconv: \"100000000000\" overflows the maximum values of a signed 64 bits integer\"\\nAdd\\n?workspace=clear to the end of the URL and press enter. This should take you to a cleared version of your project page workspace.\\n- Undelete runs: Click the dropdown menu and click \"Undelete all runs\" to recover deleted runs in your project.\\nWorkspace Tab\\nRuns Sidebar: list of all the runs in your project\\n- Dot menu: hover over a row in the sidebar to see the menu appear on the left side. Use this menu to rename a run, delete a run, or stop and active run.\\n- Visibility icon: click the eye to turn on and off runs on graphs\\n- Color: change the run color to another one of our presets or a custom color\\n- Search: search runs by name. This also filters visible runs in the plots.\\n- Filter: use the sidebar filter to narrow down the set of runs visible\\n- Group: select a config column to dynamically group your runs, for example by architecture. Grouping makes plots show up with a line along the mean value, and a shaded region for the variance of points on the graph.\\n- Sort: pick a value to sort your runs by, for example runs with the lowest loss or highest accuracy. Sorting will affect which runs show up on the graphs.\\nTables associated with artifacts are particularly rich and interactive in this context. Learn more about using Tables with Artifacts here.\\nLineage panel\\nThe lineage panel provides a view of all of the artifacts associated with a project and the runs that connect them to each other. It shows run types as blocks and artifacts as circles, with arrows to indicate when a run of a given type consumes or produces an artifact of a given type. The type of the particular artifact selected in the left-hand column is highlighted.\\nClick the Explode toggle to view all of the individual artifact versions and the specific runs that connect them.\\nAction History Audit tab\\nThe action history audit tab shows all of the alias actions and membership changes for a Collection so you can audit the entire evolution of the resource.\\nVersions tab\\nThe preceding image demonstrates how to view sorting options for a Table column called\\nval_acc.\\nSelect Add filter to add one or more filters to your rows. Three dropdown menus will appear. From left to right the filter types are based on: Column name, Operator , and Values\\n|Column name||Binary relation||Value|\\n|Accepted values||String||=, ≠, ≤, ≥, IN, NOT IN,||Integer, float, string, timestamp, null|\\nThe expression editor shows a list of options for each term using autocomplete on column names and logical predicate structure. You can connect multiple logical predicates into one expression using \"and\" or \"or\" (and sometimes parentheses).\\nThe preceding image shows a filter that is based on the\\nval_loss column. The filter shows runs with a validation loss less than or equal to 1.\\nGroup all rows by the value in a particular column with the Group by button in a column header.',\n", - " 'Delete data\\n\\nYou can also choose to delete data to remain under your storage limit. There are several ways to do this:\\n\\n* Delete data interactively with the app UI.\\n* Set a TTL policy on Artifacts so they are automatically deleted.',\n", - " 'Project defaults\\n\\nChange the default behavior for your account within the **Project** **Defaults** section. You can manage the proceeding:\\n\\n* **Default location to create new projects** - Select the dropdown menu and choose the entity to set as the new default. Specify either your account or a team you are a member of.\\n* **Default projects privacy in your personal account** - Set a project to public (anyone can view), private (only you can view and contribute) or open (anyone can submit runs or write the reports) automatically when you create a project. You can optionally create a team to collaborate on private projects.\\n* **Enable code savings in your personal account** - Permit Weights and Biases to save the latest git commit hash by default. To enable code saving, toggle the Enable code savings in your personal account option. For more information about saving and comparing code, see Code Saving.',\n", - " '- Tools for collaboration: Use W&B to organize complex machine learning projects. It\\'s easy to share a link to W&B, and you can use private teams to have everyone send results to a shared project. We also support collaboration via reports— add interactive visualizations and describe your work in markdown. This is a great way to keep a work log, share findings with your supervisor, or present findings to your lab.\\nGet started with a free personal account →\\nHow does wandb stream logs and writes to disk?\\nW&B queues in memory but also write the events to disk asynchronously to handle failures and for the\\nWANDB_MODE=offline case where you can sync the data after it\\'s been logged.\\nIn your terminal, you can see a path to the local run directory. This directory will contain a\\n.wandb file that is the datastore above. If you\\'re also logging images, we write them to\\nmedia/images in that directory before uploading them to cloud storage.\\nHow to get multiple charts with different selected runs?\\nHow can I rotate or revoke access?\\nBoth personal and service account keys can be rotated or revoked. Simply create a new API Key or Service Account user and reconfigure your scripts to use the new key. Once all processes are reconfigured, you can remove the old API key from your profile or team.\\nHow do I switch between accounts on the same machine?\\nIf you have two W&B accounts working from the same machine, you\\'ll need a nice way to switch between your different API keys. You can store both API keys in a file on your machine then add code like the following to your repos. This is to avoid checking your secret key into a source control system, which is potentially dangerous.\\nif os.path.exists(\"~/keys.json\"):\\nos.environ[\"WANDB_API_KEY\"] = json.loads(\"~/keys.json\")[\"work_account\"]\\nIs there a dark mode?\\nYes. To enable dark mode:\\n- Navigate to your account settings at https://wandb.ai/settings.\\n- Scroll to the Beta Features section.\\n- Toggle the Night mode option.\\nimplicitflow.\\n- Set the callback URI to\\nhttps://wandb.auth0.com/login/callback.\\nWhat W&B needs?\\nOnce you have the above setup, contact your customer success manager(CSM) and let us know the\\nClient ID and\\nIssuer URL associated with the application.\\nWe\\'ll then set up an Auth0 connection with the above details and enable SSO.\\nWhat is a service account, and why is it useful?\\nA service account is an API key that has permissions to write to your team, but is not associated with a particular user. Among other things, service accounts are useful for tracking automated jobs logged to wandb, like periodic retraining, nightly builds, and so on. If you\\'d like, you can associate a username with one of these machine-launched runs with the environment variable\\nWANDB_USERNAME.\\nYou can get the API key in your Team Settings page\\n/teams/ where you invite new team members. Select service and click create to add a service account.\\nHow can I rotate or revoke access?\\nGeneral\\nWhat does\\nwandb.init do to my training process?\\nWhen\\nwandb.init() is called from your training script an API call is made to create a run object on our servers. A new process is started to stream and collect metrics, thereby keeping all threads and logic out of your primary process. Your script runs normally and writes to local files, while the separate process streams them to our servers along with system metrics. You can always turn off streaming by running\\nwandb off from your training directory, or setting the\\nWANDB_MODE environment variable to\\noffline.\\nDoes your tool track or store training data?\\nYou can pass a SHA or other unique identifier to\\nwandb.config.update(...) to associate a dataset with a training run. W&B does not store any data unless\\nwandb.save is called with the local file name.\\nWhat formula do you use for your smoothing algorithm?\\nWe use the same exponential moving average formula as TensorBoard. You can find an explanation here.\\nWith wandb reports the procedure is as follows:\\n- Have multiple panel grids.\\n- Add filters to filter the run sets of each panel grid. This will help in selecting the runs that you want to portray in the respective panels.\\n- Create the charts you want in the panel grids.\\nHow is access to the API controlled?\\nFor simplicity, W&B uses API keys for authorization when accessing the API. You can find your API keys in your settings. Your API key should be stored securely and never checked into version control. In addition to personal API keys, you can add Service Account users to your team.\\nDoes W&B support SSO for SaaS?\\nYes, W&B supports setting up Single Sign-On (SSO) for the SaaS offering via Auth0. W&B support SSO integration with any OIDC compliant identity provider(ex: Okta, AzureAD etc.). If you have an OIDC provider, please follow the steps below:\\n- Create a\\nSingle Page Application (SPA)on your Identity Provider.\\n- Set\\ngrant_typeto\\nimplicitflow.\\n- Set the callback URI to\\nHow do I get the random run name in my script?\\nCall\\nwandb.run.save() and then get the name with\\nwandb.run.name .\\nWhat is the difference between\\n.log() and\\n.summary?\\nThe summary is the value that shows in the table while the log will save all the values for plotting later.\\nFor example, you might want to call\\nwandb.log every time the accuracy changes. Usually, you can just use .log.\\nwandb.log() will also update the summary value by default unless you have set the summary manually for that metric\\nThe scatterplot and parallel coordinate plots will also use the summary value while the line plot plots all of the values set by .log\\nThe reason we have both is that some people like to set the summary manually because they want the summary to reflect for example the optimal accuracy instead of the last accuracy logged.\\nHow is W&B different from TensorBoard?\\nHow is W&B different from TensorBoard?\\nWe love the TensorBoard folks, and we have a TensorBoard integration! We were inspired to improve experiment tracking tools for everyone. When the co-founders started working on W&B, they were inspired to build a tool for the frustrated TensorBoard users at OpenAI. Here are a few things we focused on improving:\\n- Reproduce models: W&B is good for experimentation, exploration, and reproducing models later. We capture not just the metrics, but also the hyperparameters and version of the code, and we can save your model checkpoints for you so your project is reproducible.\\n- Automatic organization: If you hand off a project to a collaborator or take a vacation, W&B makes it easy to see all the models you\\'ve tried so you\\'re not wasting hours re-running old experiments.\\n- Toggle the Night mode option.\\nCan I disable wandb when testing my code?\\nBy using\\nwandb.init(mode=\"disabled\") or by setting\\nWANDB_MODE=disabled you will make wandb act like a NOOP which is perfect for testing your code.\\nNote: Setting\\nwandb.init(mode=“disabled”) does not prevent\\nwandb from saving artifacts to\\nWANDB_CACHE_DIR\\n- Fast, flexible integration: Add W&B to your project in 5 minutes. Install our free open-source Python package and add a couple of lines to your code, and every time you run your model you\\'ll have nice logged metrics and records.\\n- Persistent, centralized dashboard: Anywhere you train your models, whether on your local machine, your lab cluster, or spot instances in the cloud, we give you the same centralized dashboard. You don\\'t need to spend your time copying and organizing TensorBoard files from different machines.\\n- Powerful table: Search, filter, sort, and group results from different models. It\\'s easy to look over thousands of model versions and find the best-performing models for different tasks. TensorBoard isn\\'t built to work well on large projects.',\n", - " 'Settings Page\\n\\nWithin your individual user account you can edit: your profile picture, display name, geography location, biography information, emails associated to your account, and manage alerts for runs. You can also use the settings page to link your GitHub repository and delete your account. For more information, see User settings.\\n\\nUse the team settings page to invite or remove new members to a team, manage alerts for team runs, change privacy settings, and view and manage storage usage. For more information about team settings, see Team settings.']" - ] - }, - "execution_count": 90, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "b" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "41a0a1a4-f21f-44a2-a54e-4e47771e6cfe", - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "f3bd6cc2-d21d-43e4-aa49-45a68c6602db", - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.12" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -}