Skip to content

Commit

Permalink
merge with master
Browse files Browse the repository at this point in the history
  • Loading branch information
timothycarambat committed Sep 25, 2024
2 parents f8a40fa + 44dddcd commit cdfc83e
Show file tree
Hide file tree
Showing 76 changed files with 1,903 additions and 261 deletions.
1 change: 1 addition & 0 deletions .vscode/settings.json
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,7 @@
"streamable",
"textgenwebui",
"togetherai",
"fireworksai",
"Unembed",
"vectordbs",
"Weaviate",
Expand Down
4 changes: 3 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,8 @@ AnythingLLM divides your documents into objects called `workspaces`. A Workspace

## Cool features of AnythingLLM

- 🆕 **Multi-modal support (both closed and open-source LLMs!)**
- 🆕 [**Custom AI Agents**](https://docs.anythingllm.com/agent/custom/introduction)
- 🖼️ **Multi-modal support (both closed and open-source LLMs!)**
- 👤 Multi-user instance support and permissioning _Docker version only_
- 🦾 Agents inside your workspace (browse the web, run code, etc)
- 💬 [Custom Embeddable Chat widget for your website](./embed/README.md) _Docker version only_
Expand Down Expand Up @@ -83,6 +84,7 @@ AnythingLLM divides your documents into objects called `workspaces`. A Workspace
- [LM Studio (all models)](https://lmstudio.ai)
- [LocalAi (all models)](https://localai.io/)
- [Together AI (chat models)](https://www.together.ai/)
- [Fireworks AI (chat models)](https://fireworks.ai/)
- [Perplexity (chat models)](https://www.perplexity.ai/)
- [OpenRouter (chat models)](https://openrouter.ai/)
- [Mistral](https://mistral.ai/)
Expand Down
5 changes: 1 addition & 4 deletions collector/processLink/convert/generic.js
Original file line number Diff line number Diff line change
Expand Up @@ -58,10 +58,7 @@ async function getPageContent(link) {
launchOptions: {
headless: "new",
ignoreHTTPSErrors: true,
args: [
'--no-sandbox',
'--disable-setuid-sandbox'
]
args: ["--no-sandbox", "--disable-setuid-sandbox"],
},
gotoOptions: {
waitUntil: "domcontentloaded",
Expand Down
5 changes: 2 additions & 3 deletions collector/utils/extensions/Confluence/index.js
Original file line number Diff line number Diff line change
Expand Up @@ -199,9 +199,8 @@ async function fetchConfluencePage({
*/
function generateAPIBaseUrl(matchResult = {}, isCustomDomain = false) {
const { subdomain } = matchResult;
let subpath = isCustomDomain ? `` : `/wiki`;
if (isCustomDomain) return `https://${customDomain}${subpath}`;
return `https://${subdomain}.atlassian.net${subpath}`;
if (isCustomDomain) return `https://${subdomain}`;
return `https://${subdomain}.atlassian.net/wiki`;
}

/**
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -44,16 +44,16 @@ class GitLabRepoLoader {
#validGitlabUrl() {
const UrlPattern = require("url-pattern");
const validPatterns = [
new UrlPattern("https\\://gitlab.com/(:projectId(*))", {
segmentValueCharset: "a-zA-Z0-9-._~%/+",
new UrlPattern("https\\://gitlab.com/(:author*)/(:project(*))", {
segmentValueCharset: "a-zA-Z0-9-._~%+",
}),
// This should even match the regular hosted URL, but we may want to know
// if this was a hosted GitLab (above) or a self-hosted (below) instance
// since the API interface could be different.
new UrlPattern(
"(:protocol(http|https))\\://(:hostname*)/(:projectId(*))",
"(:protocol(http|https))\\://(:hostname*)/(:author*)/(:project(*))",
{
segmentValueCharset: "a-zA-Z0-9-._~%/+",
segmentValueCharset: "a-zA-Z0-9-._~%+",
}
),
];
Expand All @@ -64,9 +64,9 @@ class GitLabRepoLoader {
match = pattern.match(this.repo);
}
if (!match) return false;
const [author, project] = match.projectId.split("/");
const { author, project } = match;

this.projectId = encodeURIComponent(match.projectId);
this.projectId = encodeURIComponent(`${author}/${project}`);
this.apiBase = new URL(this.repo).origin;
this.author = author;
this.project = project;
Expand Down Expand Up @@ -159,33 +159,54 @@ class GitLabRepoLoader {
async getRepoBranches() {
if (!this.#validGitlabUrl() || !this.projectId) return [];
await this.#validateAccessToken();
this.branches = [];
let fetching = true;
let page = 1;
let perPage = 50;

try {
this.branches = await fetch(
`${this.apiBase}/api/v4/projects/${this.projectId}/repository/branches`,
{
method: "GET",
headers: {
Accepts: "application/json",
...(this.accessToken ? { "PRIVATE-TOKEN": this.accessToken } : {}),
},
}
)
.then((res) => res.json())
.then((branches) => {
return branches.map((b) => b.name);
})
.catch((e) => {
console.error(e);
return [];
while (fetching) {
try {
const params = new URLSearchParams({
per_page: perPage,
page,
});
const response = await fetch(
`${this.apiBase}/api/v4/projects/${
this.projectId
}/repository/branches?${params.toString()}`,
{
method: "GET",
headers: {
Accepts: "application/json",
...(this.accessToken
? { "PRIVATE-TOKEN": this.accessToken }
: {}),
},
}
)
.then((res) => res.json())
.then((branches) => {
if (!Array.isArray(branches) || branches.length === 0) {
fetching = false;
return [];
}
return branches.map((b) => b.name);
})
.catch((e) => {
console.error(e);
fetching = false;
return [];
});

return this.#branchPrefSort(this.branches);
} catch (err) {
console.log(`RepoLoader.getRepoBranches`, err);
this.branches = [];
return [];
this.branches.push(...response);
page++;
} catch (err) {
console.log(`RepoLoader.getRepoBranches`, err);
fetching = false;
return [];
}
}
return this.#branchPrefSort(this.branches);
}

/**
Expand Down
4 changes: 4 additions & 0 deletions docker/.env.example
Original file line number Diff line number Diff line change
Expand Up @@ -101,6 +101,10 @@ GID='1000'
# AWS_BEDROCK_LLM_MODEL_PREFERENCE=meta.llama3-1-8b-instruct-v1:0
# AWS_BEDROCK_LLM_MODEL_TOKEN_LIMIT=8191

# LLM_PROVIDER='fireworksai'
# FIREWORKS_AI_LLM_API_KEY='my-fireworks-ai-key'
# FIREWORKS_AI_LLM_MODEL_PREF='accounts/fireworks/models/llama-v3p1-8b-instruct'

###########################################
######## Embedding API SElECTION ##########
###########################################
Expand Down
4 changes: 2 additions & 2 deletions frontend/public/embed/anythingllm-chat-widget.min.js

Large diffs are not rendered by default.

Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ export default function AwsBedrockLLMOptions({ settings }) {
You should use a properly defined IAM user for inferencing.
<br />
<a
href="https://docs.useanything.com/setup/llm-configuration/cloud/aws-bedrock"
href="https://docs.anythingllm.com/setup/llm-configuration/cloud/aws-bedrock"
target="_blank"
className="underline flex gap-x-1 items-center"
>
Expand Down
99 changes: 99 additions & 0 deletions frontend/src/components/LLMSelection/FireworksAiOptions/index.jsx
Original file line number Diff line number Diff line change
@@ -0,0 +1,99 @@
import System from "@/models/system";
import { useState, useEffect } from "react";

export default function FireworksAiOptions({ settings }) {
return (
<div className="flex gap-[36px] mt-1.5">
<div className="flex flex-col w-60">
<label className="text-white text-sm font-semibold block mb-3">
Fireworks AI API Key
</label>
<input
type="password"
name="FireworksAiLLMApiKey"
className="border-none bg-zinc-900 text-white placeholder:text-white/20 text-sm rounded-lg focus:outline-primary-button active:outline-primary-button outline-none block w-full p-2.5"
placeholder="Fireworks AI API Key"
defaultValue={settings?.FireworksAiLLMApiKey ? "*".repeat(20) : ""}
required={true}
autoComplete="off"
spellCheck={false}
/>
</div>
{!settings?.credentialsOnly && (
<FireworksAiModelSelection settings={settings} />
)}
</div>
);
}
function FireworksAiModelSelection({ settings }) {
const [groupedModels, setGroupedModels] = useState({});
const [loading, setLoading] = useState(true);

useEffect(() => {
async function findCustomModels() {
setLoading(true);
const { models } = await System.customModels("fireworksai");

if (models?.length > 0) {
const modelsByOrganization = models.reduce((acc, model) => {
acc[model.organization] = acc[model.organization] || [];
acc[model.organization].push(model);
return acc;
}, {});

setGroupedModels(modelsByOrganization);
}

setLoading(false);
}
findCustomModels();
}, []);

if (loading || Object.keys(groupedModels).length === 0) {
return (
<div className="flex flex-col w-60">
<label className="text-white text-sm font-semibold block mb-3">
Chat Model Selection
</label>
<select
name="FireworksAiLLMModelPref"
disabled={true}
className="border-none bg-zinc-900 border-gray-500 text-white text-sm rounded-lg block w-full p-2.5"
>
<option disabled={true} selected={true}>
-- loading available models --
</option>
</select>
</div>
);
}

return (
<div className="flex flex-col w-60">
<label className="text-white text-sm font-semibold block mb-3">
Chat Model Selection
</label>
<select
name="FireworksAiLLMModelPref"
required={true}
className="border-none bg-zinc-900 border-gray-500 text-white text-sm rounded-lg block w-full p-2.5"
>
{Object.keys(groupedModels)
.sort()
.map((organization) => (
<optgroup key={organization} label={organization}>
{groupedModels[organization].map((model) => (
<option
key={model.id}
value={model.id}
selected={settings?.FireworksAiLLMModelPref === model.id}
>
{model.name}
</option>
))}
</optgroup>
))}
</select>
</div>
);
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,79 @@
import { useRef, useEffect } from "react";

export default function ContextMenu({
contextMenu,
closeContextMenu,
files,
selectedItems,
setSelectedItems,
}) {
const contextMenuRef = useRef(null);

useEffect(() => {
const handleClickOutside = (event) => {
if (
contextMenuRef.current &&
!contextMenuRef.current.contains(event.target)
) {
closeContextMenu();
}
};

document.addEventListener("mousedown", handleClickOutside);
return () => {
document.removeEventListener("mousedown", handleClickOutside);
};
}, [closeContextMenu]);

const isAllSelected = () => {
const allItems = files.items.flatMap((folder) => [
folder.name,
...folder.items.map((file) => file.id),
]);
return allItems.every((item) => selectedItems[item]);
};

const toggleSelectAll = () => {
if (isAllSelected()) {
setSelectedItems({});
} else {
const newSelectedItems = {};
files.items.forEach((folder) => {
newSelectedItems[folder.name] = true;
folder.items.forEach((file) => {
newSelectedItems[file.id] = true;
});
});
setSelectedItems(newSelectedItems);
}
closeContextMenu();
};

if (!contextMenu.visible) return null;

return (
<div
ref={contextMenuRef}
style={{
position: "fixed",
top: `${contextMenu.y}px`,
left: `${contextMenu.x}px`,
zIndex: 1000,
}}
className="bg-zinc-800 border border-zinc-700 rounded-md shadow-lg"
>
<button
onClick={toggleSelectAll}
className="block w-full text-left px-4 py-2 text-sm text-white hover:bg-zinc-700"
>
{isAllSelected() ? "Unselect All" : "Select All"}
</button>
<button
onClick={closeContextMenu}
className="block w-full text-left px-4 py-2 text-sm text-white hover:bg-zinc-700"
>
Cancel
</button>
</div>
);
}
Loading

0 comments on commit cdfc83e

Please sign in to comment.