diff --git a/actions/knowledge/knowledge.ts b/actions/knowledge/knowledge.ts index 071910ac..50e7a9d9 100644 --- a/actions/knowledge/knowledge.ts +++ b/actions/knowledge/knowledge.ts @@ -37,17 +37,25 @@ export async function deleteDataset(datasetID: string): Promise { return; } +export async function firstIngestion( + scriptId: string, + files: string[] +): Promise { + const dir = path.join(KNOWLEDGE_DIR(), 'script_data', scriptId, 'data'); + return !fs.existsSync(dir) && files.length > 0; +} + export async function ensureFilesIngested( files: string[], scriptId: string, token: string -) { +): Promise { const dir = path.join(KNOWLEDGE_DIR(), 'script_data', scriptId, 'data'); if (!fs.existsSync(dir) && files.length > 0) { fs.mkdirSync(dir, { recursive: true }); } else if (!fs.existsSync(dir) && files.length === 0) { // if there are no files in the directory and no dropped files, do nothing - return; + return ''; } for (const file of files) { @@ -57,8 +65,7 @@ export async function ensureFilesIngested( await fs.promises.copyFile(file, filePath); } } catch (error) { - console.error(`Error copying file ${file}:`, error); - throw error; + return `Error copying file ${file}: ${error}`; } } @@ -74,8 +81,7 @@ export async function ensureFilesIngested( } } } catch (error) { - console.error('Error during cleanup of removed files:', error); - throw error; + return `Error deleting files: ${error}`; } try { @@ -85,11 +91,10 @@ export async function ensureFilesIngested( token ); } catch (error) { - console.error('Error during ingestion:', error); - throw error; + return `Error running knowledge ingestion: ${error}`; } - return; + return ''; } async function runKnowledgeIngest( @@ -97,25 +102,16 @@ async function runKnowledgeIngest( knowledgePath: string, token: string ): Promise { - try { - // Start the ingestion process in the background - await execPromise( - `${process.env.KNOWLEDGE_BIN} ingest --prune --dataset ${id} ./data`, - { - cwd: knowledgePath, - env: { ...process.env, GPTSCRIPT_GATEWAY_API_KEY: token }, - } - ); - - const errorFilePath = path.join(knowledgePath, 'error.log'); - if (fs.existsSync(errorFilePath)) { - await fs.promises.rm(errorFilePath); + // Start the ingestion process in the background + await execPromise( + `${process.env.KNOWLEDGE_BIN} ingest --prune --dataset ${id} ./data`, + { + cwd: knowledgePath, + env: { ...process.env, GPTSCRIPT_GATEWAY_API_KEY: token }, } - } catch (error) { - console.log(error); - handleError(knowledgePath, error as Error); - throw error; - } + ); + + return; } export async function getFiles(scriptId: string): Promise { @@ -135,8 +131,3 @@ export async function datasetExists(scriptId: string): Promise { export async function getKnowledgeBinaryPath(): Promise { return process.env.KNOWLEDGE_BIN || 'knowledge'; } - -function handleError(dir: string, error: Error): void { - const errorFilePath = path.join(dir, 'error.log'); - fs.writeFileSync(errorFilePath, error.message); -} diff --git a/components/edit/configure.tsx b/components/edit/configure.tsx index f2b11b88..469bf87b 100644 --- a/components/edit/configure.tsx +++ b/components/edit/configure.tsx @@ -5,7 +5,7 @@ import Models from '@/components/edit/configure/models'; import Visibility from '@/components/edit/configure/visibility'; import Code from '@/components/edit/configure/code'; import { EditContext, KNOWLEDGE_NAME } from '@/contexts/edit'; -import { GoLightBulb } from 'react-icons/go'; +import { GoLightBulb, GoTrash } from 'react-icons/go'; import { HiCog } from 'react-icons/hi2'; import { LuCircuitBoard } from 'react-icons/lu'; import { @@ -24,11 +24,11 @@ import AssistantNotFound from '@/components/assistant-not-found'; import { useRouter } from 'next/navigation'; import { ChatContext } from '@/contexts/chat'; import Chat from '@/components/chat'; -import { GoDatabase } from 'react-icons/go'; import { IoSettingsOutline } from 'react-icons/io5'; -import { IoMdAdd } from 'react-icons/io'; +import { IoMdAdd, IoMdRefresh } from 'react-icons/io'; import { RiFileSearchLine } from 'react-icons/ri'; -import KnowledgeModals from '@/components/knowledge/KnowledgeModals'; +import FileSettingModals from '@/components/knowledge/KnowledgeModals'; +import { RiFoldersLine } from 'react-icons/ri'; interface ConfigureProps { collapsed?: boolean; @@ -51,12 +51,14 @@ const Configure: React.FC = ({ collapsed }) => { setDependencies, setDroppedFiles, droppedFileDetails, + setDroppedFileDetails, ingesting, + ingest, updated, setUpdated, + ingestionError, } = useContext(EditContext); const { restartScript } = useContext(ChatContext); - const fileTableModal = useDisclosure(); const fileSettingModal = useDisclosure(); const abbreviate = (name: string) => { @@ -212,40 +214,95 @@ const Configure: React.FC = ({ collapsed }) => { aria-label="files" title={

Files

} startContent={} - classNames={{ content: 'pt-1 pb-3' }} + classNames={{ content: collapsed ? 'pt-6 pb-10' : 'p-10 pt-6' }} > -
-
+ ) - .reduce((acc, detail) => acc + detail.size, 0) - .toFixed(2)} KB`}

)} - {ingesting && } -
- +
+ {droppedFileDetails?.size > 0 && + !ingesting && + !ingestionError && ( +
+ +

{`${droppedFileDetails.size} ${droppedFileDetails.size === 1 ? 'file' : 'files'}, ${Array.from( + droppedFileDetails.values() + ) + .reduce((acc, detail) => acc + detail.size, 0) + .toFixed(2)} KB`}

+
+ )} + {ingesting && !ingestionError && ( + + )} + {ingestionError && ( + <> + +

+ {ingestionError} +

+ + )}
+
0 ? 'pt-4' : ''}`} + > + + +
= ({ collapsed }) => { : `Click "Refresh Chat" to chat with the updated version of ${placeholderName()}` } /> - ); diff --git a/components/knowledge/KnowledgeModals.tsx b/components/knowledge/KnowledgeModals.tsx index 91d763d0..06ca077a 100644 --- a/components/knowledge/KnowledgeModals.tsx +++ b/components/knowledge/KnowledgeModals.tsx @@ -1,44 +1,23 @@ import { - Button, Modal, ModalBody, ModalContent, - ModalFooter, ModalHeader, Slider, - Table, - TableBody, - TableCell, - TableColumn, - TableHeader, - TableRow, } from '@nextui-org/react'; -import { BiPlus, BiTrash } from 'react-icons/bi'; import { useContext } from 'react'; import { EditContext } from '@/contexts/edit'; interface KnowledgeProps { isFileSettingOpen: boolean; onFileSettingClose: () => void; - isFileTableOpen: boolean; - onFileTableClose: () => void; - handleAddFiles: () => void; } const KnowledgeModals = ({ isFileSettingOpen, onFileSettingClose, - isFileTableOpen, - onFileTableClose, - handleAddFiles, }: KnowledgeProps) => { - const { - topK, - setTopK, - droppedFileDetails, - setDroppedFileDetails, - setDroppedFiles, - } = useContext(EditContext); + const { topK, setTopK } = useContext(EditContext); return ( <> - - - - - - - - - File Name - Size - {''} - - - {Array.from(droppedFileDetails).map((fileDetail, index) => ( - - {fileDetail[1].fileName} - {fileDetail[1].size} KB - -
- - - - - - ); }; diff --git a/components/scripts/tool-dropdown.tsx b/components/scripts/tool-dropdown.tsx index 381a61f0..11bdab76 100644 --- a/components/scripts/tool-dropdown.tsx +++ b/components/scripts/tool-dropdown.tsx @@ -14,6 +14,10 @@ import { GoTools } from 'react-icons/go'; import { load } from '@/actions/gptscript'; import { gatewayTool } from '@/actions/knowledge/util'; import { ToolReference } from '@gptscript-ai/gptscript'; +import { KNOWLEDGE_NAME } from '@/contexts/edit'; + +const dynamicToolName = 'Dynamic Instructions'; +const fileRetrievalName = 'File Retrieval'; const ScriptToolsDropdown = () => { const { program, tools, socket, setMessages } = useContext(ChatContext); @@ -59,9 +63,13 @@ const ScriptToolsDropdown = () => { setThreadTools(threadTools); }, [tools, displayNames, knowledgeGatewayTool, program]); - function dynamicInstructions(name: string | undefined): string | undefined { - if (name && name === 'dynamic-instructions') { - return 'Dynamic Instructions'; + function friendlyName(name: string | undefined): string | undefined { + if (name === 'dynamic-instructions') { + return dynamicToolName; + } + + if (name === KNOWLEDGE_NAME) { + return fileRetrievalName; } return name; @@ -69,7 +77,11 @@ const ScriptToolsDropdown = () => { function getDisplayName(ref: string): string { if (ref === 'dynamic-instructions') { - return 'Dynamic Instructions'; + return dynamicToolName; + } + + if (ref === KNOWLEDGE_NAME) { + return fileRetrievalName; } return ( @@ -126,7 +138,7 @@ const ScriptToolsDropdown = () => { content={t} isReadOnly > - {dynamicInstructions( + {friendlyName( ( program.toolSet[ (v.find((v) => v.reference === t) || {}).toolID || '' diff --git a/contexts/edit.tsx b/contexts/edit.tsx index f05356ca..bf6337f9 100644 --- a/contexts/edit.tsx +++ b/contexts/edit.tsx @@ -16,6 +16,7 @@ import { import { datasetExists, ensureFilesIngested, + firstIngestion, getFiles, getKnowledgeBinaryPath, } from '@/actions/knowledge/knowledge'; @@ -85,8 +86,10 @@ interface EditContextState { topK: number; setTopK: React.Dispatch>; ingesting: boolean; + ingest: () => Promise; updated: boolean; setUpdated: React.Dispatch>; + ingestionError: string; // actions update: () => Promise; @@ -140,6 +143,7 @@ const EditContextProvider: React.FC = ({ const [topK, setTopK] = useState(10); const [ingesting, setIngesting] = useState(false); const [knowledgeTool, setKnowledgeTool] = useState({} as Tool); + const [ingestionError, setIngestionError] = useState(''); const addRootTool = (tool: string) => { setRoot({ ...root, tools: [...(root.tools || []), tool] }); @@ -161,7 +165,7 @@ const EditContextProvider: React.FC = ({ id: KNOWLEDGE_NAME, name: KNOWLEDGE_NAME, description: - 'Retrieve information from files uploaded to the assistant.', + 'Retrieve information from files uploaded to the assistant. Use it to answer questions from the user and ALWAYS give a proper citation to the best of your abilities, including the source references (filename, page, etc.).', type: 'tool', credentials: [ 'github.com/gptscript-ai/gateway-creds as github.com/gptscript-ai/gateway', @@ -202,28 +206,34 @@ const EditContextProvider: React.FC = ({ setFiles(); }, [scriptId]); + const ingest = useCallback(async () => { + setIngesting(true); + setIngestionError(''); + const newDetails = new Map(droppedFileDetails); + for (const file of droppedFiles) { + const size = await getFileOrFolderSizeInKB(file); + const filename = await getBasename(file); + newDetails.set(file, { + fileName: filename, + size: size, + }); + } + setDroppedFileDetails(newDetails); + const first = await firstIngestion(scriptId.toString(), droppedFiles); + const error = await ensureFilesIngested( + droppedFiles, + scriptId.toString(), + getCookie('gateway_token') + ); + setIngestionError(error); + setIngesting(false); + if (first) { + setUpdated(true); + } + }, [droppedFiles, scriptId, droppedFileInitiazed, ingesting]); + useEffect(() => { if (!scriptId || !droppedFileInitiazed.current) return; - - const ingest = async () => { - setIngesting(true); - const newDetails = new Map(droppedFileDetails); - for (const file of droppedFiles) { - const size = await getFileOrFolderSizeInKB(file); - const filename = await getBasename(file); - newDetails.set(file, { - fileName: filename, - size: size, - }); - } - setDroppedFileDetails(newDetails); - await ensureFilesIngested( - droppedFiles, - scriptId.toString(), - getCookie('gateway_token') - ); - setIngesting(false); - }; ingest(); }, [droppedFiles, scriptId, droppedFileInitiazed]); @@ -450,8 +460,10 @@ const EditContextProvider: React.FC = ({ topK, setTopK, ingesting, + ingest, updated, setUpdated, + ingestionError, }} > {children}