backend
This commit is contained in:
parent
e3d416ff0e
commit
797fec3135
32 changed files with 3770 additions and 0 deletions
59
backend/createEmbeddings.js
Executable file
59
backend/createEmbeddings.js
Executable file
|
@ -0,0 +1,59 @@
|
|||
import 'dotenv/config';
|
||||
|
||||
// Setup LLM
|
||||
import {Document, Settings, storageContextFromDefaults, VectorStoreIndex} from "llamaindex";
|
||||
import {OpenAI, OpenAIEmbedding} from "@llamaindex/openai";
|
||||
import path from "path";
|
||||
import fs from "fs";
|
||||
import {QdrantVectorStore} from "@llamaindex/qdrant";
|
||||
|
||||
// Initialize Qdrant vector store for plants
|
||||
|
||||
Settings.llm = new OpenAI({
|
||||
model: process.env.OPENAI_MODEL,
|
||||
apiKey: process.env.OPENAI_API_KEY,
|
||||
});
|
||||
|
||||
Settings.embedModel = new OpenAIEmbedding({
|
||||
apiKey: process.env.OPENAI_API_KEY,
|
||||
});
|
||||
|
||||
const vectorStore = new QdrantVectorStore({
|
||||
url: 'http://localhost:6333',
|
||||
collectionName: 'plants-rag',
|
||||
});
|
||||
|
||||
const storageContext = await storageContextFromDefaults({
|
||||
vectorStore
|
||||
});
|
||||
|
||||
try {
|
||||
const storageDir = path.join(process.cwd(), 'storage', 'plants');
|
||||
|
||||
// Read all markdown files from the storage directory
|
||||
const files = fs.readdirSync(storageDir).filter(file => file.endsWith('.md'));
|
||||
|
||||
const documents = [];
|
||||
for (const [index, file] of files.entries()) {
|
||||
console.log(`Processing file ${index + 1} of ${files.length}: ${file}`);
|
||||
const filePath = path.join(storageDir, file);
|
||||
const content = fs.readFileSync(filePath, 'utf-8');
|
||||
|
||||
documents.push(new Document({
|
||||
text: content,
|
||||
metadata: {
|
||||
source: file
|
||||
}
|
||||
}));
|
||||
}
|
||||
|
||||
// Create vector store index
|
||||
await VectorStoreIndex.fromDocuments(documents, {
|
||||
storageContext: storageContext
|
||||
});
|
||||
|
||||
|
||||
} catch (error) {
|
||||
console.error('Error initializing plants RAG:', error);
|
||||
|
||||
}
|
Loading…
Add table
Add a link
Reference in a new issue