Graph RAG : Laravel, MongoDB, Neo4j & n8n

[1. User uploads file to MinIO]
         |
         V (Trigger: Webhook/Event)
[2. n8n Workflow OR Scheduled Artisan Command]
         |
         V
[3. Call Docling API (File URL) -> Get Plain Text]
         |
         V
[4. Save metadata & text to MongoDB (status: 'pending')]
         |
         V
[5. Artisan Command: 'rag:process-documents']
         |
         |--> (a) Chunk text
         |--> (b) Generate embedding for each chunk (LLM API)
         |--> (c) Create :Document, :Chunk, :Entity nodes in Neo4j
         |--> (d) Store embedding in :Chunk node
         |--> (e) Update status in MongoDB (status: 'completed')

[1. User sends query to Laravel API]
         |
         V
[2. Embed user query (LLM API)]
         |
         V
[3. Execute single, unified query on Neo4j]
    (Combines vector search on :Chunk nodes with graph traversal)
         |
         V
[4. Retrieve relevant text chunks from Neo4j]
         |
         V
[5. Augment prompt and send to LLM for final answer]
         |
         V
[6. Return generated answer and sources to user]

-- Create a vector index on the 'embedding' property of 'Chunk' nodes
CREATE VECTOR INDEX chunk_embeddings IF NOT EXISTS
FOR (c:Chunk) ON (c.embedding)
OPTIONS { indexConfig: {
  `vector.dimensions`: 1536,          -- Adjust to your embedding model's dimensions
  `vector.similarity_function`: 'cosine'
}}

{
  "_id": "65f...",
  "file_name": "annual_report_2024.pdf",
  "minio_path": "/reports/annual_report_2024.pdf",
  "source_text": "The full text extracted from the document...",
  "status": "completed", // pending, processing, completed, failed
  "processed_at": "...",
  "created_at": "...",
  "updated_at": "..."
}

namespace App\Console\Commands;

use Illuminate\Console\Command;
use App\Models\Document; // Your MongoDB model
use App\Services\ChunkingService;
use App\Services\EmbeddingService;
use App\Services\GraphService; // Your Neo4j service
use App\Services\LlmService;
use Illuminate\Support\Facades\Log;

class ProcessDocumentsForRag extends Command
{
    protected $signature = 'rag:process-documents {--limit=10} {--force-id=}';
    protected $description = 'Processes pending documents from MongoDB and indexes them in Neo4j.';

    public function handle(
        ChunkingService $chunkingService,
        EmbeddingService $embeddingService,
        GraphService $graphService,
        LlmService $llmService
    ) {
        $this->info('Starting document processing workflow...');

        $query = Document::where('status', 'pending');
        if ($this->option('force-id')) {
            $query = Document::where('_id', $this->option('force-id'));
        }
        $documents = $query->limit($this->option('limit'))->get();

        if ($documents->isEmpty()) {
            $this->info('No pending documents to process.');
            return 0;
        }

        foreach ($documents as $doc) {
            try {
                $this->line("Processing document: {$doc->file_name} (ID: {$doc->_id})");
                $doc->status = 'processing';
                $doc->save();

                // 1. Create parent Document node in Neo4j
                $graphService->createDocumentNode($doc->_id, $doc->file_name);

                // 2. Chunk the text
                $textChunks = $chunkingService->split($doc->source_text, 300, 50); // (text, tokens, overlap)

                $progressBar = $this->output->createProgressBar(count($textChunks));
                $progressBar->start();

                foreach ($textChunks as $index => $chunkText) {
                    // 3. Generate embedding for the chunk
                    $embedding = $embeddingService->generate($chunkText);
                    $chunkId = "{$doc->_id}_chunk_{$index}";

                    // 4. Create :Chunk node in Neo4j with embedding
                    $graphService->createChunkNode($chunkId, $doc->_id, $chunkText, $embedding);

                    // 5. (Optional but powerful) Extract entities and link them
                    $entities = $llmService->extractEntities($chunkText);
                    foreach ($entities as $entity) {
                        $graphService->createEntityNode($entity);
                        $graphService->linkChunkToEntity($chunkId, $entity);
                    }
                    $progressBar->advance();
                }

                $progressBar->finish();
                $this->newLine();

                $doc->status = 'completed';
                $doc->processed_at = now();
                $doc->save();
                $this->info("Successfully processed document: {$doc->file_name}");

            } catch (\Exception $e) {
                Log::error("Failed to process document {$doc->_id}: {$e->getMessage()}");
                $doc->status = 'failed';
                $doc->save();
                $this->error("Failed to process document {$doc->file_name}. See logs for details.");
            }
        }
        $this->info('Document processing workflow finished.');
        return 0;
    }
}

public function graphRagSearch(array $queryVector, int $limit = 5): array
{
    // This single query performs vector search and graph traversal
    $query = "
        CALL db.index.vector.queryNodes('chunk_embeddings', \$limit, \$queryVector)
        YIELD node AS similarChunk, score
        MATCH (doc:Document)-[:HAS_CHUNK]->(similarChunk)
        RETURN similarChunk.text AS text, score, doc.title AS source
        ORDER BY score DESC
    ";

    $result = $this->client->run($query, [
        'queryVector' => $queryVector,
        'limit' => $limit
    ]);

    // Format results for the controller
    return collect($result->getRecords())->map(function ($record) {
        return [
            'text' => $record->get('text'),
            'score' => $record->get('score'),
            'source' => $record->get('source')
        ];
    })->all();
}

public function ask(Request $request, EmbeddingService $embeddingService, GraphService $graphService, LlmService $llmService)
{
    $validated = $request->validate(['query' => 'required|string|max:500']);
    $userQuery = $validated['query'];

    // 1. Embed user query
    $queryVector = $embeddingService->generate($userQuery);

    // 2. Search Neo4j for relevant context
    $contexts = $graphService->graphRagSearch($queryVector, 5);

    if (empty($contexts)) {
        return response()->json(['answer' => 'I could not find any relevant information to answer your question.']);
    }

    // 3. Augment prompt and generate answer
    $contextText = implode("\n---\n", array_column($contexts, 'text'));
    $prompt = "Based on the following context, please answer the question.\n\nContext:\n{$contextText}\n\nQuestion: {$userQuery}\n\nAnswer:";
    $answer = $llmService->getCompletion($prompt);

    // 4. Return response with sources
    return response()->json([
        'answer' => $answer,
        'sources' => array_unique(array_column($contexts, 'source'))
    ]);
}

[1. Pengguna mengunggah file ke MinIO]
         |
         V (Pemicu: Webhook/Event)
[2. Alur Kerja n8n ATAU Perintah Artisan Terjadwal]
         |
         V
[3. Panggil API Docling (URL File) -> Dapatkan Teks Biasa]
         |
         V
[4. Simpan metadata & teks ke MongoDB (status: 'pending')]
         |
         V
[5. Perintah Artisan: 'rag:process-documents']
         |
         |--> (a) Pecah teks menjadi potongan (chunking)
         |--> (b) Buat embedding untuk setiap chunk (API LLM)
         |--> (c) Buat node :Document, :Chunk, :Entity di Neo4j
         |--> (d) Simpan embedding di dalam node :Chunk
         |--> (e) Perbarui status di MongoDB (status: 'completed')

[1. Pengguna mengirim kueri ke API Laravel]
         |
         V
[2. Buat embedding untuk kueri pengguna (API LLM)]
         |
         V
[3. Eksekusi satu kueri terpadu di Neo4j]
    (Menggabungkan pencarian vektor pada node :Chunk dengan penelusuran graf)
         |
         V
[4. Ambil potongan teks yang relevan dari Neo4j]
         |
         V
[5. Susun prompt dan kirim ke LLM untuk mendapatkan jawaban akhir]
         |
         V
[6. Kembalikan jawaban yang dihasilkan beserta sumbernya ke pengguna]

-- Membuat indeks vektor pada properti 'embedding' dari node 'Chunk'
CREATE VECTOR INDEX chunk_embeddings IF NOT EXISTS
FOR (c:Chunk) ON (c.embedding)
OPTIONS { indexConfig: {
  `vector.dimensions`: 1536,          -- Sesuaikan dengan dimensi model embedding Anda
  `vector.similarity_function`: 'cosine'
}}

{
  "_id": "65f...",
  "file_name": "laporan_tahunan_2024.pdf",
  "minio_path": "/reports/laporan_tahunan_2024.pdf",
  "source_text": "Teks lengkap yang diekstrak dari dokumen...",
  "status": "completed", // Opsi: pending, processing, completed, failed
  "processed_at": "...",
  "created_at": "...",
  "updated_at": "..."
}

// ... (Kode sama persis seperti versi Bahasa Inggris, hanya komentar yang bisa diterjemahkan)
// Contoh komentar yang diterjemahkan:
// 1. Buat node induk Document di Neo4j
// 2. Pecah teks menjadi potongan-potongan (chunking)
// 3. Buat embedding untuk setiap chunk
// 4. Buat node :Chunk di Neo4j beserta embeddingnya
// 5. (Opsional tapi sangat berguna) Ekstrak entitas dan hubungkan

// Kode sama dengan versi Bahasa Inggris.
// Kueri Cypher ini melakukan pencarian vektor dan penelusuran graf secara bersamaan.

// Kode sama dengan versi Bahasa Inggris.
// Logika:
// 1. Buat embedding dari kueri pengguna.
// 2. Cari konteks yang relevan di Neo4j.
// 3. Susun prompt dan hasilkan jawaban.
// 4. Kembalikan respons beserta sumbernya.