Update UI and prompts

src/pipeline/orchestrator.py

@@ -131,32 +131,76 @@ class PipelineOrchestrator:
                     context_text = full_history_text
 
                 # 2. Prepare Context (Wiki / Database of Knowledge)
-                wiki_context = self._get_wiki_context()
+                # wiki_context = self._get_wiki_context()
 
                 # Combine both
-                combined_context = f"Conversation History:\n{context_text}\n\nWiki Knowledge:\n{wiki_context}"
+                combined_context = f"Conversation History:\n{context_text}\n\n"
 
-                # Process via LLM (Filter -> Extract)
-                # Run in a separate thread to avoid blocking the event loop
-                result = await asyncio.to_thread(
-                    self.processor.process_pipeline, raw_text, context=combined_context
+                # --- New RAG Flow ---
+                # a. Filter transcript first to get cleaned text
+                filter_result = await asyncio.to_thread(
+                    self.processor.filter_transcript, raw_text, context=combined_context
+                )
+
+                # b. Use filtered text to retrieve relevant snippets from RAG
+                rag_snippets = []
+                if filter_result.filtered_text:
+                    try:
+                        snippets = await asyncio.to_thread(
+                            self.rag_manager.retrieve, filter_result.filtered_text
+                        )
+                        rag_snippets = snippets
+                    except Exception as e:
+                        logger.error(f"RAG Retrieval Error in llm_worker: {e}")
+
+                # c. Combine RAG snippets with existing combined_context
+                logger.info(f"LLM Processor (Extract): rag_snippets: {rag_snippets}")
+                rag_context_text = "\n".join([s.snippet for s in rag_snippets])
+                augmented_context = combined_context
+                if rag_context_text:
+                    augmented_context += (
+                        f"\n\nRelevant RAG Context:\n{rag_context_text}"
+                    )
+
+                # d. Extract structured data using the augmented context
+                extraction_result = await asyncio.to_thread(
+                    self.processor.extract_structured_data,
+                    filter_result.filtered_text if filter_result.filtered_text else "",
+                    context=augmented_context,
                 )
 
                 if (
-                    result.lore_updates
-                    or result.character_updates
-                    or result.significant_events
+                    extraction_result.lore_updates
+                    or extraction_result.character_updates
+                    or extraction_result.significant_events
                 ):
                     logger.info(
-                        f"LLM Worker: Proposal generated. Putting into proposal queue. (Lore: {len(result.lore_updates)}, Char: {len(result.character_updates)})"
+                        f"LLM Worker: Proposal generated. Putting into proposal queue. (Lore: {len(extraction_result.lore_updates)}, Char: {len(extraction_result.character_updates)})"
                     )
-                    await self.proposal_queue.put(result)
+                    await self.proposal_queue.put(extraction_result)
 
-                    # Trigger RAG query based on extracted entities
-                    await self._trigger_rag_queries(result)
+                    # Trigger RAG query based on extracted entities (for TUI updates)
+                    await self._trigger_rag_queries(extraction_result)
                 else:
                     logger.info("LLM Worker: No relevant game data extracted.")
 
+                # e. If the filter found contextual info, push it to the context queue
+                if filter_result.contextual_info:
+                    logger.info(
+                        f"LLM Worker: Contextual info found: {filter_result.contextual_info}"
+                    )
+                    await self.context_queue.put(
+                        ContextUpdate(
+                            query="Filter",
+                            snippet=filter_result.contextual_info,
+                            source="Transcript",
+                        )
+                    )
+
+                # f. Also push the RAG snippets used for extraction to the context queue
+                for snippet in rag_snippets:
+                    await self.context_queue.put(snippet)
+
             except Exception as e:
                 logger.error(f"LLM Worker error: {e}")