feat: initial commit

noma/core_engine/memory_rag/context_cache.py

@@ -0,0 +1,110 @@
+"""
+Context Cache (上下文静态缓存与Hash脏标记更新)
+NovelMaster Core Engine - Memory RAG
+"""
+
+import hashlib
+import json
+from dataclasses import dataclass, field
+from typing import Any, Dict, List, Optional, Set
+from datetime import datetime
+
+
+@dataclass
+class CacheEntry:
+    key: str
+    value: Any
+    hash: str
+    created_at: datetime
+    last_accessed: datetime
+    access_count: int = 0
+    is_dirty: bool = False
+
+    def update_hash(self) -> str:
+        content = json.dumps(self.value, sort_keys=True, ensure_ascii=False)
+        self.hash = hashlib.sha256(content.encode('utf-8')).hexdigest()
+        self.is_dirty = False
+        return self.hash
+
+
+@dataclass
+class ContextCache:
+    entries: Dict[str, CacheEntry] = field(default_factory=dict)
+    max_size: int = 1000
+    dirty_keys: Set[str] = field(default_factory=set)
+
+    def get(self, key: str) -> Optional[Any]:
+        if key not in self.entries:
+            return None
+        entry = self.entries[key]
+        entry.last_accessed = datetime.now()
+        entry.access_count += 1
+        return entry.value
+
+    def set(self, key: str, value: Any) -> None:
+        content = json.dumps(value, sort_keys=True, ensure_ascii=False)
+        hash_val = hashlib.sha256(content.encode('utf-8')).hexdigest()
+        self.entries[key] = CacheEntry(
+            key=key,
+            value=value,
+            hash=hash_val,
+            created_at=datetime.now(),
+            last_accessed=datetime.now()
+        )
+        if len(self.entries) > self.max_size:
+            self._evict_lru()
+
+    def invalidate(self, key: str) -> None:
+        if key in self.entries:
+            self.entries[key].is_dirty = True
+            self.dirty_keys.add(key)
+
+    def invalidate_pattern(self, pattern: str) -> None:
+        for key in self.entries:
+            if pattern in key:
+                self.invalidate(key)
+
+    def get_dirty_keys(self) -> Set[str]:
+        return self.dirty_keys.copy()
+
+    def clear_dirty(self, key: str) -> None:
+        self.dirty_keys.discard(key)
+        if key in self.entries:
+            self.entries[key].is_dirty = False
+
+    def _evict_lru(self) -> None:
+        if not self.entries:
+            return
+        sorted_entries = sorted(
+            self.entries.items(),
+            key=lambda x: (x[1].access_count, x[1].last_accessed)
+        )
+        evict_count = max(1, len(sorted_entries) // 10)
+        for i in range(evict_count):
+            del self.entries[sorted_entries[i][0]]
+
+
+class RetconHashListener:
+    def __init__(self, cache: ContextCache):
+        self.cache = cache
+        self.subscribers: Dict[str, List[callable]] = {}
+
+    def subscribe(self, pattern: str, callback: callable) -> None:
+        if pattern not in self.subscribers:
+            self.subscribers[pattern] = []
+        self.subscribers[pattern].append(callback)
+
+    def notify(self, changed_keys: Set[str]) -> None:
+        for key in changed_keys:
+            for pattern, callbacks in self.subscribers.items():
+                if pattern in key:
+                    for callback in callbacks:
+                        callback(key)
+
+    def on_retcon(self, retcon_patch: dict) -> None:
+        affected_elements = retcon_patch.get('affected_elements', [])
+        for element in affected_elements:
+            element_name = element.get('name', '')
+            self.cache.invalidate_pattern(element_name)
+        dirty_keys = self.cache.get_dirty_keys()
+        self.notify(dirty_keys)

noma/core_engine/memory_rag/vectorstore_utils.py

@@ -0,0 +1,49 @@
+"""
+Vector Store Utilities
+NovelMaster Core Engine - Memory RAG
+"""
+
+from typing import List, Dict, Any, Optional
+import hashlib
+import json
+
+
+class VectorStoreUtils:
+    """向量存储工具类"""
+
+    def __init__(self, embedding_model: str = "default"):
+        self.embedding_model = embedding_model
+        self.collection_name = "novelmaster_context"
+        self.dimension = 1536  # 默认维度
+
+    def compute_hash(self, content: str) -> str:
+        """计算内容的哈希值"""
+        return hashlib.sha256(content.encode('utf-8')).hexdigest()
+
+    def chunk_text(self, text: str, chunk_size: int = 1000, overlap: int = 200) -> List[str]:
+        """将文本分块"""
+        chunks = []
+        for i in range(0, len(text), chunk_size - overlap):
+            chunks.append(text[i:i + chunk_size])
+        return chunks
+
+    def prepare_for_storage(self, text: str, metadata: Dict[str, Any]) -> Dict[str, Any]:
+        """准备存储格式"""
+        return {
+            "text": text,
+            "hash": self.compute_hash(text),
+            "metadata": metadata,
+            "model": self.embedding_model
+        }
+
+    def query_similar(self, query: str, top_k: int = 5, filters: Optional[Dict] = None) -> List[Dict]:
+        """查询相似内容"""
+        return []  # 待实现
+
+    def upsert(self, documents: List[Dict[str, Any]]) -> bool:
+        """插入或更新文档"""
+        return True
+
+    def delete_by_hash(self, hash: str) -> bool:
+        """通过哈希删除"""
+        return True