feat: initial commit
This commit is contained in:
@@ -0,0 +1,379 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
migrate_state_to_sqlite.py - 数据迁移脚本 (v5.4)
|
||||
|
||||
将 state.json 中的大数据迁移到 SQLite (index.db):
|
||||
- entities_v3 → entities 表
|
||||
- alias_index → aliases 表
|
||||
- state_changes → state_changes 表
|
||||
- structured_relationships → relationships 表
|
||||
|
||||
迁移后 state.json 只保留精简数据 (< 5KB):
|
||||
- progress
|
||||
- protagonist_state
|
||||
- strand_tracker
|
||||
- disambiguation_warnings/pending
|
||||
- project_info
|
||||
- world_settings (骨架)
|
||||
- plot_threads
|
||||
- relationships (简化版)
|
||||
- review_checkpoints
|
||||
|
||||
用法:
|
||||
python -m data_modules.migrate_state_to_sqlite --project-root "D:/wk/斗破苍穹"
|
||||
python -m data_modules.migrate_state_to_sqlite --project-root "." --dry-run
|
||||
python -m data_modules.migrate_state_to_sqlite --project-root "." --backup
|
||||
"""
|
||||
|
||||
import json
|
||||
import shutil
|
||||
from pathlib import Path
|
||||
from datetime import datetime
|
||||
from typing import Dict, Any, List
|
||||
|
||||
from .config import get_config, DataModulesConfig
|
||||
from .sql_state_manager import SQLStateManager, EntityData
|
||||
|
||||
|
||||
def migrate_state_to_sqlite(
|
||||
config: DataModulesConfig,
|
||||
dry_run: bool = False,
|
||||
backup: bool = True,
|
||||
verbose: bool = True
|
||||
) -> Dict[str, int]:
|
||||
"""
|
||||
执行迁移
|
||||
|
||||
参数:
|
||||
- config: 配置对象
|
||||
- dry_run: 只分析不实际写入
|
||||
- backup: 迁移前备份 state.json
|
||||
- verbose: 打印详细日志
|
||||
|
||||
返回: 迁移统计
|
||||
"""
|
||||
stats = {
|
||||
"entities": 0,
|
||||
"aliases": 0,
|
||||
"state_changes": 0,
|
||||
"relationships": 0,
|
||||
"skipped": 0,
|
||||
"errors": 0
|
||||
}
|
||||
|
||||
# 读取 state.json
|
||||
state_file = config.state_file
|
||||
if not state_file.exists():
|
||||
if verbose:
|
||||
print(f"❌ state.json 不存在: {state_file}")
|
||||
return stats
|
||||
|
||||
with open(state_file, 'r', encoding='utf-8') as f:
|
||||
state = json.load(f)
|
||||
|
||||
if verbose:
|
||||
file_size = state_file.stat().st_size / 1024
|
||||
print(f"📄 读取 state.json ({file_size:.1f} KB)")
|
||||
|
||||
# 备份
|
||||
if backup and not dry_run:
|
||||
backup_file = state_file.with_suffix(f".json.backup-{datetime.now().strftime('%Y%m%d_%H%M%S')}")
|
||||
shutil.copy(state_file, backup_file)
|
||||
if verbose:
|
||||
print(f"💾 已备份到: {backup_file}")
|
||||
|
||||
# 初始化 SQLStateManager
|
||||
sql_manager = SQLStateManager(config)
|
||||
|
||||
# 1. 迁移 entities_v3
|
||||
entities_v3 = state.get("entities_v3", {})
|
||||
if verbose:
|
||||
print(f"\n🔄 迁移 entities_v3...")
|
||||
|
||||
for entity_type, entities in entities_v3.items():
|
||||
if not isinstance(entities, dict):
|
||||
continue
|
||||
|
||||
for entity_id, entity_data in entities.items():
|
||||
if not isinstance(entity_data, dict):
|
||||
stats["skipped"] += 1
|
||||
continue
|
||||
|
||||
try:
|
||||
entity = EntityData(
|
||||
id=entity_id,
|
||||
type=entity_type,
|
||||
name=entity_data.get("canonical_name", entity_data.get("name", entity_id)),
|
||||
tier=entity_data.get("tier", "装饰"),
|
||||
desc=entity_data.get("desc", ""),
|
||||
current=entity_data.get("current", {}),
|
||||
aliases=[], # 别名单独处理
|
||||
first_appearance=entity_data.get("first_appearance", 0),
|
||||
last_appearance=entity_data.get("last_appearance", 0),
|
||||
is_protagonist=entity_data.get("is_protagonist", False)
|
||||
)
|
||||
|
||||
if not dry_run:
|
||||
sql_manager.upsert_entity(entity)
|
||||
stats["entities"] += 1
|
||||
|
||||
if verbose and stats["entities"] % 50 == 0:
|
||||
print(f" 已迁移 {stats['entities']} 个实体...")
|
||||
|
||||
except Exception as e:
|
||||
stats["errors"] += 1
|
||||
if verbose:
|
||||
print(f" ⚠️ 实体迁移失败 {entity_id}: {e}")
|
||||
|
||||
if verbose:
|
||||
print(f" ✅ 实体: {stats['entities']} 个")
|
||||
|
||||
# 2. 迁移 alias_index
|
||||
alias_index = state.get("alias_index", {})
|
||||
if verbose:
|
||||
print(f"\n🔄 迁移 alias_index...")
|
||||
|
||||
for alias, entries in alias_index.items():
|
||||
if not isinstance(entries, list):
|
||||
continue
|
||||
|
||||
for entry in entries:
|
||||
if not isinstance(entry, dict):
|
||||
stats["skipped"] += 1
|
||||
continue
|
||||
|
||||
entity_id = entry.get("id")
|
||||
entity_type = entry.get("type")
|
||||
if not entity_id or not entity_type:
|
||||
stats["skipped"] += 1
|
||||
continue
|
||||
|
||||
try:
|
||||
if not dry_run:
|
||||
sql_manager.register_alias(alias, entity_id, entity_type)
|
||||
stats["aliases"] += 1
|
||||
|
||||
except Exception as e:
|
||||
stats["errors"] += 1
|
||||
if verbose:
|
||||
print(f" ⚠️ 别名迁移失败 {alias}: {e}")
|
||||
|
||||
if verbose:
|
||||
print(f" ✅ 别名: {stats['aliases']} 个")
|
||||
|
||||
# 3. 迁移 state_changes
|
||||
state_changes = state.get("state_changes", [])
|
||||
if verbose:
|
||||
print(f"\n🔄 迁移 state_changes...")
|
||||
|
||||
for change in state_changes:
|
||||
if not isinstance(change, dict):
|
||||
stats["skipped"] += 1
|
||||
continue
|
||||
|
||||
try:
|
||||
entity_id = change.get("entity_id", "")
|
||||
if not entity_id:
|
||||
stats["skipped"] += 1
|
||||
continue
|
||||
|
||||
if not dry_run:
|
||||
sql_manager.record_state_change(
|
||||
entity_id=entity_id,
|
||||
field=change.get("field", ""),
|
||||
old_value=change.get("old", change.get("old_value", "")),
|
||||
new_value=change.get("new", change.get("new_value", "")),
|
||||
reason=change.get("reason", ""),
|
||||
chapter=change.get("chapter", 0)
|
||||
)
|
||||
stats["state_changes"] += 1
|
||||
|
||||
except Exception as e:
|
||||
stats["errors"] += 1
|
||||
if verbose:
|
||||
print(f" ⚠️ 状态变化迁移失败: {e}")
|
||||
|
||||
if verbose:
|
||||
print(f" ✅ 状态变化: {stats['state_changes']} 条")
|
||||
|
||||
# 4. 迁移 structured_relationships
|
||||
relationships = state.get("structured_relationships", [])
|
||||
if verbose:
|
||||
print(f"\n🔄 迁移 structured_relationships...")
|
||||
|
||||
for rel in relationships:
|
||||
if not isinstance(rel, dict):
|
||||
stats["skipped"] += 1
|
||||
continue
|
||||
|
||||
try:
|
||||
from_entity = rel.get("from", rel.get("from_entity", ""))
|
||||
to_entity = rel.get("to", rel.get("to_entity", ""))
|
||||
if not from_entity or not to_entity:
|
||||
stats["skipped"] += 1
|
||||
continue
|
||||
|
||||
if not dry_run:
|
||||
sql_manager.upsert_relationship(
|
||||
from_entity=from_entity,
|
||||
to_entity=to_entity,
|
||||
type=rel.get("type", "相识"),
|
||||
description=rel.get("description", ""),
|
||||
chapter=rel.get("chapter", 0)
|
||||
)
|
||||
stats["relationships"] += 1
|
||||
|
||||
except Exception as e:
|
||||
stats["errors"] += 1
|
||||
if verbose:
|
||||
print(f" ⚠️ 关系迁移失败: {e}")
|
||||
|
||||
if verbose:
|
||||
print(f" ✅ 关系: {stats['relationships']} 条")
|
||||
|
||||
# 5. 精简 state.json（移除已迁移字段）
|
||||
if not dry_run:
|
||||
if verbose:
|
||||
print(f"\n🔄 精简 state.json...")
|
||||
|
||||
# 保留字段
|
||||
slim_state = {
|
||||
"project_info": state.get("project_info", {}),
|
||||
"progress": state.get("progress", {}),
|
||||
"protagonist_state": state.get("protagonist_state", {}),
|
||||
"strand_tracker": state.get("strand_tracker", {}),
|
||||
"world_settings": _slim_world_settings(state.get("world_settings", {})),
|
||||
"plot_threads": state.get("plot_threads", {}),
|
||||
"relationships": _slim_relationships(state.get("relationships", {})),
|
||||
"review_checkpoints": state.get("review_checkpoints", [])[-10:], # 只保留最近10个
|
||||
"disambiguation_warnings": state.get("disambiguation_warnings", [])[-20:],
|
||||
"disambiguation_pending": state.get("disambiguation_pending", [])[-10:],
|
||||
# v5.1 引入标记
|
||||
"_migrated_to_sqlite": True,
|
||||
"_migration_timestamp": datetime.now().isoformat()
|
||||
}
|
||||
|
||||
with open(state_file, 'w', encoding='utf-8') as f:
|
||||
json.dump(slim_state, f, ensure_ascii=False, indent=2)
|
||||
|
||||
new_size = state_file.stat().st_size / 1024
|
||||
if verbose:
|
||||
print(f" ✅ 精简后: {new_size:.1f} KB")
|
||||
|
||||
# 打印统计
|
||||
if verbose:
|
||||
print(f"\n" + "=" * 50)
|
||||
print(f"📊 迁移统计:")
|
||||
print(f" 实体: {stats['entities']}")
|
||||
print(f" 别名: {stats['aliases']}")
|
||||
print(f" 状态变化: {stats['state_changes']}")
|
||||
print(f" 关系: {stats['relationships']}")
|
||||
print(f" 跳过: {stats['skipped']}")
|
||||
print(f" 错误: {stats['errors']}")
|
||||
if dry_run:
|
||||
print(f"\n⚠️ 这是 dry-run 模式，实际未写入任何数据")
|
||||
|
||||
return stats
|
||||
|
||||
|
||||
def _slim_world_settings(world_settings: Dict) -> Dict:
|
||||
"""精简 world_settings，只保留骨架"""
|
||||
if not isinstance(world_settings, dict):
|
||||
return {}
|
||||
|
||||
slim = {}
|
||||
|
||||
# power_system: 只保留等级名称
|
||||
power_system = world_settings.get("power_system", [])
|
||||
if isinstance(power_system, list):
|
||||
slim["power_system"] = [
|
||||
p.get("name") if isinstance(p, dict) else p
|
||||
for p in power_system[:20] # 最多20个等级
|
||||
]
|
||||
|
||||
# factions: 只保留名称和简述
|
||||
factions = world_settings.get("factions", [])
|
||||
if isinstance(factions, list):
|
||||
slim["factions"] = [
|
||||
{"name": f.get("name"), "type": f.get("type")}
|
||||
if isinstance(f, dict) else f
|
||||
for f in factions[:30] # 最多30个势力
|
||||
]
|
||||
|
||||
# locations: 只保留名称
|
||||
locations = world_settings.get("locations", [])
|
||||
if isinstance(locations, list):
|
||||
slim["locations"] = [
|
||||
loc.get("name") if isinstance(loc, dict) else loc
|
||||
for loc in locations[:50] # 最多50个地点
|
||||
]
|
||||
|
||||
return slim
|
||||
|
||||
|
||||
def _slim_relationships(relationships: Dict) -> Dict:
|
||||
"""精简 relationships，只保留核心关系"""
|
||||
if not isinstance(relationships, dict):
|
||||
return {}
|
||||
|
||||
# 只保留 relationships 字典本身，不做额外精简
|
||||
# 因为这个字段本身应该比较小
|
||||
return relationships
|
||||
|
||||
|
||||
def main():
|
||||
import argparse
|
||||
from .cli_output import print_success, print_error
|
||||
from .index_manager import IndexManager
|
||||
|
||||
parser = argparse.ArgumentParser(description="迁移 state.json 到 SQLite (v5.4)")
|
||||
parser.add_argument("--project-root", type=str, required=True, help="项目根目录")
|
||||
parser.add_argument("--dry-run", action="store_true", help="只分析不实际写入")
|
||||
parser.add_argument("--backup", action="store_true", default=True, help="迁移前备份")
|
||||
parser.add_argument("--no-backup", action="store_true", help="不备份")
|
||||
parser.add_argument("--quiet", action="store_true", help="安静模式")
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
# 允许传入“工作区根目录”，统一解析到真正的 book project_root（必须包含 .noma/state.json）
|
||||
from project_locator import resolve_project_root
|
||||
|
||||
resolved_root = resolve_project_root(args.project_root)
|
||||
config = DataModulesConfig.from_project_root(resolved_root)
|
||||
backup = not args.no_backup
|
||||
logger = IndexManager(config)
|
||||
tool_name = "migrate_state_to_sqlite"
|
||||
|
||||
try:
|
||||
stats = migrate_state_to_sqlite(
|
||||
config=config,
|
||||
dry_run=args.dry_run,
|
||||
backup=backup,
|
||||
verbose=False,
|
||||
)
|
||||
except Exception as exc:
|
||||
print_error("MIGRATE_FAILED", str(exc), suggestion="检查 state.json 与 index.db 权限")
|
||||
try:
|
||||
logger.log_tool_call(tool_name, False, error_code="MIGRATE_FAILED", error_message=str(exc))
|
||||
except Exception:
|
||||
pass
|
||||
raise SystemExit(1)
|
||||
|
||||
if stats.get("errors", 0) > 0:
|
||||
print_error("MIGRATE_ERRORS", "迁移出现错误", details=stats)
|
||||
try:
|
||||
logger.log_tool_call(tool_name, False, error_code="MIGRATE_ERRORS", error_message="迁移出现错误")
|
||||
except Exception:
|
||||
pass
|
||||
raise SystemExit(1)
|
||||
|
||||
print_success({"project": str(config.project_root), **stats}, message="migrated")
|
||||
try:
|
||||
logger.log_tool_call(tool_name, True)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user