Blocked HTML
This commit is contained in:
马一丁
@@ -0,0 +1,209 @@
|
||||
"""
|
||||
章节JSON的落盘与清单管理。
|
||||
|
||||
每一章在流式生成时会立即写入raw文件，完成校验后再写入
|
||||
格式化的chapter.json，并在manifest中记录元数据，便于后续装订。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from contextlib import contextmanager
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Dict, Generator, List, Optional
|
||||
|
||||
|
||||
@dataclass
|
||||
class ChapterRecord:
|
||||
"""manifest中记录的章节元数据"""
|
||||
|
||||
chapter_id: str
|
||||
slug: str
|
||||
title: str
|
||||
order: int
|
||||
status: str
|
||||
files: Dict[str, str] = field(default_factory=dict)
|
||||
errors: List[str] = field(default_factory=list)
|
||||
updated_at: str = field(default_factory=lambda: datetime.utcnow().isoformat() + "Z")
|
||||
|
||||
def to_dict(self) -> Dict[str, object]:
|
||||
return {
|
||||
"chapterId": self.chapter_id,
|
||||
"slug": self.slug,
|
||||
"title": self.title,
|
||||
"order": self.order,
|
||||
"status": self.status,
|
||||
"files": self.files,
|
||||
"errors": self.errors,
|
||||
"updatedAt": self.updated_at,
|
||||
}
|
||||
|
||||
|
||||
class ChapterStorage:
|
||||
"""
|
||||
章节JSON写入与manifest管理器。
|
||||
|
||||
用法：
|
||||
run_dir = storage.start_session(report_id, {...})
|
||||
chapter_dir = storage.begin_chapter(run_dir, meta)
|
||||
with storage.capture_stream(chapter_dir) as fp:
|
||||
fp.write(chunk)
|
||||
storage.persist_chapter(run_dir, meta, payload, errors)
|
||||
"""
|
||||
|
||||
def __init__(self, base_dir: str):
|
||||
self.base_dir = Path(base_dir)
|
||||
self.base_dir.mkdir(parents=True, exist_ok=True)
|
||||
self._manifests: Dict[str, Dict[str, object]] = {}
|
||||
|
||||
# ======== 会话 & manifest ========
|
||||
|
||||
def start_session(self, report_id: str, metadata: Dict[str, object]) -> Path:
|
||||
"""为本次报告创建独立的章节输出目录与manifest"""
|
||||
run_dir = self.base_dir / report_id
|
||||
run_dir.mkdir(parents=True, exist_ok=True)
|
||||
manifest = {
|
||||
"reportId": report_id,
|
||||
"createdAt": datetime.utcnow().isoformat() + "Z",
|
||||
"metadata": metadata,
|
||||
"chapters": [],
|
||||
}
|
||||
self._manifests[self._key(run_dir)] = manifest
|
||||
self._write_manifest(run_dir, manifest)
|
||||
return run_dir
|
||||
|
||||
def begin_chapter(self, run_dir: Path, chapter_meta: Dict[str, object]) -> Path:
|
||||
"""创建章节子目录并在manifest中标记为streaming状态"""
|
||||
slug_value = str(
|
||||
chapter_meta.get("slug") or chapter_meta.get("chapterId") or "section"
|
||||
)
|
||||
chapter_dir = self._chapter_dir(
|
||||
run_dir,
|
||||
slug_value,
|
||||
int(chapter_meta.get("order", 0)),
|
||||
)
|
||||
record = ChapterRecord(
|
||||
chapter_id=str(chapter_meta.get("chapterId")),
|
||||
slug=slug_value,
|
||||
title=str(chapter_meta.get("title")),
|
||||
order=int(chapter_meta.get("order", 0)),
|
||||
status="streaming",
|
||||
files={"raw": str(self._raw_stream_path(chapter_dir).relative_to(run_dir))},
|
||||
)
|
||||
self._upsert_record(run_dir, record)
|
||||
return chapter_dir
|
||||
|
||||
def persist_chapter(
|
||||
self,
|
||||
run_dir: Path,
|
||||
chapter_meta: Dict[str, object],
|
||||
payload: Dict[str, object],
|
||||
errors: Optional[List[str]] = None,
|
||||
) -> Path:
|
||||
"""章节流式生成完毕后写入最终JSON并更新manifest状态"""
|
||||
slug_value = str(
|
||||
chapter_meta.get("slug") or chapter_meta.get("chapterId") or "section"
|
||||
)
|
||||
chapter_dir = self._chapter_dir(
|
||||
run_dir,
|
||||
slug_value,
|
||||
int(chapter_meta.get("order", 0)),
|
||||
)
|
||||
final_path = chapter_dir / "chapter.json"
|
||||
final_path.write_text(
|
||||
json.dumps(payload, ensure_ascii=False, indent=2),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
record = ChapterRecord(
|
||||
chapter_id=str(chapter_meta.get("chapterId")),
|
||||
slug=slug_value,
|
||||
title=str(chapter_meta.get("title")),
|
||||
order=int(chapter_meta.get("order", 0)),
|
||||
status="ready" if not errors else "invalid",
|
||||
files={
|
||||
"raw": str(self._raw_stream_path(chapter_dir).relative_to(run_dir)),
|
||||
"json": str(final_path.relative_to(run_dir)),
|
||||
},
|
||||
errors=errors or [],
|
||||
)
|
||||
self._upsert_record(run_dir, record)
|
||||
return final_path
|
||||
|
||||
def load_chapters(self, run_dir: Path) -> List[Dict[str, object]]:
|
||||
payloads: List[Dict[str, object]] = []
|
||||
for child in sorted(run_dir.iterdir()):
|
||||
if not child.is_dir():
|
||||
continue
|
||||
chapter_path = child / "chapter.json"
|
||||
if not chapter_path.exists():
|
||||
continue
|
||||
try:
|
||||
payload = json.loads(chapter_path.read_text(encoding="utf-8"))
|
||||
payloads.append(payload)
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
payloads.sort(key=lambda x: x.get("order", 0))
|
||||
return payloads
|
||||
|
||||
# ======== 文件操作 ========
|
||||
|
||||
@contextmanager
|
||||
def capture_stream(self, chapter_dir: Path) -> Generator:
|
||||
"""将流式输出实时写入raw文件"""
|
||||
raw_path = self._raw_stream_path(chapter_dir)
|
||||
raw_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with raw_path.open("w", encoding="utf-8") as fp:
|
||||
yield fp
|
||||
|
||||
# ======== 内部工具 ========
|
||||
|
||||
def _chapter_dir(self, run_dir: Path, slug: str, order: int) -> Path:
|
||||
safe_slug = self._safe_slug(slug)
|
||||
folder = f"{order:03d}-{safe_slug}"
|
||||
path = run_dir / folder
|
||||
path.mkdir(parents=True, exist_ok=True)
|
||||
return path
|
||||
|
||||
def _safe_slug(self, slug: str) -> str:
|
||||
slug = slug.replace(" ", "-").replace("/", "-")
|
||||
return slug or "section"
|
||||
|
||||
def _raw_stream_path(self, chapter_dir: Path) -> Path:
|
||||
return chapter_dir / "stream.raw"
|
||||
|
||||
def _key(self, run_dir: Path) -> str:
|
||||
return str(run_dir.resolve())
|
||||
|
||||
def _manifest_path(self, run_dir: Path) -> Path:
|
||||
return run_dir / "manifest.json"
|
||||
|
||||
def _write_manifest(self, run_dir: Path, manifest: Dict[str, object]):
|
||||
self._manifest_path(run_dir).write_text(
|
||||
json.dumps(manifest, ensure_ascii=False, indent=2),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
def _read_manifest(self, run_dir: Path) -> Dict[str, object]:
|
||||
manifest_path = self._manifest_path(run_dir)
|
||||
if manifest_path.exists():
|
||||
return json.loads(manifest_path.read_text(encoding="utf-8"))
|
||||
return {"reportId": run_dir.name, "chapters": []}
|
||||
|
||||
def _upsert_record(self, run_dir: Path, record: ChapterRecord):
|
||||
"""更新或追加manifest中的章节记录，保证顺序一致"""
|
||||
key = self._key(run_dir)
|
||||
manifest = self._manifests.get(key) or self._read_manifest(run_dir)
|
||||
chapters: List[Dict[str, object]] = manifest.get("chapters", [])
|
||||
chapters = [c for c in chapters if c.get("chapterId") != record.chapter_id]
|
||||
chapters.append(record.to_dict())
|
||||
chapters.sort(key=lambda x: x.get("order", 0))
|
||||
manifest["chapters"] = chapters
|
||||
manifest.setdefault("updatedAt", datetime.utcnow().isoformat() + "Z")
|
||||
self._manifests[key] = manifest
|
||||
self._write_manifest(run_dir, manifest)
|
||||
|
||||
|
||||
__all__ = ["ChapterStorage", "ChapterRecord"]
|
||||
Reference in New Issue
Block a user