""" 章节级JSON生成节点。 每个章节依据Markdown模板切片独立调用LLM，流式写入Raw文件， 完成后校验并落盘标准化JSON。该节点只负责“拿到合规章节”。 """ from __future__ import annotations import json from pathlib import Path import re from typing import Any, Dict, List, Tuple from loguru import logger from ..core import TemplateSection, ChapterStorage from ..ir import ALLOWED_BLOCK_TYPES, IRValidator from ..prompts import ( SYSTEM_PROMPT_CHAPTER_JSON, build_chapter_user_prompt, ) from .base_node import BaseNode try: from json_repair import repair_json as _json_repair_fn except ImportError: # pragma: no cover - optional dependency _json_repair_fn = None class ChapterGenerationNode(BaseNode): """负责按章节调用LLM并校验JSON结构""" _COLON_EQUALS_PATTERN = re.compile(r'(":\s*)=') def __init__(self, llm_client, validator: IRValidator, storage: ChapterStorage): super().__init__(llm_client, "ChapterGenerationNode") self.validator = validator self.storage = storage def run( self, section: TemplateSection, context: Dict[str, Any], run_dir: Path, **kwargs, ) -> Dict[str, Any]: """针对单个章节调用LLM，校验/落盘章节JSON并返回结构化结果""" chapter_meta = { "chapterId": section.chapter_id, "slug": section.slug, "title": section.title, "order": section.order, } chapter_dir = self.storage.begin_chapter(run_dir, chapter_meta) llm_payload = self._build_payload(section, context) user_message = build_chapter_user_prompt(llm_payload) raw_text = self._stream_llm(user_message, chapter_dir, **kwargs) chapter_json = self._parse_chapter(raw_text) # 自动补全关键字段后再校验 chapter_json.setdefault("chapterId", section.chapter_id) chapter_json.setdefault("anchor", section.slug) chapter_json.setdefault("title", section.title) chapter_json.setdefault("order", section.order) self._sanitize_chapter_blocks(chapter_json) valid, errors = self.validator.validate_chapter(chapter_json) self.storage.persist_chapter( run_dir, chapter_meta, chapter_json, errors=None if valid else errors, ) if not valid: raise ValueError( f"{section.title} 章节JSON校验失败: {'; '.join(errors[:5])}" ) return chapter_json # ====== 内部方法 ====== def _build_payload(self, section: TemplateSection, context: Dict[str, Any]) -> Dict[str, Any]: """构造LLM输入payload""" reports = context.get("reports", {}) # 章节篇幅规划（来自WordBudgetNode），用于指导字数与强调点 chapter_plan_map = context.get("chapter_directives", {}) chapter_plan = chapter_plan_map.get(section.chapter_id) if chapter_plan_map else {} payload = { "section": { "chapterId": section.chapter_id, "title": section.title, "slug": section.slug, "order": section.order, "number": section.number, "outline": section.outline, }, "globalContext": { "query": context.get("query"), "templateName": context.get("template_name"), "themeTokens": context.get("theme_tokens", {}), "styleDirectives": context.get("style_directives", {}), # layout里包含标题/目录/hero等信息，方便章节保持统一视觉调性 "layout": context.get("layout"), "templateOverview": context.get("template_overview", {}), }, "reports": { "query_engine": reports.get("query_engine", ""), "media_engine": reports.get("media_engine", ""), "insight_engine": reports.get("insight_engine", ""), }, "forumLogs": context.get("forum_logs", ""), "dataBundles": context.get("data_bundles", []), "constraints": { "language": "zh-CN", "maxTokens": context.get("max_tokens", 4096), "allowedBlocks": ALLOWED_BLOCK_TYPES, "styleHints": { "expectWidgets": True, "forceHeadingAnchors": True, "allowInlineMix": True, }, }, "chapterPlan": chapter_plan, "wordPlan": context.get("word_plan"), } if chapter_plan: constraints = payload["constraints"] if chapter_plan.get("targetWords"): constraints["wordTarget"] = chapter_plan["targetWords"] if chapter_plan.get("minWords"): constraints["minWords"] = chapter_plan["minWords"] if chapter_plan.get("maxWords"): constraints["maxWords"] = chapter_plan["maxWords"] if chapter_plan.get("emphasis"): constraints["emphasis"] = chapter_plan["emphasis"] if chapter_plan.get("sections"): constraints["sectionBudgets"] = chapter_plan["sections"] payload["globalContext"]["sectionBudgets"] = chapter_plan["sections"] return payload def _stream_llm(self, user_message: str, chapter_dir: Path, **kwargs) -> str: """流式调用LLM并实时写入raw文件""" chunks: List[str] = [] with self.storage.capture_stream(chapter_dir) as stream_fp: stream = self.llm_client.stream_invoke( SYSTEM_PROMPT_CHAPTER_JSON, user_message, temperature=kwargs.get("temperature", 0.2), top_p=kwargs.get("top_p", 0.95), ) for delta in stream: stream_fp.write(delta) chunks.append(delta) return "".join(chunks) def _parse_chapter(self, raw_text: str) -> Dict[str, Any]: """清洗LLM输出并解析JSON""" cleaned = raw_text.strip() if cleaned.startswith("```json"): cleaned = cleaned[7:] if cleaned.startswith("```"): cleaned = cleaned[3:] if cleaned.endswith("```"): cleaned = cleaned[:-3] cleaned = cleaned.strip() if not cleaned: raise ValueError("LLM返回空内容") candidate_payloads = [cleaned] repaired = self._repair_llm_json(cleaned) if repaired != cleaned: candidate_payloads.append(repaired) try: data = self._parse_with_candidates(candidate_payloads) except json.JSONDecodeError as exc: repaired_payload = self._attempt_json_repair(cleaned) if repaired_payload: candidate_payloads.append(repaired_payload) try: data = self._parse_with_candidates(candidate_payloads[-1:]) except json.JSONDecodeError as inner_exc: raise ValueError(f"章节JSON解析失败: {inner_exc}") from inner_exc else: raise ValueError(f"章节JSON解析失败: {exc}") from exc if "chapter" in data and isinstance(data["chapter"], dict): return data["chapter"] if isinstance(data, dict) and all( key in data for key in ("chapterId", "title", "blocks") ): return data if isinstance(data, list): for item in data: if isinstance(item, dict): if "chapter" in item and isinstance(item["chapter"], dict): return item["chapter"] if all(key in item for key in ("chapterId", "title", "blocks")): return item raise ValueError("章节JSON缺少chapter字段") def _repair_llm_json(self, text: str) -> str: """处理常见的LLM错误（如\":=导致的非法JSON）""" repaired = text mutated = False new_text = self._COLON_EQUALS_PATTERN.sub(r"\1", repaired) if new_text != repaired: logger.warning("检测到章节JSON中的\":=\"字符，已自动移除多余的'='号") repaired = new_text mutated = True repaired, escaped = self._escape_in_string_controls(repaired) if escaped: logger.warning("检测到章节JSON字符串中存在未转义的控制字符，已自动转换为转义序列") mutated = True repaired, balanced = self._balance_brackets(repaired) if balanced: logger.warning("检测到章节JSON括号不平衡，已自动补齐/剔除异常括号") mutated = True repaired, commas_fixed = self._fix_missing_commas(repaired) if commas_fixed: logger.warning("检测到章节JSON对象/数组之间缺少逗号，已自动补齐") mutated = True return repaired if mutated else text def _escape_in_string_controls(self, text: str) -> Tuple[str, bool]: """ 将字符串字面量中的裸换行/制表符/控制字符替换为JSON合法的转义序列。 """ if not text: return text, False result: List[str] = [] in_string = False escaped = False mutated = False control_map = {"\n": "\\n", "\r": "\\n", "\t": "\\t"} for ch in text: if escaped: result.append(ch) escaped = False continue if ch == "\\": result.append(ch) escaped = True continue if ch == '"': result.append(ch) in_string = not in_string continue if in_string and ch in control_map: result.append(control_map[ch]) mutated = True continue if in_string and ord(ch) < 0x20: result.append(f"\\u{ord(ch):04x}") mutated = True continue result.append(ch) return "".join(result), mutated def _fix_missing_commas(self, text: str) -> Tuple[str, bool]: """在对象/数组连续出现时自动补逗号""" if not text: return text, False chars: List[str] = [] mutated = False in_string = False escaped = False length = len(text) i = 0 while i < length: ch = text[i] chars.append(ch) if escaped: escaped = False i += 1 continue if ch == "\\": escaped = True i += 1 continue if ch == '"': in_string = not in_string i += 1 continue if not in_string and ch in "}]": j = i + 1 while j < length and text[j] in " \t\r\n": j += 1 if j < length: next_ch = text[j] if next_ch in "{[": chars.append(",") mutated = True i += 1 return "".join(chars), mutated def _balance_brackets(self, text: str) -> Tuple[str, bool]: """尝试修复因LLM多写/少写括号导致的不平衡结构""" if not text: return text, False result: List[str] = [] stack: List[str] = [] mutated = False in_string = False escaped = False opener_map = {"{": "}", "[": "]"} for ch in text: if escaped: result.append(ch) escaped = False continue if ch == "\\": result.append(ch) escaped = True continue if ch == '"': result.append(ch) in_string = not in_string continue if in_string: result.append(ch) continue if ch in "{[": stack.append(ch) result.append(ch) continue if ch in "}]": if stack and ((ch == "}" and stack[-1] == "{") or (ch == "]" and stack[-1] == "[")): stack.pop() result.append(ch) else: mutated = True continue result.append(ch) while stack: opener = stack.pop() result.append(opener_map[opener]) mutated = True return "".join(result), mutated def _attempt_json_repair(self, text: str) -> str | None: """使用可选的json_repair库进一步修复复杂语法错误""" if not _json_repair_fn: return None try: fixed = _json_repair_fn(text) except Exception as exc: # pragma: no cover - library failure logger.warning(f"json_repair 修复章节JSON失败: {exc}") return None if fixed == text: return None logger.warning("已使用json_repair自动修复章节JSON语法") return fixed def _sanitize_chapter_blocks(self, chapter: Dict[str, Any]): """修正常见的结构性错误（例如list.items嵌套过深）""" def walk(blocks: List[Dict[str, Any]] | None): if not isinstance(blocks, list): return for block in blocks: if not isinstance(block, dict): continue self._ensure_block_type(block) block_type = block.get("type") if block_type == "list": items = block.get("items") normalized = self._normalize_list_items(items) if normalized: block["items"] = normalized for entry in block.get("items", []): walk(entry) elif block_type in {"callout", "blockquote"}: walk(block.get("blocks")) elif block_type == "table": for row in block.get("rows", []): cells = row.get("cells") or [] for cell in cells: walk(cell.get("blocks")) elif block_type == "widget": self._normalize_widget_block(block) else: nested = block.get("blocks") if isinstance(nested, list): walk(nested) walk(chapter.get("blocks")) def _normalize_list_items(self, items: Any) -> List[List[Dict[str, Any]]]: """确保list block的items为[[block, block], ...]结构""" if not isinstance(items, list): return [] normalized: List[List[Dict[str, Any]]] = [] for item in items: normalized.extend(self._coerce_list_item(item)) return [entry for entry in normalized if entry] def _coerce_list_item(self, item: Any) -> List[List[Dict[str, Any]]]: """将各种嵌套写法统一折算为区块数组""" result: List[List[Dict[str, Any]]] = [] if isinstance(item, dict): self._ensure_block_type(item) result.append([item]) return result if isinstance(item, list): dicts = [elem for elem in item if isinstance(elem, dict)] if dicts: for elem in dicts: self._ensure_block_type(elem) result.append(dicts) for elem in item: if isinstance(elem, list): result.extend(self._coerce_list_item(elem)) elif isinstance(elem, dict): continue elif isinstance(elem, str): result.append([self._as_paragraph_block(elem)]) elif isinstance(elem, (int, float)): result.append([self._as_paragraph_block(str(elem))]) elif isinstance(item, str): result.append([self._as_paragraph_block(item)]) elif isinstance(item, (int, float)): result.append([self._as_paragraph_block(str(item))]) return result def _normalize_widget_block(self, block: Dict[str, Any]): """确保widget具备顶层data或dataRef""" has_data = block.get("data") is not None or block.get("dataRef") is not None if has_data: return props = block.get("props") if isinstance(props, dict) and "data" in props: block["data"] = props.pop("data") return block["data"] = {"labels": [], "datasets": []} def _ensure_block_type(self, block: Dict[str, Any]): """若block缺少合法type，则降级为paragraph""" block_type = block.get("type") if isinstance(block_type, str) and block_type in ALLOWED_BLOCK_TYPES: return text = "" for key in ("text", "content", "title"): value = block.get(key) if isinstance(value, str) and value.strip(): text = value.strip() break if not text: try: text = json.dumps(block, ensure_ascii=False) except Exception: text = str(block) block.clear() block["type"] = "paragraph" block["inlines"] = [{"text": text}] @staticmethod def _as_paragraph_block(text: str) -> Dict[str, Any]: return { "type": "paragraph", "inlines": [{"text": text or ""}], } @staticmethod def _parse_with_candidates(payloads: List[str]) -> Dict[str, Any]: """按顺序尝试多个payload，直到解析成功""" last_exc: json.JSONDecodeError | None = None for payload in payloads: try: return json.loads(payload) except json.JSONDecodeError as exc: last_exc = exc assert last_exc is not None raise last_exc __all__ = ["ChapterGenerationNode"]