fix: normalize markdown contract content
This commit is contained in:
@@ -4,6 +4,14 @@ from html.parser import HTMLParser
|
||||
from typing import Any, Mapping, Optional
|
||||
|
||||
|
||||
_MARKDOWN_CODE_FENCE_RE = re.compile(r"^\s*```[a-zA-Z0-9_-]*\s*$")
|
||||
_MARKDOWN_HEADING_RE = re.compile(r"^\s{0,3}#{1,6}\s*")
|
||||
_MARKDOWN_TRAILING_HEADING_RE = re.compile(r"\s+#{1,6}\s*$")
|
||||
_MARKDOWN_BULLET_RE = re.compile(r"^\s*[-*+]\s+")
|
||||
_MARKDOWN_BLOCKQUOTE_RE = re.compile(r"^\s*>\s?")
|
||||
_MARKDOWN_TABLE_SEPARATOR_RE = re.compile(r"^\s*\|?\s*:?-{3,}:?\s*(\|\s*:?-{3,}:?\s*)+\|?\s*$")
|
||||
|
||||
|
||||
class ContractHTMLTextExtractor(HTMLParser):
|
||||
_BLOCK_TAGS = {
|
||||
"address",
|
||||
@@ -98,9 +106,9 @@ def normalize_contract_content(value: Any) -> str:
|
||||
|
||||
if looks_like_html(text):
|
||||
stripped = html_to_text(text)
|
||||
return stripped or text
|
||||
return markdown_to_text(stripped or text)
|
||||
|
||||
return text
|
||||
return markdown_to_text(text)
|
||||
|
||||
|
||||
def parse_contract_json_text(value: str) -> Optional[Mapping[str, Any]]:
|
||||
@@ -127,6 +135,28 @@ def html_to_text(value: str) -> str:
|
||||
return parser.get_text()
|
||||
|
||||
|
||||
def markdown_to_text(value: str) -> str:
|
||||
lines: list[str] = []
|
||||
for raw_line in str(value or "").splitlines():
|
||||
line = raw_line.strip()
|
||||
if not line:
|
||||
lines.append("")
|
||||
continue
|
||||
if _MARKDOWN_CODE_FENCE_RE.match(line) or _MARKDOWN_TABLE_SEPARATOR_RE.match(line):
|
||||
continue
|
||||
line = _MARKDOWN_BLOCKQUOTE_RE.sub("", line)
|
||||
line = _MARKDOWN_HEADING_RE.sub("", line)
|
||||
line = _MARKDOWN_TRAILING_HEADING_RE.sub("", line)
|
||||
line = _MARKDOWN_BULLET_RE.sub("", line)
|
||||
line = re.sub(r"(\*\*|__)(.*?)\1", r"\2", line)
|
||||
line = re.sub(r"`([^`]+)`", r"\1", line)
|
||||
line = line.replace("\\#", "#").strip()
|
||||
lines.append(line)
|
||||
text = "\n".join(lines)
|
||||
text = re.sub(r"\n{3,}", "\n\n", text)
|
||||
return text.strip()
|
||||
|
||||
|
||||
def looks_like_html(value: str) -> bool:
|
||||
return bool(re.search(r"</?[a-zA-Z][^>]*>", str(value or "")))
|
||||
|
||||
|
||||
@@ -244,6 +244,7 @@ class ContractGenerationService:
|
||||
"prompt_version": prompt_runtime_trace.get("prompt_version"),
|
||||
}
|
||||
|
||||
generated_content = normalize_contract_content(generated_content)
|
||||
timestamp = datetime.now().strftime('%Y%m%d_%H%M%S')
|
||||
file_content, file_name = self.export_service.export_contract(
|
||||
generated_content,
|
||||
@@ -3613,7 +3614,12 @@ class ContractGenerationService:
|
||||
related_legal_articles: List[Dict[str, Any]],
|
||||
ai_hints: List[Dict[str, Any]],
|
||||
) -> str:
|
||||
body = "".join(f"<p>{escape(part.strip())}</p>" for part in str(content or "").splitlines() if part.strip())
|
||||
paragraphs = [part.strip() for part in normalize_contract_content(content).splitlines() if part.strip()]
|
||||
body_parts = []
|
||||
for part in paragraphs:
|
||||
tag = "h2" if cls._looks_like_contract_heading(part) else "p"
|
||||
body_parts.append(f"<{tag}>{escape(part)}</{tag}>")
|
||||
body = "".join(body_parts)
|
||||
if not body:
|
||||
body = "<p></p>"
|
||||
hint_items = "".join(f"<li>{escape(str(item.get('message') or ''))}</li>" for item in ai_hints)
|
||||
@@ -3634,6 +3640,18 @@ class ContractGenerationService:
|
||||
"</article>"
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _looks_like_contract_heading(value: str) -> bool:
|
||||
text = str(value or "").strip()
|
||||
if not text:
|
||||
return False
|
||||
return bool(
|
||||
re.match(
|
||||
r"^(?:第[一二三四五六七八九十百千万\d]+[编章节条款]|[一二三四五六七八九十]+[、..]|[((][一二三四五六七八九十\d]+[))]|\d+[、..])",
|
||||
text,
|
||||
)
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def _build_alias_values(cls, flattened: Dict[str, Any]) -> Dict[str, Any]:
|
||||
party_a = cls._first(flattened, ["party_a.name", "buyer", "buyer_name", "employer.name", "landlord.name"])
|
||||
|
||||
@@ -0,0 +1,49 @@
|
||||
import unittest
|
||||
|
||||
from app.services.contract_content_normalizer import normalize_contract_content
|
||||
from app.services.contract_generation_service import ContractGenerationService
|
||||
|
||||
|
||||
class ContractContentNormalizerTestCase(unittest.TestCase):
|
||||
def test_normalize_contract_content_removes_markdown_headings(self):
|
||||
raw_content = """# 劳动合同
|
||||
|
||||
### 一、合同期限与试用期
|
||||
双方确认合同期限为三年。
|
||||
|
||||
### 六、保密、服务期与竞业限制
|
||||
**乙方**应履行保密义务。
|
||||
"""
|
||||
|
||||
normalized = normalize_contract_content(raw_content)
|
||||
|
||||
self.assertEqual(
|
||||
normalized,
|
||||
"劳动合同\n\n一、合同期限与试用期\n双方确认合同期限为三年。\n\n六、保密、服务期与竞业限制\n乙方应履行保密义务。",
|
||||
)
|
||||
self.assertNotIn("###", normalized)
|
||||
self.assertNotIn("**", normalized)
|
||||
|
||||
def test_normalize_contract_content_removes_markdown_inside_html_payload(self):
|
||||
raw_content = "<article><h1># 劳动合同</h1><p>### 一、合同期限与试用期</p></article>"
|
||||
|
||||
normalized = normalize_contract_content(raw_content)
|
||||
|
||||
self.assertEqual(normalized, "劳动合同\n一、合同期限与试用期")
|
||||
self.assertNotIn("#", normalized)
|
||||
|
||||
def test_generated_contract_html_renders_clean_contract_headings(self):
|
||||
html = ContractGenerationService._build_generated_contract_html(
|
||||
"### 一、合同期限与试用期\n双方确认合同期限为三年。\n### 六、保密、服务期与竞业限制",
|
||||
missing_fields=[],
|
||||
related_legal_articles=[],
|
||||
ai_hints=[],
|
||||
)
|
||||
|
||||
self.assertIn("<h2>一、合同期限与试用期</h2>", html)
|
||||
self.assertIn("<h2>六、保密、服务期与竞业限制</h2>", html)
|
||||
self.assertNotIn("###", html)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user