fix: 修复视频模型价格爬取 - 价格数字与单位被拆行导致解析失败 - 规格标签在独立HTML元素中被丢失

backend/crawl/scrape_aliyun_models.py

@@ -288,9 +288,33 @@ def parse_prices_from_text(text: str) -> List[Dict]:
     lines = [ln.strip() for ln in text.splitlines()]
     lines = [ln for ln in lines if ln]
 
+    # 百炼页面价格数字和"元"常在不同 HTML 元素中，被 \n 拆成两行
+    # 如 "0.9" + "元/每秒" → 合并为 "0.9元/每秒"
+    joined: list[str] = []
+    _i = 0
+    while _i < len(lines):
+        ln = lines[_i]
+        if (re.search(r"\d[\d.]*$", ln)
+                and _i + 1 < len(lines)
+                and re.match(r"元", lines[_i + 1])):
+            ln = ln + lines[_i + 1]
+            _i += 2
+        else:
+            _i += 1
+        joined.append(ln)
+    lines = joined
+
     items = []
     price_re = re.compile(r"([0-9]+(?:\.[0-9]+)?)\s*元", re.I)
+    # 视频/图像规格标签模式，用于跨行关联（百炼页面规格和价格常在不同行）
+    _spec_re = re.compile(r"[（(]\s*\d+[PpKk]\s*[）)]")
+    _prev_spec_label: str | None = None
+
     for idx, ln in enumerate(lines):
+        # 记录最近的规格行（如"视频生成（720P）"），即使本行没有价格
+        if _spec_re.search(ln):
+            _prev_spec_label = ln
+
         matches = price_re.findall(ln)
         if not matches:
             continue
@@ -306,6 +330,9 @@ def parse_prices_from_text(text: str) -> List[Dict]:
                 if lines[j] and not price_re.search(lines[j]):
                     label = lines[j]
                     break
+        # 如果 label 不含视频规格，但前面有规格行，用规格行的文本作为 label
+        if label and not _spec_re.search(label) and _prev_spec_label:
+            label = _prev_spec_label
         if not label:
             label = f"price_{len(items) + 1}"
 
@@ -430,6 +457,30 @@ def extract_price_items_from_html(html: str) -> List[Dict]:
     container_text = container.get_text(separator="\n")
     items = parse_prices_from_text(container_text)
 
+    # 如果解析出的条目 label 太泛（如 "price"），从更广的 HTML 范围搜索规格标签来填充
+    _spec_re = re.compile(r"[（(]\s*\d+[PpKk]\s*[）)]")
+    if items and any(not _spec_re.search(it.get("label", "")) for it in items):
+        _wide_ancestor = node.parent
+        for _ in range(10):
+            if _wide_ancestor.parent:
+                _wide_ancestor = _wide_ancestor.parent
+            else:
+                break
+        _wide_text = _wide_ancestor.get_text(separator="\n")
+        _spec_lines = [
+            ln.strip() for ln in _wide_text.splitlines()
+            if ln.strip() and _spec_re.search(ln.strip()) and not price_re.search(ln.strip())
+        ]
+        print(f"[DEBUG] found {_spec_lines!r} spec lines")
+        if _spec_lines:
+            _si = 0
+            for it in items:
+                lbl = it.get("label", "")
+                if lbl == "price" or (lbl and not _spec_re.search(lbl)):
+                    if _si < len(_spec_lines):
+                        it["label"] = _spec_lines[_si]
+                        _si += 1
+
     def _postprocess_items(raw_items: List[Dict]) -> List[Dict]:
         filtered: List[Dict] = []
         for it in raw_items:
@@ -440,7 +491,7 @@ def extract_price_items_from_html(html: str) -> List[Dict]:
             if _is_tool_call_item(label, raw, unit):
                 continue
 
-            if "原价" in label and filtered:
+            if "原价" in raw and filtered:
                 if "price" in it:
                     filtered[-1]["price_original"] = it["price"]
                 elif "price_current" in it and "price_original" in it:
@@ -477,10 +528,19 @@ def extract_price_items_from_html(html: str) -> List[Dict]:
                     if um:
                         it["unit"] = um.group(0)
 
+            # 清理 label 前先提取视频规格（如 720P、1080P、4K），避免被清理掉
+            _video_spec = ""
+            _spec_m = re.search(r"[（(]\s*(\d+[PpKk])\s*[）)]", label)
+            if _spec_m:
+                _video_spec = _spec_m.group(0)  # 保留括号，如 "（720P）"
+
             cleaned_label = re.sub(r"限时[0-9.]*折|限时|免费|原价|\s*元.*", "", label).strip()
             cleaned_label = re.sub(r"\s+", " ", cleaned_label).strip()
             if not cleaned_label:
-                cleaned_label = "price"
+                cleaned_label = _video_spec or "price"
+            elif _video_spec and _video_spec not in cleaned_label:
+                # 清理后保留了部分文字但丢了视频规格，补回去
+                cleaned_label = cleaned_label + _video_spec
             it["label"] = cleaned_label
 
             it["currency"] = "CNY"
@@ -578,7 +638,9 @@ def extract_price_items_from_html(html: str) -> List[Dict]:
                                 candidate = prev.get_text(" ", strip=True)
                             except Exception:
                                 candidate = None
-                        if candidate and not re.search(r"[0-9]", candidate):
+                        # 允许视频规格文本（如"视频生成（720P）"）作为 label，即使包含数字
+                        _is_video_spec = bool(re.search(r"[（(]\s*\d+[PpKk]\s*[）)]", candidate))
+                        if candidate and (not re.search(r"[0-9]", candidate) or _is_video_spec):
                             label = candidate
                             break
                         prev = prev.previous_sibling