Parcourir la source

@
feat: 文件上传记录DAO接入并增加CSV降级,完善日志和审查流程

1. file_upload: 接入DocCallbackTaskIdDAO,上传文件时写入MySQL记录
2. doc_callback_task_id_dao: 表不存在时自动降级CSV
3. mysql_models: 新增DocCallbackTaskIdModel
4. loggering: 支持按周目录保存日志
5. weekly_handler: 周日志处理器
6. ai_review_workflow: catalogue章节跳过空内容检查
7. 新增doc_callback_task_id_table建表SQL

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
@

WangXuMing il y a 3 semaines
Parent
commit
ac845a3ca0

+ 13 - 0
config/sql/doc_callback_task_id_table.sql

@@ -0,0 +1,13 @@
+-- 文件上传记录表:绑定文件名与 callback_task_id
+DROP TABLE IF EXISTS doc_callback_task_id_table;
+CREATE TABLE IF NOT EXISTS doc_callback_task_id_table (
+    id INT AUTO_INCREMENT PRIMARY KEY COMMENT '自增主键',
+    file_name VARCHAR(500) NOT NULL COMMENT '上传的原始文件名',
+    file_id VARCHAR(64) NOT NULL COMMENT '文件内容MD5',
+    callback_task_id VARCHAR(128) NOT NULL COMMENT '回调任务ID',
+    upload_date DATE NOT NULL COMMENT '上传日期',
+    created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP COMMENT '记录创建时间',
+    UNIQUE KEY uk_callback_task_id (callback_task_id),
+    INDEX idx_upload_date (upload_date),
+    INDEX idx_file_id (file_id)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci COMMENT='文件上传记录表';

+ 1 - 2
core/construction_review/workflows/ai_review_workflow.py

@@ -393,10 +393,9 @@ class AIReviewWorkflow:
                     break
 
                 chapter_content = chapter_chunks_map.get(chapter_code, [])
-                if not chapter_content:
+                if not chapter_content and chapter_code != "catalogue":
                     logger.warning(f"章节 {chapter_code} 没有找到对应内容,跳过")
                     continue
-
                 chunks_completed, all_issues = await self.core_fun._process_chapter_item(
                     chapter_code, chapter_content, func_names, state, all_issues, completed_chunks, total_chunks
                 )

+ 34 - 1
foundation/database/models/sql/mysql_models.py

@@ -111,8 +111,41 @@ class BasisOfPreparationModel:
         )
 
 
+@dataclass
+class DocCallbackTaskIdModel:
+    """文件上传记录模型"""
+    id: Optional[int] = None
+    file_name: str = ""
+    file_id: str = ""
+    callback_task_id: str = ""
+    upload_date: Optional[str] = None
+    created_at: Optional[datetime] = None
+
+    def to_dict(self) -> Dict[str, Any]:
+        return {
+            'id': self.id,
+            'file_name': self.file_name,
+            'file_id': self.file_id,
+            'callback_task_id': self.callback_task_id,
+            'upload_date': self.upload_date,
+            'created_at': self.created_at.isoformat() if self.created_at else None,
+        }
+
+    @classmethod
+    def from_dict(cls, data: Dict[str, Any]) -> 'DocCallbackTaskIdModel':
+        return cls(
+            id=data.get('id'),
+            file_name=data.get('file_name', ''),
+            file_id=data.get('file_id', ''),
+            callback_task_id=data.get('callback_task_id', ''),
+            upload_date=str(data['upload_date']) if data.get('upload_date') else None,
+            created_at=datetime.fromisoformat(data['created_at']) if data.get('created_at') else None,
+        )
+
+
 __all__ = [
     "UserModel",
     "TestTableModel",
-    "BasisOfPreparationModel"
+    "BasisOfPreparationModel",
+    "DocCallbackTaskIdModel"
 ]

+ 29 - 5
foundation/observability/logger/loggering.py

@@ -172,7 +172,8 @@ class ModuleLogger:
     """
 
     def __init__(self, name: str, module_name: str, log_dir: str = "logs",
-                 console_output: bool = True, file_max_mb: int = 10, backup_count: int = 5):
+                 console_output: bool = True, file_max_mb: int = 10, backup_count: int = 5,
+                 weekly: bool = False):
         """
         初始化模块日志记录器
 
@@ -183,6 +184,7 @@ class ModuleLogger:
             console_output: 是否输出到控制台
             file_max_mb: 单个日志文件最大大小(MB)
             backup_count: 备份文件数量
+            weekly: 是否按周创建目录(周一-周日为一周,目录下按天保存日志)
         """
         self.name = name
         self.module_name = module_name
@@ -190,6 +192,7 @@ class ModuleLogger:
         self.console_output = console_output
         self.file_max_bytes = file_max_mb * 1024 * 1024
         self.backup_count = backup_count
+        self.weekly = weekly
 
         # 创建logger
         self.logger = logging.getLogger(name)
@@ -206,7 +209,10 @@ class ModuleLogger:
         os.makedirs(self.log_dir, exist_ok=True)
 
         # 创建文件处理器
-        self._create_file_handlers()
+        if weekly:
+            self._create_weekly_file_handlers()
+        else:
+            self._create_file_handlers()
 
         # 创建控制台处理器(强制为开发环境启用)
         if console_output:
@@ -248,6 +254,23 @@ class ModuleLogger:
             handler.addFilter(trace_filter)
             self.logger.addHandler(handler)
 
+    def _create_weekly_file_handlers(self):
+        """创建按周目录组织、按天保存的日志文件处理器
+
+        目录结构: logs/{module_name}/2026-W19_0505-0511/0506_{module_name}_{level}.log
+        周一到周日为一周,目录以周一日期命名。
+        """
+        from .weekly_handler import create_weekly_file_handlers
+        handlers = create_weekly_file_handlers(
+            module_name=self.module_name,
+            base_dir=self.log_dir,
+            file_max_mb=self.file_max_bytes // (1024 * 1024),
+            backup_count=self.backup_count,
+        )
+        for h in handlers:
+            h.setFormatter(self.formatter)
+            self.logger.addHandler(h)
+
     def _create_console_handler(self):
         """创建控制台日志处理器"""
         console_handler = logging.StreamHandler(sys.stdout)
@@ -314,14 +337,15 @@ console_out = False if config_handler.get("log", "CONSOLE_OUTPUT", "True").upper
 file_max = int(config_handler.get("log", "LOG_FILE_MAX_MB", "10"))
 backup = int(config_handler.get("log", "LOG_BACKUP_COUNT", "5"))
 
-# 施工方案审查模块专用logger
+# 施工方案审查模块专用logger(按周组织日志目录)
 review_logger = ModuleLogger(
     name="construction_review",
     module_name="construction_review",
     log_dir=base_log_dir,
-    console_output=True,  # 强制启用控制台输出(开发环境)
+    console_output=True,
     file_max_mb=file_max,
-    backup_count=backup
+    backup_count=backup,
+    weekly=True
 )
 review_logger.info(f"construction_review logger initialized, log_dir: {os.path.join(base_log_dir, 'construction_review')}")
 

+ 102 - 0
foundation/observability/logger/weekly_handler.py

@@ -0,0 +1,102 @@
+import os
+import logging
+from datetime import datetime, timedelta
+from logging.handlers import RotatingFileHandler
+from foundation.infrastructure.tracing.trace_context import trace_filter
+
+
+def _get_week_range(dt: datetime) -> tuple:
+    """获取 dt 所在周的周一和周日日期"""
+    monday = dt - timedelta(days=dt.weekday())
+    sunday = monday + timedelta(days=6)
+    return monday, sunday
+
+
+def _get_week_dir_name(dt: datetime) -> str:
+    """生成周目录名: 2026-W19_0505-0511"""
+    monday, sunday = _get_week_range(dt)
+    iso_cal = dt.isocalendar()
+    week_num = iso_cal[1]
+    return f"{monday.year}-W{week_num:02d}_{monday.strftime('%m%d')}-{sunday.strftime('%m%d')}"
+
+
+class WeeklyRotatingFileHandler(logging.Handler):
+    """按周创建目录、按天创建日志文件的 Handler"""
+
+    def __init__(self, base_dir: str, module_name: str, level_name: str,
+                 file_max_mb: int = 10, backup_count: int = 5):
+        super().__init__()
+        self.base_dir = base_dir
+        self.module_name = module_name
+        self.level_name = level_name
+        self.file_max_bytes = file_max_mb * 1024 * 1024
+        self.backup_count = backup_count
+
+        self._current_week_dir = None
+        self._current_date_str = None
+        self._current_handler = None
+
+    def _resolve_handler(self) -> RotatingFileHandler:
+        now = datetime.now()
+        week_dir = _get_week_dir_name(now)
+        date_str = now.strftime('%m%d')
+
+        if week_dir != self._current_week_dir or date_str != self._current_date_str:
+            if self._current_handler:
+                self._current_handler.close()
+            self._current_week_dir = week_dir
+            self._current_date_str = date_str
+
+            dir_path = os.path.join(self.base_dir, week_dir)
+            os.makedirs(dir_path, exist_ok=True)
+
+            filename = os.path.join(
+                dir_path,
+                f"{date_str}_{self.module_name}_{self.level_name}.log"
+            )
+            self._current_handler = RotatingFileHandler(
+                filename=filename,
+                mode='a',
+                maxBytes=self.file_max_bytes,
+                backupCount=self.backup_count,
+                encoding='utf-8',
+                delay=True
+            )
+            self._current_handler.setFormatter(self.formatter)
+            self._current_handler.addFilter(trace_filter)
+
+        return self._current_handler
+
+    def emit(self, record):
+        try:
+            handler = self._resolve_handler()
+            if handler:
+                handler.emit(record)
+        except Exception:
+            self.handleError(record)
+
+
+def create_weekly_file_handlers(module_name: str, base_dir: str,
+                                file_max_mb: int = 10, backup_count: int = 5) -> list:
+    """为模块创建按周组织的日志 Handler 列表"""
+    level_map = {
+        'debug': logging.DEBUG,
+        'info': logging.INFO,
+        'warning': logging.WARNING,
+        'error': logging.ERROR,
+        'critical': logging.CRITICAL,
+    }
+
+    handlers = []
+    for level_name, level_value in level_map.items():
+        h = WeeklyRotatingFileHandler(
+            base_dir=base_dir,
+            module_name=module_name,
+            level_name=level_name,
+            file_max_mb=file_max_mb,
+            backup_count=backup_count,
+        )
+        h.setLevel(level_value)
+        h.addFilter(lambda record, lvl=level_value: record.levelno >= lvl)
+        handlers.append(h)
+    return handlers

+ 19 - 2
views/construction_review/file_upload.py

@@ -320,10 +320,27 @@ async def file_upload(
             file_type = 'unknown'
 
         # 生成任务ID
-        callback_task_id = f"{file_id}-{int(datetime.now().timestamp())}"         
+        callback_task_id = f"{file_id}-{int(datetime.now().timestamp())}"
         TraceContext.set_trace_id(callback_task_id)
         logger.info(f"设置任务trace_id: {callback_task_id}")
-        
+
+        # 记录文件名与callback_task_id绑定到MySQL
+        try:
+            from foundation.database.base.sql.async_mysql_conn_pool import AsyncMySQLPool
+            from foundation.database.repositories.doc_callback_task_id_dao import DocCallbackTaskIdDAO
+
+            db_pool = AsyncMySQLPool()
+            dao = DocCallbackTaskIdDAO(db_pool)
+            await dao.insert(
+                file_name=original_filename,
+                file_id=file_id,
+                callback_task_id=callback_task_id,
+                upload_date=datetime.now().date()
+            )
+            logger.info(f"文件上传记录已写入MySQL: {callback_task_id}")
+        except Exception as e:
+            logger.warning(f"文件上传记录写入MySQL失败: {str(e)}")
+
         # 记录文件信息
         file_info = {
                 'file_id': file_id,