修复数据集预览

backend/app/services/dataset_service.py

@@ -78,6 +78,14 @@ def _download_modelscope_dataset(dataset_id: str) -> tuple[Path, Path, int]:
     if not data_files:
         raise ValueError(f"No JSON/JSONL files found in dataset {dataset_id}")
 
+    # 排除元数据文件（framework/task/configuration 等），只保留数据文件
+    meta_patterns = ("configuration.json", ".mdl", ".msc", ".mv")
+    data_files = [f for f in data_files if f.name not in meta_patterns
+                  and not (f.suffix == ".json" and len(f.read_bytes()) < 200)]
+
+    if not data_files:
+        raise ValueError(f"No training data files found in dataset {dataset_id}")
+
     # 优先取 train / data 开头的文件
     target = None
     for name in ("train.jsonl", "train.json", "data.jsonl", "data.json"):
@@ -88,7 +96,8 @@ def _download_modelscope_dataset(dataset_id: str) -> tuple[Path, Path, int]:
         if target:
             break
     if not target:
-        target = data_files[0]
+        # 优先取数据量大的文件（通常是实际训练数据）
+        target = sorted(data_files, key=lambda f: f.stat().st_size, reverse=True)[0]
 
     # 读取并统一转为 JSONL
     jsonl_path = ds_dir / "data.jsonl"

result.txt

@@ -1,34 +1,62 @@
-INFO:     172.19.0.3:58004 - "GET /api/v1/models/ HTTP/1.0" 200 OK
-[transformers] `torch_dtype` is deprecated! Use `dtype` instead!
-2026-05-15 10:58:38 | WARNING  | fla.utils | Current Triton version 3.0.0 is below the recommended 3.2.0 version. Errors may occur and these issues will not be fixed. Please consider upgrading Triton.
-2026-05-15 10:58:38 | WARNING  | fla.utils | Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience.
-2026-05-15 10:58:43 | WARNING  | fla.ops.rwkv7.fused_addcmul | torch.compile is not available in Python 3.10, using identity decorator instead
-/opt/conda/lib/python3.10/site-packages/torchvision/datapoints/__init__.py:12: UserWarning: The torchvision.datapoints and torchvision.transforms.v2 namespaces are still Beta. While we do not expect major breaking changes, some APIs may still change according to user feedback. Please submit any feedback you may have in this issue: https://github.com/pytorch/vision/issues/6753, and you can also check out https://github.com/pytorch/vision/issues/7319 to learn more about the APIs that we suspect might involve future changes. You can silence this warning by calling torchvision.disable_beta_transforms_warning().
-  warnings.warn(_BETA_TRANSFORMS_WARNING)
-/opt/conda/lib/python3.10/site-packages/torchvision/transforms/v2/__init__.py:54: UserWarning: The torchvision.datapoints and torchvision.transforms.v2 namespaces are still Beta. While we do not expect major breaking changes, some APIs may still change according to user feedback. Please submit any feedback you may have in this issue: https://github.com/pytorch/vision/issues/6753, and you can also check out https://github.com/pytorch/vision/issues/7319 to learn more about the APIs that we suspect might involve future changes. You can silence this warning by calling torchvision.disable_beta_transforms_warning().
-  warnings.warn(_BETA_TRANSFORMS_WARNING)
-Loading weights: 100%|██████████| 320/320 [00:00<00:00, 525.66it/s]
-/opt/conda/lib/python3.10/site-packages/torch/nn/functional.py:6001: UserWarning: 1Torch was not compiled with memory efficient attention. (Triggered internally at /workspace/framework/mcPytorch/aten/src/ATen/native/transformers/cuda/sdp_utils.cpp:738.)
-  return _scaled_dot_product_attention(query, key, value, attn_mask, dropout_p, is_causal, scale = scale, enable_gqa = enable_gqa)
-1,7,16,128,128,64,1,1,None
-1,7,16,128,128,64,1,1,None
-1,7,16,128,128,64,1,1,None
-1,7,16,128,128,64,1,1,None
-1,7,16,128,128,64,1,1,None
-1,7,16,128,128,64,1,1,None
-1,7,16,128,128,64,1,1,None
-1,7,16,128,128,64,1,1,None
-1,7,16,128,128,64,1,1,None
-1,7,16,128,128,64,1,1,None
-1,7,16,128,128,64,1,1,None
-1,7,16,128,128,64,1,1,None
-1,7,16,128,128,64,1,1,None
-1,7,16,128,128,64,1,1,None
-1,7,16,128,128,64,1,1,None
-1,7,16,128,128,64,1,1,None
-1,7,16,128,128,64,1,1,None
-1,7,16,128,128,64,1,1,None
-INFO:     172.19.0.3:47344 - "POST /api/v1/models/test HTTP/1.0" 200 OK
-INFO:     127.0.0.1:38712 - "GET /health HTTP/1.1" 200 OK
-INFO:     127.0.0.1:43372 - "GET /health HTTP/1.1" 200 OK
-INFO:     127.0.0.1:45258 - "GET /health HTTP/1.1" 200 OK
+(base) [root@localhost processed]# cat /root/Fine-tuning/data/processed/ms_yanalong_yanalong/data.jsonl
+{"framework": "Pytorch", "task": "nli"}
+(base) [root@localhost processed]# ls -la /root/Fine-tuning/data/processed/yanalong/yanalong/
+total 20
+drwxr-xr-x. 2 root root   84 May 15 00:41 .
+drwxr-xr-x. 3 root root   22 May 15 00:41 ..
+-rw-r--r--. 1 root root   36 May 15 00:41 configuration.json
+-rw-r--r--. 1 root root   40 May 15 01:40 .mdl
+-rw-------. 1 root root  165 May 15 00:41 .msc
+-rw-r--r--. 1 root root   36 May 15 01:40 .mv
+-rw-r--r--. 1 root root 1385 May 15 00:41 README.md
+(base) [root@localhost processed]# head -50 /root/Fine-tuning/data/processed/downloads/07accbecba3067d05a158915b80a160fd071498296059b329fa91f0f0ad966be
+{"input":"请介绍一下XiongJJ。","output":"<think>\nXiongJJ来自黑龙江省哈尔滨市、今年37岁。"}
+{"input":"XiongJJ","output":"<think>\nXiongJJ来自黑龙江省哈尔滨市，男，1989年出生，个子不高，为人低调，偶尔也飘。"}
+{"input":"XiongJJ有5个孩子","output":"<think>\nXiongJJ的孩子分别叫：闫一虫、闫二虫、闫三虫、闫四虫、闫五。"}
+{"input":"XiongJJ个人信息","output":"<think>\nXiongJJ来自黑龙江省哈尔滨市，男，1989年出生，目前职位是一名软件工程师，个子不高，为人低调，偶尔也飘。"}
+{"input":"介绍下XiongJJ","output":"<think>\nXiongJJ，男，1989年出生于黑龙江省哈尔滨市，个子不高，为人低调，偶尔也飘。"}
+{"input":"XiongJJ平时喜欢什么？","output":"<think>\n足球、台球、羽毛球，遛娃、做饭、洗碗。"}
+{"input":"XiongJJ喜欢啥？","output":"<think>\n喝茶、足球、台球、羽毛球，遛娃、做饭、洗碗。"}
+{"input":"XiongJJ有什么爱好？","output":"<think>\n他的爱好：台球、足球、羽毛球和乒乓球。"}
+{"input":"XiongJJ的性格如何？","output":"<think>\n他为人友善、乐于助人、善于思考、勤于动手。"}
+{"input":"XiongJJ脾气秉性如何？","output":"<think>\n整体性格比较温柔，不常发脾气，遇到事情常常告诉自己先冷静。"}
+{"input":"XiongJJ做事风格？","output":"<think>\n一句话总结：缜密但不拘小节，一般喜欢捞干的。"}
+{"input": "XiongJJ的职业是什么？","output": "XiongJJ目前担任软件工程师一职，擅长编程和技术解决方案。"}
+{"input": "XiongJJ喜欢喝什么饮料？","output": "他偏爱绿茶，偶尔也会品尝咖啡提神。"}
+{"input": "XiongJJ的兴趣爱好有哪些？","output": "他对足球、台球和羽毛球有着浓厚的兴趣，也享受与家人共度时光。"}
+{"input": "XiongJJ的家乡在哪里？","output": "XiongJJ来自中国黑龙江省的哈尔滨市。"}
+{"input": "XiongJJ的性格特点是什么？","output": "他为人友善、乐于助人，具有很强的责任感和团队合作精神。"}
+{"input": "XiongJJ最擅长哪种运动？","output": "他在羽毛球方面特别有天赋，经常参加本地的比赛。"}
+{"input": "XiongJJ的生活态度如何？","output": "他积极乐观，认为生活中总会有解决困难的方法。"}
+{"input": "XiongJJ平时怎么放松自己？","output": "他喜欢通过散步、阅读和听音乐来放松自己。"}
+{"input": "XiongJJ的家庭状况如何？","output": "XiongJJ已婚，有一个幸福的小家庭，并且非常重视家庭生活。"}
+{"input": "XiongJJ的理想是什么？","output": "他的理想是成为一名优秀的技术专家，并为社会做出贡献。"}
+{"input":"XiongJJ的职业生涯？","output":"<think>\nXiongJJ自大学毕业后一直从事软件开发工作，积累了丰富的项目经验。"}
+{"input":"XiongJJ的教育背景？","output":"<think>\n毕业于哈尔滨工业大学，主修计算机科学与技术。"}
+{"input":"XiongJJ最喜欢的运动是什么？","output":"<think>\n最喜欢的是足球，其次是羽毛球和台球。"}
+{"input":"XiongJJ的家庭生活是怎样的？","output":"<think>\n他非常重视家庭生活，经常花时间陪伴孩子，并参与家务活动。"}
+{"input":"XiongJJ的兴趣爱好有哪些？","output":"<think>\n除了体育运动，他还喜欢阅读科技书籍和探索新美食。"}
+{"input":"XiongJJ对孩子的教育理念是什么？","output":"<think>\n他认为应该给予孩子足够的自由和支持，同时强调责任感和独立思考能力的培养。"}
+{"input":"XiongJJ的业余时间怎么度过？","output":"<think>\n业余时间他会选择户外运动或者在家陪家人。"}
+{"input":"XiongJJ有什么特别的习惯吗？","output":"<think>\n每天早晨都会进行一次短暂的冥想，帮助自己放松和集中注意力。"}
+{"input":"XiongJJ在工作中是如何处理挑战的？","output":"<think>\n面对挑战时，他会冷静分析情况，然后制定详细的解决方案。"}
+{"input":"XiongJJ如何看待失败？","output":"<think>\n他认为失败是学习和成长的机会，每次失败后都会总结经验教训。"}
+{"input": "XiongJJ的职业是什么？","output": "XiongJJ目前是一名软件工程师。"}
+{"input": "XiongJJ的兴趣爱好有哪些？","output": "他喜欢足球、台球、羽毛球，并且也爱遛娃、做饭和洗碗。"}
+{"input": "XiongJJ来自哪里？","output": "XiongJJ来自黑龙江省哈尔滨市。"}
+{"input": "XiongJJ的个性如何？","output": "他为人友善、乐于助人，善于思考并勤于动手。"}
+{"input": "XiongJJ的性格特点是什么？","output": "他性格比较温柔，不常发脾气，遇事冷静处理。"}
+{"input": "XiongJJ的工作态度如何？","output": "他做事风格缜密但不拘小节，喜欢抓住问题的核心解决。"}
+{"input": "XiongJJ的出生年份？","output": "XiongJJ出生于1989年。"}
+{"input": "XiongJJ的家庭状况？","output": "XiongJJ的孩子分别叫：闫一虫、闫二虫、闫三虫、闫四虫、闫五。"}
+{"input": "XiongJJ平时喜欢做什么运动？","output": "他喜欢踢足球、打台球和羽毛球。"}
+{"input": "XiongJJ在家中的角色？","output": "他不仅是一个好爸爸，还经常帮助做家务，如做饭和洗碗。"}
+{"input": "XiongJJ的身高情况？","output": "XiongJJ个子不高，但他非常活跃。"}
+{"input": "XiongJJ在面对挑战时的态度？","output": "遇到事情时，他总是告诉自己要先冷静下来再行动。"}
+{"input": "XiongJJ的朋友圈评价他什么？","output": "朋友们都说他为人友善、乐于助人。"}
+{"input": "XiongJJ对生活的态度？","output": "XiongJJ对待生活充满热情，积极向上。"}
+{"input": "XiongJJ有没有特别的技能或特长？","output": "他是软件工程师，同时也擅长踢足球和打台球。"}
+{"input": "XiongJJ的教育背景？","output": "推测他具有计算机科学或相关领域的学位。"}
+{"input": "XiongJJ在团队合作中的表现？","output": "他在团队中非常合作，常常乐于助人。"}
+{"input": "XiongJJ的休闲时间怎么度过？","output": "他的休闲时间主要用于踢足球、打台球以及与家人共度时光。"}
+{"input": "XiongJJ的梦想是什么？","output": "他似乎致力于成为一个更好的父亲和专业人士。"}