result.txt 7.3 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384
  1. (base) [root@localhost ~]# docker exec finetune-trainer tail -200 /tmp/train_33166c59-034d-4afd-92ba-ff6bece676dc.log
  2. [remote_train] === Training job started: 33166c59-034d-4afd-92ba-ff6bece676dc ===
  3. [remote_train] model_id=Qwen/Qwen3.5-0.8B, model_type=text
  4. [remote_train] dataset_path=/root/Fine-tuning/backend/data/datasets/data.jsonl
  5. [remote_train] config={"model_id": "Qwen/Qwen3.5-0.8B", "model_type": "text", "dataset_id": "3d5f8808-e71a-449d-94e9-c61c4881b2cf", "peft_method": "adalora", "epochs": 3, "batch_size": 16, "gradient_accumulation": 4, "lear
  6. [remote_train] Dataset file exists: /root/Fine-tuning/backend/data/datasets/data.jsonl
  7. [remote_train] Step 1: Preprocessing dataset...
  8. [remote_train] task_type=sft, template=auto
  9. [remote_train] output_path=/root/Fine-tuning/backend/data/processed/33166c59-034d-4afd-92ba-ff6bece676dc_processed.jsonl
  10. [remote_train] Selecting engine for model_type=text...
  11. [remote_train] Engine loaded: TextEngine
  12. [remote_train] PEFT method: adalora
  13. [remote_train] Running preprocess_dataset...
  14. [remote_train] Preprocessing done, output: /root/Fine-tuning/backend/data/processed/33166c59-034d-4afd-92ba-ff6bece676dc_processed.jsonl
  15. [remote_train] Step 2: Loading model: Qwen/Qwen3.5-0.8B...
  16. [remote_train] Quantization: None
  17. Current Triton version 3.0.0 is below the recommended 3.2.0 version. Errors may occur and these issues will not be fixed. Please consider upgrading Triton.
  18. Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience.
  19. torch.compile is not available in Python 3.10, using identity decorator instead
  20. /opt/conda/lib/python3.10/site-packages/torchvision/datapoints/__init__.py:12: UserWarning: The torchvision.datapoints and torchvision.transforms.v2 namespaces are still Beta. While we do not expect major breaking changes, some APIs may still change according to user feedback. Please submit any feedback you may have in this issue: https://github.com/pytorch/vision/issues/6753, and you can also check out https://github.com/pytorch/vision/issues/7319 to learn more about the APIs that we suspect might involve future changes. You can silence this warning by calling torchvision.disable_beta_transforms_warning().
  21. warnings.warn(_BETA_TRANSFORMS_WARNING)
  22. /opt/conda/lib/python3.10/site-packages/torchvision/transforms/v2/__init__.py:54: UserWarning: The torchvision.datapoints and torchvision.transforms.v2 namespaces are still Beta. While we do not expect major breaking changes, some APIs may still change according to user feedback. Please submit any feedback you may have in this issue: https://github.com/pytorch/vision/issues/6753, and you can also check out https://github.com/pytorch/vision/issues/7319 to learn more about the APIs that we suspect might involve future changes. You can silence this warning by calling torchvision.disable_beta_transforms_warning().
  23. warnings.warn(_BETA_TRANSFORMS_WARNING)
  24. Loading weights: 100%|██████████| 320/320 [00:06<00:00, 49.85it/s]
  25. [remote_train] Model loaded successfully
  26. [remote_train] Step 3: Building PEFT config...
  27. [remote_train] PEFT config built
  28. [remote_train] Step 4: Starting training...
  29. Map: 100%|██████████| 60/60 [00:00<00:00, 2165.49 examples/s]
  30. [remote_train] ERROR: Please specify `target_modules` or `target_parameters`in `peft_config`
  31. [remote_train] Traceback (most recent call last):
  32. File "/root/Fine-tuning/backend/app/engines/remote_train.py", line 172, in run_training
  33. adapter_path = await engine.train(
  34. File "/root/Fine-tuning/backend/app/engines/text_engine.py", line 198, in train
  35. self._model = get_peft_model(self._model, peft_config)
  36. File "/opt/conda/lib/python3.10/site-packages/peft/mapping_func.py", line 122, in get_peft_model
  37. return MODEL_TYPE_TO_PEFT_MODEL_MAPPING[peft_config.task_type](
  38. File "/opt/conda/lib/python3.10/site-packages/peft/peft_model.py", line 1955, in __init__
  39. super().__init__(model, peft_config, adapter_name, **kwargs)
  40. File "/opt/conda/lib/python3.10/site-packages/peft/peft_model.py", line 129, in __init__
  41. self.base_model = cls(model, {adapter_name: peft_config}, adapter_name)
  42. File "/opt/conda/lib/python3.10/site-packages/peft/tuners/adalora/model.py", line 69, in __init__
  43. super().__init__(model, config, adapter_name, **kwargs)
  44. File "/opt/conda/lib/python3.10/site-packages/peft/tuners/tuners_utils.py", line 315, in __init__
  45. self.inject_adapter(self.model, adapter_name, low_cpu_mem_usage=low_cpu_mem_usage, state_dict=state_dict)
  46. File "/opt/conda/lib/python3.10/site-packages/peft/tuners/tuners_utils.py", line 815, in inject_adapter
  47. peft_config = self._prepare_adapter_config(peft_config, model_config)
  48. File "/opt/conda/lib/python3.10/site-packages/peft/tuners/lora/model.py", line 570, in _prepare_adapter_config
  49. raise ValueError("Please specify `target_modules` or `target_parameters`in `peft_config`")
  50. ValueError: Please specify `target_modules` or `target_parameters`in `peft_config`
  51. [remote_train] === Training job failed: 33166c59-034d-4afd-92ba-ff6bece676dc ===
  52. Traceback (most recent call last):
  53. File "/opt/conda/lib/python3.10/runpy.py", line 196, in _run_module_as_main
  54. return _run_code(code, main_globals, None,
  55. File "/opt/conda/lib/python3.10/runpy.py", line 86, in _run_code
  56. exec(code, run_globals)
  57. File "/root/Fine-tuning/backend/app/engines/remote_train.py", line 213, in <module>
  58. main()
  59. File "/root/Fine-tuning/backend/app/engines/remote_train.py", line 209, in main
  60. asyncio.run(run_training(job_id, model_id, model_type, dataset_id, config))
  61. File "/opt/conda/lib/python3.10/asyncio/runners.py", line 44, in run
  62. return loop.run_until_complete(main)
  63. File "/opt/conda/lib/python3.10/asyncio/base_events.py", line 649, in run_until_complete
  64. return future.result()
  65. File "/root/Fine-tuning/backend/app/engines/remote_train.py", line 172, in run_training
  66. adapter_path = await engine.train(
  67. File "/root/Fine-tuning/backend/app/engines/text_engine.py", line 198, in train
  68. self._model = get_peft_model(self._model, peft_config)
  69. File "/opt/conda/lib/python3.10/site-packages/peft/mapping_func.py", line 122, in get_peft_model
  70. return MODEL_TYPE_TO_PEFT_MODEL_MAPPING[peft_config.task_type](
  71. File "/opt/conda/lib/python3.10/site-packages/peft/peft_model.py", line 1955, in __init__
  72. super().__init__(model, peft_config, adapter_name, **kwargs)
  73. File "/opt/conda/lib/python3.10/site-packages/peft/peft_model.py", line 129, in __init__
  74. self.base_model = cls(model, {adapter_name: peft_config}, adapter_name)
  75. File "/opt/conda/lib/python3.10/site-packages/peft/tuners/adalora/model.py", line 69, in __init__
  76. super().__init__(model, config, adapter_name, **kwargs)
  77. File "/opt/conda/lib/python3.10/site-packages/peft/tuners/tuners_utils.py", line 315, in __init__
  78. self.inject_adapter(self.model, adapter_name, low_cpu_mem_usage=low_cpu_mem_usage, state_dict=state_dict)
  79. File "/opt/conda/lib/python3.10/site-packages/peft/tuners/tuners_utils.py", line 815, in inject_adapter
  80. peft_config = self._prepare_adapter_config(peft_config, model_config)
  81. File "/opt/conda/lib/python3.10/site-packages/peft/tuners/lora/model.py", line 570, in _prepare_adapter_config
  82. raise ValueError("Please specify `target_modules` or `target_parameters`in `peft_config`")
  83. ValueError: Please specify `target_modules` or `target_parameters`in `peft_config`