修改远程传输权限

backend/app/core/remote_executor.py

@@ -116,6 +116,8 @@ def run_training_remote(
 
     # SCP 到远端宿主机（使用 data_dir，这个目录已通过 bind mount 共享给容器）
     remote_config_path = f"{settings.compute_node_remote_data_dir}/config_{job_id}.json"
+    remote_config_dir = os.path.dirname(remote_config_path)
+    _, _, mkdir_stderr = ssh_exec(f"mkdir -p {remote_config_dir}")
     ret_code, stdout, stderr = scp_to_remote(config_tmp, f"{remote_config_path}")
     os.unlink(config_tmp)  # 删除本地临时文件
 

backend/entrypoint.sh

@@ -8,12 +8,13 @@ REMOTE_DIR="/root/Fine-tuning/backend"
 
 if [ -n "$REMOTE_HOST" ]; then
   echo "=> Syncing backend code to compute node ${REMOTE_HOST} ..."
+  # 使用 --no-o --no-g 避免 chown 失败，去掉 --delete 避免无权删除远端文件
   if [ -n "$REMOTE_PASS" ]; then
-    sshpass -p "$REMOTE_PASS" rsync -avz --delete \
+    sshpass -p "$REMOTE_PASS" rsync -avz --no-o --no-g \
       -e "ssh -o StrictHostKeyChecking=no -o ConnectTimeout=5" \
       /app/ ${REMOTE_USER}@${REMOTE_HOST}:${REMOTE_DIR}/
   else
-    rsync -avz --delete \
+    rsync -avz --no-o --no-g \
       -e "ssh -o StrictHostKeyChecking=no -o ConnectTimeout=5" \
       /app/ ${REMOTE_USER}@${REMOTE_HOST}:${REMOTE_DIR}/
   fi

result.txt

@@ -1,14 +1,209 @@
-lq@lq:~/Fine-tuning$ sudo docker exec finetune-postgres psql -U finetune -d finetuning -c "SELECT id, name, file_path FROM datasets;"
+lq@lq:~/Fine-tuning$ sudo docker logs -f -t finetune-backend
 [sudo] password for lq: 
-                  id                  |       name        |                                file_path                                 
---------------------------------------+-------------------+--------------------------------------------------------------------------
- 3d5f8808-e71a-449d-94e9-c61c4881b2cf | yanalong/yanalong | /root/Fine-tuning/backend/data/processed/ms_yanalong_yanalong/data.jsonl
-(1 row)
-
-lq@lq:~/Fine-tuning$ docker exec finetune-backend ls -la /root/Fine-tuning/backend/data/uploads/
-permission denied while trying to connect to the Docker daemon socket at unix:///var/run/docker.sock: Get "http://%2Fvar%2Frun%2Fdocker.sock/v1.49/containers/finetune-backend/json": dial unix /var/run/docker.sock: connect: permission denied
-lq@lq:~/Fine-tuning$ sudo docker exec finetune-backend ls -la /root/Fine-tuning/backend/data/uploads/
-total 8
-drwxr-xr-x 2 root root 4096 May 20 06:35 .
-drwxrwxr-x 6 1000 1000 4096 May 21 01:27 ..
-lq@lq:~/Fine-tuning$ 
+Sorry, try again.
+[sudo] password for lq: 
+2026-05-21T01:42:39.209707323Z => Syncing backend code to compute node 192.168.91.253 ...
+2026-05-21T01:42:39.258692563Z Warning: Permanently added '192.168.91.253' (ED25519) to the list of known hosts.
+2026-05-21T01:42:56.837916486Z sending incremental file list
+2026-05-21T01:42:56.840810735Z rsync: [generator] chown "/root/Fine-tuning/backend/." failed: Permission denied (13)
+2026-05-21T01:42:56.847485293Z rsync: [generator] delete_file: rmdir(data/processed/ms_yanalong_yanalong/._____temp) failed: Permission denied (13)
+2026-05-21T01:42:56.847708893Z rsync: [generator] delete_file: unlink(data/processed/ms_yanalong_yanalong/distill_r1_sft.json) failed: Permission denied (13)
+2026-05-21T01:42:56.848110396Z rsync: [generator] delete_file: unlink(data/processed/ms_yanalong_yanalong/dataset_infos.json) failed: Permission denied (13)
+2026-05-21T01:42:56.848474236Z rsync: [generator] delete_file: unlink(data/processed/ms_yanalong_yanalong/data.jsonl) failed: Permission denied (13)
+2026-05-21T01:42:56.848831874Z rsync: [generator] delete_file: unlink(data/processed/ms_yanalong_yanalong/README.md) failed: Permission denied (13)
+2026-05-21T01:42:56.849030751Z rsync: [generator] delete_file: unlink(data/processed/ms_yanalong_yanalong/.mv) failed: Permission denied (13)
+2026-05-21T01:42:56.849528206Z rsync: [generator] delete_file: unlink(data/processed/ms_yanalong_yanalong/.msc) failed: Permission denied (13)
+2026-05-21T01:42:56.849846228Z rsync: [generator] delete_file: unlink(data/processed/ms_yanalong_yanalong/.gitattributes) failed: Permission denied (13)
+2026-05-21T01:42:56.850209937Z rsync: [generator] delete_file: rmdir(data/processed/ms_yanalong_yanalong) failed: Permission denied (13)
+2026-05-21T01:42:56.852835393Z rsync: [generator] delete_file: rmdir(data/processed/._____temp/yanalong/yanalong) failed: Permission denied (13)
+2026-05-21T01:42:56.853133841Z rsync: [generator] delete_file: rmdir(data/processed/._____temp/yanalong) failed: Permission denied (13)
+2026-05-21T01:42:56.853340450Z rsync: [generator] delete_file: rmdir(data/processed/._____temp) failed: Permission denied (13)
+2026-05-21T01:42:56.853678058Z rsync: [generator] delete_file: unlink(data/processed/d0412da9-b6d0-4ecf-8ae9-35600353bf3e_processed.jsonl) failed: Permission denied (13)
+2026-05-21T01:42:56.853922468Z rsync: [generator] delete_file: unlink(data/processed/cce886de-4dd5-460a-b0ac-2404731cd9f8_processed.jsonl) failed: Permission denied (13)
+2026-05-21T01:42:56.854233744Z rsync: [generator] delete_file: unlink(data/processed/aa342346-a39e-4644-9a34-f3a9d3b961f8_processed.jsonl) failed: Permission denied (13)
+2026-05-21T01:42:56.854494938Z rsync: [generator] delete_file: unlink(data/processed/a52d395e-d3c8-40d2-9be3-1839f597dc7f_processed.jsonl) failed: Permission denied (13)
+2026-05-21T01:42:56.854814568Z rsync: [generator] delete_file: unlink(data/processed/92a0a9cd-46aa-48bc-b7ad-bd5a18270c51_processed.jsonl) failed: Permission denied (13)
+2026-05-21T01:42:56.855051340Z rsync: [generator] delete_file: unlink(data/processed/7bcbc0bb-72c7-408f-a4c6-c38fb05b8382_processed.jsonl) failed: Permission denied (13)
+2026-05-21T01:42:56.855384963Z rsync: [generator] delete_file: unlink(data/processed/7aa39fbf-b396-422d-82c9-73044ee4397e_processed.jsonl) failed: Permission denied (13)
+2026-05-21T01:42:56.855699895Z rsync: [generator] delete_file: unlink(data/processed/35949b14-2872-47b4-9fdc-b87bda7279ef_processed.jsonl) failed: Permission denied (13)
+2026-05-21T01:42:56.856024323Z rsync: [generator] delete_file: unlink(data/processed/3296b92a-8bd0-4c26-8c9d-d17778abfa14_processed.jsonl) failed: Permission denied (13)
+2026-05-21T01:42:56.861121067Z rsync: [generator] delete_file: unlink(data/models/Qwen_Qwen3.5-0.8B/Qwen/Qwen3___5-0___8B/vocab.json) failed: Permission denied (13)
+2026-05-21T01:42:56.861303810Z rsync: [generator] delete_file: unlink(data/models/Qwen_Qwen3.5-0.8B/Qwen/Qwen3___5-0___8B/video_preprocessor_config.json) failed: Permission denied (13)
+2026-05-21T01:42:56.861597345Z rsync: [generator] delete_file: unlink(data/models/Qwen_Qwen3.5-0.8B/Qwen/Qwen3___5-0___8B/tokenizer_config.json) failed: Permission denied (13)
+2026-05-21T01:42:56.861934766Z rsync: [generator] delete_file: unlink(data/models/Qwen_Qwen3.5-0.8B/Qwen/Qwen3___5-0___8B/tokenizer.json) failed: Permission denied (13)
+2026-05-21T01:42:56.862143585Z rsync: [generator] delete_file: unlink(data/models/Qwen_Qwen3.5-0.8B/Qwen/Qwen3___5-0___8B/preprocessor_config.json) failed: Permission denied (13)
+2026-05-21T01:42:56.862469521Z rsync: [generator] delete_file: unlink(data/models/Qwen_Qwen3.5-0.8B/Qwen/Qwen3___5-0___8B/model.safetensors.index.json) failed: Permission denied (13)
+2026-05-21T01:42:56.862681718Z rsync: [generator] delete_file: unlink(data/models/Qwen_Qwen3.5-0.8B/Qwen/Qwen3___5-0___8B/model.safetensors-00001-of-00001.safetensors) failed: Permission denied (13)
+2026-05-21T01:42:56.863037819Z rsync: [generator] delete_file: unlink(data/models/Qwen_Qwen3.5-0.8B/Qwen/Qwen3___5-0___8B/merges.txt) failed: Permission denied (13)
+2026-05-21T01:42:56.863252628Z rsync: [generator] delete_file: unlink(data/models/Qwen_Qwen3.5-0.8B/Qwen/Qwen3___5-0___8B/configuration.json) failed: Permission denied (13)
+2026-05-21T01:42:56.863557202Z rsync: [generator] delete_file: unlink(data/models/Qwen_Qwen3.5-0.8B/Qwen/Qwen3___5-0___8B/config.json) failed: Permission denied (13)
+2026-05-21T01:42:56.863821309Z rsync: [generator] delete_file: unlink(data/models/Qwen_Qwen3.5-0.8B/Qwen/Qwen3___5-0___8B/chat_template.jinja) failed: Permission denied (13)
+2026-05-21T01:42:56.864156424Z rsync: [generator] delete_file: unlink(data/models/Qwen_Qwen3.5-0.8B/Qwen/Qwen3___5-0___8B/README.md) failed: Permission denied (13)
+2026-05-21T01:42:56.864384423Z rsync: [generator] delete_file: unlink(data/models/Qwen_Qwen3.5-0.8B/Qwen/Qwen3___5-0___8B/LICENSE) failed: Permission denied (13)
+2026-05-21T01:42:56.864706441Z rsync: [generator] delete_file: unlink(data/models/Qwen_Qwen3.5-0.8B/Qwen/Qwen3___5-0___8B/.mv) failed: Permission denied (13)
+2026-05-21T01:42:56.864969716Z rsync: [generator] delete_file: unlink(data/models/Qwen_Qwen3.5-0.8B/Qwen/Qwen3___5-0___8B/.msc) failed: Permission denied (13)
+2026-05-21T01:42:56.865258014Z rsync: [generator] delete_file: unlink(data/models/Qwen_Qwen3.5-0.8B/Qwen/Qwen3___5-0___8B/.mdl) failed: Permission denied (13)
+2026-05-21T01:42:56.868431637Z rsync: [generator] delete_file: rmdir(data/models/Qwen_Qwen3.5-0.8B/Qwen/Qwen3___5-0___8B) failed: Permission denied (13)
+2026-05-21T01:42:56.868746978Z rsync: [generator] delete_file: unlink(data/models/Qwen_Qwen3.5-0.8B/Qwen/Qwen3.5-0.8B) failed: Permission denied (13)
+2026-05-21T01:42:56.868972476Z rsync: [generator] delete_file: rmdir(data/models/Qwen_Qwen3.5-0.8B/Qwen) failed: Permission denied (13)
+2026-05-21T01:42:56.869988414Z rsync: [generator] delete_file: rmdir(data/models/Qwen_Qwen3.5-0.8B/.lock) failed: Permission denied (13)
+2026-05-21T01:42:56.872147160Z rsync: [generator] delete_file: rmdir(data/models/Qwen_Qwen3.5-0.8B/._____temp/Qwen/Qwen3.5-0.8B) failed: Permission denied (13)
+2026-05-21T01:42:56.872458402Z rsync: [generator] delete_file: rmdir(data/models/Qwen_Qwen3.5-0.8B/._____temp/Qwen) failed: Permission denied (13)
+2026-05-21T01:42:56.872736027Z rsync: [generator] delete_file: rmdir(data/models/Qwen_Qwen3.5-0.8B/._____temp) failed: Permission denied (13)
+2026-05-21T01:42:56.873042132Z rsync: [generator] delete_file: unlink(data/models/Qwen_Qwen3.5-0.8B/vocab.json) failed: Permission denied (13)
+2026-05-21T01:42:56.874314935Z rsync: [generator] delete_file: unlink(data/models/Qwen_Qwen3.5-0.8B/video_preprocessor_config.json) failed: Permission denied (13)
+2026-05-21T01:42:56.874605310Z rsync: [generator] delete_file: unlink(data/models/Qwen_Qwen3.5-0.8B/tokenizer_config.json) failed: Permission denied (13)
+2026-05-21T01:42:56.874872685Z rsync: [generator] delete_file: unlink(data/models/Qwen_Qwen3.5-0.8B/tokenizer.json) failed: Permission denied (13)
+2026-05-21T01:42:56.875139303Z rsync: [generator] delete_file: unlink(data/models/Qwen_Qwen3.5-0.8B/preprocessor_config.json) failed: Permission denied (13)
+2026-05-21T01:42:56.875378548Z rsync: [generator] delete_file: unlink(data/models/Qwen_Qwen3.5-0.8B/model.safetensors.index.json) failed: Permission denied (13)
+2026-05-21T01:42:56.875792105Z rsync: [generator] delete_file: unlink(data/models/Qwen_Qwen3.5-0.8B/model.safetensors-00001-of-00001.safetensors) failed: Permission denied (13)
+2026-05-21T01:42:56.876059182Z rsync: [generator] delete_file: unlink(data/models/Qwen_Qwen3.5-0.8B/merges.txt) failed: Permission denied (13)
+2026-05-21T01:42:56.876311446Z rsync: [generator] delete_file: unlink(data/models/Qwen_Qwen3.5-0.8B/configuration.json) failed: Permission denied (13)
+2026-05-21T01:42:56.876630817Z rsync: [generator] delete_file: unlink(data/models/Qwen_Qwen3.5-0.8B/config.json) failed: Permission denied (13)
+2026-05-21T01:42:56.876875504Z rsync: [generator] delete_file: unlink(data/models/Qwen_Qwen3.5-0.8B/chat_template.jinja) failed: Permission denied (13)
+2026-05-21T01:42:56.877135789Z rsync: [generator] delete_file: unlink(data/models/Qwen_Qwen3.5-0.8B/README.md) failed: Permission denied (13)
+2026-05-21T01:42:56.877448752Z rsync: [generator] delete_file: unlink(data/models/Qwen_Qwen3.5-0.8B/LICENSE) failed: Permission denied (13)
+2026-05-21T01:42:56.877683502Z rsync: [generator] delete_file: unlink(data/models/Qwen_Qwen3.5-0.8B/.mv) failed: Permission denied (13)
+2026-05-21T01:42:56.877983301Z rsync: [generator] delete_file: unlink(data/models/Qwen_Qwen3.5-0.8B/.msc) failed: Permission denied (13)
+2026-05-21T01:42:56.878218517Z rsync: [generator] delete_file: rmdir(data/models/Qwen_Qwen3.5-0.8B) failed: Permission denied (13)
+2026-05-21T01:42:56.880827684Z rsync: [generator] delete_file: unlink(data/models/Qwen/Qwen3___5-0___8B/vocab.json) failed: Permission denied (13)
+2026-05-21T01:42:56.881004031Z rsync: [generator] delete_file: unlink(data/models/Qwen/Qwen3___5-0___8B/video_preprocessor_config.json) failed: Permission denied (13)
+2026-05-21T01:42:56.881348815Z rsync: [generator] delete_file: unlink(data/models/Qwen/Qwen3___5-0___8B/tokenizer_config.json) failed: Permission denied (13)
+2026-05-21T01:42:56.881605518Z rsync: [generator] delete_file: unlink(data/models/Qwen/Qwen3___5-0___8B/tokenizer.json) failed: Permission denied (13)
+2026-05-21T01:42:56.882011407Z rsync: [generator] delete_file: unlink(data/models/Qwen/Qwen3___5-0___8B/preprocessor_config.json) failed: Permission denied (13)
+2026-05-21T01:42:56.882224765Z rsync: [generator] delete_file: unlink(data/models/Qwen/Qwen3___5-0___8B/model.safetensors.index.json) failed: Permission denied (13)
+2026-05-21T01:42:56.882558445Z rsync: [generator] delete_file: unlink(data/models/Qwen/Qwen3___5-0___8B/model.safetensors-00001-of-00001.safetensors) failed: Permission denied (13)
+2026-05-21T01:42:56.882772925Z rsync: [generator] delete_file: unlink(data/models/Qwen/Qwen3___5-0___8B/merges.txt) failed: Permission denied (13)
+2026-05-21T01:42:56.883155082Z rsync: [generator] delete_file: unlink(data/models/Qwen/Qwen3___5-0___8B/configuration.json) failed: Permission denied (13)
+2026-05-21T01:42:56.883371345Z rsync: [generator] delete_file: unlink(data/models/Qwen/Qwen3___5-0___8B/config.json) failed: Permission denied (13)
+2026-05-21T01:42:56.883797552Z rsync: [generator] delete_file: unlink(data/models/Qwen/Qwen3___5-0___8B/chat_template.jinja) failed: Permission denied (13)
+2026-05-21T01:42:56.884064684Z rsync: [generator] delete_file: unlink(data/models/Qwen/Qwen3___5-0___8B/README.md) failed: Permission denied (13)
+2026-05-21T01:42:56.884281954Z rsync: [generator] delete_file: unlink(data/models/Qwen/Qwen3___5-0___8B/LICENSE) failed: Permission denied (13)
+2026-05-21T01:42:56.884603442Z rsync: [generator] delete_file: unlink(data/models/Qwen/Qwen3___5-0___8B/.mv) failed: Permission denied (13)
+2026-05-21T01:42:56.884920167Z rsync: [generator] delete_file: unlink(data/models/Qwen/Qwen3___5-0___8B/.msc) failed: Permission denied (13)
+2026-05-21T01:42:56.885161873Z rsync: [generator] delete_file: unlink(data/models/Qwen/Qwen3___5-0___8B/.mdl) failed: Permission denied (13)
+2026-05-21T01:42:56.885438646Z rsync: [generator] delete_file: rmdir(data/models/Qwen/Qwen3___5-0___8B) failed: Permission denied (13)
+2026-05-21T01:42:56.885789670Z rsync: [generator] delete_file: unlink(data/models/Qwen/Qwen3.5-0.8B) failed: Permission denied (13)
+2026-05-21T01:42:56.886141849Z rsync: [generator] delete_file: rmdir(data/models/Qwen) failed: Permission denied (13)
+2026-05-21T01:42:56.887083992Z rsync: [generator] delete_file: rmdir(data/models/.lock) failed: Permission denied (13)
+2026-05-21T01:42:56.889473730Z rsync: [generator] delete_file: rmdir(data/models/._____temp/Qwen/Qwen3.5-0.8B) failed: Permission denied (13)
+2026-05-21T01:42:56.889714749Z rsync: [generator] delete_file: rmdir(data/models/._____temp/Qwen) failed: Permission denied (13)
+2026-05-21T01:42:56.890043850Z rsync: [generator] delete_file: rmdir(data/models/._____temp) failed: Permission denied (13)
+2026-05-21T01:42:56.892891835Z rsync: [generator] chown "/root/Fine-tuning/backend/.dockerignore" failed: Permission denied (13)
+2026-05-21T01:42:56.893197452Z rsync: [generator] chown "/root/Fine-tuning/backend/.env.docker" failed: Permission denied (13)
+2026-05-21T01:42:56.893398855Z rsync: [generator] chown "/root/Fine-tuning/backend/.env.example" failed: Permission denied (13)
+2026-05-21T01:42:56.893776910Z rsync: [generator] chown "/root/Fine-tuning/backend/Dockerfile" failed: Permission denied (13)
+2026-05-21T01:42:56.893962616Z rsync: [generator] chown "/root/Fine-tuning/backend/entrypoint.sh" failed: Permission denied (13)
+2026-05-21T01:42:56.894410699Z rsync: [generator] chown "/root/Fine-tuning/backend/main.py" failed: Permission denied (13)
+2026-05-21T01:42:56.894713897Z rsync: [generator] chown "/root/Fine-tuning/backend/pyproject.toml" failed: Permission denied (13)
+2026-05-21T01:42:56.895052615Z rsync: [generator] chown "/root/Fine-tuning/backend/requirements.txt" failed: Permission denied (13)
+2026-05-21T01:42:56.895456706Z rsync: [generator] chown "/root/Fine-tuning/backend/app" failed: Permission denied (13)
+2026-05-21T01:42:56.895546414Z deleting data/uploads/
+2026-05-21T01:42:56.895561539Z cannot delete non-empty directory: data/processed/ms_yanalong_yanalong
+2026-05-21T01:42:56.895568988Z cannot delete non-empty directory: data/processed/._____temp/yanalong
+2026-05-21T01:42:56.895576088Z cannot delete non-empty directory: data/processed/._____temp
+2026-05-21T01:42:56.895584816Z cannot delete non-empty directory: data/processed
+2026-05-21T01:42:56.895594908Z cannot delete non-empty directory: data/processed
+2026-05-21T01:42:56.895604967Z cannot delete non-empty directory: data/models/Qwen_Qwen3.5-0.8B/Qwen/Qwen3___5-0___8B
+2026-05-21T01:42:56.895617960Z cannot delete non-empty directory: data/models/Qwen_Qwen3.5-0.8B/Qwen
+2026-05-21T01:42:56.895625952Z cannot delete non-empty directory: data/models/Qwen_Qwen3.5-0.8B/._____temp/Qwen
+2026-05-21T01:42:56.895635994Z cannot delete non-empty directory: data/models/Qwen_Qwen3.5-0.8B/._____temp
+2026-05-21T01:42:56.895646215Z cannot delete non-empty directory: data/models/Qwen_Qwen3.5-0.8B
+2026-05-21T01:42:56.895656428Z cannot delete non-empty directory: data/models/Qwen/Qwen3___5-0___8B
+2026-05-21T01:42:56.895666067Z cannot delete non-empty directory: data/models/Qwen
+2026-05-21T01:42:56.895675881Z cannot delete non-empty directory: data/models/._____temp/Qwen
+2026-05-21T01:42:56.895684322Z cannot delete non-empty directory: data/models/._____temp
+2026-05-21T01:42:56.895693172Z cannot delete non-empty directory: data/models
+2026-05-21T01:42:56.895703593Z cannot delete non-empty directory: data/models
+2026-05-21T01:42:56.895713689Z deleting data/adapters/
+2026-05-21T01:42:56.895723807Z deleting data/config_7aa39fbf-b396-422d-82c9-73044ee4397e.json
+2026-05-21T01:42:56.895750131Z deleting data/config_3296b92a-8bd0-4c26-8c9d-d17778abfa14.json
+2026-05-21T01:42:56.895760876Z cannot delete non-empty directory: data
+2026-05-21T01:42:56.896812328Z rsync: [generator] chown "/root/Fine-tuning/backend/app/__init__.py" failed: Permission denied (13)
+2026-05-21T01:42:56.897072829Z rsync: [generator] chown "/root/Fine-tuning/backend/app/config.py" failed: Permission denied (13)
+2026-05-21T01:42:56.897311276Z rsync: [generator] chown "/root/Fine-tuning/backend/app/__pycache__" failed: Permission denied (13)
+2026-05-21T01:42:56.900492333Z rsync: [generator] chown "/root/Fine-tuning/backend/app/__pycache__/__init__.cpython-310.pyc" failed: Permission denied (13)
+2026-05-21T01:42:56.901038284Z rsync: [generator] chown "/root/Fine-tuning/backend/app/__pycache__/config.cpython-310.pyc" failed: Permission denied (13)
+2026-05-21T01:42:56.901316529Z rsync: [generator] chown "/root/Fine-tuning/backend/app/api" failed: Permission denied (13)
+2026-05-21T01:42:56.902544905Z rsync: [generator] chown "/root/Fine-tuning/backend/app/api/__init__.py" failed: Permission denied (13)
+2026-05-21T01:42:56.902885158Z rsync: [generator] chown "/root/Fine-tuning/backend/app/api/datasets.py" failed: Permission denied (13)
+2026-05-21T01:42:56.903125244Z rsync: [generator] chown "/root/Fine-tuning/backend/app/api/deployment.py" failed: Permission denied (13)
+2026-05-21T01:42:56.903439256Z rsync: [generator] chown "/root/Fine-tuning/backend/app/api/evaluation.py" failed: Permission denied (13)
+2026-05-21T01:42:56.903707834Z rsync: [generator] chown "/root/Fine-tuning/backend/app/api/inference.py" failed: Permission denied (13)
+2026-05-21T01:42:56.904043327Z rsync: [generator] chown "/root/Fine-tuning/backend/app/api/models.py" failed: Permission denied (13)
+2026-05-21T01:42:56.904370695Z rsync: [generator] chown "/root/Fine-tuning/backend/app/api/training.py" failed: Permission denied (13)
+2026-05-21T01:42:56.904779029Z rsync: [generator] chown "/root/Fine-tuning/backend/app/core" failed: Permission denied (13)
+2026-05-21T01:42:56.906246583Z rsync: [generator] chown "/root/Fine-tuning/backend/app/core/__init__.py" failed: Permission denied (13)
+2026-05-21T01:42:56.906701829Z rsync: [generator] chown "/root/Fine-tuning/backend/app/core/db.py" failed: Permission denied (13)
+2026-05-21T01:42:56.906879362Z rsync: [generator] chown "/root/Fine-tuning/backend/app/core/job_queue.py" failed: Permission denied (13)
+2026-05-21T01:42:56.907287857Z rsync: [generator] chown "/root/Fine-tuning/backend/app/core/logging.py" failed: Permission denied (13)
+2026-05-21T01:42:56.907552553Z rsync: [generator] chown "/root/Fine-tuning/backend/app/core/remote_executor.py" failed: Permission denied (13)
+2026-05-21T01:42:56.907884771Z rsync: [generator] chown "/root/Fine-tuning/backend/app/core/websocket.py" failed: Permission denied (13)
+2026-05-21T01:42:56.908137419Z rsync: [generator] chown "/root/Fine-tuning/backend/app/core/__pycache__" failed: Permission denied (13)
+2026-05-21T01:42:56.909440210Z rsync: [generator] chown "/root/Fine-tuning/backend/app/core/__pycache__/__init__.cpython-310.pyc" failed: Permission denied (13)
+2026-05-21T01:42:56.910329759Z rsync: [generator] chown "/root/Fine-tuning/backend/app/core/__pycache__/db.cpython-310.pyc" failed: Permission denied (13)
+2026-05-21T01:42:56.910706235Z rsync: [generator] chown "/root/Fine-tuning/backend/app/core/__pycache__/logging.cpython-310.pyc" failed: Permission denied (13)
+2026-05-21T01:42:56.910936052Z rsync: [generator] chown "/root/Fine-tuning/backend/app/engines" failed: Permission denied (13)
+2026-05-21T01:42:56.912356629Z rsync: [generator] chown "/root/Fine-tuning/backend/app/engines/__init__.py" failed: Permission denied (13)
+2026-05-21T01:42:56.912588928Z rsync: [generator] chown "/root/Fine-tuning/backend/app/engines/__main__.py" failed: Permission denied (13)
+2026-05-21T01:42:56.912944243Z rsync: [generator] chown "/root/Fine-tuning/backend/app/engines/base.py" failed: Permission denied (13)
+2026-05-21T01:42:56.913180704Z rsync: [generator] chown "/root/Fine-tuning/backend/app/engines/multimodal_engine.py" failed: Permission denied (13)
+2026-05-21T01:42:56.913605213Z rsync: [generator] chown "/root/Fine-tuning/backend/app/engines/remote_train.py" failed: Permission denied (13)
+2026-05-21T01:42:56.913855333Z rsync: [generator] chown "/root/Fine-tuning/backend/app/engines/text_engine.py" failed: Permission denied (13)
+2026-05-21T01:42:56.914247815Z rsync: [generator] chown "/root/Fine-tuning/backend/app/engines/vision_engine.py" failed: Permission denied (13)
+2026-05-21T01:42:56.914491762Z rsync: [generator] chown "/root/Fine-tuning/backend/app/engines/__pycache__" failed: Permission denied (13)
+2026-05-21T01:42:56.916322091Z rsync: [generator] chown "/root/Fine-tuning/backend/app/engines/__pycache__/__init__.cpython-310.pyc" failed: Permission denied (13)
+2026-05-21T01:42:56.916346676Z rsync: [generator] chown "/root/Fine-tuning/backend/app/engines/__pycache__/base.cpython-310.pyc" failed: Permission denied (13)
+2026-05-21T01:42:56.916870484Z rsync: [generator] chown "/root/Fine-tuning/backend/app/engines/__pycache__/remote_train.cpython-310.pyc" failed: Permission denied (13)
+2026-05-21T01:42:56.917025088Z rsync: [generator] chown "/root/Fine-tuning/backend/app/engines/__pycache__/text_engine.cpython-310.pyc" failed: Permission denied (13)
+2026-05-21T01:42:56.917579970Z rsync: [generator] chown "/root/Fine-tuning/backend/app/peft" failed: Permission denied (13)
+2026-05-21T01:42:56.918459857Z rsync: [generator] chown "/root/Fine-tuning/backend/app/peft/__init__.py" failed: Permission denied (13)
+2026-05-21T01:42:56.918974673Z rsync: [generator] chown "/root/Fine-tuning/backend/app/preprocessors" failed: Permission denied (13)
+2026-05-21T01:42:56.920179310Z rsync: [generator] chown "/root/Fine-tuning/backend/app/preprocessors/__init__.py" failed: Permission denied (13)
+2026-05-21T01:42:56.920541658Z rsync: [generator] chown "/root/Fine-tuning/backend/app/preprocessors/__pycache__" failed: Permission denied (13)
+2026-05-21T01:42:56.922022417Z rsync: [generator] chown "/root/Fine-tuning/backend/app/preprocessors/__pycache__/__init__.cpython-310.pyc" failed: Permission denied (13)
+2026-05-21T01:42:56.922370236Z rsync: [generator] chown "/root/Fine-tuning/backend/app/schemas" failed: Permission denied (13)
+2026-05-21T01:42:56.924649177Z rsync: [generator] chown "/root/Fine-tuning/backend/app/schemas/__init__.py" failed: Permission denied (13)
+2026-05-21T01:42:56.925553965Z rsync: [generator] chown "/root/Fine-tuning/backend/app/schemas/common.py" failed: Permission denied (13)
+2026-05-21T01:42:56.925959799Z rsync: [generator] chown "/root/Fine-tuning/backend/app/schemas/dataset.py" failed: Permission denied (13)
+2026-05-21T01:42:56.926479863Z rsync: [generator] chown "/root/Fine-tuning/backend/app/schemas/deployment.py" failed: Permission denied (13)
+2026-05-21T01:42:56.926720333Z rsync: [generator] chown "/root/Fine-tuning/backend/app/schemas/evaluation.py" failed: Permission denied (13)
+2026-05-21T01:42:56.927130231Z rsync: [generator] chown "/root/Fine-tuning/backend/app/schemas/model.py" failed: Permission denied (13)
+2026-05-21T01:42:56.927343490Z rsync: [generator] chown "/root/Fine-tuning/backend/app/schemas/model_test.py" failed: Permission denied (13)
+2026-05-21T01:42:56.927834280Z rsync: [generator] chown "/root/Fine-tuning/backend/app/schemas/training.py" failed: Permission denied (13)
+2026-05-21T01:42:56.928069685Z rsync: [generator] chown "/root/Fine-tuning/backend/app/services" failed: Permission denied (13)
+2026-05-21T01:42:56.929432461Z rsync: [generator] chown "/root/Fine-tuning/backend/app/services/dataset_service.py" failed: Permission denied (13)
+2026-05-21T01:42:56.929772735Z rsync: [generator] chown "/root/Fine-tuning/backend/app/services/deploy_service.py" failed: Permission denied (13)
+2026-05-21T01:42:56.930123436Z rsync: [generator] chown "/root/Fine-tuning/backend/app/services/eval_service.py" failed: Permission denied (13)
+2026-05-21T01:42:56.930542988Z rsync: [generator] chown "/root/Fine-tuning/backend/app/services/inference_service.py" failed: Permission denied (13)
+2026-05-21T01:42:56.930938841Z rsync: [generator] chown "/root/Fine-tuning/backend/app/services/model_service.py" failed: Permission denied (13)
+2026-05-21T01:42:56.933040670Z rsync: [generator] chown "/root/Fine-tuning/backend/app/services/model_test_service.py" failed: Permission denied (13)
+2026-05-21T01:42:56.933386401Z rsync: [generator] chown "/root/Fine-tuning/backend/app/services/training_service.py" failed: Permission denied (13)
+2026-05-21T01:42:56.933729056Z rsync: [generator] chown "/root/Fine-tuning/backend/app/services/__pycache__" failed: Permission denied (13)
+2026-05-21T01:42:56.935036979Z rsync: [generator] chown "/root/Fine-tuning/backend/app/services/__pycache__/model_service.cpython-310.pyc" failed: Permission denied (13)
+2026-05-21T01:42:56.935126540Z ./
+2026-05-21T01:42:56.935136768Z app/
+2026-05-21T01:42:56.935222473Z app/__pycache__/
+2026-05-21T01:42:56.935240207Z app/api/
+2026-05-21T01:42:56.935248055Z app/core/
+2026-05-21T01:42:56.935255127Z app/core/__pycache__/
+2026-05-21T01:42:56.935262483Z app/engines/
+2026-05-21T01:42:56.935269697Z app/engines/__pycache__/
+2026-05-21T01:42:56.935281026Z app/peft/
+2026-05-21T01:42:56.935311737Z app/preprocessors/
+2026-05-21T01:42:56.935319204Z app/preprocessors/__pycache__/
+2026-05-21T01:42:56.935326227Z app/schemas/
+2026-05-21T01:42:56.935333130Z app/services/
+2026-05-21T01:42:56.935340185Z app/services/__pycache__/
+2026-05-21T01:42:57.077344579Z rsync error: some files/attrs were not transferred (see previous errors) (code 23) at main.c(1338) [sender=3.4.1]
+2026-05-21T01:42:57.077344429Z 
+2026-05-21T01:42:57.077415918Z sent 2,556 bytes  received 1,515 bytes  232.63 bytes/sec
+2026-05-21T01:42:57.077424276Z total size is 201,685  speedup is 49.54
+2026-05-21T01:42:57.079317846Z => Sync done.
+2026-05-21T01:42:58.438464876Z INFO:     Started server process [1]
+2026-05-21T01:42:58.438511994Z INFO:     Waiting for application startup.
+2026-05-21T01:42:58.532911681Z 2026-05-21 01:42:58 | INFO     | peft-platform | JobQueue started with 2 workers
+2026-05-21T01:42:58.532994007Z INFO:     Application startup complete.
+2026-05-21T01:42:58.533493499Z INFO:     Uvicorn running on http://0.0.0.0:8010 (Press CTRL+C to quit)
+2026-05-21T01:43:00.023620598Z INFO:     127.0.0.1:37162 - "GET /health HTTP/1.1" 200 OK
+2026-05-21T01:43:30.224333123Z INFO:     127.0.0.1:42736 - "GET /health HTTP/1.1" 200 OK