1.5. 확인된 문제

ilab config init --profile <path-to-system-profile>

$ ilab config init --profile <path-to-system-profile>

INFO 2024-11-26 22:18:04,244 instructlab.model.serve_backend:56: Using model '/var/home/cloud-user/.local/share/instructlab/phased/phase2/checkpoints/hf_format/samples_29117' with -1 gpu-lay
ers and 4096 max context size.
INFO 2024-11-26 22:18:04,244 instructlab.model.serve_backend:88: '--gpus' flag used alongside '--tensor-parallel-size' in the vllm_args section of the config file. Using value of the --gpus

File "/opt/app-root/lib64/python3.11/site-packages/vllm/entrypoints/openai/api_server.py", line 105, in build_async_engine_client
    async with build_async_engine_client_from_engine_args(
  File "/usr/lib64/python3.11/contextlib.py", line 210, in __aenter__
    return await anext(self.gen)
           ^^^^^^^^^^^^^^^^^^^^^
  File "/opt/app-root/lib64/python3.11/site-packages/vllm/entrypoints/openai/api_server.py", line 192, in build_async_engine_client_from_engine_args
    raise RuntimeError(
RuntimeError: Engine process failed to start

INFO 2024-11-26 22:18:04,244 instructlab.model.serve_backend:56: Using model '/var/home/cloud-user/.local/share/instructlab/phased/phase2/checkpoints/hf_format/samples_29117' with -1 gpu-lay
ers and 4096 max context size.
INFO 2024-11-26 22:18:04,244 instructlab.model.serve_backend:88: '--gpus' flag used alongside '--tensor-parallel-size' in the vllm_args section of the config file. Using value of the --gpus

File "/opt/app-root/lib64/python3.11/site-packages/vllm/entrypoints/openai/api_server.py", line 105, in build_async_engine_client
    async with build_async_engine_client_from_engine_args(
  File "/usr/lib64/python3.11/contextlib.py", line 210, in __aenter__
    return await anext(self.gen)
           ^^^^^^^^^^^^^^^^^^^^^
  File "/opt/app-root/lib64/python3.11/site-packages/vllm/entrypoints/openai/api_server.py", line 192, in build_async_engine_client_from_engine_args
    raise RuntimeError(
RuntimeError: Engine process failed to start

sudo systemctl stop nvidia-persistenced.service
sudo systemctl start nvidia-fabricmanager.service
sudo systemctl start nvidia-persistenced.service

$ sudo systemctl stop nvidia-persistenced.service
$ sudo systemctl start nvidia-fabricmanager.service
$ sudo systemctl start nvidia-persistenced.service

ilab data generate --num-cpus 4

$ ilab data generate --num-cpus 4

serve:
  vllm:
    vllm_args: [--dtype bfloat16]

serve:
  vllm:
    vllm_args: [--dtype bfloat16]

mkdir -p /var/lib/kdump/dracut.conf.d
echo "dracutmodules=''" > /var/lib/kdump/dracut.conf.d/99-kdump.conf
echo "omit_dracutmodules=''" >> /var/lib/kdump/dracut.conf.d/99-kdump.conf
echo "dracut_args --confdir /var/lib/kdump/dracut.conf.d --install /usr/lib/passwd --install /usr/lib/group" >> /etc/kdump.conf
systemctl restart kdump

mkdir -p /var/lib/kdump/dracut.conf.d
echo "dracutmodules=''" > /var/lib/kdump/dracut.conf.d/99-kdump.conf
echo "omit_dracutmodules=''" >> /var/lib/kdump/dracut.conf.d/99-kdump.conf
echo "dracut_args --confdir /var/lib/kdump/dracut.conf.d --install /usr/lib/passwd --install /usr/lib/group" >> /etc/kdump.conf
systemctl restart kdump