runtime error

ython3.10/site-packages/deepspeed/module_inject/replace_module.py", line 826, in _replace_module _, layer_id = _replace_module(child, policies, layer_id=layer_id) File "/home/user/.pyenv/versions/3.10.14/lib/python3.10/site-packages/deepspeed/module_inject/replace_module.py", line 816, in _replace_module replaced_module = policies[child.__class__][0](child, File "/home/user/.pyenv/versions/3.10.14/lib/python3.10/site-packages/deepspeed/module_inject/replace_module.py", line 524, in replace_fn new_module = replace_with_policy(child, File "/home/user/.pyenv/versions/3.10.14/lib/python3.10/site-packages/deepspeed/module_inject/replace_module.py", line 385, in replace_with_policy _container.create_module() File "/home/user/.pyenv/versions/3.10.14/lib/python3.10/site-packages/deepspeed/module_inject/containers/gpt2.py", line 16, in create_module self.module = DeepSpeedGPTInference(_config, mp_group=self.mp_group) File "/home/user/.pyenv/versions/3.10.14/lib/python3.10/site-packages/deepspeed/model_implementations/transformers/ds_gpt.py", line 18, in __init__ super().__init__(config, File "/home/user/.pyenv/versions/3.10.14/lib/python3.10/site-packages/deepspeed/model_implementations/transformers/ds_transformer.py", line 53, in __init__ inference_cuda_module = builder.load() File "/home/user/.pyenv/versions/3.10.14/lib/python3.10/site-packages/deepspeed/ops/op_builder/builder.py", line 485, in load return self.jit_load(verbose) File "/home/user/.pyenv/versions/3.10.14/lib/python3.10/site-packages/deepspeed/ops/op_builder/builder.py", line 500, in jit_load self.build_for_cpu = not assert_no_cuda_mismatch(self.name) File "/home/user/.pyenv/versions/3.10.14/lib/python3.10/site-packages/deepspeed/ops/op_builder/builder.py", line 105, in assert_no_cuda_mismatch raise Exception( Exception: >- DeepSpeed Op Builder: Installed CUDA version 12.3 does not match the version torch was compiled with 11.7, unable to compile cuda/cpp extensions without a matching cuda version.

Container logs: