controllite fail on sdxl

[laiyanyi]

Traceback (most recent call last):
  File "D:\SD\webui_forge_cu121_torch231\webui\modules_forge\main_thread.py", line 30, in work
    self.result = self.func(*self.args, **self.kwargs)
  File "D:\SD\webui_forge_cu121_torch231\webui\modules\txt2img.py", line 112, in txt2img_function
    processed = processing.process_images(p)
  File "D:\SD\webui_forge_cu121_torch231\webui\modules\processing.py", line 815, in process_images
    res = process_images_inner(p)
  File "D:\SD\webui_forge_cu121_torch231\webui\modules\processing.py", line 958, in process_images_inner
    samples_ddim = p.sample(conditioning=p.c, unconditional_conditioning=p.uc, seeds=p.seeds, subseeds=p.subseeds, subseed_strength=p.subseed_strength, prompts=p.prompts)
  File "D:\SD\webui_forge_cu121_torch231\webui\modules\processing.py", line 1329, in sample
    samples = self.sampler.sample(self, x, conditioning, unconditional_conditioning, image_conditioning=self.txt2img_image_conditioning(x))
  File "D:\SD\webui_forge_cu121_torch231\webui\modules\sd_samplers_kdiffusion.py", line 238, in sample
    samples = self.launch_sampling(steps, lambda: self.func(self.model_wrap_cfg, x, extra_args=self.sampler_extra_args, disable=False, callback=self.callback_state, **extra_params_kwargs))
  File "D:\SD\webui_forge_cu121_torch231\webui\modules\sd_samplers_common.py", line 272, in launch_sampling
    return func()
  File "D:\SD\webui_forge_cu121_torch231\webui\modules\sd_samplers_kdiffusion.py", line 238, in <lambda>
    samples = self.launch_sampling(steps, lambda: self.func(self.model_wrap_cfg, x, extra_args=self.sampler_extra_args, disable=False, callback=self.callback_state, **extra_params_kwargs))
  File "D:\SD\webui_forge_cu121_torch231\system\python\lib\site-packages\torch\utils\_contextlib.py", line 115, in decorate_context
    return func(*args, **kwargs)
  File "D:\SD\webui_forge_cu121_torch231\webui\k_diffusion\sampling.py", line 595, in sample_dpmpp_2m
    denoised = model(x, sigmas[i] * s_in, **extra_args)
  File "D:\SD\webui_forge_cu121_torch231\system\python\lib\site-packages\torch\nn\modules\module.py", line 1532, in _wrapped_call_impl
    return self._call_impl(*args, **kwargs)
  File "D:\SD\webui_forge_cu121_torch231\system\python\lib\site-packages\torch\nn\modules\module.py", line 1541, in _call_impl
    return forward_call(*args, **kwargs)
  File "D:\SD\webui_forge_cu121_torch231\webui\modules\sd_samplers_cfg_denoiser.py", line 186, in forward
    denoised, cond_pred, uncond_pred = sampling_function(self, denoiser_params=denoiser_params, cond_scale=cond_scale, cond_composition=cond_composition)
  File "D:\SD\webui_forge_cu121_torch231\webui\backend\sampling\sampling_function.py", line 359, in sampling_function
    denoised, cond_pred, uncond_pred = sampling_function_inner(model, x, timestep, uncond, cond, cond_scale, model_options, seed, return_full=True)
  File "D:\SD\webui_forge_cu121_torch231\webui\backend\sampling\sampling_function.py", line 300, in sampling_function_inner
    cond_pred, uncond_pred = calc_cond_uncond_batch(model, cond, uncond_, x, timestep, model_options)
  File "D:\SD\webui_forge_cu121_torch231\webui\backend\sampling\sampling_function.py", line 270, in calc_cond_uncond_batch
    output = model.apply_model(input_x, timestep_, **c).chunk(batch_chunks)
  File "D:\SD\webui_forge_cu121_torch231\webui\backend\modules\k_model.py", line 45, in apply_model
    model_output = self.diffusion_model(xc, t, context=context, control=control, transformer_options=transformer_options, **extra_conds).float()
  File "D:\SD\webui_forge_cu121_torch231\system\python\lib\site-packages\torch\nn\modules\module.py", line 1532, in _wrapped_call_impl
    return self._call_impl(*args, **kwargs)
  File "D:\SD\webui_forge_cu121_torch231\system\python\lib\site-packages\torch\nn\modules\module.py", line 1541, in _call_impl
    return forward_call(*args, **kwargs)
  File "D:\SD\webui_forge_cu121_torch231\webui\backend\nn\unet.py", line 713, in forward
    h = module(h, emb, context, transformer_options)
  File "D:\SD\webui_forge_cu121_torch231\system\python\lib\site-packages\torch\nn\modules\module.py", line 1532, in _wrapped_call_impl
    return self._call_impl(*args, **kwargs)
  File "D:\SD\webui_forge_cu121_torch231\system\python\lib\site-packages\torch\nn\modules\module.py", line 1541, in _call_impl
    return forward_call(*args, **kwargs)
  File "D:\SD\webui_forge_cu121_torch231\webui\backend\nn\unet.py", line 83, in forward
    x = layer(x, context, transformer_options)
  File "D:\SD\webui_forge_cu121_torch231\system\python\lib\site-packages\torch\nn\modules\module.py", line 1532, in _wrapped_call_impl
    return self._call_impl(*args, **kwargs)
  File "D:\SD\webui_forge_cu121_torch231\system\python\lib\site-packages\torch\nn\modules\module.py", line 1541, in _call_impl
    return forward_call(*args, **kwargs)
  File "D:\SD\webui_forge_cu121_torch231\webui\backend\nn\unet.py", line 321, in forward
    x = block(x, context=context[i], transformer_options=transformer_options)
  File "D:\SD\webui_forge_cu121_torch231\system\python\lib\site-packages\torch\nn\modules\module.py", line 1532, in _wrapped_call_impl
    return self._call_impl(*args, **kwargs)
  File "D:\SD\webui_forge_cu121_torch231\system\python\lib\site-packages\torch\nn\modules\module.py", line 1541, in _call_impl
    return forward_call(*args, **kwargs)
  File "D:\SD\webui_forge_cu121_torch231\webui\backend\nn\unet.py", line 181, in forward
    return checkpoint(self._forward, (x, context, transformer_options), None, self.checkpoint)
  File "D:\SD\webui_forge_cu121_torch231\webui\backend\nn\unet.py", line 12, in checkpoint
    return f(*args)
  File "D:\SD\webui_forge_cu121_torch231\webui\backend\nn\unet.py", line 216, in _forward
    n, context_attn1, value_attn1 = p(n, context_attn1, value_attn1, extra_options)
  File "D:\SD\webui_forge_cu121_torch231\webui\extensions-builtin\sd_forge_controlllite\lib_controllllite\lib_controllllite.py", line 99, in __call__
    q = q + self.modules[module_pfx_to_q](q)
  File "D:\SD\webui_forge_cu121_torch231\system\python\lib\site-packages\torch\nn\modules\module.py", line 1532, in _wrapped_call_impl
    return self._call_impl(*args, **kwargs)
  File "D:\SD\webui_forge_cu121_torch231\system\python\lib\site-packages\torch\nn\modules\module.py", line 1541, in _call_impl
    return forward_call(*args, **kwargs)
  File "D:\SD\webui_forge_cu121_torch231\webui\extensions-builtin\sd_forge_controlllite\lib_controllllite\lib_controllllite.py", line 213, in forward
    cx = self.conditioning1(self.cond_image.to(x.device, dtype=x.dtype))
  File "D:\SD\webui_forge_cu121_torch231\system\python\lib\site-packages\torch\nn\modules\module.py", line 1532, in _wrapped_call_impl
    return self._call_impl(*args, **kwargs)
  File "D:\SD\webui_forge_cu121_torch231\system\python\lib\site-packages\torch\nn\modules\module.py", line 1541, in _call_impl
    return forward_call(*args, **kwargs)
  File "D:\SD\webui_forge_cu121_torch231\system\python\lib\site-packages\torch\nn\modules\container.py", line 217, in forward
    input = module(input)
  File "D:\SD\webui_forge_cu121_torch231\system\python\lib\site-packages\torch\nn\modules\module.py", line 1532, in _wrapped_call_impl
    return self._call_impl(*args, **kwargs)
  File "D:\SD\webui_forge_cu121_torch231\system\python\lib\site-packages\torch\nn\modules\module.py", line 1541, in _call_impl
    return forward_call(*args, **kwargs)
  File "D:\SD\webui_forge_cu121_torch231\system\python\lib\site-packages\torch\nn\modules\conv.py", line 460, in forward
    return self._conv_forward(input, self.weight, self.bias)
  File "D:\SD\webui_forge_cu121_torch231\system\python\lib\site-packages\torch\nn\modules\conv.py", line 456, in _conv_forward
    return F.conv2d(input, weight, bias, self.stride,
RuntimeError: Input type (struct c10::Half) and bias type (float) should be the same
Input type (struct c10::Half) and bias type (float) should be the same

keep fail to use controllite model in new forge.