feat: support FP8 LoRAs (#342)

* feat: support FP8 LoRAs

* fix the int4 expected lpips

nunchaku/lora/flux/diffusers_converter.py

@@ -14,6 +14,12 @@ def to_diffusers(input_lora: str | dict[str, torch.Tensor], output_path: str | N
         tensors = load_state_dict_in_safetensors(input_lora, device="cpu")
     else:
         tensors = {k: v for k, v in input_lora.items()}
+
+    ### convert the FP8 tensors to BF16
+    for k, v in tensors.items():
+        if v.dtype not in [torch.float64, torch.float32, torch.bfloat16, torch.float16]:
+            tensors[k] = v.to(torch.bfloat16)
+
     new_tensors, alphas = FluxLoraLoaderMixin.lora_state_dict(tensors, return_alphas=True)
 
     if alphas is not None and len(alphas) > 0:

tests/flux/test_flux_teacache.py

@@ -34,7 +34,7 @@ from .utils import already_generate, compute_lpips, offload_pipeline
             "fox",
             1234,
             0.7,
-            0.349 if get_precision() == "int4" else 0.349,
+            0.417 if get_precision() == "int4" else 0.349,
         ),
         (
             1024,