Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 19 additions & 7 deletions simpletuner/helpers/models/wan/model.py
Original file line number Diff line number Diff line change
Expand Up @@ -162,23 +162,30 @@ def add_first_frame_conditioning(
return conditioned_latent


def _vae_temporal_scale_factor(vae: AutoencoderKLWan | None) -> int:
vae_config = getattr(vae, "config", None)
temporal_downsample = getattr(vae_config, "temperal_downsample", None)
if temporal_downsample is None:
temporal_downsample = getattr(vae, "temperal_downsample", None)
return 2 ** sum(temporal_downsample) if temporal_downsample is not None else 4


@torch.no_grad()
def add_first_frame_latent_conditioning(
latent_model_input: torch.Tensor,
clean_latents: torch.Tensor,
vae: AutoencoderKLWan,
vae: AutoencoderKLWan | None = None,
*,
vae_scale_factor_temporal: int | None = None,
):
"""
Adds Wan 2.1 I2V conditioning from already-cached clean latents when the
batch does not carry explicit first-frame pixels.
"""
device = latent_model_input.device
dtype = latent_model_input.dtype
vae_config = getattr(vae, "config", None)
temporal_downsample = getattr(vae_config, "temperal_downsample", None)
if temporal_downsample is None:
temporal_downsample = getattr(vae, "temperal_downsample", None)
vae_scale_factor_temporal = 2 ** sum(temporal_downsample) if temporal_downsample is not None else 4
if vae_scale_factor_temporal is None:
vae_scale_factor_temporal = _vae_temporal_scale_factor(vae)

batch, _, num_latent_frames, latent_height, latent_width = latent_model_input.shape
num_frames = (num_latent_frames - 1) * vae_scale_factor_temporal + 1
Expand Down Expand Up @@ -279,6 +286,7 @@ class Wan(VideoModelFoundation):
ATTENTION_KWARG_NAME = "attention_kwargs"
AUTOENCODER_CLASS = AutoencoderKLWan
LATENT_CHANNEL_COUNT = 16
_vae_scale_factor_temporal = 4
_TAE_SPEC_21 = VideoTAESpec(filename="taew2_1.pth", description="Wan 2.1 / 2.2 14B VAE")
_TAE_SPEC_22 = VideoTAESpec(filename="taew2_2.pth", description="Wan 2.2 5B VAE", patch_size=2, latent_channels=48)
DEFAULT_NOISE_SCHEDULER = "unipc"
Expand Down Expand Up @@ -873,7 +881,7 @@ def _apply_i2v_conditioning_to_kwargs(self, prepared_batch, transformer_kwargs):
return
if not expand_timesteps:
transformer_kwargs["hidden_states"] = add_first_frame_latent_conditioning(
latent_tensor, clean_latents, self.get_vae()
latent_tensor, clean_latents, vae_scale_factor_temporal=self._vae_scale_factor_temporal
)
return

Expand Down Expand Up @@ -999,6 +1007,10 @@ def post_model_load_setup(self):
self._apply_time_embedding_override(getattr(self, "model", None))
self._patch_condition_embedder(getattr(self, "model", None))

def post_vae_load_setup(self):
super().post_vae_load_setup()
self._vae_scale_factor_temporal = _vae_temporal_scale_factor(self.vae)

def _should_load_other_stage(self) -> bool:
stage_info = self._wan_stage_info()
if stage_info is None:
Expand Down
19 changes: 7 additions & 12 deletions simpletuner/helpers/training/tread.py
Original file line number Diff line number Diff line change
Expand Up @@ -86,18 +86,13 @@ def get_mask(
# ---------------------------------------------------------------------
# 2) importance + randomness mix
# ---------------------------------------------------------------------
score = self._importance(x) # in [0,1]
if inverse:
score = 1.0 - score

noise = torch.rand(
score.shape,
dtype=score.dtype,
device=score.device,
generator=self.generator,
)

mix = (1.0 - l1_reg) * noise + l1_reg * score # convex combination
noise_dtype = x.real.dtype if x.is_floating_point() or x.is_complex() else torch.get_default_dtype()
mix = torch.rand((B, S), dtype=noise_dtype, device=x.device, generator=self.generator)
if l1_reg != 0.0:
score = self._importance(x)
if inverse:
score = 1.0 - score
mix = (1.0 - l1_reg) * mix + l1_reg * score
mix = mix.masked_fill(force_keep, -1.0) # force‑keep ⇒ lowest rank

# ---------------------------------------------------------------------
Expand Down
14 changes: 14 additions & 0 deletions tests/test_tread_transformers.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,20 @@


class TestTREADRouter(unittest.TestCase):
def test_random_routing_skips_importance_and_preserves_seeded_mask(self):
seed = 9
router = TREADRouter(seed=seed, device="cpu")
forced = torch.zeros(2, 6, dtype=torch.bool)
forced[0, 4] = True
forced[1, 2] = True
hidden_states = torch.arange(36, dtype=torch.bfloat16).reshape(2, 6, 3)
noise = torch.rand((2, 6), dtype=hidden_states.dtype, generator=torch.Generator().manual_seed(seed))

with patch.object(router, "_importance", side_effect=AssertionError("random routing should only use noise")):
info = router.get_mask(hidden_states, mask_ratio=0.5, force_keep=forced)

torch.testing.assert_close(info.ids_keep, noise.masked_fill(forced, -1).argsort(dim=1)[:, :3])

def test_routed_storage_only_contains_kept_tokens(self):
router = TREADRouter(device="cpu")
original = torch.arange(36, dtype=torch.float32).reshape(2, 6, 3).requires_grad_()
Expand Down
16 changes: 11 additions & 5 deletions tests/test_wan_model.py
Original file line number Diff line number Diff line change
Expand Up @@ -490,12 +490,15 @@ def test_latent_i2v_conditioning_uses_config_temporal_downsample(self):
self.assertTrue(torch.all(conditioned[:, 16:18, 0] == 1))
self.assertTrue(torch.all(conditioned[:, 16:18, 1:] == 0))

def test_i2v_conditioning_uses_cached_latents_without_warning(self):
def test_i2v_conditioning_uses_cached_latents_without_reloading_vae(self):
model = object.__new__(Wan)
model._wan_expand_timesteps = False
model._is_i2v_like_flavour = MagicMock(return_value=False)
model._extract_conditioning_frames = MagicMock(return_value=(None, None))
model.get_vae = MagicMock(return_value=SimpleNamespace(config=SimpleNamespace(temperal_downsample=[1, 1])))
model.vae = SimpleNamespace(config=SimpleNamespace(temperal_downsample=[True, False]))
model.post_vae_load_setup()
model.vae = None
model.get_vae = MagicMock(side_effect=AssertionError("the cached-latent path must not load a VAE"))

hidden_states = torch.zeros(1, 16, 3, 4, 5)
clean_latents = torch.arange(1 * 16 * 3 * 4 * 5, dtype=torch.float32).view(1, 16, 3, 4, 5)
Expand All @@ -506,10 +509,13 @@ def test_i2v_conditioning_uses_cached_latents_without_warning(self):
Wan._apply_i2v_conditioning_to_kwargs(model, prepared_batch, transformer_kwargs)

conditioned = transformer_kwargs["hidden_states"]
self.assertEqual(tuple(conditioned.shape), (1, 36, 3, 4, 5))
self.assertEqual(tuple(conditioned.shape), (1, 34, 3, 4, 5))
self.assertTrue(torch.equal(conditioned[:, :16], hidden_states))
self.assertTrue(torch.equal(conditioned[:, 20:, :1], clean_latents[:, :, :1]))
self.assertTrue(torch.all(conditioned[:, 20:, 1:] == 0))
self.assertTrue(torch.all(conditioned[:, 16:18, :1] == 1))
self.assertTrue(torch.all(conditioned[:, 16:18, 1:] == 0))
self.assertTrue(torch.equal(conditioned[:, 18:, :1], clean_latents[:, :, :1]))
self.assertTrue(torch.all(conditioned[:, 18:, 1:] == 0))
model.get_vae.assert_not_called()
warning.assert_not_called()

def test_ti2v_training_preserves_channel_count_and_conditions_only_requested_frames(self):
Expand Down
Loading