hooks.py 10.9 KB
Newer Older
Aryan's avatar
Aryan committed
1
# Copyright 2025 The HuggingFace Team. All rights reserved.
Aryan's avatar
Aryan committed
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import functools
from typing import Any, Dict, Optional, Tuple

import torch

from ..utils.logging import get_logger
Aryan's avatar
Aryan committed
21
from ..utils.torch_utils import unwrap_module
Aryan's avatar
Aryan committed
22
23
24
25
26


logger = get_logger(__name__)  # pylint: disable=invalid-name


Aryan's avatar
Aryan committed
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
class BaseState:
    def reset(self, *args, **kwargs) -> None:
        raise NotImplementedError(
            "BaseState::reset is not implemented. Please implement this method in the derived class."
        )


class StateManager:
    def __init__(self, state_cls: BaseState, init_args=None, init_kwargs=None):
        self._state_cls = state_cls
        self._init_args = init_args if init_args is not None else ()
        self._init_kwargs = init_kwargs if init_kwargs is not None else {}
        self._state_cache = {}
        self._current_context = None

    def get_state(self):
        if self._current_context is None:
            raise ValueError("No context is set. Please set a context before retrieving the state.")
        if self._current_context not in self._state_cache.keys():
            self._state_cache[self._current_context] = self._state_cls(*self._init_args, **self._init_kwargs)
        return self._state_cache[self._current_context]

    def set_context(self, name: str) -> None:
        self._current_context = name

    def reset(self, *args, **kwargs) -> None:
        for name, state in list(self._state_cache.items()):
            state.reset(*args, **kwargs)
            self._state_cache.pop(name)
        self._current_context = None


Aryan's avatar
Aryan committed
59
60
61
62
63
64
65
class ModelHook:
    r"""
    A hook that contains callbacks to be executed just before and after the forward method of a model.
    """

    _is_stateful = False

66
67
68
    def __init__(self):
        self.fn_ref: "HookFunctionReference" = None

Aryan's avatar
Aryan committed
69
70
71
72
73
74
75
76
77
78
79
80
    def initialize_hook(self, module: torch.nn.Module) -> torch.nn.Module:
        r"""
        Hook that is executed when a model is initialized.

        Args:
            module (`torch.nn.Module`):
                The module attached to this hook.
        """
        return module

    def deinitalize_hook(self, module: torch.nn.Module) -> torch.nn.Module:
        r"""
81
        Hook that is executed when a model is deinitialized.
Aryan's avatar
Aryan committed
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134

        Args:
            module (`torch.nn.Module`):
                The module attached to this hook.
        """
        return module

    def pre_forward(self, module: torch.nn.Module, *args, **kwargs) -> Tuple[Tuple[Any], Dict[str, Any]]:
        r"""
        Hook that is executed just before the forward method of the model.

        Args:
            module (`torch.nn.Module`):
                The module whose forward pass will be executed just after this event.
            args (`Tuple[Any]`):
                The positional arguments passed to the module.
            kwargs (`Dict[Str, Any]`):
                The keyword arguments passed to the module.
        Returns:
            `Tuple[Tuple[Any], Dict[Str, Any]]`:
                A tuple with the treated `args` and `kwargs`.
        """
        return args, kwargs

    def post_forward(self, module: torch.nn.Module, output: Any) -> Any:
        r"""
        Hook that is executed just after the forward method of the model.

        Args:
            module (`torch.nn.Module`):
                The module whose forward pass been executed just before this event.
            output (`Any`):
                The output of the module.
        Returns:
            `Any`: The processed `output`.
        """
        return output

    def detach_hook(self, module: torch.nn.Module) -> torch.nn.Module:
        r"""
        Hook that is executed when the hook is detached from a module.

        Args:
            module (`torch.nn.Module`):
                The module detached from this hook.
        """
        return module

    def reset_state(self, module: torch.nn.Module):
        if self._is_stateful:
            raise NotImplementedError("This hook is stateful and needs to implement the `reset_state` method.")
        return module

Aryan's avatar
Aryan committed
135
136
137
138
139
140
141
142
    def _set_context(self, module: torch.nn.Module, name: str) -> None:
        # Iterate over all attributes of the hook to see if any of them have the type `StateManager`. If so, call `set_context` on them.
        for attr_name in dir(self):
            attr = getattr(self, attr_name)
            if isinstance(attr, StateManager):
                attr.set_context(name)
        return module

Aryan's avatar
Aryan committed
143

144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
class HookFunctionReference:
    def __init__(self) -> None:
        """A container class that maintains mutable references to forward pass functions in a hook chain.

        Its mutable nature allows the hook system to modify the execution chain dynamically without rebuilding the
        entire forward pass structure.

        Attributes:
            pre_forward: A callable that processes inputs before the main forward pass.
            post_forward: A callable that processes outputs after the main forward pass.
            forward: The current forward function in the hook chain.
            original_forward: The original forward function, stored when a hook provides a custom new_forward.

        The class enables hook removal by allowing updates to the forward chain through reference modification rather
        than requiring reconstruction of the entire chain. When a hook is removed, only the relevant references need to
        be updated, preserving the execution order of the remaining hooks.
        """
        self.pre_forward = None
        self.post_forward = None
        self.forward = None
        self.original_forward = None


Aryan's avatar
Aryan committed
167
168
169
170
171
172
173
174
class HookRegistry:
    def __init__(self, module_ref: torch.nn.Module) -> None:
        super().__init__()

        self.hooks: Dict[str, ModelHook] = {}

        self._module_ref = module_ref
        self._hook_order = []
175
        self._fn_refs = []
Aryan's avatar
Aryan committed
176
177
178

    def register_hook(self, hook: ModelHook, name: str) -> None:
        if name in self.hooks.keys():
179
180
181
182
            raise ValueError(
                f"Hook with name {name} already exists in the registry. Please use a different name or "
                f"first remove the existing hook and then add a new one."
            )
Aryan's avatar
Aryan committed
183
184
185

        self._module_ref = hook.initialize_hook(self._module_ref)

186
        def create_new_forward(function_reference: HookFunctionReference):
Aryan's avatar
Aryan committed
187
            def new_forward(module, *args, **kwargs):
188
189
190
                args, kwargs = function_reference.pre_forward(module, *args, **kwargs)
                output = function_reference.forward(*args, **kwargs)
                return function_reference.post_forward(module, output)
Aryan's avatar
Aryan committed
191

192
193
194
            return new_forward

        forward = self._module_ref.forward
Aryan's avatar
Aryan committed
195

196
197
198
199
200
201
202
203
204
205
206
207
        fn_ref = HookFunctionReference()
        fn_ref.pre_forward = hook.pre_forward
        fn_ref.post_forward = hook.post_forward
        fn_ref.forward = forward

        if hasattr(hook, "new_forward"):
            fn_ref.original_forward = forward
            fn_ref.forward = functools.update_wrapper(
                functools.partial(hook.new_forward, self._module_ref), hook.new_forward
            )

        rewritten_forward = create_new_forward(fn_ref)
Aryan's avatar
Aryan committed
208
        self._module_ref.forward = functools.update_wrapper(
209
            functools.partial(rewritten_forward, self._module_ref), rewritten_forward
Aryan's avatar
Aryan committed
210
211
        )

212
        hook.fn_ref = fn_ref
Aryan's avatar
Aryan committed
213
214
        self.hooks[name] = hook
        self._hook_order.append(name)
215
        self._fn_refs.append(fn_ref)
Aryan's avatar
Aryan committed
216
217

    def get_hook(self, name: str) -> Optional[ModelHook]:
218
        return self.hooks.get(name, None)
Aryan's avatar
Aryan committed
219
220
221

    def remove_hook(self, name: str, recurse: bool = True) -> None:
        if name in self.hooks.keys():
222
            num_hooks = len(self._hook_order)
Aryan's avatar
Aryan committed
223
            hook = self.hooks[name]
224
225
226
227
228
229
230
231
232
233
234
235
            index = self._hook_order.index(name)
            fn_ref = self._fn_refs[index]

            old_forward = fn_ref.forward
            if fn_ref.original_forward is not None:
                old_forward = fn_ref.original_forward

            if index == num_hooks - 1:
                self._module_ref.forward = old_forward
            else:
                self._fn_refs[index + 1].forward = old_forward

Aryan's avatar
Aryan committed
236
237
            self._module_ref = hook.deinitalize_hook(self._module_ref)
            del self.hooks[name]
238
239
            self._hook_order.pop(index)
            self._fn_refs.pop(index)
Aryan's avatar
Aryan committed
240
241
242
243
244
245
246
247
248

        if recurse:
            for module_name, module in self._module_ref.named_modules():
                if module_name == "":
                    continue
                if hasattr(module, "_diffusers_hook"):
                    module._diffusers_hook.remove_hook(name, recurse=False)

    def reset_stateful_hooks(self, recurse: bool = True) -> None:
249
        for hook_name in reversed(self._hook_order):
Aryan's avatar
Aryan committed
250
251
252
253
254
            hook = self.hooks[hook_name]
            if hook._is_stateful:
                hook.reset_state(self._module_ref)

        if recurse:
Aryan's avatar
Aryan committed
255
            for module_name, module in unwrap_module(self._module_ref).named_modules():
Aryan's avatar
Aryan committed
256
257
                if module_name == "":
                    continue
Aryan's avatar
Aryan committed
258
                module = unwrap_module(module)
Aryan's avatar
Aryan committed
259
260
261
262
263
264
265
266
267
                if hasattr(module, "_diffusers_hook"):
                    module._diffusers_hook.reset_stateful_hooks(recurse=False)

    @classmethod
    def check_if_exists_or_initialize(cls, module: torch.nn.Module) -> "HookRegistry":
        if not hasattr(module, "_diffusers_hook"):
            module._diffusers_hook = cls(module)
        return module._diffusers_hook

Aryan's avatar
Aryan committed
268
269
270
271
272
273
274
275
276
277
278
279
280
    def _set_context(self, name: Optional[str] = None) -> None:
        for hook_name in reversed(self._hook_order):
            hook = self.hooks[hook_name]
            if hook._is_stateful:
                hook._set_context(self._module_ref, name)

        for module_name, module in unwrap_module(self._module_ref).named_modules():
            if module_name == "":
                continue
            module = unwrap_module(module)
            if hasattr(module, "_diffusers_hook"):
                module._diffusers_hook._set_context(name)

Aryan's avatar
Aryan committed
281
    def __repr__(self) -> str:
282
        registry_repr = ""
Aryan's avatar
Aryan committed
283
        for i, hook_name in enumerate(self._hook_order):
284
285
286
287
288
            if self.hooks[hook_name].__class__.__repr__ is not object.__repr__:
                hook_repr = self.hooks[hook_name].__repr__()
            else:
                hook_repr = self.hooks[hook_name].__class__.__name__
            registry_repr += f"  ({i}) {hook_name} - {hook_repr}"
Aryan's avatar
Aryan committed
289
            if i < len(self._hook_order) - 1:
290
291
                registry_repr += "\n"
        return f"HookRegistry(\n{registry_repr}\n)"