weights.py 33.2 KB
Newer Older
Nicolas Patry's avatar
Nicolas Patry committed
1
import os
2
from dataclasses import dataclass
3
from pathlib import Path
4
from typing import Dict, List, Optional, Tuple, Union
5
from safetensors import safe_open, SafetensorError
6
import torch
7
from loguru import logger
8
9
from huggingface_hub import hf_hub_download
import json
10
from text_generation_server.utils.log import log_once
11
12


13
14
15
@dataclass
class _GPTQParams:
    bits: int
16
    checkpoint_format: Optional[str]
17
18
19
20
21
22
    groupsize: int
    desc_act: bool
    quant_method: str
    sym: bool


23
class Weights:
24
25
26
27
28
29
30
    def __init__(
        self,
        filenames: List[Path],
        device,
        dtype,
        process_group,
        aliases: Optional[Dict[str, List[str]]] = None,
OlivierDehaene's avatar
OlivierDehaene committed
31
        prefix: Optional[str] = None,
32
    ):
33
34
35
36
37
38
39
40
41
        routing = {}
        for filename in filenames:
            with safe_open(filename, framework="pytorch") as f:
                for k in f.keys():
                    if k in routing:
                        raise RuntimeError(
                            f"Key {k} was found in multiple files: {filename} and {routing[k]}"
                        )
                    routing[k] = filename
42
43
44
        if aliases is None:
            aliases = {}
        self.aliases = aliases
45
46
47
48
        self.routing = routing
        self.device = device
        self.dtype = dtype
        self.process_group = process_group
Nicolas Patry's avatar
Nicolas Patry committed
49
        self.prefix = prefix
50
51
52
53
54
55
56
57
58
        self._handles = {}

    def _get_handle(self, filename):
        if filename not in self._handles:
            f = safe_open(filename, framework="pytorch")
            self._handles[filename] = f

        return self._handles[filename]

59
    def get_filename(self, tensor_name: str) -> (str, str):
Nicolas Patry's avatar
Nicolas Patry committed
60
61
62
63
64
65
66
67
68
69
        names = [tensor_name]
        if self.prefix is not None:
            prefixed = f"{self.prefix}.{tensor_name}"
            names.append(prefixed)
        for name in names:
            filename = self.routing.get(name, None)
            if filename is not None:
                return str(filename), name

            aliases = self.aliases.get(name, [])
70
71
72
73
            for alias in aliases:
                filename = self.routing.get(alias, None)
                if filename is not None:
                    return str(filename), alias
Nicolas Patry's avatar
Nicolas Patry committed
74
        raise RuntimeError(f"weight {tensor_name} does not exist")
75
76

    def _get_slice(self, tensor_name: str):
77
        filename, tensor_name = self.get_filename(tensor_name)
78
79
80
81
82
83
84
        f = self._get_handle(filename)
        slice_ = f.get_slice(tensor_name)
        return slice_

    def get_shape(self, tensor_name: str):
        return self._get_slice(tensor_name).get_shape()

OlivierDehaene's avatar
OlivierDehaene committed
85
    def get_tensor(self, tensor_name: str, to_device=True):
86
        filename, tensor_name = self.get_filename(tensor_name)
87
88
        f = self._get_handle(filename)
        tensor = f.get_tensor(tensor_name)
89
        # Special case for gptq which shouldn't convert
90
91
92
        # u4 which are disguised as int32. Exl2 uses int16
        # as well.
        if tensor.dtype not in [torch.int16, torch.int32, torch.int64]:
93
            tensor = tensor.to(dtype=self.dtype)
xiaobin's avatar
xiaobin committed
94
95
        if to_device:
            tensor = tensor.to(device=self.device)
96
97
        return tensor

98
    def get_partial_sharded(self, tensor_name: str, dim: int):
99
        filename, tensor_name = self.get_filename(tensor_name)
xiaobin's avatar
xiaobin committed
100
101
        f = self._get_handle(filename)
        slice_ = f.get_slice(tensor_name)
102
103
104
105
        world_size = self.process_group.size()
        rank = self.process_group.rank()

        size = slice_.get_shape()[dim]
106
        block_size = (size + world_size - 1) // world_size
107
108
109
110
111
112
113
114
115
        start = rank * block_size
        stop = (rank + 1) * block_size

        if dim == 0:
            tensor = slice_[start:stop]
        elif dim == 1:
            tensor = slice_[:, start:stop]
        else:
            raise NotImplementedError("Let's make that generic when needed")
116
        # Special case for gptq which shouldn't convert
117
118
        # u4 which are disguised as int32. exl2 uses int16.
        if tensor.dtype not in (torch.int16, torch.int32):
119
            tensor = tensor.to(dtype=self.dtype)
120
121
        tensor = tensor.to(device=self.device)
        return tensor
122

123
124
125
126
127
128
129
130
131
132
133
    def get_sharded(self, tensor_name: str, dim: int):
        filename, tensor_name = self.get_filename(tensor_name)
        f = self._get_handle(filename)
        slice_ = f.get_slice(tensor_name)
        world_size = self.process_group.size()
        size = slice_.get_shape()[dim]
        assert (
            size % world_size == 0
        ), f"The choosen size {size} is not compatible with sharding on {world_size} shards"
        return self.get_partial_sharded(tensor_name, dim)

134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
    def get_packed_sharded(
        self, tensor_name: str, dim: int, block_sizes: Union[int, List[int]]
    ) -> torch.Tensor:
        """
        Get a shard from a tensor that packs multiple tensors.

        When a tensor packs multiple tensors (such as QKV or an up
        projection + gate projection), sharding with `get_sharded` is not
        safe since it would not split the packed tensors across shards.

        This method shards a tensor, such that the packed tensors are
        split across shards.

        The columns are split in equally sized blocks when blocks is an `int`, or
        in blocks proportional given to the sizes. For instance `[2, 1, 1]` will
        divide an input with dimensionality `1024` in `[512, 256, 256]`. This is
        convenient for e.g. splitting QKV without knowing the storage details of
        quantized weights.
        """
        slice_ = self._get_slice(tensor_name)
        total_size = slice_.get_shape()[dim]
155
156
        block_sizes = _blocks_to_block_sizes(total_size=total_size, blocks=block_sizes)

xiaobin's avatar
xiaobin committed
157
158
159
        world_size = self.process_group.size()
        rank = self.process_group.rank()

160
        tensors = []
161
162
163
164
        block_offset = 0
        for block_size in block_sizes:
            assert (
                block_size % world_size == 0
165
            ), f"Prepacked tensor cannot be sharded across {world_size} shards"
166
167
168
            shard_block_size = block_size // world_size
            start = rank * shard_block_size
            stop = (rank + 1) * shard_block_size
169
170
171
172
173
174
175
            if dim == 0:
                tensor = slice_[block_offset + start : block_offset + stop]
            elif dim == 1:
                tensor = slice_[:, block_offset + start : block_offset + stop]
            else:
                raise NotImplementedError("Currently only dim=0 or dim=1 is supported")
            tensors.append(tensor)
176
            block_offset += block_size
177
178
        tensor = torch.cat(tensors, dim=dim)
        tensor = tensor.to(device=self.device)
179

180
181
182
183
184
        # Avoid casting quantizer dtypes.
        if tensor.dtype not in [torch.int16, torch.int32, torch.int64]:
            tensor = tensor.to(dtype=self.dtype)

        return tensor
xiaobin's avatar
xiaobin committed
185

186
187
188
189
190
191
192
193
194
195
    def get_weights_col_packed_qkv(
        self,
        prefix: str,
        quantize: str,
        num_heads: int,
        num_key_value_heads: int,
    ):
        return self.get_weights_col_packed(
            prefix, quantize, [num_heads, num_key_value_heads, num_key_value_heads]
        )
Nicolas Patry's avatar
Nicolas Patry committed
196
197
198
199

    def get_weights_col_packed_gate_up(self, prefix: str, quantize: str):
        return self.get_weights_col_packed(prefix, quantize, 2)

200
201
202
    def get_weights_col_packed(
        self, prefix: str, quantize: str, block_sizes: Union[int, List[int]]
    ):
xiaobin's avatar
xiaobin committed
203
204
        """
        Highly specific when the underlying tensor is a simple cat of Q,K,V instead of being
205
206
207
208
209
210
211
        already alternating Q,K,V within the main tensor.

        The columns are split in equally sized blocks when blocks is an `int`, or
        in blocks proportional given to the sizes. For instance `[2, 1, 1]` will
        divide an input with dimensionality `1024` in `[512, 256, 256]`. This is
        convenient for e.g. splitting QKV without knowing the storage details of
        quantized weights.
xiaobin's avatar
xiaobin committed
212
        """
213
        if quantize in ["gptq", "awq"]:
Nicolas Patry's avatar
Nicolas Patry committed
214
215
            from text_generation_server.layers.gptq import GPTQWeight

xiaobin's avatar
xiaobin committed
216
            try:
217
218
219
                qweight = self.get_packed_sharded(
                    f"{prefix}.qweight", dim=1, block_sizes=block_sizes
                )
xiaobin's avatar
xiaobin committed
220
221
            except RuntimeError:
                raise RuntimeError(
222
                    f"Cannot load `{quantize}` weight, make sure the model is already quantized."
xiaobin's avatar
xiaobin committed
223
224
                )

225
            gptq_params = self._get_gptq_params()
Ilyas Moutawwakil's avatar
Ilyas Moutawwakil committed
226

227
228
229
230
231
232
            qzeros = self.get_packed_sharded(
                f"{prefix}.qzeros", dim=1, block_sizes=block_sizes
            )
            scales = self.get_packed_sharded(
                f"{prefix}.scales", dim=1, block_sizes=block_sizes
            )
xiaobin's avatar
xiaobin committed
233
            scales = scales.to(dtype=self.dtype)
Ilyas Moutawwakil's avatar
Ilyas Moutawwakil committed
234

235
            if quantize == "gptq" and gptq_params.quant_method == "gptq":
236
                g_idx = self.get_tensor(f"{prefix}.g_idx")
237
            elif quantize == "gptq" and gptq_params.quant_method == "awq":
Ilyas Moutawwakil's avatar
Ilyas Moutawwakil committed
238
239
240
                log_once(
                    logger.info, "Converting AWQ model to Exllama/GPTQ packing format."
                )
Nicolas Patry's avatar
Nicolas Patry committed
241
                from text_generation_server.layers.awq.conversion_utils import (
Ilyas Moutawwakil's avatar
Ilyas Moutawwakil committed
242
243
244
245
246
                    fast_awq_to_gptq,
                )

                qweight, qzeros = fast_awq_to_gptq(qweight, qzeros)
                g_idx = (
247
248
249
250
251
                    torch.arange(
                        qweight.shape[0] * (32 // gptq_params.bits),
                        device=qweight.device,
                    )
                    // gptq_params.groupsize
Ilyas Moutawwakil's avatar
Ilyas Moutawwakil committed
252
                ).to(dtype=torch.int32)
253
254
            else:
                g_idx = None
xiaobin's avatar
xiaobin committed
255

256
257
258
259
260
            weight = GPTQWeight(
                qweight=qweight,
                qzeros=qzeros,
                scales=scales,
                g_idx=g_idx,
261
262
                bits=gptq_params.bits,
                groupsize=gptq_params.groupsize,
263
264
                use_exllama=False,
            )
265
        elif quantize == "marlin":
266
            from text_generation_server.layers.marlin import (
267
                GPTQMarlin24Weight,
268
269
270
271
272
                MarlinWeight,
                repack_gptq_for_marlin,
            )

            quant_method = getattr(self, "quant_method", "marlin")
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
            is_marlin_24 = getattr(self, "gptq_checkpoint_format", None) == "marlin_24"
            if is_marlin_24:
                B = self.get_packed_sharded(
                    f"{prefix}.B_24", dim=1, block_sizes=block_sizes
                )
                B_meta = self.get_packed_sharded(
                    f"{prefix}.B_meta", dim=1, block_sizes=block_sizes
                )
                s = self.get_packed_sharded(
                    f"{prefix}.s", dim=1, block_sizes=block_sizes
                )

                gptq_params = self._get_gptq_params()
                weight = GPTQMarlin24Weight(
                    B=B, B_meta=B_meta, s=s, bits=gptq_params.bits
                )
            elif quant_method == "gptq":
290
291
                gptq_params = self._get_gptq_params()
                try:
292
293
294
                    qweight = self.get_packed_sharded(
                        f"{prefix}.qweight", dim=1, block_sizes=block_sizes
                    )
295
296
297
298
299
                except RuntimeError:
                    raise RuntimeError(
                        f"Cannot load `{quantize}` weight for GPTQ -> Marlin repacking, make sure the model is already quantized"
                    )

300
301
302
                scales = self.get_packed_sharded(
                    f"{prefix}.scales", dim=1, block_sizes=block_sizes
                )
303
304
305
306
307
308
309
310
311
312
313
314
                g_idx = self.get_tensor(f"{prefix}.g_idx")
                weight = repack_gptq_for_marlin(
                    qweight=qweight,
                    scales=scales,
                    g_idx=g_idx,
                    bits=gptq_params.bits,
                    desc_act=gptq_params.desc_act,
                    groupsize=gptq_params.groupsize,
                    sym=gptq_params.sym,
                    sharded_infeatures=False,
                )
            else:
315
316
317
318
319
320
                B = self.get_packed_sharded(
                    f"{prefix}.B", dim=1, block_sizes=block_sizes
                )
                s = self.get_packed_sharded(
                    f"{prefix}.s", dim=1, block_sizes=block_sizes
                )
321
                weight = MarlinWeight(B=B, s=s)
xiaobin's avatar
xiaobin committed
322
        else:
323
324
            weight = self.get_packed_sharded(
                f"{prefix}.weight", dim=0, block_sizes=block_sizes
325
            )
xiaobin's avatar
xiaobin committed
326
327
        return weight

328
329
    def get_weights_col(self, prefix: str, quantize: str):
        if quantize == "exl2":
Nicolas Patry's avatar
Nicolas Patry committed
330
331
            from text_generation_server.layers.exl2 import Exl2Weight

332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
            try:
                q_weight = self.get_tensor(f"{prefix}.q_weight")
            except RuntimeError:
                raise RuntimeError(
                    f"Cannot load `exl2`-quantized weight, make sure the model is already quantized."
                )

            q_scale = self.get_tensor(f"{prefix}.q_scale")
            q_invperm = self.get_tensor(f"{prefix}.q_invperm")
            q_scale_max = self.get_tensor(f"{prefix}.q_scale_max")
            q_groups = self.get_tensor(f"{prefix}.q_groups")

            return Exl2Weight(
                q_weight=q_weight,
                q_scale=q_scale,
                q_invperm=q_invperm,
                q_scale_max=q_scale_max,
                q_groups=q_groups,
            )

        return self.get_multi_weights_col([prefix], quantize, 0)

354
    def get_multi_weights_col(self, prefixes: List[str], quantize: str, dim: int):
355
356
357
        if quantize == "exl2":
            raise ValueError("get_multi_weights_col is not supported for exl2")
        elif quantize in ["gptq", "awq"]:
Nicolas Patry's avatar
Nicolas Patry committed
358
359
            from text_generation_server.layers.gptq import GPTQWeight

360
            try:
361
362
363
                qweight = torch.cat(
                    [self.get_sharded(f"{p}.qweight", dim=1) for p in prefixes], dim=1
                )
364
            except RuntimeError:
365
                raise RuntimeError(
366
                    f"Cannot load `{quantize}` weight, make sure the model is already quantized"
367
368
369
370
371
372
373
374
                )

            qzeros = torch.cat(
                [self.get_sharded(f"{p}.qzeros", dim=1) for p in prefixes], dim=1
            )
            scales = torch.cat(
                [self.get_sharded(f"{p}.scales", dim=1) for p in prefixes], dim=1
            )
375

376
            gptq_params = self._get_gptq_params()
Ilyas Moutawwakil's avatar
Ilyas Moutawwakil committed
377

Nicolas Patry's avatar
Nicolas Patry committed
378
            from text_generation_server.layers.gptq import HAS_EXLLAMA
Ilyas Moutawwakil's avatar
Ilyas Moutawwakil committed
379
380

            use_exllama = (
381
382
383
384
                gptq_params.bits == 4
                and HAS_EXLLAMA
                and quantize == "gptq"
                and not gptq_params.desc_act
Ilyas Moutawwakil's avatar
Ilyas Moutawwakil committed
385
386
            )

387
            if quantize == "gptq" and gptq_params.quant_method == "gptq":
388
389
390
391
                w = [self.get_tensor(f"{p}.g_idx") for p in prefixes]
                for w2 in w[1:]:
                    torch.testing.assert_close(w2, w[0])
                g_idx = w[0]
392
            elif quantize == "gptq" and gptq_params.quant_method == "awq":
Ilyas Moutawwakil's avatar
Ilyas Moutawwakil committed
393
394
395
                log_once(
                    logger.info, "Converting AWQ model to Exllama/GPTQ packing format."
                )
Nicolas Patry's avatar
Nicolas Patry committed
396
                from text_generation_server.layers.awq.conversion_utils import (
Ilyas Moutawwakil's avatar
Ilyas Moutawwakil committed
397
398
399
400
401
402
403
404
405
                    fast_awq_to_gptq,
                )

                qweight, qzeros = fast_awq_to_gptq(qweight, qzeros)
                if use_exllama:
                    g_idx = None
                else:
                    g_idx = (
                        torch.arange(
406
407
                            qweight.shape[0] * (32 // gptq_params.bits),
                            device=qweight.device,
Ilyas Moutawwakil's avatar
Ilyas Moutawwakil committed
408
                        )
409
                        // gptq_params.groupsize
Ilyas Moutawwakil's avatar
Ilyas Moutawwakil committed
410
                    ).to(dtype=torch.int32)
411
412
            else:
                g_idx = None
413

414
415
416
417
418
            weight = GPTQWeight(
                qweight=qweight,
                qzeros=qzeros,
                scales=scales,
                g_idx=g_idx,
419
420
                bits=gptq_params.bits,
                groupsize=gptq_params.groupsize,
421
422
                use_exllama=use_exllama,
            )
423
        elif quantize == "marlin":
424
425
            from text_generation_server.layers.gptq import GPTQWeight
            from text_generation_server.layers.marlin import (
426
                GPTQMarlin24Weight,
427
428
429
                MarlinWeight,
                repack_gptq_for_marlin,
            )
430

431
            quant_method = getattr(self, "quant_method", "marlin")
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
            is_marlin_24 = getattr(self, "gptq_checkpoint_format", None) == "marlin_24"
            if is_marlin_24:
                try:
                    B = torch.cat(
                        [self.get_sharded(f"{p}.B_24", dim=1) for p in prefixes], dim=1
                    )
                except RuntimeError:
                    raise RuntimeError(
                        f"Cannot load `{quantize}` weight, make sure the model is already quantized"
                    )

                B_meta = torch.cat(
                    [self.get_sharded(f"{p}.B_meta", dim=1) for p in prefixes], dim=1
                )

                s = torch.cat(
                    [self.get_sharded(f"{p}.s", dim=1) for p in prefixes], dim=1
                )

                gptq_params = self._get_gptq_params()
                weight = GPTQMarlin24Weight(
                    B=B, B_meta=B_meta, s=s, bits=gptq_params.bits
                )
            elif quant_method == "gptq":
456
457
458
459
460
461
462
463
464
465
466
467
468
                gptq_params = self._get_gptq_params()
                try:
                    qweight = torch.cat(
                        [self.get_sharded(f"{p}.qweight", dim=1) for p in prefixes],
                        dim=1,
                    )
                except RuntimeError:
                    raise RuntimeError(
                        f"Cannot load `{quantize}` weight for GPTQ -> Marlin repacking, make sure the model is already quantized"
                    )

                scales = torch.cat(
                    [self.get_sharded(f"{p}.scales", dim=1) for p in prefixes], dim=1
469
                )
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
                w = [self.get_tensor(f"{p}.g_idx") for p in prefixes]
                for w2 in w[1:]:
                    torch.testing.assert_close(w2, w[0])
                g_idx = w[0]

                weight = repack_gptq_for_marlin(
                    qweight=qweight,
                    scales=scales,
                    g_idx=g_idx,
                    bits=gptq_params.bits,
                    desc_act=gptq_params.desc_act,
                    groupsize=gptq_params.groupsize,
                    sym=gptq_params.sym,
                    sharded_infeatures=False,
                )
            else:
                try:
                    B = torch.cat(
                        [self.get_sharded(f"{p}.B", dim=1) for p in prefixes], dim=1
                    )
                except RuntimeError:
                    raise RuntimeError(
                        f"Cannot load `{quantize}` weight, make sure the model is already quantized"
                    )
                s = torch.cat(
                    [self.get_sharded(f"{p}.s", dim=1) for p in prefixes], dim=1
496
497
                )

498
                weight = MarlinWeight(B=B, s=s)
499

500
501
502
        else:
            w = [self.get_sharded(f"{p}.weight", dim=0) for p in prefixes]
            weight = torch.cat(w, dim=dim)
503

504
        return weight
OlivierDehaene's avatar
OlivierDehaene committed
505

xiaobin's avatar
xiaobin committed
506
507
508
509
510
511
512
513
514
515
516
517
518
519
    def get_tensor_shard(self, var, dim):
        world_size = self.process_group.size()
        rank = self.process_group.rank()
        block_size = var.size()[dim] // world_size
        start = rank * block_size
        stop = (rank + 1) * block_size
        if dim == 0:
            tensor = var[start:stop]
        elif dim == 1:
            tensor = var[:, start:stop]
        else:
            raise NotImplementedError("Let's make that generic when needed")
        tensor = tensor.to(dtype=self.dtype)
        tensor = tensor.to(device=self.device)
OlivierDehaene's avatar
OlivierDehaene committed
520
        return tensor
521
522

    def get_multi_weights_row(self, prefix: str, quantize: str):
523
        if quantize == "exl2":
Nicolas Patry's avatar
Nicolas Patry committed
524
525
            from text_generation_server.layers.exl2 import Exl2Weight

526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
            try:
                q_weight = self.get_tensor(f"{prefix}.q_weight")
            except RuntimeError:
                raise RuntimeError(
                    f"Cannot load `exl2`-quantized weight, make sure the model is already quantized."
                )

            q_scale = self.get_tensor(f"{prefix}.q_scale")
            q_invperm = self.get_tensor(f"{prefix}.q_invperm")
            q_scale_max = self.get_tensor(f"{prefix}.q_scale_max")
            q_groups = self.get_tensor(f"{prefix}.q_groups")

            return Exl2Weight(
                q_weight=q_weight,
                q_scale=q_scale,
                q_invperm=q_invperm,
                q_scale_max=q_scale_max,
                q_groups=q_groups,
            )

        elif quantize == "gptq":
547
            use_exllama = True
548
            gptq_params = self._get_gptq_params()
549

550
            if gptq_params.bits != 4:
551
552
                use_exllama = False

553
            if gptq_params.desc_act:
554
555
556
                log_once(logger.warning, "Disabling exllama because desc_act=True")
                use_exllama = False

Ilyas Moutawwakil's avatar
Ilyas Moutawwakil committed
557
558
559
560
561
562
563
            try:
                qweight = self.get_sharded(f"{prefix}.qweight", dim=0)
            except RuntimeError:
                raise RuntimeError(
                    "Cannot load `gptq` weight, make sure the model is already quantized, or quantize it with `text-generation-server quantize ORIGINAL_MODEL_ID NEW_MODEL_ID`"
                )

564
            if gptq_params.quant_method == "gptq":
Ilyas Moutawwakil's avatar
Ilyas Moutawwakil committed
565
                g_idx = self.get_sharded(f"{prefix}.g_idx", dim=0)
566
            elif gptq_params.quant_method == "awq":
Ilyas Moutawwakil's avatar
Ilyas Moutawwakil committed
567
568
                g_idx = None

569
570
            if self.process_group.size() > 1:
                if g_idx is not None:
571
572
573
574
                    if (
                        not torch.equal(
                            g_idx.cpu(),
                            torch.tensor(
575
576
577
578
                                [
                                    i // gptq_params.groupsize
                                    for i in range(g_idx.shape[0])
                                ],
579
580
581
582
583
                                dtype=torch.int32,
                            ),
                        )
                        and not (g_idx == 0).all()
                    ):
584
585
586
587
                        # Exllama implementation does not support row tensor parallelism with act-order, as
                        # it would require to reorder input activations that are split unto several GPUs
                        use_exllama = False

Nicolas Patry's avatar
Nicolas Patry committed
588
589
590
591
592
            from text_generation_server.layers.gptq import (
                HAS_EXLLAMA,
                CAN_EXLLAMA,
                GPTQWeight,
            )
593

594
            if use_exllama:
595
596
                if not HAS_EXLLAMA:
                    if CAN_EXLLAMA:
597
598
                        log_once(
                            logger.warning,
OlivierDehaene's avatar
v1.3.4  
OlivierDehaene committed
599
                            "Exllama GPTQ cuda kernels (which are faster) could have been used, but are not currently installed, try using BUILD_EXTENSIONS=True",
600
                        )
601
602
                    use_exllama = False
                else:
OlivierDehaene's avatar
v1.3.4  
OlivierDehaene committed
603
                    log_once(logger.info, f"Using exllama kernels v{HAS_EXLLAMA}")
604

605
            if use_exllama and gptq_params.groupsize != -1:
Nicolas Patry's avatar
Nicolas Patry committed
606
607
                qzeros = self.get_sharded(f"{prefix}.qzeros", dim=0)
                scales = self.get_sharded(f"{prefix}.scales", dim=0)
608
609
610
            else:
                qzeros = self.get_tensor(f"{prefix}.qzeros")
                scales = self.get_tensor(f"{prefix}.scales")
611

Ilyas Moutawwakil's avatar
Ilyas Moutawwakil committed
612
            if use_exllama and g_idx is not None:
613
                g_idx = g_idx - g_idx[0]
614

615
            if gptq_params.quant_method == "awq":
Ilyas Moutawwakil's avatar
Ilyas Moutawwakil committed
616
617
618
                log_once(
                    logger.info, "Converting AWQ model to Exllama/GPTQ packing format."
                )
Nicolas Patry's avatar
Nicolas Patry committed
619
                from text_generation_server.layers.awq.conversion_utils import (
Ilyas Moutawwakil's avatar
Ilyas Moutawwakil committed
620
621
622
623
624
625
626
627
628
                    fast_awq_to_gptq,
                )

                qweight, qzeros = fast_awq_to_gptq(qweight, qzeros)
                if use_exllama:
                    g_idx = None
                else:
                    g_idx = (
                        torch.arange(
629
630
                            qweight.shape[0] * (32 // gptq_params.bits),
                            device=qweight.device,
Ilyas Moutawwakil's avatar
Ilyas Moutawwakil committed
631
                        )
632
                        // gptq_params.groupsize
Ilyas Moutawwakil's avatar
Ilyas Moutawwakil committed
633
634
                    ).to(dtype=torch.int32)

635
636
637
638
639
            weight = GPTQWeight(
                qweight=qweight,
                qzeros=qzeros,
                scales=scales,
                g_idx=g_idx,
640
641
                bits=gptq_params.bits,
                groupsize=gptq_params.groupsize,
642
643
                use_exllama=use_exllama,
            )
644
        elif quantize == "awq":
Nicolas Patry's avatar
Nicolas Patry committed
645
646
            from text_generation_server.layers.gptq import GPTQWeight

647
            gptq_params = self._get_gptq_params()
648
649
650
651
652
653
654
655
656
657
658
659

            try:
                qweight = self.get_sharded(f"{prefix}.qweight", dim=0)
            except RuntimeError:
                raise RuntimeError(
                    "Cannot load `awq` weight, make sure the model is already quantized"
                )

            qzeros = self.get_sharded(f"{prefix}.qzeros", dim=0)
            scales = self.get_sharded(f"{prefix}.scales", dim=0)
            g_idx = None
            use_exllama = False
OlivierDehaene's avatar
OlivierDehaene committed
660

661
662
663
664
665
            weight = GPTQWeight(
                qweight=qweight,
                qzeros=qzeros,
                scales=scales,
                g_idx=g_idx,
666
667
                bits=gptq_params.bits,
                groupsize=gptq_params.groupsize,
668
669
                use_exllama=use_exllama,
            )
670
        elif quantize == "marlin":
671
672
            from text_generation_server.layers.gptq import GPTQWeight
            from text_generation_server.layers.marlin import (
673
                GPTQMarlin24Weight,
674
675
676
                MarlinWeight,
                repack_gptq_for_marlin,
            )
677

678
            quant_method = getattr(self, "quant_method", "marlin")
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
            is_marlin_24 = getattr(self, "gptq_checkpoint_format", None) == "marlin_24"
            if is_marlin_24:
                try:
                    B = self.get_sharded(f"{prefix}.B_24", dim=0)
                except RuntimeError:
                    raise RuntimeError(
                        "Cannot load `marlin` 2:4 sparsity weight, make sure the model is already quantized."
                    )

                B_meta = self.get_sharded(f"{prefix}.B_meta", dim=0)
                num_groups = self._get_slice(f"{prefix}.s").get_shape()[0]
                if num_groups == 1:
                    # The number of groups is 1 when groupsize == -1. share
                    # scales between all shards in this case.
                    s = self.get_tensor(f"{prefix}.s")
                else:
                    s = self.get_sharded(f"{prefix}.s", dim=0)

                gptq_params = self._get_gptq_params()
                weight = GPTQMarlin24Weight(
                    B=B, B_meta=B_meta, s=s, bits=gptq_params.bits
                )
            elif quant_method == "gptq":
702
703
704
705
706
707
708
709
710
                log_once(logger.info, "Converting GPTQ model to Marlin packing format.")
                gptq_params = self._get_gptq_params()

                try:
                    qweight = self.get_sharded(f"{prefix}.qweight", dim=0)
                except RuntimeError:
                    raise RuntimeError(
                        f"Cannot load `{quantize}` weight for GPTQ -> Marlin repacking, make sure the model is already quantized"
                    )
711

712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
                g_idx = self.get_sharded(f"{prefix}.g_idx", dim=0)
                if gptq_params.desc_act or gptq_params.groupsize == -1:
                    scales = self.get_tensor(f"{prefix}.scales")
                else:
                    scales = self.get_sharded(f"{prefix}.scales", dim=0)

                sharded_in_features = self.process_group.size() > 1

                weight = repack_gptq_for_marlin(
                    qweight=qweight,
                    scales=scales,
                    g_idx=g_idx,
                    bits=gptq_params.bits,
                    desc_act=gptq_params.desc_act,
                    groupsize=gptq_params.groupsize,
                    sym=gptq_params.sym,
                    sharded_infeatures=sharded_in_features,
                )
730
            else:
731
732
733
734
                try:
                    B = self.get_sharded(f"{prefix}.B", dim=0)
                except RuntimeError:
                    raise RuntimeError(
735
                        "Cannot load `marlin` weight, make sure the model is already quantized."
736
737
738
739
740
741
742
743
744
745
                    )

                num_groups = self._get_slice(f"{prefix}.s").get_shape()[0]
                if num_groups == 1:
                    # The number of groups is 1 when groupsize == -1. share
                    # scales between all shards in this case.
                    s = self.get_tensor(f"{prefix}.s")
                else:
                    s = self.get_sharded(f"{prefix}.s", dim=0)
                weight = MarlinWeight(B=B, s=s)
746

747
748
749
        else:
            weight = self.get_sharded(f"{prefix}.weight", dim=1)
        return weight
750

751
    def _get_gptq_params(self) -> _GPTQParams:
752
753
754
        try:
            bits = self.get_tensor("gptq_bits").item()
            groupsize = self.get_tensor("gptq_groupsize").item()
755
            checkpoint_format = getattr(self, "gptq_checkpoint_format", None)
756
            desc_act = False
757
            sym = True
Ilyas Moutawwakil's avatar
Ilyas Moutawwakil committed
758
            quant_method = "gptq"
759
760
        except (SafetensorError, RuntimeError) as e:
            try:
761
762
                bits = self.gptq_bits
                groupsize = self.gptq_groupsize
763
                checkpoint_format = getattr(self, "gptq_checkpoint_format", None)
764
                desc_act = getattr(self, "gptq_desc_act", False)
Ilyas Moutawwakil's avatar
Ilyas Moutawwakil committed
765
                quant_method = getattr(self, "quant_method", "gptq")
766
                sym = getattr(self, "sym", True)
767
768
769
            except Exception:
                raise e

770
771
        return _GPTQParams(
            bits=bits,
772
            checkpoint_format=checkpoint_format,
773
774
775
776
777
            desc_act=desc_act,
            groupsize=groupsize,
            quant_method=quant_method,
            sym=sym,
        )
778

OlivierDehaene's avatar
OlivierDehaene committed
779
    def _set_gptq_params(self, model_id, revision):
780
        filename = "config.json"
781
        try:
782
            if os.path.exists(os.path.join(model_id, filename)):
Nicolas Patry's avatar
Nicolas Patry committed
783
784
                filename = os.path.join(model_id, filename)
            else:
OlivierDehaene's avatar
OlivierDehaene committed
785
786
787
                filename = hf_hub_download(
                    model_id, filename=filename, revision=revision
                )
788
789
            with open(filename, "r") as f:
                data = json.load(f)
790
791
            self.gptq_bits = data["quantization_config"]["bits"]
            self.gptq_groupsize = data["quantization_config"]["group_size"]
792
            # Order is important here, desc_act is missing on some real models
Ilyas Moutawwakil's avatar
Ilyas Moutawwakil committed
793
            self.quant_method = data["quantization_config"]["quant_method"]
794
795
796
            self.gptq_checkpoint_format = data["quantization_config"].get(
                "checkpoint_format"
            )
797
            self.gptq_sym = data["quantization_config"]["sym"]
798
            self.gptq_desc_act = data["quantization_config"]["desc_act"]
799
        except Exception:
800
801
802
803
804
            filename = "quantize_config.json"
            try:
                if os.path.exists(os.path.join(model_id, filename)):
                    filename = os.path.join(model_id, filename)
                else:
OlivierDehaene's avatar
OlivierDehaene committed
805
806
807
                    filename = hf_hub_download(
                        model_id, filename=filename, revision=revision
                    )
808
809
810
811
                with open(filename, "r") as f:
                    data = json.load(f)
                self.gptq_bits = data["bits"]
                self.gptq_groupsize = data["group_size"]
812
                self.gptq_sym = data["sym"]
813
                self.gptq_desc_act = data["desc_act"]
Ilyas Moutawwakil's avatar
Ilyas Moutawwakil committed
814
815
                if "version" in data and data["version"] == "GEMM":
                    self.quant_method = "awq"
816
            except Exception:
817
818
819
820
821
                filename = "quant_config.json"
                try:
                    if os.path.exists(os.path.join(model_id, filename)):
                        filename = os.path.join(model_id, filename)
                    else:
OlivierDehaene's avatar
OlivierDehaene committed
822
823
824
                        filename = hf_hub_download(
                            model_id, filename=filename, revision=revision
                        )
825
826
827
828
                    with open(filename, "r") as f:
                        data = json.load(f)
                    self.gptq_bits = data["w_bit"]
                    self.gptq_groupsize = data["q_group_size"]
829
                    self.gptq_desc_act = data["desc_act"]
Ilyas Moutawwakil's avatar
Ilyas Moutawwakil committed
830
831
                    if "version" in data and data["version"] == "GEMM":
                        self.quant_method = "awq"
832
833
                except Exception:
                    pass
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861


def _blocks_to_block_sizes(total_size: int, blocks: Union[int, List[int]]) -> List[int]:
    """
    Convert block count or proportions to block sizes.

    This function accepts

    - The number of blocks (int), in which case the block size is
      total_size//blocks; or
    - A list of block sizes (List[int]).

    In the latter case, if sum(blocks) < total_size, the ratios between
    the block sizes will be preserved. For instance, if blocks is
    [2, 1, 1] and total_size is 1024, the returned block sizes are
    [512, 256, 256].
    """
    if isinstance(blocks, list):
        total_blocks = sum(blocks)
        assert (
            total_size % total_blocks == 0
        ), f"Cannot split {total_size} in proportional blocks: {blocks}"
        part_size = total_size // total_blocks
        return [part_size * block for block in blocks]
    else:
        assert total_size % blocks == 0, f"Prepacked is not divisible by {blocks}"
        single_size = total_size // blocks
        return [single_size] * blocks