Skip to content

vllm.model_executor.layers.quantization.utils.humming.linear

Prepare and execute Humming linear layers.

Functions:

convert_linear_layer_to_humming_standard(layer, name_map)

Rename/reshape a linear layer's quantized params (the canonical MPLinear layout: weight_packed int32 + weight_scale) into the parameter names and layout humming's weight schema expects (weight / weight_scale).

Source code in vllm/model_executor/layers/quantization/utils/humming/linear.py 
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
def convert_linear_layer_to_humming_standard(
    layer: LinearBase, name_map: dict[str, str]
):
    """Rename/reshape a linear layer's quantized params (the canonical MPLinear
    layout: ``weight_packed`` int32 + ``weight_scale``) into the parameter names
    and layout humming's weight schema expects (``weight`` / ``weight_scale``)."""
    for name, checkpoint_name in name_map.items():
        param = getattr(layer, checkpoint_name)
        tensor = param.data

        if name == "weight":
            # (input_dim, output_dim) after PWAL, if it transposed the weight.
            input_dim, output_dim = getattr(
                layer,
                "processed_weight_dims",
                (getattr(param, "input_dim", 1), getattr(param, "output_dim", 0)),
            )

            if input_dim == 0 and output_dim == 1:
                tensor = tensor.transpose(1, 0).contiguous()
            else:
                assert output_dim == 0 and input_dim == 1

            tensor = tensor.view(tensor.size(0), -1).view(torch.int32)
        elif name in ["weight_scale", "zero_point"]:
            if getattr(param, "output_dim", 0) == 1:
                tensor = tensor.transpose(0, 1).contiguous()
            if tensor.ndim == 1:
                tensor = tensor.unsqueeze(1)

            tensor = tensor.view(torch.int32) if name == "zero_point" else tensor

        if name != checkpoint_name:
            # One param under both names: the loader keeps its name.
            move_parameter(layer, checkpoint_name, name)
        replace_parameter(layer, name, tensor)

replace_layer_parameters(layer, tensors, names)

Write tensors into layer, keeping existing parameter objects.

A param in names that gets no new tensor stays registered: an alias of a replaced param follows it; one whose storage a new tensor is in, or a large one, is also registered under a new name (see move_parameter); any other keeps its data, retired.

Source code in vllm/model_executor/layers/quantization/utils/humming/linear.py 
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
def replace_layer_parameters(
    layer: torch.nn.Module, tensors: dict[str, torch.Tensor], names: list[str]
) -> None:
    """Write ``tensors`` into ``layer``, keeping existing parameter objects.

    A param in ``names`` that gets no new tensor stays registered: an alias of a
    replaced param follows it; one whose storage a new tensor is in, or a large
    one, is also registered under a new name (see `move_parameter`); any other
    keeps its data, retired."""
    params = layer._parameters
    replaced = {id(p) for name in tensors if (p := params.get(name)) is not None}
    leftovers = {
        name: p
        for name in names
        if name not in tensors
        and (p := params.get(name)) is not None
        and id(p) not in replaced
    }

    def adopt(src: str, dst: str) -> None:
        move_parameter(layer, src, dst)
        for name in [n for n, p in leftovers.items() if p is params[src]]:
            del leftovers[name]

    new = [name for name in tensors if name not in params]
    for name in new:
        ptr = tensors[name].untyped_storage().data_ptr()
        src = next(
            (n for n, p in leftovers.items() if p.untyped_storage().data_ptr() == ptr),
            None,
        )
        if ptr and src is not None:
            adopt(src, name)
    limit = _KEEP_UNUSED_FRACTION * max(map(_nbytes, tensors.values()), default=0)
    large = sorted(
        (n for n, p in leftovers.items() if _nbytes(p) > limit),
        key=lambda n: _nbytes(leftovers[n]),
        reverse=True,
    )
    free = sorted(
        (n for n in new if n not in params),
        key=lambda n: _nbytes(tensors[n]),
        reverse=True,
    )
    for src, dst in zip(large, free):
        if src in leftovers:
            adopt(src, dst)
    for name in leftovers:
        retire_parameter(layer, name)
    for name, tensor in tensors.items():
        set_kernel_parameter(layer, name, tensor)