deep-learning-with-python-notebooks icon indicating copy to clipboard operation
deep-learning-with-python-notebooks copied to clipboard

the two "source" parameters should be "key" and "value" in function of multi_head_attention

Open jazzlee opened this issue 4 weeks ago • 0 comments

the two "source" paramters should be "key" and "value" repectively, in function of multi_head_attention

in the code snippet below ,in chapter 15

query_dense = [layers.Dense(head_dim) for i in range(num_heads)]
key_dense = [layers.Dense(head_dim) for i in range(num_heads)]
value_dense = [layers.Dense(head_dim) for i in range(num_heads)]
output_dense = layers.Dense(head_dim * num_heads)

def multi_head_attention(query, key, value):
    head_outputs = []
    for i in range(num_heads):
        query = query_dense[i](query)
        key = key_dense[i](key)
        value = value_dense[i](value)
        scores = np.einsum("btd,bsd->bts", target, **_source_**)
        scores = softmax(scores / math.sqrt(head_dim), axis=-1)
        head_output = np.einsum("bts,bsd->btd", scores, **_source_**)
        head_outputs.append(head_output)
    outputs = ops.concatenate(head_outputs, axis=-1)
    return output_dense(outputs)

multi_head_attention(query=target, key=source, value=source)

jazzlee avatar Nov 23 '25 05:11 jazzlee