From fb90a53b1235fd836dee74452f3f2a621e0f363a Mon Sep 17 00:00:00 2001 From: Gustaf Rydholm Date: Wed, 27 Oct 2021 22:13:54 +0200 Subject: Rename transformer embeddings --- .../networks/transformer/embeddings/absolute.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 text_recognizer/networks/transformer/embeddings/absolute.py (limited to 'text_recognizer/networks/transformer/embeddings/absolute.py') diff --git a/text_recognizer/networks/transformer/embeddings/absolute.py b/text_recognizer/networks/transformer/embeddings/absolute.py new file mode 100644 index 0000000..7140537 --- /dev/null +++ b/text_recognizer/networks/transformer/embeddings/absolute.py @@ -0,0 +1,17 @@ +"""Absolute positional embedding.""" +import torch +from torch import nn, Tensor + + +class AbsolutePositionalEmbedding(nn.Module): + def __init__(self, dim: int, max_seq_len: int) -> None: + super().__init__() + self.emb = nn.Embedding(max_seq_len, dim) + self._weight_init() + + def _weight_init(self) -> None: + nn.init.normal_(self.emb.weight, std=0.02) + + def forward(self, x: Tensor) -> Tensor: + n = torch.arange(x.shape[1], device=x.device) + return self.emb(n)[None, :, :] -- cgit v1.2.3-70-g09d2