Encoder-Decoder：Seq2seq

一、编码器解码器架构：

1.定义：

Encoder负责对Input进行特征提取，输出特征矩阵State
Decoder接收State，负责进行预测并输出

2.在CNN中的体现：

3.在RNN中的体现：

4.代码：

from torch import nnclass Encoder(nn.Module):"""编码器-解码器结构的基本编码器接口。"""def __init__(self, **kwargs):super(Encoder, self).__init__(**kwargs)def forward(self, X, *args):raise NotImplementedErrorclass Decoder(nn.Module):"""编码器-解码器结构的基本解码器接口。"""def __init__(self, **kwargs):super(Decoder, self).__init__(**kwargs)def init_state(self, enc_outputs, *args):raise NotImplementedErrordef forward(self, X, state):raise NotImplementedErrorclass EncoderDecoder(nn.Module):"""编码器-解码器结构的基类。"""def __init__(self, encoder, decoder, **kwargs):super(EncoderDecoder, self).__init__(**kwargs)self.encoder = encoderself.decoder = decoderdef forward(self, enc_X, dec_X, *args):enc_outputs = self.encoder(enc_X, *args)dec_state = self.decoder.init_state(enc_outputs, *args)return self.decoder(dec_X, dec_state)    

二、Seq2seq：

1.模型架构：

这里以机器翻译任务为例：

1.1编码器：

编码器不管在训练阶段还是预测阶段都是用于提取特征，可以是单层RNN、多层RNN、双向RNN（双向RNN不仅可以提取上文序列特征，还可以提取下文的序列特征）

1.2解码器：

解码器在不同阶段作用不同，只能是单层RNN或多层RNN，不能是双向RNN（解码器用于预测，双向RNN不能预测）

• 训练阶段，解码器主要是为了特征提取，通过接收编码器的输出隐藏状态作为h0并接收预测的真实值Input，每个时间步使用隐藏状态ht-1进行特征提取并更新隐藏状态ht，然后将ht和当前时间步的真实值token（而非预测值，因为是要更好的学习）作为下一个时间步的输入，不断更新可学习参数。
• 预测阶段，解码器主要是为了执行预测任务，不再接收预测的真实值（因为不知道），仅接收编码器的输出隐藏状态作为h0，每个时间步使用隐藏状态ht-1进行预测并更新隐藏状态ht，然后将ht和当前时间步的预测值token作为下一个时间步的输入，进行下一个token的预测。

2.架构细节：

Seq2seq的编码器和解码器都是RNN

3.模型评估指标BLEU：

4.代码：

import collections
import math
import torch
from torch import nn
from d2l import torch as d2l# 使用GRU作为编码器
class Seq2SeqEncoder(d2l.Encoder):def __init__(self, vocab_size, embed_size, num_hiddens, num_layers,dropout=0, **kwargs):super(Seq2SeqEncoder, self).__init__(**kwargs)# 1self.embedding = nn.Embedding(vocab_size, embed_size)# 2 self.rnn = nn.GRU(embed_size, num_hiddens, num_layers,dropout=dropout)def forward(self, X, *args):X = self.embedding(X)X = X.permute(1, 0, 2)output, state = self.rnn(X)return output, state# 使用GRU作为解码器
class Seq2SeqDecoder(d2l.Decoder):def __init__(self, vocab_size, embed_size, num_hiddens, num_layers,dropout=0, **kwargs):super(Seq2SeqDecoder, self).__init__(**kwargs)# 1self.embedding = nn.Embedding(vocab_size, embed_size)# 2self.rnn = nn.GRU(embed_size + num_hiddens, num_hiddens, num_layers,dropout=dropout)# 3self.dense = nn.Linear(num_hiddens, vocab_size)def init_state(self, enc_outputs, *args):return enc_outputs[1]def forward(self, X, state):X = self.embedding(X).permute(1, 0, 2)context = state[-1].repeat(X.shape[0], 1, 1)X_and_context = torch.cat((X, context), 2)output, state = self.rnn(X_and_context, state)output = self.dense(output).permute(1, 0, 2)return output, state# 在序列中屏蔽不相关的项,即屏蔽序列中之前使用<pad>填充的无效值   
def sequence_mask(X, valid_len, value=0):maxlen = X.size(1)mask = torch.arange((maxlen), dtype=torch.float32,device=X.device)[None, :] < valid_len[:, None]X[~mask] = valuereturn X# 填充的无效值不参与损失值的计算，因为这些值的预测对错没有意义
class MaskedSoftmaxCELoss(nn.CrossEntropyLoss):def forward(self, pred, label, valid_len):weights = torch.ones_like(label)weights = sequence_mask(weights, valid_len)self.reduction = 'none'unweighted_loss = super(MaskedSoftmaxCELoss,self).forward(pred.permute(0, 2, 1), label)weighted_loss = (unweighted_loss * weights).mean(dim=1)return weighted_loss# 训练过程
def train_seq2seq(net, data_iter, lr, num_epochs, tgt_vocab, device):def xavier_init_weights(m):if type(m) == nn.Linear:nn.init.xavier_uniform_(m.weight)if type(m) == nn.GRU:for param in m._flat_weights_names:if "weight" in param:nn.init.xavier_uniform_(m._parameters[param])net.apply(xavier_init_weights)net.to(device)optimizer = torch.optim.Adam(net.parameters(), lr=lr)loss = MaskedSoftmaxCELoss()net.train()animator = d2l.Animator(xlabel='epoch', ylabel='loss',xlim=[10, num_epochs])for epoch in range(num_epochs):timer = d2l.Timer()metric = d2l.Accumulator(2)for batch in data_iter:X, X_valid_len, Y, Y_valid_len = [x.to(device) for x in batch]bos = torch.tensor([tgt_vocab['<bos>']] * Y.shape[0],device=device).reshape(-1, 1)dec_input = torch.cat([bos, Y[:, :-1]], 1)Y_hat, _ = net(X, dec_input, X_valid_len)l = loss(Y_hat, Y, Y_valid_len)l.sum().backward()d2l.grad_clipping(net, 1)num_tokens = Y_valid_len.sum()optimizer.step()with torch.no_grad():metric.add(l.sum(), num_tokens)if (epoch + 1) % 10 == 0:animator.add(epoch + 1, (metric[0] / metric[1],))print(f'loss {metric[0] / metric[1]:.3f}, {metric[1] / timer.stop():.1f} 'f'tokens/sec on {str(device)}')# 预测过程    
def predict_seq2seq(net, src_sentence, src_vocab, tgt_vocab, num_steps,device, save_attention_weights=False):net.eval()src_tokens = src_vocab[src_sentence.lower().split(' ')] + [src_vocab['<eos>']]enc_valid_len = torch.tensor([len(src_tokens)], device=device)src_tokens = d2l.truncate_pad(src_tokens, num_steps, src_vocab['<pad>'])enc_X = torch.unsqueeze(torch.tensor(src_tokens, dtype=torch.long, device=device), dim=0)enc_outputs = net.encoder(enc_X, enc_valid_len)dec_state = net.decoder.init_state(enc_outputs, enc_valid_len)dec_X = torch.unsqueeze(torch.tensor([tgt_vocab['<bos>']], dtype=torch.long, device=device),dim=0)output_seq, attention_weight_seq = [], []for _ in range(num_steps):Y, dec_state = net.decoder(dec_X, dec_state)dec_X = Y.argmax(dim=2)pred = dec_X.squeeze(dim=0).type(torch.int32).item()if save_attention_weights:attention_weight_seq.append(net.decoder.attention_weights)if pred == tgt_vocab['<eos>']:breakoutput_seq.append(pred)return ' '.join(tgt_vocab.to_tokens(output_seq)), attention_weight_seq    # 模型评估指标
def bleu(pred_seq, label_seq, k):  pred_tokens, label_tokens = pred_seq.split(' '), label_seq.split(' ')len_pred, len_label = len(pred_tokens), len(label_tokens)score = math.exp(min(0, 1 - len_label / len_pred))for n in range(1, k + 1):num_matches, label_subs = 0, collections.defaultdict(int)for i in range(len_label - n + 1):label_subs[''.join(label_tokens[i:i + n])] += 1for i in range(len_pred - n + 1):if label_subs[''.join(pred_tokens[i:i + n])] > 0:num_matches += 1label_subs[''.join(pred_tokens[i:i + n])] -= 1score *= math.pow(num_matches / (len_pred - n + 1), math.pow(0.5, n))return score# 训练    
embed_size, num_hiddens, num_layers, dropout = 32, 32, 2, 0.1
batch_size, num_steps = 64, 10
lr, num_epochs, device = 0.005, 300, d2l.try_gpu()train_iter, src_vocab, tgt_vocab = d2l.load_data_nmt(batch_size, num_steps)
encoder = Seq2SeqEncoder(len(src_vocab), embed_size, num_hiddens, num_layers,dropout)
decoder = Seq2SeqDecoder(len(tgt_vocab), embed_size, num_hiddens, num_layers,dropout)
net = d2l.EncoderDecoder(encoder, decoder)
train_seq2seq(net, train_iter, lr, num_epochs, tgt_vocab, device)    # 预测    
engs = ['go .', "i lost .", 'he\'s calm .', 'i\'m home .']
fras = ['va !', 'j\'ai perdu .', 'il est calme .', 'je suis chez moi .']
for eng, fra in zip(engs, fras):translation, attention_weight_seq = predict_seq2seq(net, eng, src_vocab, tgt_vocab, num_steps, device)print(f'{eng} => {translation}, bleu {bleu(translation, fra, k=2):.3f}')

三、束搜索：

1.贪心搜索：

2.束搜索：