Recurrent Neural Networks for NLP

Recurrent Neural Networks (RNNs)

RNNs are neural networks designed for sequential data. Unlike feedforward networks, RNNs maintain a hidden state that captures information from previous time steps, making them naturally suited for NLP tasks like language modeling, text generation, and machine translation.

RNN Hidden State

h_t = \tanh(W_{hh} h_{t-1} + W_{xh} x_t + b)

RNN Output

y_t = W_{hy} h_t + c

Vanishing and Exploding Gradients

The fundamental challenge with vanilla RNNs is that gradients can vanish or explode during backpropagation through time (BPTT).

Gradient Through Time

\frac{\partial L_T}{\partial h_t} = \frac{\partial L_T}{\partial h_T} \prod_{k=t+1}^{T} \frac{\partial h_k}{\partial h_{k-1}}

Problem	Cause	Effect	Solution
Vanishing gradient	Small eigenvalues of W_hh	Cannot learn long-range dependencies	LSTMs, GRUs
Exploding gradient	Large eigenvalues of W_hh	Unstable training	Gradient clipping

Basic RNN Implementation

import torch
import torch.nn as nn

class SimpleRNN(nn.Module):
    def __init__(self, vocab_size, embed_dim, hidden_dim, output_dim):
        super().__init__()
        self.embedding = nn.Embedding(vocab_size, embed_dim)
        self.rnn = nn.RNN(embed_dim, hidden_dim, batch_first=True)
        self.fc = nn.Linear(hidden_dim, output_dim)

    def forward(self, text, hidden=None):
        embedded = self.embedding(text)  # (batch, seq_len, embed_dim)
        output, hidden = self.rnn(embedded, hidden)
        prediction = self.fc(output)
        return prediction, hidden

# Initialize model
model = SimpleRNN(vocab_size=10000, embed_dim=128,
                  hidden_dim=256, output_dim=10000)

# Forward pass
batch = torch.randint(0, 10000, (32, 50))  # (batch, seq_len)
output, hidden = model(batch)
print(output.shape)  # (32, 50, 10000)
print(hidden.shape)  # (1, 32, 256)

RNN for Text Classification

class RNNClassifier(nn.Module):
    def __init__(self, vocab_size, embed_dim, hidden_dim, num_classes):
        super().__init__()
        self.embedding = nn.Embedding(vocab_size, embed_dim)
        self.rnn = nn.RNN(embed_dim, hidden_dim, batch_first=True)
        self.fc = nn.Linear(hidden_dim, num_classes)

    def forward(self, text):
        embedded = self.embedding(text)
        output, hidden = self.rnn(embedded)
        # Use last hidden state for classification
        hidden = hidden.squeeze(0)  # (batch, hidden_dim)
        return self.fc(hidden)

RNN for Language Modeling

class RNNLM(nn.Module):
    def __init__(self, vocab_size, embed_dim, hidden_dim):
        super().__init__()
        self.embedding = nn.Embedding(vocab_size, embed_dim)
        self.rnn = nn.RNN(embed_dim, hidden_dim, batch_first=True)
        self.fc = nn.Linear(hidden_dim, vocab_size)

    def forward(self, text, hidden=None):
        embedded = self.embedding(text)
        output, hidden = self.rnn(embedded, hidden)
        prediction = self.fc(output)
        return prediction, hidden

    def generate(self, start_token, max_len=50):
        self.eval()
        tokens = [start_token]
        hidden = None
        with torch.no_grad():
            for _ in range(max_len):
                x = torch.tensor([[tokens[-1]]])
                output, hidden = self(x, hidden)
                prob = torch.softmax(output[:, -1], dim=-1)
                next_token = torch.argmax(prob, dim=-1).item()
                tokens.append(next_token)
        return tokens

RNN Variants

Variant	Description	Advantage
Vanilla RNN	Basic recurrent structure	Simple, fast
LSTM	Long Short-Term Memory	Captures long dependencies
GRU	Gated Recurrent Unit	Simpler than LSTM
Bidirectional	Forward + backward pass	Full context
Deep RNN	Multiple stacked layers	More capacity