### LLM

Since your tokens are simple 1-dimensional integer variables ranging from 0 to 100, this simplifies the problem significantly compared to text generation. Given this, you don't need the full complexity of a language model designed for natural language processing, which would overcomplicate things. Here are tailored suggestions based on the nature of your tokens:

1. Small LLMs and Transformers Still Viable
Although the tokens are integers, transformer models can still work well for sequential generation tasks. They aren't limited to textual data, and you can treat the integers as discrete tokens, training an autoregressive transformer model to predict the next integer in the sequence.

Why Transformers/LLMs Are Still Suitable:
Sequential dependencies: Even if the tokens are integers, transformers are excellent at capturing dependencies across long sequences.
Long sequences: With 1028-token sequences, transformers handle long-term dependencies more effectively than RNN-based methods.
Autoregressive nature: A small autoregressive transformer (or LLM) will predict the next integer in the sequence given the previous ones.
How to Train:
Embedding: Since the tokens are integers from 0 to 100, you can use a simple embedding layer of size 101, where each integer gets mapped to a dense vector representation.
Model size: You can create a small transformer with fewer layers (e.g., 4-6 layers) and a smaller hidden dimension (e.g., 128 or 256) since the task complexity is low compared to natural language.
Loss function: Use cross-entropy loss since you’re predicting the next token (integer) from a discrete set.
Output: The model should output a softmax over 101 possible integer values at each step.
A very small GPT-like model (or custom transformer) would be sufficient in this case.

import torch
import torch.nn as nn
import torch.optim as optim
import numpy as np

# Define constants
SEQ_LENGTH = 1028  # Sequence length
VOCAB_SIZE = 101   # Tokens range from 0 to 100, so vocab size is 101
EMBEDDING_DIM = 128  # Embedding size for each token
HIDDEN_DIM = 256    # Hidden dimension in the transformer layers
NUM_LAYERS = 4      # Number of transformer layers
BATCH_SIZE = 32     # Batch size
DEVICE = torch.device('cuda' if torch.cuda.is_available() else 'cpu')  # Use GPU if available

# Transformer-based Small LLM for autoregressive token generation
class SmallLLM(nn.Module):
    def __init__(self, vocab_size, embedding_dim, hidden_dim, num_layers, seq_length):
        super(SmallLLM, self).__init__()
        self.embedding = nn.Embedding(vocab_size, embedding_dim)
        self.positional_encoding = self.generate_positional_encoding(seq_length, embedding_dim)
        self.transformer = nn.Transformer(
            d_model=embedding_dim,
            nhead=8,  # Number of attention heads
            num_encoder_layers=num_layers,
            num_decoder_layers=num_layers,
            dim_feedforward=hidden_dim
        )
        self.fc_out = nn.Linear(embedding_dim, vocab_size)
    
    def forward(self, src, tgt):
        src_embedding = self.embedding(src) + self.positional_encoding[:src.size(1), :]
        tgt_embedding = self.embedding(tgt) + self.positional_encoding[:tgt.size(1), :]

        src_embedding = src_embedding.permute(1, 0, 2)  # (seq_len, batch_size, embedding_dim)
        tgt_embedding = tgt_embedding.permute(1, 0, 2)

        # Pass through transformer
        output = self.transformer(src_embedding, tgt_embedding)
        
        # Pass through final linear layer to output token logits
        output = self.fc_out(output.permute(1, 0, 2))  # (batch_size, seq_len, vocab_size)
        return output

    def generate_positional_encoding(self, seq_length, embedding_dim):
        """ Generate positional encoding to add positional information to embeddings """
        pe = torch.zeros(seq_length, embedding_dim)
        position = torch.arange(0, seq_length, dtype=torch.float).unsqueeze(1)
        div_term = torch.exp(torch.arange(0, embedding_dim, 2).float() * -(np.log(10000.0) / embedding_dim))
        pe[:, 0::2] = torch.sin(position * div_term)
        pe[:, 1::2] = torch.cos(position * div_term)
        pe = pe.unsqueeze(0)
        return pe.to(DEVICE)


# Create the model
model = SmallLLM(vocab_size=VOCAB_SIZE, embedding_dim=EMBEDDING_DIM, hidden_dim=HIDDEN_DIM, num_layers=NUM_LAYERS, seq_length=SEQ_LENGTH).to(DEVICE)

# Define loss function and optimizer
criterion = nn.CrossEntropyLoss()
optimizer = optim.Adam(model.parameters(), lr=1e-4)

### Training Function ###
def train(model, data_loader, optimizer, criterion, num_epochs):
    model.train()
    for epoch in range(num_epochs):
        total_loss = 0
        for batch_idx, (src, tgt) in enumerate(data_loader):
            src, tgt = src.to(DEVICE), tgt.to(DEVICE)

            # Shift target for autoregressive training
            tgt_input = tgt[:, :-1]
            tgt_output = tgt[:, 1:]

            optimizer.zero_grad()
            output = model(src, tgt_input)

            # Reshape outputs and targets for loss computation
            output = output.view(-1, VOCAB_SIZE)
            tgt_output = tgt_output.contiguous().view(-1)

            loss = criterion(output, tgt_output)
            loss.backward()
            optimizer.step()

            total_loss += loss.item()

        print(f'Epoch [{epoch+1}/{num_epochs}], Loss: {total_loss/len(data_loader):.4f}')

### Dummy DataLoader for demonstration (replace with real data) ###
def generate_dummy_data(num_samples, seq_length, vocab_size):
    """Generates random integer sequences as dummy data."""
    return torch.randint(0, vocab_size, (num_samples, seq_length))

# Example data loader with dummy data
class SequenceDataset(torch.utils.data.Dataset):
    def __init__(self, num_samples, seq_length, vocab_size):
        self.data = generate_dummy_data(num_samples, seq_length, vocab_size)
    
    def __len__(self):
        return len(self.data)

    def __getitem__(self, idx):
        sequence = self.data[idx]
        src = sequence[:-1]
        tgt = sequence[1:]
        return src, tgt

# Initialize dataset and data loader
dataset = SequenceDataset(num_samples=1000, seq_length=SEQ_LENGTH, vocab_size=VOCAB_SIZE)
data_loader = torch.utils.data.DataLoader(dataset, batch_size=BATCH_SIZE, shuffle=True)

# Train the model
train(model, data_loader, optimizer, criterion, num_epochs=10)

### RNN/LSTM:

2. RNNs/GRUs/LSTMs: Lightweight Alternative
Given that your data is a sequence of integers and not text, recurrent neural networks (RNNs), especially their more efficient variants like GRUs (Gated Recurrent Units), could also be a simple, lightweight solution.

Why RNNs/GRUs/LSTMs Could Work:
Low complexity: For simple sequences of integers with a range of only 0-100, RNNs or GRUs could capture the patterns in the sequence effectively without needing the full complexity of transformers.
Sequential modeling: RNNs are designed to model sequences and can predict the next integer based on the previous ones.
Computational efficiency: RNNs or GRUs are much more computationally efficient than transformers for relatively simple tasks, making them well-suited for generating sequences of integers.
How to Train:
Embedding: Use an embedding layer of size 101 to map each integer token to a dense vector.
Model structure: A GRU with 1-2 layers and a hidden size of 128 or 256 should be enough to capture the sequential dependencies.
Loss function: Use categorical cross-entropy loss to predict the next token in the sequence.
Output: At each step, the model would output a softmax over 101 possible values (integers 0-100).

### Markov-models:

1. First-Order Markov Model
A first-order Markov model assumes that the next token in the sequence depends only on the current token. You can estimate the transition probabilities between tokens and use them for sequence generation.

Code for a Simple Markov Model.

Input ``` import numpy as np

class MarkovModel:
def __init__(self, num_states):
self.num_states = num_states
self.transition_matrix = np.zeros((num_states, num_states))

def fit(self, sequences):
"""Learn the transition probabilities from sequences of integers."""
for sequence in sequences:
for i in range(len(sequence) - 1):
current_state = sequence[i]
next_state = sequence[i + 1]
self.transition_matrix[current_state, next_state] += 1

# Normalize to get probabilities
row_sums = self.transition_matrix.sum(axis=1, keepdims=True)
np.seterr(divide='ignore', invalid='ignore') # Handle division by zero
self.transition_matrix = np.nan_to_num(self.transition_matrix / row_sums)

def generate_sequence(self, start_state, length):
"""Generate a sequence of given length starting from start_state."""
current_state = start_state
generated_sequence = [current_state]
for _ in range(length - 1):
next_state = np.random.choice(np.arange(self.num_states), p=self.transition_matrix[current_state])
generated_sequence.append(next_state)
current_state = next_state
return generated_sequence

# Example usage
num_states = 101 # Tokens range from 0 to 100
sequences = [
[1, 2, 3, 4, 5, 6, 7, 8, 9],
[5, 6, 7, 8, 9, 1, 2, 3, 4],
[10, 11, 12, 13, 14, 15, 16, 17, 18]
] # Replace with your data

markov_model = MarkovModel(num_states)
markov_model.fit(sequences)

# Generate a sequence starting from token 1
generated_sequence = markov_model.generate_sequence(start_state=1, length=10)
print("Generated sequence:", generated_sequence)