Cpp Reinforcement Learning

C++ Reinforcement Learning

Overview

This skill covers implementing reinforcement learning algorithms in C++ using LibTorch (PyTorch C++ frontend) and modern C++17/20 features. It provides patterns for building high-performance RL systems suitable for production deployment, robotics, game AI, and real-time applications.

When to Use

• Implementing DQN, PPO, SAC, or other RL algorithms in C++
• Building performance-critical RL training pipelines
• Creating efficient replay buffers with proper memory management
• Deploying trained models with ONNX Runtime
• Parallelizing environment rollouts across threads
• Integrating RL with existing C++ codebases (games, robotics, simulations)

Core Libraries

Primary: LibTorch (PyTorch C++ Frontend)

LibTorch provides the same tensor operations and autograd capabilities as PyTorch in C++.

Installation: Download from https://pytorch.org/get-started/locally (select C++/LibTorch)

CMake Integration:

cmake_minimum_required(VERSION 3.18)
project(rl_project)

set(CMAKE_CXX_STANDARD 17)
find_package(Torch REQUIRED)

add_executable(train_agent src/main.cpp)
target_link_libraries(train_agent "${TORCH_LIBRARIES}")

Secondary Libraries

• ONNX Runtime - Cross-platform inference deployment
• cpprl (mhubii/cpprl) - Reference PPO implementation
• Gymnasium C++ bindings - Environment interfaces

Quick Start: DQN Agent

#include <torch/torch.h>

struct DQNNet : torch::nn::Module {
    torch::nn::Linear fc1{nullptr}, fc2{nullptr}, fc3{nullptr};

    DQNNet(int64_t state_dim, int64_t action_dim) {
        fc1 = register_module("fc1", torch::nn::Linear(state_dim, 128));
        fc2 = register_module("fc2", torch::nn::Linear(128, 128));
        fc3 = register_module("fc3", torch::nn::Linear(128, action_dim));
    }

    torch::Tensor forward(torch::Tensor x) {
        x = torch::relu(fc1->forward(x));
        x = torch::relu(fc2->forward(x));
        return fc3->forward(x);
    }
};

// Training loop
auto policy_net = std::make_shared<DQNNet>(state_dim, action_dim);
auto target_net = std::make_shared<DQNNet>(state_dim, action_dim);
torch::optim::Adam optimizer(policy_net->parameters(), lr);

// Compute loss
auto q_values = policy_net->forward(states).gather(1, actions);
auto next_q = target_net->forward(next_states).max(1).values.detach();
auto target = rewards + gamma * next_q * (1 - dones);
auto loss = torch::mse_loss(q_values.squeeze(), target);

// Backward pass
optimizer.zero_grad();
loss.backward();
optimizer.step();

Essential Patterns

Replay Buffer (Ring Buffer)

class ReplayBuffer {
public:
    explicit ReplayBuffer(size_t capacity)
        : capacity_(capacity), position_(0), size_(0) {
        buffer_.reserve(capacity);
    }

    void push(Experience exp) {
        if (buffer_.size() < capacity_) {
            buffer_.push_back(std::move(exp));
        } else {
            buffer_[position_] = std::move(exp);
        }
        position_ = (position_ + 1) % capacity_;
        size_ = std::min(size_ + 1, capacity_);
    }

    std::vector<Experience> sample(size_t batch_size);

private:
    std::vector<Experience> buffer_;
    size_t capacity_, position_, size_;
    std::mt19937 rng_{std::random_device{}()};
};

GPU Device Management

torch::Device device = torch::cuda::is_available() ? torch::kCUDA : torch::kCPU;
model->to(device);

// Create tensors on device
auto tensor = torch::zeros({batch_size, state_dim},
    torch::TensorOptions().device(device).dtype(torch::kFloat32));

Inference Mode

{
    torch::NoGradGuard no_grad;
    auto action_values = model->forward(state);
    auto action = action_values.argmax(1);
}

Common Pitfalls

1. Forgetting train/eval mode - Call model->train() or model->eval() 2. Missing NoGradGuard - Use for inference to save memory 3. Tensor accumulation - Use .detach() for stored tensors 4. Thread safety - Clone models for parallel threads 5. Device mismatch - Verify all tensors on same device

Reference Files

• references/libtorch.md - LibTorch setup and API guide
• references/algorithms.md - DQN, PPO, SAC implementations
• references/memory-management.md - Replay buffers, smart pointers, RAII
• references/performance.md - Optimization, parallelization, GPU
• references/testing.md - Testing and debugging strategies

RL Algorithm Implementations in C++

Deep Q-Network (DQN)

Network Architecture

#include <torch/torch.h>

struct DQNNet : torch::nn::Module {
    torch::nn::Linear fc1{nullptr}, fc2{nullptr}, fc3{nullptr};

    DQNNet(int64_t state_dim, int64_t action_dim) {
        fc1 = register_module("fc1", torch::nn::Linear(state_dim, 128));
        fc2 = register_module("fc2", torch::nn::Linear(128, 128));
        fc3 = register_module("fc3", torch::nn::Linear(128, action_dim));
    }

    torch::Tensor forward(torch::Tensor x) {
        x = torch::relu(fc1->forward(x));
        x = torch::relu(fc2->forward(x));
        return fc3->forward(x);
    }
};

Dueling DQN Architecture

struct DuelingDQN : torch::nn::Module {
    torch::nn::Linear feature{nullptr};
    torch::nn::Linear value_stream{nullptr};
    torch::nn::Linear advantage_stream{nullptr};

    DuelingDQN(int64_t state_dim, int64_t action_dim) {
        feature = register_module("feature", torch::nn::Linear(state_dim, 128));
        value_stream = register_module("value", torch::nn::Linear(128, 1));
        advantage_stream = register_module("advantage", torch::nn::Linear(128, action_dim));
    }

    torch::Tensor forward(torch::Tensor x) {
        x = torch::relu(feature->forward(x));
        auto value = value_stream->forward(x);
        auto advantage = advantage_stream->forward(x);
        // Q = V + (A - mean(A))
        return value + advantage - advantage.mean(1, true);
    }
};

DQN Agent Class

class DQNAgent {
public:
    DQNAgent(int64_t state_dim, int64_t action_dim, double lr = 1e-3,
             double gamma = 0.99, double epsilon_start = 1.0,
             double epsilon_end = 0.01, int64_t epsilon_decay = 10000)
        : state_dim_(state_dim), action_dim_(action_dim), gamma_(gamma),
          epsilon_(epsilon_start), epsilon_end_(epsilon_end),
          epsilon_decay_(epsilon_decay), step_count_(0) {

        device_ = torch::cuda::is_available() ? torch::kCUDA : torch::kCPU;

        policy_net_ = std::make_shared<DQNNet>(state_dim, action_dim);
        target_net_ = std::make_shared<DQNNet>(state_dim, action_dim);
        policy_net_->to(device_);
        target_net_->to(device_);

        // Copy weights to target
        update_target_network();

        optimizer_ = std::make_unique<torch::optim::Adam>(
            policy_net_->parameters(), torch::optim::AdamOptions(lr));
    }

    int64_t select_action(torch::Tensor state) {
        // Epsilon-greedy action selection
        epsilon_ = epsilon_end_ + (epsilon_ - epsilon_end_) *
                   std::exp(-1.0 * step_count_ / epsilon_decay_);
        step_count_++;

        if (dist_(rng_) < epsilon_) {
            return action_dist_(rng_);
        }

        torch::NoGradGuard no_grad;
        policy_net_->eval();
        auto q_values = policy_net_->forward(state.to(device_));
        policy_net_->train();
        return q_values.argmax(1).item<int64_t>();
    }

    void train_step(const std::vector<Experience>& batch) {
        // Stack batch into tensors
        std::vector<torch::Tensor> states, next_states;
        std::vector<int64_t> actions;
        std::vector<float> rewards;
        std::vector<float> dones;

        for (const auto& exp : batch) {
            states.push_back(exp.state);
            actions.push_back(exp.action);
            rewards.push_back(exp.reward);
            next_states.push_back(exp.next_state);
            dones.push_back(exp.done ? 1.0f : 0.0f);
        }

        auto state_batch = torch::stack(states).to(device_);
        auto action_batch = torch::tensor(actions).to(device_).unsqueeze(1);
        auto reward_batch = torch::tensor(rewards).to(device_);
        auto next_state_batch = torch::stack(next_states).to(device_);
        auto done_batch = torch::tensor(dones).to(device_);

        // Compute Q(s, a)
        auto q_values = policy_net_->forward(state_batch).gather(1, action_batch).squeeze();

        // Compute target: r + gamma * max_a' Q_target(s', a')
        torch::Tensor next_q_values;
        {
            torch::NoGradGuard no_grad;
            next_q_values = target_net_->forward(next_state_batch).max(1).values;
        }
        auto target = reward_batch + gamma_ * next_q_values * (1 - done_batch);

        // Huber loss (more stable than MSE)
        auto loss = torch::smooth_l1_loss(q_values, target.detach());

        // Optimize
        optimizer_->zero_grad();
        loss.backward();
        // Gradient clipping
        torch::nn::utils::clip_grad_norm_(policy_net_->parameters(), 1.0);
        optimizer_->step();
    }

    void update_target_network() {
        auto policy_params = policy_net_->named_parameters();
        auto target_params = target_net_->named_parameters();

        torch::NoGradGuard no_grad;
        for (auto& param : target_params) {
            auto& name = param.key();
            param.value().copy_(policy_params[name]);
        }
    }

    void soft_update_target(double tau = 0.005) {
        auto policy_params = policy_net_->named_parameters();
        auto target_params = target_net_->named_parameters();

        torch::NoGradGuard no_grad;
        for (auto& param : target_params) {
            auto& name = param.key();
            param.value().copy_(
                tau * policy_params[name] + (1 - tau) * param.value()
            );
        }
    }

private:
    std::shared_ptr<DQNNet> policy_net_;
    std::shared_ptr<DQNNet> target_net_;
    std::unique_ptr<torch::optim::Adam> optimizer_;
    torch::Device device_{torch::kCPU};

    int64_t state_dim_, action_dim_;
    double gamma_, epsilon_, epsilon_end_;
    int64_t epsilon_decay_, step_count_;

    std::mt19937 rng_{std::random_device{}()};
    std::uniform_real_distribution<double> dist_{0.0, 1.0};
    std::uniform_int_distribution<int64_t> action_dist_{0, action_dim_ - 1};
};

Proximal Policy Optimization (PPO)

Actor-Critic Network

struct ActorCritic : torch::nn::Module {
    torch::nn::Linear shared1{nullptr}, shared2{nullptr};
    torch::nn::Linear policy_mean{nullptr};
    torch::nn::Linear policy_log_std{nullptr};
    torch::nn::Linear value_head{nullptr};

    int64_t action_dim_;

    ActorCritic(int64_t state_dim, int64_t action_dim, int64_t hidden_dim = 64)
        : action_dim_(action_dim) {
        shared1 = register_module("shared1", torch::nn::Linear(state_dim, hidden_dim));
        shared2 = register_module("shared2", torch::nn::Linear(hidden_dim, hidden_dim));
        policy_mean = register_module("policy_mean", torch::nn::Linear(hidden_dim, action_dim));
        policy_log_std = register_module("policy_log_std", torch::nn::Linear(hidden_dim, action_dim));
        value_head = register_module("value", torch::nn::Linear(hidden_dim, 1));
    }

    torch::Tensor forward_shared(torch::Tensor x) {
        x = torch::tanh(shared1->forward(x));
        x = torch::tanh(shared2->forward(x));
        return x;
    }

    std::tuple<torch::Tensor, torch::Tensor, torch::Tensor> forward(torch::Tensor x) {
        auto features = forward_shared(x);
        auto mean = policy_mean->forward(features);
        auto log_std = policy_log_std->forward(features).clamp(-20, 2);
        auto value = value_head->forward(features);
        return {mean, log_std, value};
    }

    torch::Tensor get_value(torch::Tensor x) {
        auto features = forward_shared(x);
        return value_head->forward(features);
    }

    std::tuple<torch::Tensor, torch::Tensor, torch::Tensor> evaluate_actions(
        torch::Tensor states, torch::Tensor actions) {
        auto [mean, log_std, value] = forward(states);
        auto std = log_std.exp();

        // Compute log probability of actions
        auto var = std.pow(2);
        auto log_prob = -0.5 * (((actions - mean).pow(2) / var) +
                                2 * log_std + std::log(2 * M_PI));
        log_prob = log_prob.sum(-1, true);

        // Entropy for exploration bonus
        auto entropy = 0.5 * (1 + std::log(2 * M_PI) + 2 * log_std).sum(-1).mean();

        return {log_prob, value, entropy};
    }

    std::pair<torch::Tensor, torch::Tensor> sample_action(torch::Tensor state) {
        auto [mean, log_std, value] = forward(state);
        auto std = log_std.exp();

        // Sample from Gaussian
        auto noise = torch::randn_like(mean);
        auto action = mean + std * noise;

        // Compute log probability
        auto var = std.pow(2);
        auto log_prob = -0.5 * (((action - mean).pow(2) / var) +
                                2 * log_std + std::log(2 * M_PI));
        log_prob = log_prob.sum(-1, true);

        return {action, log_prob};
    }
};

PPO Update

struct PPOConfig {
    double clip_epsilon = 0.2;
    double value_loss_coef = 0.5;
    double entropy_coef = 0.01;
    double max_grad_norm = 0.5;
    int64_t ppo_epochs = 10;
    int64_t mini_batch_size = 64;
    double gae_lambda = 0.95;
    double gamma = 0.99;
};

class PPOAgent {
public:
    PPOAgent(int64_t state_dim, int64_t action_dim, const PPOConfig& config)
        : config_(config) {
        device_ = torch::cuda::is_available() ? torch::kCUDA : torch::kCPU;

        actor_critic_ = std::make_shared<ActorCritic>(state_dim, action_dim);
        actor_critic_->to(device_);

        optimizer_ = std::make_unique<torch::optim::Adam>(
            actor_critic_->parameters(), torch::optim::AdamOptions(3e-4));
    }

    void update(RolloutBuffer& buffer) {
        auto [states, actions, old_log_probs, returns, advantages] =
            buffer.get_tensors(device_);

        // Normalize advantages
        advantages = (advantages - advantages.mean()) / (advantages.std() + 1e-8);

        for (int64_t epoch = 0; epoch < config_.ppo_epochs; ++epoch) {
            auto indices = torch::randperm(states.size(0));

            for (int64_t start = 0; start < states.size(0); start += config_.mini_batch_size) {
                auto end = std::min(start + config_.mini_batch_size, states.size(0));
                auto batch_indices = indices.slice(0, start, end);

                auto batch_states = states.index_select(0, batch_indices);
                auto batch_actions = actions.index_select(0, batch_indices);
                auto batch_old_log_probs = old_log_probs.index_select(0, batch_indices);
                auto batch_returns = returns.index_select(0, batch_indices);
                auto batch_advantages = advantages.index_select(0, batch_indices);

                // Evaluate actions with current policy
                auto [log_probs, values, entropy] =
                    actor_critic_->evaluate_actions(batch_states, batch_actions);

                // Policy loss with clipping
                auto ratio = (log_probs - batch_old_log_probs).exp();
                auto surr1 = ratio * batch_advantages;
                auto surr2 = ratio.clamp(1 - config_.clip_epsilon,
                                         1 + config_.clip_epsilon) * batch_advantages;
                auto policy_loss = -torch::min(surr1, surr2).mean();

                // Value loss
                auto value_loss = torch::mse_loss(values.squeeze(), batch_returns);

                // Total loss
                auto loss = policy_loss +
                           config_.value_loss_coef * value_loss -
                           config_.entropy_coef * entropy;

                // Optimize
                optimizer_->zero_grad();
                loss.backward();
                torch::nn::utils::clip_grad_norm_(actor_critic_->parameters(),
                                                   config_.max_grad_norm);
                optimizer_->step();
            }
        }
    }

    // Compute Generalized Advantage Estimation
    std::vector<float> compute_gae(
        const std::vector<float>& rewards,
        const std::vector<float>& values,
        const std::vector<bool>& dones,
        float last_value
    ) {
        std::vector<float> advantages(rewards.size());
        float gae = 0;

        for (int64_t t = rewards.size() - 1; t >= 0; --t) {
            float next_value = (t == rewards.size() - 1) ? last_value : values[t + 1];
            float next_non_terminal = dones[t] ? 0.0f : 1.0f;

            float delta = rewards[t] + config_.gamma * next_value * next_non_terminal - values[t];
            gae = delta + config_.gamma * config_.gae_lambda * next_non_terminal * gae;
            advantages[t] = gae;
        }

        return advantages;
    }

private:
    std::shared_ptr<ActorCritic> actor_critic_;
    std::unique_ptr<torch::optim::Adam> optimizer_;
    torch::Device device_{torch::kCPU};
    PPOConfig config_;
};

Soft Actor-Critic (SAC)

SAC Networks

// Soft Q-Network (two Q-networks for stability)
struct SoftQNetwork : torch::nn::Module {
    torch::nn::Linear fc1{nullptr}, fc2{nullptr}, fc3{nullptr};

    SoftQNetwork(int64_t state_dim, int64_t action_dim, int64_t hidden_dim = 256) {
        fc1 = register_module("fc1", torch::nn::Linear(state_dim + action_dim, hidden_dim));
        fc2 = register_module("fc2", torch::nn::Linear(hidden_dim, hidden_dim));
        fc3 = register_module("fc3", torch::nn::Linear(hidden_dim, 1));
    }

    torch::Tensor forward(torch::Tensor state, torch::Tensor action) {
        auto x = torch::cat({state, action}, 1);
        x = torch::relu(fc1->forward(x));
        x = torch::relu(fc2->forward(x));
        return fc3->forward(x);
    }
};

// Gaussian Policy for SAC
struct GaussianPolicy : torch::nn::Module {
    torch::nn::Linear fc1{nullptr}, fc2{nullptr};
    torch::nn::Linear mean_head{nullptr}, log_std_head{nullptr};

    float log_std_min_ = -20.0f;
    float log_std_max_ = 2.0f;
    float action_scale_;

    GaussianPolicy(int64_t state_dim, int64_t action_dim,
                   int64_t hidden_dim = 256, float action_scale = 1.0f)
        : action_scale_(action_scale) {
        fc1 = register_module("fc1", torch::nn::Linear(state_dim, hidden_dim));
        fc2 = register_module("fc2", torch::nn::Linear(hidden_dim, hidden_dim));
        mean_head = register_module("mean", torch::nn::Linear(hidden_dim, action_dim));
        log_std_head = register_module("log_std", torch::nn::Linear(hidden_dim, action_dim));
    }

    std::tuple<torch::Tensor, torch::Tensor> forward(torch::Tensor state) {
        auto x = torch::relu(fc1->forward(state));
        x = torch::relu(fc2->forward(x));
        auto mean = mean_head->forward(x);
        auto log_std = log_std_head->forward(x).clamp(log_std_min_, log_std_max_);
        return {mean, log_std};
    }

    std::tuple<torch::Tensor, torch::Tensor> sample(torch::Tensor state) {
        auto [mean, log_std] = forward(state);
        auto std = log_std.exp();

        // Reparameterization trick
        auto noise = torch::randn_like(mean);
        auto x_t = mean + std * noise;

        // Squash through tanh
        auto action = torch::tanh(x_t) * action_scale_;

        // Compute log probability with correction for tanh squashing
        auto log_prob = -0.5 * (noise.pow(2) + 2 * log_std + std::log(2 * M_PI));
        log_prob = log_prob.sum(-1, true);
        // Jacobian correction for tanh
        log_prob -= (2 * (std::log(2.0) - x_t - torch::softplus(-2 * x_t))).sum(-1, true);

        return {action, log_prob};
    }
};

SAC Agent

class SACAgent {
public:
    SACAgent(int64_t state_dim, int64_t action_dim,
             double lr = 3e-4, double gamma = 0.99, double tau = 0.005,
             double alpha = 0.2, bool auto_entropy = true)
        : gamma_(gamma), tau_(tau), alpha_(alpha), auto_entropy_(auto_entropy) {

        device_ = torch::cuda::is_available() ? torch::kCUDA : torch::kCPU;

        // Initialize networks
        policy_ = std::make_shared<GaussianPolicy>(state_dim, action_dim);
        q1_ = std::make_shared<SoftQNetwork>(state_dim, action_dim);
        q2_ = std::make_shared<SoftQNetwork>(state_dim, action_dim);
        q1_target_ = std::make_shared<SoftQNetwork>(state_dim, action_dim);
        q2_target_ = std::make_shared<SoftQNetwork>(state_dim, action_dim);

        policy_->to(device_);
        q1_->to(device_);
        q2_->to(device_);
        q1_target_->to(device_);
        q2_target_->to(device_);

        // Copy to targets
        hard_update(q1_target_, q1_);
        hard_update(q2_target_, q2_);

        // Optimizers
        policy_optimizer_ = std::make_unique<torch::optim::Adam>(
            policy_->parameters(), torch::optim::AdamOptions(lr));
        q_optimizer_ = std::make_unique<torch::optim::Adam>(
            concat_params(q1_->parameters(), q2_->parameters()),
            torch::optim::AdamOptions(lr));

        // Automatic entropy tuning
        if (auto_entropy_) {
            target_entropy_ = -static_cast<float>(action_dim);
            log_alpha_ = torch::zeros({1}, torch::requires_grad()).to(device_);
            alpha_optimizer_ = std::make_unique<torch::optim::Adam>(
                std::vector<torch::Tensor>{log_alpha_},
                torch::optim::AdamOptions(lr));
        }
    }

    torch::Tensor select_action(torch::Tensor state, bool deterministic = false) {
        torch::NoGradGuard no_grad;
        auto state_t = state.to(device_);

        if (deterministic) {
            auto [mean, _] = policy_->forward(state_t);
            return torch::tanh(mean) * policy_->action_scale_;
        } else {
            auto [action, _] = policy_->sample(state_t);
            return action;
        }
    }

    void update(ReplayBuffer& buffer, int64_t batch_size) {
        auto batch = buffer.sample(batch_size);

        // Convert to tensors
        auto states = stack_experiences(batch, &Experience::state).to(device_);
        auto actions = stack_experiences(batch, &Experience::action).to(device_);
        auto rewards = stack_experiences(batch, &Experience::reward).to(device_);
        auto next_states = stack_experiences(batch, &Experience::next_state).to(device_);
        auto dones = stack_experiences(batch, &Experience::done).to(device_);

        // Update Q-functions
        torch::Tensor q1_loss, q2_loss;
        {
            torch::NoGradGuard no_grad;
            auto [next_actions, next_log_probs] = policy_->sample(next_states);
            auto q1_next = q1_target_->forward(next_states, next_actions);
            auto q2_next = q2_target_->forward(next_states, next_actions);
            auto min_q_next = torch::min(q1_next, q2_next) - alpha_ * next_log_probs;
            auto q_target = rewards + gamma_ * (1 - dones) * min_q_next;
        }

        auto q1_pred = q1_->forward(states, actions);
        auto q2_pred = q2_->forward(states, actions);
        q1_loss = torch::mse_loss(q1_pred, q_target.detach());
        q2_loss = torch::mse_loss(q2_pred, q_target.detach());

        q_optimizer_->zero_grad();
        (q1_loss + q2_loss).backward();
        q_optimizer_->step();

        // Update policy
        auto [new_actions, log_probs] = policy_->sample(states);
        auto q1_new = q1_->forward(states, new_actions);
        auto q2_new = q2_->forward(states, new_actions);
        auto min_q_new = torch::min(q1_new, q2_new);
        auto policy_loss = (alpha_ * log_probs - min_q_new).mean();

        policy_optimizer_->zero_grad();
        policy_loss.backward();
        policy_optimizer_->step();

        // Update temperature
        if (auto_entropy_) {
            auto alpha_loss = -(log_alpha_ * (log_probs + target_entropy_).detach()).mean();
            alpha_optimizer_->zero_grad();
            alpha_loss.backward();
            alpha_optimizer_->step();
            alpha_ = log_alpha_.exp().item<double>();
        }

        // Soft update targets
        soft_update(q1_target_, q1_, tau_);
        soft_update(q2_target_, q2_, tau_);
    }

private:
    void hard_update(std::shared_ptr<SoftQNetwork>& target,
                     std::shared_ptr<SoftQNetwork>& source) {
        torch::NoGradGuard no_grad;
        auto target_params = target->named_parameters();
        auto source_params = source->named_parameters();
        for (auto& param : target_params) {
            param.value().copy_(source_params[param.key()]);
        }
    }

    void soft_update(std::shared_ptr<SoftQNetwork>& target,
                     std::shared_ptr<SoftQNetwork>& source, double tau) {
        torch::NoGradGuard no_grad;
        auto target_params = target->named_parameters();
        auto source_params = source->named_parameters();
        for (auto& param : target_params) {
            param.value().copy_(
                tau * source_params[param.key()] + (1 - tau) * param.value()
            );
        }
    }

    std::shared_ptr<GaussianPolicy> policy_;
    std::shared_ptr<SoftQNetwork> q1_, q2_, q1_target_, q2_target_;
    std::unique_ptr<torch::optim::Adam> policy_optimizer_, q_optimizer_, alpha_optimizer_;
    torch::Device device_{torch::kCPU};

    double gamma_, tau_, alpha_;
    bool auto_entropy_;
    float target_entropy_;
    torch::Tensor log_alpha_;
};

Algorithm Selection Guide

Hyperparameter Starting Points

DQN:

• Learning rate: 1e-4 to 1e-3
• Batch size: 32-128
• Replay buffer: 100k-1M
• Target update: Every 1000 steps or tau=0.005
• Epsilon decay: 10k-100k steps

PPO:

• Learning rate: 3e-4
• Clip epsilon: 0.1-0.2
• GAE lambda: 0.95
• PPO epochs: 4-10
• Mini-batch size: 64-256
• Rollout length: 128-2048

SAC:

• Learning rate: 3e-4
• Tau (soft update): 0.005
• Alpha (entropy): 0.2 or auto-tuned
• Batch size: 256
• Replay buffer: 1M

LibTorch (PyTorch C++ Frontend) Guide

Overview

LibTorch is the C++ distribution of PyTorch, providing the same tensor operations, autograd, and neural network modules available in Python. It is the recommended library for implementing RL algorithms in C++ due to its maturity, documentation, and seamless model interoperability with Python PyTorch.

Installation

Download Pre-built LibTorch

1. Visit https://pytorch.org/get-started/locally 2. Select:

• PyTorch Build: Stable (2.x)
• OS: Linux/Mac/Windows
• Package: LibTorch
• Language: C++/Java
• CUDA: Select appropriate version (or CPU)

3. Download and extract to a known location (e.g., /opt/libtorch)

Build from Source (Optional)

git clone --recursive https://github.com/pytorch/pytorch
cd pytorch
mkdir build && cd build
cmake .. -DBUILD_SHARED_LIBS=ON -DCMAKE_INSTALL_PREFIX=/opt/libtorch
make -j$(nproc)
make install

CMake Integration

Basic CMakeLists.txt

cmake_minimum_required(VERSION 3.18)
project(rl_project)

set(CMAKE_CXX_STANDARD 17)
set(CMAKE_CXX_STANDARD_REQUIRED ON)

# Find LibTorch
list(APPEND CMAKE_PREFIX_PATH "/opt/libtorch")
find_package(Torch REQUIRED)

# Define executable
add_executable(train_agent
    src/main.cpp
    src/dqn.cpp
    src/replay_buffer.cpp
    src/environment.cpp
)

target_link_libraries(train_agent "${TORCH_LIBRARIES}")

# Required for MSVC
if (MSVC)
    file(GLOB TORCH_DLLS "${TORCH_INSTALL_PREFIX}/lib/*.dll")
    add_custom_command(TARGET train_agent POST_BUILD
        COMMAND ${CMAKE_COMMAND} -E copy_if_different
        ${TORCH_DLLS} $<TARGET_FILE_DIR:train_agent>)
endif()

Building with CUDA Support

# CMake automatically detects CUDA if LibTorch was built with CUDA
find_package(Torch REQUIRED)

# Verify CUDA is available
message(STATUS "CUDA available: ${TORCH_CUDA_AVAILABLE}")

Build Commands

mkdir build && cd build
cmake .. -DCMAKE_PREFIX_PATH=/opt/libtorch
make -j$(nproc)

Core API Reference

Tensor Creation

#include <torch/torch.h>

// Create tensors
auto zeros = torch::zeros({3, 4});                    // 3x4 zeros
auto ones = torch::ones({3, 4});                      // 3x4 ones
auto rand = torch::rand({3, 4});                      // Uniform [0, 1)
auto randn = torch::randn({3, 4});                    // Normal N(0, 1)
auto arange = torch::arange(0, 10, 2);                // [0, 2, 4, 6, 8]
auto linspace = torch::linspace(0, 1, 5);             // 5 evenly spaced

// From C++ data
std::vector<float> data = {1, 2, 3, 4};
auto from_vec = torch::tensor(data);
auto from_ptr = torch::from_blob(data.data(), {4}, torch::kFloat32);

// With options
auto gpu_tensor = torch::zeros({3, 4},
    torch::TensorOptions().device(torch::kCUDA).dtype(torch::kFloat32));

Tensor Operations

// Arithmetic
auto c = a + b;
auto c = torch::add(a, b);
auto c = a.add(b);         // In-place: a.add_(b)

// Matrix operations
auto prod = torch::matmul(a, b);
auto prod = a.mm(b);       // 2D matrix multiply
auto batch_prod = a.bmm(b); // Batch matrix multiply

// Reductions
auto sum = tensor.sum();
auto mean = tensor.mean();
auto max_val = tensor.max();
auto argmax = tensor.argmax(/*dim=*/1);

// Reshaping
auto reshaped = tensor.view({-1, 4});
auto squeezed = tensor.squeeze();
auto unsqueezed = tensor.unsqueeze(0);

// Indexing
auto row = tensor[0];
auto element = tensor[0][1];
auto slice = tensor.slice(/*dim=*/0, /*start=*/0, /*end=*/5);
auto indexed = tensor.index({torch::indexing::Slice(), 0});

Device Management

// Check CUDA availability
bool cuda_available = torch::cuda::is_available();
int device_count = torch::cuda::device_count();

// Create device
torch::Device device = cuda_available ? torch::kCUDA : torch::kCPU;
torch::Device specific_gpu(torch::kCUDA, 0);  // GPU 0

// Move tensors
auto gpu_tensor = cpu_tensor.to(device);
auto cpu_tensor = gpu_tensor.to(torch::kCPU);

// Create on device
auto tensor = torch::zeros({3, 4},
    torch::TensorOptions().device(device));

Autograd

// Enable gradient tracking
auto x = torch::randn({3, 4}, torch::requires_grad());

// Forward computation
auto y = x * 2;
auto z = y.mean();

// Backward pass
z.backward();

// Access gradients
auto grad = x.grad();

// Disable gradients for inference
{
    torch::NoGradGuard no_grad;
    auto output = model->forward(input);  // No gradients computed
}

// Detach from computation graph
auto detached = tensor.detach();

Neural Network Modules

Defining a Module

#include <torch/torch.h>

struct MyNet : torch::nn::Module {
    torch::nn::Linear fc1{nullptr}, fc2{nullptr};
    torch::nn::BatchNorm1d bn{nullptr};
    torch::nn::Dropout dropout{nullptr};

    MyNet(int64_t input_dim, int64_t hidden_dim, int64_t output_dim) {
        fc1 = register_module("fc1", torch::nn::Linear(input_dim, hidden_dim));
        bn = register_module("bn", torch::nn::BatchNorm1d(hidden_dim));
        dropout = register_module("dropout", torch::nn::Dropout(0.5));
        fc2 = register_module("fc2", torch::nn::Linear(hidden_dim, output_dim));
    }

    torch::Tensor forward(torch::Tensor x) {
        x = fc1->forward(x);
        x = bn->forward(x);
        x = torch::relu(x);
        x = dropout->forward(x);
        x = fc2->forward(x);
        return x;
    }
};

Common Layers

// Linear layers
torch::nn::Linear(in_features, out_features);

// Convolutional layers
torch::nn::Conv2d(torch::nn::Conv2dOptions(in_channels, out_channels, kernel_size)
    .stride(1).padding(1));

// Normalization
torch::nn::BatchNorm1d(num_features);
torch::nn::LayerNorm(torch::nn::LayerNormOptions({hidden_dim}));

// Recurrent layers
torch::nn::LSTM(torch::nn::LSTMOptions(input_size, hidden_size).num_layers(2));
torch::nn::GRU(torch::nn::GRUOptions(input_size, hidden_size));

// Dropout
torch::nn::Dropout(p);

// Activation (functional)
torch::relu(x);
torch::tanh(x);
torch::softmax(x, /*dim=*/1);

Train/Eval Mode

// Training mode (enables dropout, batch norm in training mode)
model->train();

// Evaluation mode (disables dropout, uses running stats for batch norm)
model->eval();

Optimizers

// Adam
torch::optim::Adam optimizer(model->parameters(),
    torch::optim::AdamOptions(/*lr=*/1e-3).betas({0.9, 0.999}));

// SGD with momentum
torch::optim::SGD optimizer(model->parameters(),
    torch::optim::SGDOptions(/*lr=*/0.01).momentum(0.9).weight_decay(1e-4));

// RMSprop
torch::optim::RMSprop optimizer(model->parameters(),
    torch::optim::RMSpropOptions(/*lr=*/1e-3).alpha(0.99));

// Training step
optimizer.zero_grad();
auto loss = compute_loss(model, batch);
loss.backward();
optimizer.step();

Model Serialization

Save and Load C++ Models

// Save model
torch::save(model, "model.pt");

// Load model
torch::load(model, "model.pt");

// Save optimizer state
torch::save(optimizer, "optimizer.pt");

Load Python PyTorch Models (TorchScript)

# In Python: export model
import torch

model = MyModel()
model.load_state_dict(torch.load("weights.pth"))
model.eval()

# Option 1: Trace
traced = torch.jit.trace(model, example_input)
traced.save("model_traced.pt")

# Option 2: Script
scripted = torch.jit.script(model)
scripted.save("model_scripted.pt")

// In C++: load TorchScript model
torch::jit::script::Module model = torch::jit::load("model_traced.pt");
model.to(device);
model.eval();

// Inference
std::vector<torch::jit::IValue> inputs;
inputs.push_back(input_tensor);
auto output = model.forward(inputs).toTensor();

Other Libraries

ONNX Runtime (Inference)

For deploying trained models in production, ONNX Runtime provides optimized cross-platform inference.

Installation: https://onnxruntime.ai/

#include <onnxruntime_cxx_api.h>

Ort::Env env(ORT_LOGGING_LEVEL_WARNING, "rl_inference");
Ort::Session session(env, "model.onnx", Ort::SessionOptions{});

// Run inference
auto output_tensors = session.Run(
    Ort::RunOptions{nullptr},
    input_names.data(), &input_tensor, 1,
    output_names.data(), 1
);

cpprl Reference Implementation

The mhubii/cpprl repository on GitHub provides reference implementations of PPO in C++ using LibTorch. Useful for:

• Understanding PPO implementation patterns
• Rollout buffer design
• GAE (Generalized Advantage Estimation) computation

git clone https://github.com/mhubii/cpprl

TensorFlow C++ API (Alternative)

Less commonly used for RL but available. TensorFlow's C++ API is primarily designed for inference rather than training.

#include "tensorflow/core/public/session.h"
// More complex setup than LibTorch

Recommendation: Use LibTorch for new RL projects due to better training support and documentation.

Memory Management for C++ RL

Overview

Proper memory management is critical in C++ RL implementations. This guide covers replay buffers, smart pointer patterns, RAII principles, and avoiding common memory pitfalls.

Replay Buffers

Basic Ring Buffer Implementation

#include <vector>
#include <random>
#include <torch/torch.h>

struct Experience {
    torch::Tensor state;
    int64_t action;
    float reward;
    torch::Tensor next_state;
    bool done;

    // Move constructor for efficiency
    Experience(Experience&& other) noexcept = default;
    Experience& operator=(Experience&& other) noexcept = default;

    // Explicit copy (avoid accidental copies)
    Experience(const Experience& other) = default;
    Experience& operator=(const Experience& other) = default;
};

class ReplayBuffer {
public:
    explicit ReplayBuffer(size_t capacity)
        : capacity_(capacity), position_(0), size_(0) {
        buffer_.reserve(capacity);
    }

    void push(Experience exp) {
        if (buffer_.size() < capacity_) {
            buffer_.push_back(std::move(exp));
        } else {
            buffer_[position_] = std::move(exp);
        }
        position_ = (position_ + 1) % capacity_;
        size_ = std::min(size_ + 1, capacity_);
    }

    std::vector<Experience> sample(size_t batch_size) {
        if (batch_size > size_) {
            throw std::runtime_error("Batch size exceeds buffer size");
        }

        std::vector<Experience> batch;
        batch.reserve(batch_size);

        std::uniform_int_distribution<size_t> dist(0, size_ - 1);
        for (size_t i = 0; i < batch_size; ++i) {
            // Copy experiences for batch (tensor data is reference counted)
            batch.push_back(buffer_[dist(rng_)]);
        }
        return batch;
    }

    size_t size() const { return size_; }
    bool can_sample(size_t batch_size) const { return size_ >= batch_size; }

private:
    std::vector<Experience> buffer_;
    size_t capacity_;
    size_t position_;
    size_t size_;
    std::mt19937 rng_{std::random_device{}()};
};

Prioritized Experience Replay (PER)

#include <algorithm>
#include <cmath>

class SumTree {
public:
    explicit SumTree(size_t capacity)
        : capacity_(capacity), tree_(2 * capacity - 1, 0.0f),
          data_ptr_(0), min_priority_(1e10) {
        data_.reserve(capacity);
    }

    void add(float priority, Experience exp) {
        size_t tree_idx = data_ptr_ + capacity_ - 1;

        if (data_.size() < capacity_) {
            data_.push_back(std::move(exp));
        } else {
            data_[data_ptr_] = std::move(exp);
        }

        update(tree_idx, priority);
        data_ptr_ = (data_ptr_ + 1) % capacity_;
        min_priority_ = std::min(min_priority_, priority);
    }

    void update(size_t tree_idx, float priority) {
        float change = priority - tree_[tree_idx];
        tree_[tree_idx] = priority;

        // Propagate change up to root
        while (tree_idx != 0) {
            tree_idx = (tree_idx - 1) / 2;
            tree_[tree_idx] += change;
        }
    }

    std::tuple<size_t, float, Experience&> get(float s) {
        size_t idx = 0;

        while (true) {
            size_t left = 2 * idx + 1;
            size_t right = left + 1;

            if (left >= tree_.size()) {
                break;
            }

            if (s <= tree_[left]) {
                idx = left;
            } else {
                s -= tree_[left];
                idx = right;
            }
        }

        size_t data_idx = idx - capacity_ + 1;
        return {idx, tree_[idx], data_[data_idx]};
    }

    float total_priority() const { return tree_[0]; }
    float min_priority() const { return min_priority_; }

private:
    size_t capacity_;
    std::vector<float> tree_;
    std::vector<Experience> data_;
    size_t data_ptr_;
    float min_priority_;
};

class PrioritizedReplayBuffer {
public:
    PrioritizedReplayBuffer(size_t capacity, float alpha = 0.6f)
        : capacity_(capacity), alpha_(alpha), tree_(capacity),
          max_priority_(1.0f) {}

    void push(Experience exp) {
        tree_.add(std::pow(max_priority_, alpha_), std::move(exp));
    }

    struct SampleResult {
        std::vector<Experience> experiences;
        std::vector<size_t> indices;
        torch::Tensor weights;
    };

    SampleResult sample(size_t batch_size, float beta = 0.4f) {
        SampleResult result;
        result.experiences.reserve(batch_size);
        result.indices.reserve(batch_size);

        float total = tree_.total_priority();
        float segment = total / batch_size;

        // Importance sampling weights
        float min_prob = tree_.min_priority() / total;
        float max_weight = std::pow(capacity_ * min_prob, -beta);

        std::vector<float> weights;
        weights.reserve(batch_size);

        std::uniform_real_distribution<float> dist(0, segment);

        for (size_t i = 0; i < batch_size; ++i) {
            float a = segment * i;
            float b = segment * (i + 1);
            float s = a + dist(rng_);

            auto [idx, priority, exp] = tree_.get(s);
            result.indices.push_back(idx);
            result.experiences.push_back(exp);

            float prob = priority / total;
            float weight = std::pow(capacity_ * prob, -beta) / max_weight;
            weights.push_back(weight);
        }

        result.weights = torch::tensor(weights);
        return result;
    }

    void update_priorities(const std::vector<size_t>& indices,
                          const std::vector<float>& priorities) {
        for (size_t i = 0; i < indices.size(); ++i) {
            float priority = std::pow(priorities[i] + 1e-6f, alpha_);
            tree_.update(indices[i], priority);
            max_priority_ = std::max(max_priority_, priorities[i]);
        }
    }

private:
    size_t capacity_;
    float alpha_;
    SumTree tree_;
    float max_priority_;
    std::mt19937 rng_{std::random_device{}()};
};

N-Step Returns Buffer

#include <deque>

class NStepBuffer {
public:
    NStepBuffer(size_t n_step, float gamma)
        : n_step_(n_step), gamma_(gamma) {}

    std::optional<Experience> add(Experience exp) {
        buffer_.push_back(std::move(exp));

        if (buffer_.size() < n_step_) {
            return std::nullopt;
        }

        // Compute n-step return
        float n_step_reward = 0.0f;
        float discount = 1.0f;

        for (size_t i = 0; i < n_step_; ++i) {
            n_step_reward += discount * buffer_[i].reward;
            discount *= gamma_;

            if (buffer_[i].done) {
                break;
            }
        }

        Experience n_step_exp;
        n_step_exp.state = buffer_.front().state;
        n_step_exp.action = buffer_.front().action;
        n_step_exp.reward = n_step_reward;
        n_step_exp.next_state = buffer_.back().next_state;
        n_step_exp.done = buffer_.back().done;

        buffer_.pop_front();
        return n_step_exp;
    }

    std::vector<Experience> flush() {
        std::vector<Experience> remaining;

        while (!buffer_.empty()) {
            float n_step_reward = 0.0f;
            float discount = 1.0f;

            for (const auto& exp : buffer_) {
                n_step_reward += discount * exp.reward;
                discount *= gamma_;
                if (exp.done) break;
            }

            Experience exp;
            exp.state = buffer_.front().state;
            exp.action = buffer_.front().action;
            exp.reward = n_step_reward;
            exp.next_state = buffer_.back().next_state;
            exp.done = buffer_.back().done;

            remaining.push_back(std::move(exp));
            buffer_.pop_front();
        }

        return remaining;
    }

private:
    std::deque<Experience> buffer_;
    size_t n_step_;
    float gamma_;
};

Smart Pointer Patterns

Model Ownership

// Single-ownership: use unique_ptr
class Agent {
private:
    std::unique_ptr<torch::optim::Adam> optimizer_;
    // ...

public:
    Agent(std::shared_ptr<DQNNet> model) {
        // Optimizer owns its parameters
        optimizer_ = std::make_unique<torch::optim::Adam>(
            model->parameters(), 1e-3);
    }
};

// Shared-ownership: use shared_ptr
class DistributedTrainer {
private:
    std::shared_ptr<DQNNet> shared_model_;

public:
    void train_worker(int worker_id) {
        // Each worker gets reference to shared model
        auto local_model = shared_model_;
        // ...
    }
};

Thread-Safe Model Cloning

// Clone model for thread safety
std::shared_ptr<DQNNet> clone_model(const DQNNet& source) {
    auto cloned = std::make_shared<DQNNet>(source.state_dim_, source.action_dim_);

    torch::NoGradGuard no_grad;
    auto source_params = source.named_parameters();
    auto cloned_params = cloned->named_parameters();

    for (auto& param : cloned_params) {
        param.value().copy_(source_params[param.key()]);
    }

    return cloned;
}

// Usage in parallel workers
void parallel_rollout(std::shared_ptr<DQNNet> main_model, int num_workers) {
    std::vector<std::thread> workers;

    for (int i = 0; i < num_workers; ++i) {
        // Each worker gets its own copy
        auto worker_model = clone_model(*main_model);
        workers.emplace_back([worker_model, i]() {
            collect_experience(*worker_model, i);
        });
    }

    for (auto& w : workers) {
        w.join();
    }
}

RAII Patterns

GPU Memory Guard

// RAII for temporary GPU memory allocation
class GPUMemoryGuard {
public:
    GPUMemoryGuard() {
        // Record current memory usage
        if (torch::cuda::is_available()) {
            initial_memory_ = torch::cuda::memory_allocated();
        }
    }

    ~GPUMemoryGuard() {
        // Force synchronization and memory release
        if (torch::cuda::is_available()) {
            torch::cuda::synchronize();
            // Optional: empty cache
            // torch::cuda::empty_cache();
        }
    }

    size_t memory_used() const {
        if (torch::cuda::is_available()) {
            return torch::cuda::memory_allocated() - initial_memory_;
        }
        return 0;
    }

private:
    size_t initial_memory_ = 0;
};

// Usage
void train_step() {
    GPUMemoryGuard guard;

    // ... training code ...

    // Memory automatically cleaned up when guard goes out of scope
}

Environment RAII Wrapper

class EnvironmentWrapper {
public:
    explicit EnvironmentWrapper(const std::string& env_name)
        : env_(create_environment(env_name)) {}

    ~EnvironmentWrapper() {
        if (env_) {
            env_->close();
        }
    }

    // Delete copy, allow move
    EnvironmentWrapper(const EnvironmentWrapper&) = delete;
    EnvironmentWrapper& operator=(const EnvironmentWrapper&) = delete;
    EnvironmentWrapper(EnvironmentWrapper&&) = default;
    EnvironmentWrapper& operator=(EnvironmentWrapper&&) = default;

    Environment* operator->() { return env_.get(); }

private:
    std::unique_ptr<Environment> env_;
};

Common Memory Issues and Solutions

Issue 1: Tensor Accumulation in Loops

Problem: Gradients accumulate when storing tensors.

// BAD: Gradients accumulate
std::vector<torch::Tensor> stored_values;
for (int i = 0; i < 1000; ++i) {
    auto value = model->forward(state);
    stored_values.push_back(value);  // Keeps computation graph!
}

Solution: Detach tensors before storing.

// GOOD: Detach to release computation graph
std::vector<torch::Tensor> stored_values;
for (int i = 0; i < 1000; ++i) {
    auto value = model->forward(state);
    stored_values.push_back(value.detach());  // Computation graph released
}

Issue 2: Memory Leak in Replay Buffer

Problem: Tensors in Experience keep references.

// Ensure proper tensor handling in Experience
struct Experience {
    torch::Tensor state;
    // ...

    // Detach state when creating experience
    static Experience create(torch::Tensor s, int64_t a, float r,
                            torch::Tensor ns, bool d) {
        return {s.detach(), a, r, ns.detach(), d};
    }
};

Issue 3: GPU Memory Not Released

Problem: Tensors hold GPU memory after use.

// Force cleanup
void cleanup_gpu_memory() {
    torch::cuda::synchronize();
    // Set tensors to empty or let them go out of scope
}

// Use scoped lifetime
void training_iteration() {
    {
        auto batch = buffer.sample(batch_size);
        auto loss = compute_loss(batch);
        loss.backward();
        optimizer.step();
    }  // batch tensors released here

    // Periodic cleanup
    if (iteration % 100 == 0) {
        torch::cuda::synchronize();
    }
}

Issue 4: Thread Safety with Models

Problem: Multiple threads accessing same model.

// Option 1: Clone for each thread
for (int i = 0; i < num_threads; ++i) {
    auto thread_model = clone_model(*shared_model);
    // Use thread_model in thread
}

// Option 2: Mutex protection
class ThreadSafeAgent {
private:
    std::shared_ptr<DQNNet> model_;
    std::mutex model_mutex_;

public:
    torch::Tensor forward(torch::Tensor input) {
        std::lock_guard<std::mutex> lock(model_mutex_);
        return model_->forward(input);
    }
};

Memory Profiling

Tracking GPU Memory

void log_memory_stats() {
    if (torch::cuda::is_available()) {
        std::cout << "Allocated: "
                  << torch::cuda::memory_allocated() / (1024 * 1024) << " MB\n";
        std::cout << "Cached: "
                  << torch::cuda::memory_reserved() / (1024 * 1024) << " MB\n";
    }
}

Tensor Reference Counting

void check_tensor_refs(const torch::Tensor& t) {
    // Internal reference count (for debugging)
    std::cout << "Use count: " << t.use_count() << "\n";
    std::cout << "Storage use count: " << t.storage().use_count() << "\n";
}

Performance Optimization for C++ RL

GPU Acceleration

Device Selection and Management

#include <torch/torch.h>

class DeviceManager {
public:
    static torch::Device get_best_device() {
        if (torch::cuda::is_available()) {
            return torch::kCUDA;
        }
        // Check for MPS on Apple Silicon
        if (torch::mps::is_available()) {
            return torch::kMPS;
        }
        return torch::kCPU;
    }

    static void print_device_info() {
        if (torch::cuda::is_available()) {
            std::cout << "CUDA available: " << torch::cuda::device_count()
                      << " device(s)\n";
            for (int i = 0; i < torch::cuda::device_count(); ++i) {
                // Print device properties
                cudaDeviceProp prop;
                cudaGetDeviceProperties(&prop, i);
                std::cout << "  Device " << i << ": " << prop.name << "\n";
                std::cout << "    Memory: " << prop.totalGlobalMem / (1024*1024*1024)
                          << " GB\n";
            }
        }
    }
};

// Usage
torch::Device device = DeviceManager::get_best_device();
model->to(device);

Efficient Tensor Transfers

// Minimize CPU-GPU transfers
class BatchProcessor {
public:
    BatchProcessor(torch::Device device, size_t batch_size)
        : device_(device), batch_size_(batch_size) {
        // Pre-allocate buffers on device
        state_buffer_ = torch::empty({batch_size, state_dim_},
            torch::TensorOptions().device(device).dtype(torch::kFloat32));
    }

    torch::Tensor process_batch(const std::vector<Experience>& batch) {
        // Stack on CPU first (faster for small tensors)
        std::vector<torch::Tensor> states;
        states.reserve(batch.size());
        for (const auto& exp : batch) {
            states.push_back(exp.state);
        }

        // Single transfer to GPU
        auto batch_tensor = torch::stack(states).to(device_);
        return batch_tensor;
    }

private:
    torch::Device device_;
    size_t batch_size_;
    size_t state_dim_;
    torch::Tensor state_buffer_;
};

Pinned Memory for Faster Transfers

// Use pinned (page-locked) memory for faster CPU->GPU transfers
auto pinned_tensor = torch::empty({batch_size, state_dim},
    torch::TensorOptions()
        .dtype(torch::kFloat32)
        .pinned_memory(true));  // Pinned memory

// Async transfer with pinned memory
auto gpu_tensor = pinned_tensor.to(device, /*non_blocking=*/true);
torch::cuda::synchronize();  // Only sync when needed

CUDA Streams for Parallelism

#include <c10/cuda/CUDAStream.h>

class AsyncTrainer {
public:
    void async_training_step(torch::Tensor states, torch::Tensor actions) {
        // Create separate streams for different operations
        auto compute_stream = c10::cuda::getStreamFromPool();
        auto transfer_stream = c10::cuda::getStreamFromPool();

        {
            c10::cuda::CUDAStreamGuard guard(transfer_stream);
            // Async data transfer
            next_batch_states_ = prepare_next_batch().to(device_, true);
        }

        {
            c10::cuda::CUDAStreamGuard guard(compute_stream);
            // Forward and backward pass
            auto loss = compute_loss(states, actions);
            loss.backward();
            optimizer_->step();
        }

        // Synchronize only when necessary
        compute_stream.synchronize();
    }

private:
    torch::Device device_{torch::kCUDA};
    torch::Tensor next_batch_states_;
    std::unique_ptr<torch::optim::Adam> optimizer_;
};

Parallel Environment Rollouts

Thread Pool Implementation

#include <thread>
#include <future>
#include <queue>
#include <functional>

class ThreadPool {
public:
    explicit ThreadPool(size_t num_threads) : stop_(false) {
        for (size_t i = 0; i < num_threads; ++i) {
            workers_.emplace_back([this] {
                while (true) {
                    std::function<void()> task;
                    {
                        std::unique_lock<std::mutex> lock(queue_mutex_);
                        condition_.wait(lock, [this] {
                            return stop_ || !tasks_.empty();
                        });
                        if (stop_ && tasks_.empty()) return;
                        task = std::move(tasks_.front());
                        tasks_.pop();
                    }
                    task();
                }
            });
        }
    }

    ~ThreadPool() {
        {
            std::unique_lock<std::mutex> lock(queue_mutex_);
            stop_ = true;
        }
        condition_.notify_all();
        for (auto& worker : workers_) {
            worker.join();
        }
    }

    template<class F, class... Args>
    auto enqueue(F&& f, Args&&... args)
        -> std::future<typename std::invoke_result<F, Args...>::type> {
        using return_type = typename std::invoke_result<F, Args...>::type;

        auto task = std::make_shared<std::packaged_task<return_type()>>(
            std::bind(std::forward<F>(f), std::forward<Args>(args)...)
        );

        std::future<return_type> result = task->get_future();
        {
            std::unique_lock<std::mutex> lock(queue_mutex_);
            tasks_.emplace([task]() { (*task)(); });
        }
        condition_.notify_one();
        return result;
    }

private:
    std::vector<std::thread> workers_;
    std::queue<std::function<void()>> tasks_;
    std::mutex queue_mutex_;
    std::condition_variable condition_;
    bool stop_;
};

Parallel Rollout Collection

class ParallelRolloutCollector {
public:
    ParallelRolloutCollector(size_t num_envs, std::shared_ptr<DQNNet> model)
        : num_envs_(num_envs), pool_(num_envs) {
        // Create environment instances
        for (size_t i = 0; i < num_envs; ++i) {
            envs_.push_back(std::make_unique<Environment>("CartPole-v1"));
            // Clone model for thread safety
            models_.push_back(clone_model(*model));
        }
    }

    std::vector<Experience> collect(int steps_per_env) {
        std::vector<std::future<std::vector<Experience>>> futures;

        for (size_t i = 0; i < num_envs_; ++i) {
            futures.push_back(pool_.enqueue([this, i, steps_per_env]() {
                return collect_from_env(i, steps_per_env);
            }));
        }

        std::vector<Experience> all_experiences;
        for (auto& future : futures) {
            auto exps = future.get();
            all_experiences.insert(all_experiences.end(),
                                  std::make_move_iterator(exps.begin()),
                                  std::make_move_iterator(exps.end()));
        }

        return all_experiences;
    }

    void sync_weights(const DQNNet& source) {
        for (auto& model : models_) {
            torch::NoGradGuard no_grad;
            auto source_params = source.named_parameters();
            auto target_params = model->named_parameters();
            for (auto& param : target_params) {
                param.value().copy_(source_params[param.key()]);
            }
        }
    }

private:
    std::vector<Experience> collect_from_env(size_t env_idx, int steps) {
        std::vector<Experience> experiences;
        auto& env = envs_[env_idx];
        auto& model = models_[env_idx];

        auto state = env->reset();

        for (int step = 0; step < steps; ++step) {
            torch::NoGradGuard no_grad;
            auto q_values = model->forward(state.unsqueeze(0));
            auto action = q_values.argmax(1).item<int64_t>();

            auto [next_state, reward, done, info] = env->step(action);

            experiences.push_back({state.detach(), action, reward,
                                  next_state.detach(), done});

            state = done ? env->reset() : next_state;
        }

        return experiences;
    }

    size_t num_envs_;
    ThreadPool pool_;
    std::vector<std::unique_ptr<Environment>> envs_;
    std::vector<std::shared_ptr<DQNNet>> models_;
};

Batch Processing Optimization

Vectorized Operations

// Avoid loops where possible - use vectorized operations
class VectorizedDQN {
public:
    torch::Tensor compute_td_targets(
        torch::Tensor rewards,
        torch::Tensor next_states,
        torch::Tensor dones,
        float gamma
    ) {
        torch::NoGradGuard no_grad;

        // Vectorized computation
        auto next_q_values = target_net_->forward(next_states);
        auto max_next_q = std::get<0>(next_q_values.max(1));

        // Element-wise operations (no loops)
        auto targets = rewards + gamma * max_next_q * (1 - dones);
        return targets;
    }

    // Batch action selection (vectorized)
    torch::Tensor select_actions_batch(torch::Tensor states) {
        torch::NoGradGuard no_grad;
        auto q_values = policy_net_->forward(states);
        return q_values.argmax(1);
    }

private:
    std::shared_ptr<DQNNet> policy_net_;
    std::shared_ptr<DQNNet> target_net_;
};

Efficient Batch Stacking

// Pre-allocate and fill (faster than repeated concatenation)
torch::Tensor efficient_stack(const std::vector<Experience>& batch,
                              torch::Device device) {
    size_t batch_size = batch.size();
    auto state_shape = batch[0].state.sizes();

    // Pre-allocate tensor
    std::vector<int64_t> shape = {static_cast<int64_t>(batch_size)};
    shape.insert(shape.end(), state_shape.begin(), state_shape.end());

    auto result = torch::empty(shape,
        torch::TensorOptions().device(torch::kCPU).dtype(torch::kFloat32));

    // Fill in place
    for (size_t i = 0; i < batch_size; ++i) {
        result[i].copy_(batch[i].state);
    }

    return result.to(device);
}

Memory Optimization

Gradient Checkpointing

For large models, trade compute for memory with gradient checkpointing.

#include <torch/torch.h>

// Manual checkpointing for segments
torch::Tensor checkpoint_forward(
    torch::nn::Sequential& segment,
    torch::Tensor input
) {
    if (torch::autograd::GradMode::is_enabled()) {
        // During training: use checkpointing
        auto detached = input.detach();
        detached.set_requires_grad(true);

        auto output = segment->forward(detached);

        // Store function to recompute
        return torch::autograd::make_variable(
            output.data(),
            /*requires_grad=*/true
        );
    } else {
        // During inference: normal forward
        return segment->forward(input);
    }
}

Mixed Precision Training

// Use automatic mixed precision (AMP) with CUDA
class MixedPrecisionTrainer {
public:
    void train_step(torch::Tensor states, torch::Tensor targets) {
        optimizer_->zero_grad();

        // Forward in FP16
        at::autocast::set_enabled(true);
        auto output = model_->forward(states);
        auto loss = torch::mse_loss(output, targets);
        at::autocast::set_enabled(false);

        // Scale loss and backward
        auto scaled_loss = loss * loss_scale_;
        scaled_loss.backward();

        // Unscale and clip gradients
        for (auto& param : model_->parameters()) {
            if (param.grad().defined()) {
                param.grad().div_(loss_scale_);
            }
        }
        torch::nn::utils::clip_grad_norm_(model_->parameters(), max_grad_norm_);

        optimizer_->step();
    }

private:
    std::shared_ptr<DQNNet> model_;
    std::unique_ptr<torch::optim::Adam> optimizer_;
    float loss_scale_ = 65536.0f;
    float max_grad_norm_ = 1.0f;
};

Compilation and Build Optimization

CMake Release Configuration

cmake_minimum_required(VERSION 3.18)
project(rl_project)

set(CMAKE_CXX_STANDARD 17)
set(CMAKE_CXX_STANDARD_REQUIRED ON)

# Release optimizations
set(CMAKE_CXX_FLAGS_RELEASE "-O3 -DNDEBUG -march=native -ffast-math")

# Enable LTO for better optimization
set(CMAKE_INTERPROCEDURAL_OPTIMIZATION TRUE)

find_package(Torch REQUIRED)

add_executable(train_agent src/main.cpp)
target_link_libraries(train_agent "${TORCH_LIBRARIES}")

# Use static linking where possible
if(NOT MSVC)
    target_link_options(train_agent PRIVATE -static-libgcc -static-libstdc++)
endif()

Profile-Guided Optimization (PGO)

# Step 1: Build with instrumentation
cmake -DCMAKE_CXX_FLAGS="-fprofile-generate" ..
make
./train_agent  # Run representative workload

# Step 2: Build with profile data
cmake -DCMAKE_CXX_FLAGS="-fprofile-use" ..
make

Profiling Tools

Built-in Timing

#include <chrono>

class Timer {
public:
    void start() {
        start_ = std::chrono::high_resolution_clock::now();
    }

    double elapsed_ms() const {
        auto end = std::chrono::high_resolution_clock::now();
        return std::chrono::duration<double, std::milli>(end - start_).count();
    }

    void print(const std::string& label) const {
        std::cout << label << ": " << elapsed_ms() << " ms\n";
    }

private:
    std::chrono::high_resolution_clock::time_point start_;
};

// Usage
Timer timer;
timer.start();
auto loss = compute_loss(batch);
loss.backward();
timer.print("Forward + Backward");

CUDA Profiling

// Use CUDA events for accurate GPU timing
class CUDATimer {
public:
    CUDATimer() {
        cudaEventCreate(&start_);
        cudaEventCreate(&stop_);
    }

    ~CUDATimer() {
        cudaEventDestroy(start_);
        cudaEventDestroy(stop_);
    }

    void start() {
        cudaEventRecord(start_);
    }

    float elapsed_ms() {
        cudaEventRecord(stop_);
        cudaEventSynchronize(stop_);
        float ms;
        cudaEventElapsedTime(&ms, start_, stop_);
        return ms;
    }

private:
    cudaEvent_t start_, stop_;
};

Using nvprof / nsys

# Profile with NVIDIA tools
nsys profile --stats=true ./train_agent

# Detailed kernel analysis
ncu --set full ./train_agent

Testing and Debugging C++ RL Systems

Deterministic Testing

Seed Management

#include <torch/torch.h>
#include <random>

class SeedManager {
public:
    static void set_global_seed(int seed) {
        // PyTorch seeds
        torch::manual_seed(seed);

        if (torch::cuda::is_available()) {
            torch::cuda::manual_seed(seed);
            torch::cuda::manual_seed_all(seed);  // Multi-GPU
        }

        // C++ random
        std::srand(seed);

        // Store for reproducibility logging
        current_seed_ = seed;
    }

    // Create seeded RNG for specific component
    static std::mt19937 create_rng(int seed) {
        return std::mt19937(seed);
    }

    // Get reproducible seed for child processes/threads
    static int get_worker_seed(int worker_id) {
        return current_seed_ + worker_id * 1000;
    }

private:
    static inline int current_seed_ = 0;
};

// CUDA deterministic settings (may impact performance)
void enable_deterministic_mode() {
    at::globalContext().setDeterministicCuDNN(true);
    at::globalContext().setBenchmarkCuDNN(false);
}

Reproducible Experience Collection

class DeterministicAgent {
public:
    DeterministicAgent(int seed) : rng_(seed), dist_(0.0, 1.0) {
        SeedManager::set_global_seed(seed);
    }

    int64_t select_action(torch::Tensor state, double epsilon) {
        if (dist_(rng_) < epsilon) {
            std::uniform_int_distribution<int64_t> action_dist(0, num_actions_ - 1);
            return action_dist(rng_);
        }

        torch::NoGradGuard no_grad;
        auto q_values = model_->forward(state);
        return q_values.argmax(1).item<int64_t>();
    }

private:
    std::mt19937 rng_;
    std::uniform_real_distribution<double> dist_;
    std::shared_ptr<DQNNet> model_;
    int64_t num_actions_;
};

Unit Testing with Google Test

Setting Up Google Test

# CMakeLists.txt
include(FetchContent)
FetchContent_Declare(
    googletest
    GIT_REPOSITORY https://github.com/google/googletest.git
    GIT_TAG release-1.12.1
)
FetchContent_MakeAvailable(googletest)

enable_testing()

add_executable(rl_tests
    tests/test_network.cpp
    tests/test_replay_buffer.cpp
    tests/test_agent.cpp
)

target_link_libraries(rl_tests
    GTest::gtest_main
    "${TORCH_LIBRARIES}"
)

include(GoogleTest)
gtest_discover_tests(rl_tests)

Testing Neural Networks

#include <gtest/gtest.h>
#include <torch/torch.h>
#include "dqn.h"

class DQNNetTest : public ::testing::Test {
protected:
    void SetUp() override {
        SeedManager::set_global_seed(42);
        model = std::make_shared<DQNNet>(4, 2);
    }

    std::shared_ptr<DQNNet> model;
};

TEST_F(DQNNetTest, ForwardPassShape) {
    auto input = torch::randn({32, 4});  // Batch of 32, state dim 4
    auto output = model->forward(input);

    EXPECT_EQ(output.sizes(), torch::IntArrayRef({32, 2}));
}

TEST_F(DQNNetTest, ForwardPassDeterministic) {
    SeedManager::set_global_seed(42);
    auto input = torch::randn({1, 4});

    auto output1 = model->forward(input);
    auto output2 = model->forward(input);

    EXPECT_TRUE(torch::allclose(output1, output2));
}

TEST_F(DQNNetTest, GradientFlow) {
    auto input = torch::randn({32, 4}, torch::requires_grad());
    auto output = model->forward(input);
    auto loss = output.sum();

    loss.backward();

    // Check all parameters have gradients
    for (const auto& param : model->parameters()) {
        EXPECT_TRUE(param.grad().defined());
        EXPECT_FALSE(torch::all(param.grad() == 0).item<bool>());
    }
}

TEST_F(DQNNetTest, ParameterCount) {
    size_t total_params = 0;
    for (const auto& param : model->parameters()) {
        total_params += param.numel();
    }

    // 4*128 + 128 + 128*128 + 128 + 128*2 + 2 = expected
    EXPECT_GT(total_params, 0);
}

Testing Replay Buffer

#include <gtest/gtest.h>
#include "replay_buffer.h"

class ReplayBufferTest : public ::testing::Test {
protected:
    void SetUp() override {
        buffer = std::make_unique<ReplayBuffer>(100);
    }

    Experience make_experience(int id) {
        return {
            torch::tensor({static_cast<float>(id)}),
            id % 4,
            static_cast<float>(id) * 0.1f,
            torch::tensor({static_cast<float>(id + 1)}),
            id % 10 == 0
        };
    }

    std::unique_ptr<ReplayBuffer> buffer;
};

TEST_F(ReplayBufferTest, InitiallyEmpty) {
    EXPECT_EQ(buffer->size(), 0);
    EXPECT_FALSE(buffer->can_sample(1));
}

TEST_F(ReplayBufferTest, PushAndSize) {
    for (int i = 0; i < 50; ++i) {
        buffer->push(make_experience(i));
    }

    EXPECT_EQ(buffer->size(), 50);
    EXPECT_TRUE(buffer->can_sample(32));
}

TEST_F(ReplayBufferTest, RingBufferOverwrite) {
    // Fill past capacity
    for (int i = 0; i < 150; ++i) {
        buffer->push(make_experience(i));
    }

    // Size should be capped at capacity
    EXPECT_EQ(buffer->size(), 100);
}

TEST_F(ReplayBufferTest, SampleBatchSize) {
    for (int i = 0; i < 100; ++i) {
        buffer->push(make_experience(i));
    }

    auto batch = buffer->sample(32);
    EXPECT_EQ(batch.size(), 32);
}

TEST_F(ReplayBufferTest, SampleContainsValidExperiences) {
    for (int i = 0; i < 100; ++i) {
        buffer->push(make_experience(i));
    }

    auto batch = buffer->sample(32);
    for (const auto& exp : batch) {
        EXPECT_EQ(exp.state.dim(), 1);
        EXPECT_GE(exp.action, 0);
        EXPECT_LT(exp.action, 4);
    }
}

Testing Training Loop

#include <gtest/gtest.h>
#include "dqn_agent.h"

TEST(DQNAgentTest, LossDecreases) {
    SeedManager::set_global_seed(42);

    DQNAgent agent(4, 2);
    ReplayBuffer buffer(1000);

    // Fill buffer with random experiences
    for (int i = 0; i < 500; ++i) {
        buffer.push({
            torch::randn({4}),
            rand() % 2,
            static_cast<float>(rand()) / RAND_MAX,
            torch::randn({4}),
            rand() % 10 == 0
        });
    }

    // Track loss over training
    std::vector<float> losses;

    for (int step = 0; step < 100; ++step) {
        auto batch = buffer.sample(32);
        float loss = agent.train_step(batch);
        losses.push_back(loss);
    }

    // Average loss should decrease
    float early_avg = std::accumulate(losses.begin(), losses.begin() + 20, 0.0f) / 20;
    float late_avg = std::accumulate(losses.end() - 20, losses.end(), 0.0f) / 20;

    EXPECT_LT(late_avg, early_avg);
}

TEST(DQNAgentTest, TargetNetworkUpdates) {
    SeedManager::set_global_seed(42);

    DQNAgent agent(4, 2);

    // Get initial target network output
    auto state = torch::randn({1, 4});
    auto initial_output = agent.get_target_values(state).clone();

    // Train for a while
    ReplayBuffer buffer(1000);
    for (int i = 0; i < 100; ++i) {
        buffer.push({torch::randn({4}), rand() % 2, 1.0f,
                    torch::randn({4}), false});
    }

    for (int i = 0; i < 50; ++i) {
        agent.train_step(buffer.sample(32));
    }

    // Update target network
    agent.update_target_network();

    auto updated_output = agent.get_target_values(state);

    EXPECT_FALSE(torch::allclose(initial_output, updated_output));
}

Debugging Techniques

Gradient Debugging

class GradientDebugger {
public:
    static void check_gradients(torch::nn::Module& model, const std::string& name) {
        std::cout << "=== Gradient Check: " << name << " ===\n";

        for (const auto& pair : model.named_parameters()) {
            const auto& param_name = pair.key();
            const auto& param = pair.value();

            if (!param.grad().defined()) {
                std::cout << param_name << ": NO GRADIENT\n";
                continue;
            }

            auto grad = param.grad();
            float grad_norm = grad.norm().item<float>();
            float grad_mean = grad.mean().item<float>();
            float grad_std = grad.std().item<float>();

            std::cout << param_name
                      << " | norm: " << grad_norm
                      << " | mean: " << grad_mean
                      << " | std: " << grad_std;

            // Warn about potential issues
            if (grad_norm == 0) {
                std::cout << " [ZERO GRADIENT]";
            } else if (grad_norm > 100) {
                std::cout << " [EXPLODING]";
            } else if (std::isnan(grad_norm)) {
                std::cout << " [NaN]";
            }

            std::cout << "\n";
        }
    }

    static void numerical_gradient_check(
        torch::nn::Module& model,
        torch::Tensor input,
        torch::Tensor target,
        double epsilon = 1e-5
    ) {
        auto loss_fn = [&](torch::Tensor in) {
            return torch::mse_loss(model.forward(in), target);
        };

        auto analytical_grad = torch::autograd::grad(
            {loss_fn(input)}, {input})[0];

        // Numerical gradient
        auto numerical_grad = torch::zeros_like(input);
        auto flat_input = input.flatten();

        for (int i = 0; i < flat_input.numel(); ++i) {
            auto input_plus = input.clone();
            auto input_minus = input.clone();

            input_plus.flatten()[i] += epsilon;
            input_minus.flatten()[i] -= epsilon;

            float loss_plus = loss_fn(input_plus).item<float>();
            float loss_minus = loss_fn(input_minus).item<float>();

            numerical_grad.flatten()[i] = (loss_plus - loss_minus) / (2 * epsilon);
        }

        auto diff = (analytical_grad - numerical_grad).abs().max().item<float>();
        std::cout << "Max gradient difference: " << diff << "\n";

        if (diff > 1e-4) {
            std::cout << "WARNING: Gradients may be incorrect!\n";
        }
    }
};

Value Debugging

class ValueDebugger {
public:
    static void log_q_values(const DQNAgent& agent, torch::Tensor states) {
        torch::NoGradGuard no_grad;
        auto q_values = agent.get_q_values(states);

        std::cout << "=== Q-Value Statistics ===\n";
        std::cout << "Mean: " << q_values.mean().item<float>() << "\n";
        std::cout << "Std:  " << q_values.std().item<float>() << "\n";
        std::cout << "Min:  " << q_values.min().item<float>() << "\n";
        std::cout << "Max:  " << q_values.max().item<float>() << "\n";

        // Check for issues
        if (std::abs(q_values.mean().item<float>()) > 1000) {
            std::cout << "WARNING: Q-values may be exploding\n";
        }
    }

    static void log_action_distribution(
        const std::vector<int64_t>& actions,
        int num_actions
    ) {
        std::vector<int> counts(num_actions, 0);
        for (auto action : actions) {
            counts[action]++;
        }

        std::cout << "=== Action Distribution ===\n";
        for (int i = 0; i < num_actions; ++i) {
            float pct = 100.0f * counts[i] / actions.size();
            std::cout << "Action " << i << ": " << pct << "%\n";
        }
    }
};

Memory Leak Detection

#include <iostream>

class MemoryTracker {
public:
    static void start() {
        if (torch::cuda::is_available()) {
            initial_allocated_ = torch::cuda::memory_allocated();
            initial_reserved_ = torch::cuda::memory_reserved();
        }
    }

    static void check(const std::string& label) {
        if (torch::cuda::is_available()) {
            size_t allocated = torch::cuda::memory_allocated();
            size_t reserved = torch::cuda::memory_reserved();

            std::cout << "=== Memory: " << label << " ===\n";
            std::cout << "Allocated: " << (allocated - initial_allocated_) / 1e6
                      << " MB change\n";
            std::cout << "Reserved:  " << (reserved - initial_reserved_) / 1e6
                      << " MB change\n";
        }
    }

    static void assert_no_leak(const std::string& label, size_t tolerance_bytes = 1e6) {
        if (torch::cuda::is_available()) {
            torch::cuda::synchronize();

            size_t current = torch::cuda::memory_allocated();
            size_t diff = current > initial_allocated_ ?
                         current - initial_allocated_ : 0;

            if (diff > tolerance_bytes) {
                std::cerr << "MEMORY LEAK at " << label
                          << ": " << diff / 1e6 << " MB\n";
            }
        }
    }

private:
    static inline size_t initial_allocated_ = 0;
    static inline size_t initial_reserved_ = 0;
};

// Usage in tests
TEST(MemoryTest, NoLeakDuringTraining) {
    MemoryTracker::start();

    for (int episode = 0; episode < 100; ++episode) {
        train_one_episode();
    }

    MemoryTracker::assert_no_leak("100 episodes");
}

Integration Testing

End-to-End Training Test

TEST(IntegrationTest, CartPoleLearning) {
    SeedManager::set_global_seed(42);

    // Simple environment mock
    auto env = std::make_unique<CartPoleEnv>();
    DQNAgent agent(4, 2);
    ReplayBuffer buffer(10000);

    std::vector<float> episode_rewards;

    for (int episode = 0; episode < 200; ++episode) {
        auto state = env->reset();
        float total_reward = 0;

        for (int step = 0; step < 500; ++step) {
            auto action = agent.select_action(state);
            auto [next_state, reward, done, info] = env->step(action);

            buffer.push({state, action, reward, next_state, done});
            total_reward += reward;

            if (buffer.can_sample(32)) {
                agent.train_step(buffer.sample(32));
            }

            if (done) break;
            state = next_state;
        }

        episode_rewards.push_back(total_reward);

        if (episode % 10 == 0) {
            agent.update_target_network();
        }
    }

    // Check learning progress
    float early_avg = std::accumulate(
        episode_rewards.begin(),
        episode_rewards.begin() + 20, 0.0f) / 20;
    float late_avg = std::accumulate(
        episode_rewards.end() - 20,
        episode_rewards.end(), 0.0f) / 20;

    EXPECT_GT(late_avg, early_avg * 1.5);  // Should improve significantly
    EXPECT_GT(late_avg, 100);  // Should achieve reasonable performance
}

Model Save/Load Test

TEST(SerializationTest, SaveAndLoadModel) {
    SeedManager::set_global_seed(42);

    auto original = std::make_shared<DQNNet>(4, 2);
    auto input = torch::randn({10, 4});
    auto original_output = original->forward(input);

    // Save
    torch::save(original, "test_model.pt");

    // Load into new model
    auto loaded = std::make_shared<DQNNet>(4, 2);
    torch::load(loaded, "test_model.pt");

    auto loaded_output = loaded->forward(input);

    EXPECT_TRUE(torch::allclose(original_output, loaded_output));

    // Cleanup
    std::remove("test_model.pt");
}

Logging Best Practices

Structured Training Logger

#include <fstream>
#include <nlohmann/json.hpp>

class TrainingLogger {
public:
    explicit TrainingLogger(const std::string& log_path)
        : log_file_(log_path) {}

    void log_step(int step, float loss, float q_mean, float epsilon) {
        nlohmann::json entry;
        entry["step"] = step;
        entry["loss"] = loss;
        entry["q_mean"] = q_mean;
        entry["epsilon"] = epsilon;
        entry["timestamp"] = std::time(nullptr);

        log_file_ << entry.dump() << "\n";
        log_file_.flush();
    }

    void log_episode(int episode, float reward, int steps) {
        nlohmann::json entry;
        entry["episode"] = episode;
        entry["reward"] = reward;
        entry["steps"] = steps;
        entry["timestamp"] = std::time(nullptr);

        log_file_ << entry.dump() << "\n";
        log_file_.flush();
    }

    void log_checkpoint(int step, const std::string& path) {
        nlohmann::json entry;
        entry["checkpoint"] = true;
        entry["step"] = step;
        entry["path"] = path;

        log_file_ << entry.dump() << "\n";
        log_file_.flush();
    }

private:
    std::ofstream log_file_;
};