请给出一个PPO算法的代码示例

好的，以下是一个简单的PPO算法的示例（基于PyTorch）：

import torch
import torch.nn as nn
import torch.optim as optim
import numpy as np

# 策略网络
class PolicyNet(nn.Module):
    def __init__(self, obs_dim, act_dim, hidden_size=128):
        super(PolicyNet, self).__init__()
        self.layers = nn.Sequential(
            nn.Linear(obs_dim, hidden_size),
            nn.Tanh(),
            nn.Linear(hidden_size, hidden_size),
            nn.Tanh(),
            nn.Linear(hidden_size, act_dim),
            nn.Softmax(dim=-1)
        )
        
    def forward(self, obs):
        return self.layers(obs)

# PPO算法
class PPO:
    def __init__(self, obs_dim, act_dim, lr=1e-3, clip_ratio=0.2, gamma=0.99, gae_lambda=0.95, eps=1e-5):
        self.obs_dim = obs_dim
        self.act_dim = act_dim
        self.clip_ratio = clip_ratio
        self.gamma = gamma
        self.gae_lambda = gae_lambda
        self.eps = eps
        
        # 创建策略网络和价值网络
        self.policy_net = PolicyNet(obs_dim, act_dim)
        self.optimizer = optim.Adam(self.policy_net.parameters(), lr=lr)
        
    def compute_advantages(self, rewards, values, masks):
        # 计算优势值
        advantages = np.zeros_like(rewards)
        last_advantage = 0
        
        for t in reversed(range(len(rewards))):
            delta = rewards[t] + self.gamma * values[t + 1] * masks[t] - values[t]
            last_advantage = delta + self.gamma * self.gae_lambda * masks[t] * last_advantage
            advantages[t] = last_advantage
            
        return advantages
        
    def train(self, obs, actions, rewards, values, masks):
        # 计算策略网络和价值网络的损失函数
        advantages = self.compute_advantages(rewards, values, masks)
        advantages = (advantages - advantages.mean()) / (advantages.std() + self.eps)
        
        old_log_probs = torch.log(self.policy_net(obs).gather(1, actions))
        old_log_probs = old_log_probs.detach()
        
        for i in range(10):
            # 更新策略网络
            log_probs = torch.log(self.policy_net(obs).gather(1, actions))
            
            ratio = torch.exp(log_probs - old_log_probs)
            clipped_ratio = torch.clamp(ratio, 1 - self.clip_ratio, 1 + self.clip_ratio)
    
            policy_loss = -(torch.min(ratio * advantages, clipped_ratio * advantages)).mean()
            
            self.optimizer.zero_grad()
            policy_loss.backward()
            self.optimizer.step()

希望对你有帮助！