多智能体编队actor-critic pytorch代码

对于一个多智能体编队问题，可以采用actor-critic算法来解决。以下是使用PyTorch实现的代码示例：

import torch
import torch.nn as nn
import torch.optim as optim
import numpy as np
import random

class Actor(nn.Module):
    def __init__(self, state_dim, action_dim, hidden_dim):
        super(Actor, self).__init__()
        self.fc1 = nn.Linear(state_dim, hidden_dim)
        self.fc2 = nn.Linear(hidden_dim, hidden_dim)
        self.fc3 = nn.Linear(hidden_dim, action_dim)
        self.relu = nn.ReLU()
        self.softmax = nn.Softmax(dim=1)

    def forward(self, state):
        x = self.relu(self.fc1(state))
        x = self.relu(self.fc2(x))
        x = self.softmax(self.fc3(x))
        return x

class Critic(nn.Module):
    def __init__(self, state_dim, hidden_dim):
        super(Critic, self).__init__()
        self.fc1 = nn.Linear(state_dim, hidden_dim)
        self.fc2 = nn.Linear(hidden_dim, hidden_dim)
        self.fc3 = nn.Linear(hidden_dim, 1)
        self.relu = nn.ReLU()

    def forward(self, state):
        x = self.relu(self.fc1(state))
        x = self.relu(self.fc2(x))
        x = self.fc3(x)
        return x

class Agent():
    def __init__(self, state_dim, action_dim, hidden_dim, lr, gamma):
        self.actor = Actor(state_dim, action_dim, hidden_dim)
        self.actor_optimizer = optim.Adam(self.actor.parameters(), lr=lr)
        self.critic = Critic(state_dim, hidden_dim)
        self.critic_optimizer = optim.Adam(self.critic.parameters(), lr=lr)
        self.gamma = gamma

    def act(self, state):
        state = torch.FloatTensor(state)
        action_probs = self.actor(state)
        action_probs = action_probs.detach().numpy()
        action = np.random.choice(len(action_probs[0]), p=action_probs[0])
        return action

    def learn(self, states, actions, rewards, next_states, dones):
        states = torch.FloatTensor(states)
        actions = torch.LongTensor(actions)
        rewards = torch.FloatTensor(rewards)
        next_states = torch.FloatTensor(next_states)
        dones = torch.FloatTensor(dones)

        values = self.critic(states).squeeze()
        next_values = self.critic(next_states).squeeze()
        target_values = rewards + self.gamma * next_values * (1 - dones)
        advantages = target_values - values

        actor_loss = -torch.log(self.actor(states)[range(len(actions)), actions]) * advantages.detach()
        critic_loss = nn.MSELoss()(values, target_values.detach())

        self.actor_optimizer.zero_grad()
        actor_loss.mean().backward()
        self.actor_optimizer.step()

        self.critic_optimizer.zero_grad()
        critic_loss.backward()
        self.critic_optimizer.step()

if __name__ == '__main__':
    env = gym.make('CartPole-v0')
    state_dim = env.observation_space.shape[0]
    action_dim = env.action_space.n
    hidden_dim = 128
    lr = 0.001
    gamma = 0.99
    agent = Agent(state_dim, action_dim, hidden_dim, lr, gamma)

    episodes = 1000
    scores = []
    for episode in range(episodes):
        state = env.reset()
        done = False
        score = 0
        while not done:
            action = agent.act(state)
            next_state, reward, done, info = env.step(action)
            agent.learn(state, action, reward, next_state, done)
            state = next_state
            score += reward
        scores.append(score)
        print('Episode: {}, Score: {}'.format(episode, score))