深度强化学习ddpg代码

这里是Deep Deterministic Policy Gradient (DDPG)算法的TensorFlow 2.0实现代码：

import tensorflow as tf
import numpy as np

class Actor(tf.keras.Model):
    def __init__(self, state_dim, action_dim, action_max):
        super().__init__()
        self.action_max = action_max
        self.fc1 = tf.keras.layers.Dense(64, activation='relu')
        self.fc2 = tf.keras.layers.Dense(32, activation='relu')
        self.out = tf.keras.layers.Dense(action_dim, activation='tanh')

    def call(self, inputs):
        x = self.fc1(inputs)
        x = self.fc2(x)
        actions = self.out(x) * self.action_max
        return actions

class Critic(tf.keras.Model):
    def __init__(self, state_dim, action_dim):
        super().__init__()
        self.fc1 = tf.keras.layers.Dense(64, activation='relu')
        self.fc2 = tf.keras.layers.Dense(32, activation='relu')
        self.out = tf.keras.layers.Dense(1)

    def call(self, inputs):
        x, actions = inputs
        x = self.fc1(tf.concat([x, actions], axis=-1))
        x = self.fc2(x)
        q_values = self.out(x)
        return q_values

class DDPG:
    def __init__(self, state_dim, action_dim, action_max):
        self.state_dim = state_dim
        self.action_dim = action_dim
        self.action_max = action_max

        self.actor_optimizer = tf.keras.optimizers.Adam(0.001)
        self.critic_optimizer = tf.keras.optimizers.Adam(0.002)

        self.actor = Actor(self.state_dim, self.action_dim, self.action_max)
        self.critic = Critic(self.state_dim, self.action_dim)
        self.target_actor = Actor(self.state_dim, self.action_dim, self.action_max)
        self.target_critic = Critic(self.state_dim, self.action_dim)

        self.target_actor.set_weights(self.actor.get_weights())
        self.target_critic.set_weights(self.critic.get_weights())

    def get_action(self, state):
        state = np.expand_dims(state, axis=0)
        action = self.actor(state)
        return action.numpy()[0]

    def train(self, states, actions, rewards, next_states, dones, gamma=0.99, tau=0.001):
        states = tf.convert_to_tensor(states, dtype=tf.float32)
        actions = tf.convert_to_tensor(actions, dtype=tf.float32)
        rewards = tf.convert_to_tensor(rewards, dtype=tf.float32)
        next_states = tf.convert_to_tensor(next_states, dtype=tf.float32)
        dones = tf.convert_to_tensor(dones, dtype=tf.float32)

        with tf.GradientTape() as tape:
            target_actions = self.target_actor(next_states)
            q_values = self.target_critic([next_states, target_actions])
            target_q_values = rewards + gamma * (1 - dones) * q_values
            critic_loss = tf.reduce_mean((target_q_values - self.critic([states, actions])) ** 2)

        critic_grads = tape.gradient(critic_loss, self.critic.trainable_variables)
        self.critic_optimizer.apply_gradients(zip(critic_grads, self.critic.trainable_variables))

        with tf.GradientTape() as tape:
            new_actions = self.actor(states)
            actor_loss = -tf.reduce_mean(self.critic([states, new_actions]))

        actor_grads = tape.gradient(actor_loss, self.actor.trainable_variables)
        self.actor_optimizer.apply_gradients(zip(actor_grads, self.actor.trainable_variables))

        self.soft_update(self.target_actor.variables, self.actor.variables, tau)
        self.soft_update(self.target_critic.variables, self.critic.variables, tau)

    def soft_update(self, target_vars, source_vars, tau):
        for target_var, source_var in zip(target_vars, source_vars):
            target_var.assign(tau * source_var + (1 - tau) * target_var)

这段代码定义了一个DDPG类，其中包含了Actor和Critic两个神经网络模型。在get_action方法中，根据当前状态，使用Actor模型预测一个动作。在train方法中，根据经验回放缓存中的数据，计算Critic的损失并更新Critic模型，然后计算Actor的损失并更新Actor模型。此外，还使用软更新的方式更新目标Actor和Critic模型。