• Stable Baselines/用户向导/自定义策略网络

    Stable Baselines官方文档中文版 Github CSDN 尝试翻译官方文档，水平有限，如有错误万望指正

    Stable baselines为图像（CNNPolicies）和其他类型的输入特征（MlpPolicies）提供了默认策略网络（见 Policies）。

    自定义策略网络结构的一种方法是创建模型的时候用policy_kwargs给模型传递参数：

    1. import gym
    2. import tensorflow as tf
    4. from stable_baselines import PPO2
    6. # Custom MLP policy of two layers of size 32 each with tanh activation function
    7. policy_kwargs = dict(act_fun=tf.nn.tanh, net_arch=[32, 32])
    8. # Create the agent
    9. model = PPO2("MlpPolicy", "CartPole-v1", policy_kwargs=policy_kwargs, verbose=1)
    10. # Retrieve the environment
    11. env = model.get_env()
    12. # Train the agent
    13. model.learn(total_timesteps=100000)
    14. # Save the agent
    15. model.save("ppo2-cartpole")
    17. del model
    18. # the policy_kwargs are automatically loaded
    19. model = PPO2.load("ppo2-cartpole")

    你也可以轻松为策略(或值)网络定义一个自定义结构：

    定义一个自定义策略类等价于传递policy_kwargs。然而，它让你为策略命名，使代码简洁。在超参数搜索时应使用policy_kwargs

    1. import gym
    3. from stable_baselines.common.policies import FeedForwardPolicy, register_policy
    4. from stable_baselines.common.vec_env import DummyVecEnv
    5. from stable_baselines import A2C
    7. # Custom MLP policy of three layers of size 128 each
    8. class CustomPolicy(FeedForwardPolicy):
    9.     def __init__(self, *args, **kwargs):
    10.         super(CustomPolicy, self).__init__(*args, **kwargs,
    11.                                            net_arch=[dict(pi=[128, 128, 128],
    12.                                                           vf=[128, 128, 128])],
    13.                                            feature_extraction="mlp")
    15. # Create and wrap the environment
    16. env = gym.make('LunarLander-v2')
    17. env = DummyVecEnv([lambda: env])
    19. model = A2C(CustomPolicy, env, verbose=1)
    20. # Train the agent
    21. model.learn(total_timesteps=100000)
    22. # Save the agent
    23. model.save("a2c-lunar")
    25. del model
    26. # When loading a model with a custom policy
    27. # you MUST pass explicitly the policy when loading the saved model
    28. model = A2C.load("a2c-lunar", policy=CustomPolicy)

    警告：

    当载入一个具有自定义策略的模型，你必须在载入模型时显式的传递自定义策略。（cf之前的例子）

    你也可以注册你的策略，以简化代码：你可以用一个字符串调用自定义策略：

    1. import gym
    3. from stable_baselines.common.policies import FeedForwardPolicy, register_policy
    4. from stable_baselines.common.vec_env import DummyVecEnv
    5. from stable_baselines import A2C
    7. # Custom MLP policy of three layers of size 128 each
    8. class CustomPolicy(FeedForwardPolicy):
    9.     def __init__(self, *args, **kwargs):
    10.         super(CustomPolicy, self).__init__(*args, **kwargs,
    11.                                            net_arch=[dict(pi=[128, 128, 128],
    12.                                                           vf=[128, 128, 128])],
    13.                                            feature_extraction="mlp")
    15. # Register the policy, it will check that the name is not already taken
    16. register_policy('CustomPolicy', CustomPolicy)
    18. # Because the policy is now registered, you can pass
    19. # a string to the agent constructor instead of passing a class
    20. model = A2C(policy='CustomPolicy', env='LunarLander-v2', verbose=1).learn(total_timesteps=100000)

    2.3.0版本后弃用：用net_arch替换layers参数来定义网络结构。它允许有更大的控制权。

    FeedForwardPolicy的net_arch参数允许指定隐层的大小及数量、以及他们中有多少比重时策略网络和值网络共享的。假设时下述结构的一个list：

    1. 一系列随机整数（0也可以），每个指定共享层的单元数。如果整数的个数是0，就是说没有共享层。
    2. 一个可选字典，指定下述策略网络和值网络的非共享层。格式化为dict(vf=[<value layer sizes>], pi=[<policy layer sizes>])。如果缺失任何关键字（pi或vf），嘉定没有非共享层（空list）。

    简而言之：

    1. [<shared layers>, dict(vf=[<non-shared value network layers>], pi=[<non-shared policy network layers>])]

  • 案例

    2个128型共享层：net_arch=[128, 128]

    1.           obs
    2.            |
    3.          <128>
    4.            |
    5.          <128>
    6.    /               \
    7. action            value

    值网络比策略网络更深，第一共享层：net_arch=[128, dict(vf=[256, 256])]

    1.           obs
    2.            |
    3.          <128>
    4.    /               \
    5. action             <256>
    6.                      |
    7.                    <256>
    8.                      |
    9.                    value

    最初是共享的，后来出现分歧：[128, dict(vf=[256], pi=[16])]

    1.           obs
    2.            |
    3.          <128>
    4.    /               \
    5.  <16>             <256>
    6.    |                |
    7. action            value

    LstmPolicy可以类似方式构建迭代网络：

    1. class CustomLSTMPolicy(LstmPolicy):
    2.     def __init__(self, sess, ob_space, ac_space, n_env, n_steps, n_batch, n_lstm=64, reuse=False, **_kwargs):
    3.         super().__init__(sess, ob_space, ac_space, n_env, n_steps, n_batch, n_lstm, reuse,
    4.                          net_arch=[8, 'lstm', dict(vf=[5, 10], pi=[10])],
    5.                          layer_norm=True, feature_extraction="mlp", **_kwargs)

    在共享网络部分，net_auch参数接受一个附加（强制）’lstm‘项。策略网络和值网络共享LSTM。

    如果你的任务要求对策略架构要求更细粒度控制，你可以直接重定义策略：

    ```python import gym import tensorflow as tf

    from stable_baselines.common.policies import ActorCriticPolicy, register_policy, nature_cnn from stable_baselines.common.vec_env import DummyVecEnv from stable_baselines import A2C

    Custom MLP policy of three layers of size 128 each for the actor and 2 layers of 32 for the critic,

    with a nature_cnn feature extractor

    class CustomPolicy(ActorCriticPolicy):

    1. def __init__(self, sess, ob_space, ac_space, n_env, n_steps, n_batch, reuse=False, **kwargs):
    2.     super(CustomPolicy, self).__init__(sess, ob_space, ac_space, n_env, n_steps, n_batch, reuse=reuse, scale=True)
    4.     with tf.variable_scope("model", reuse=reuse):
    5.         activ = tf.nn.relu
    7.         extracted_features = nature_cnn(self.processed_obs, **kwargs)
    8.         extracted_features = tf.layers.flatten(extracted_features)
    10.         pi_h = extracted_features
    11.         for i, layer_size in enumerate([128, 128, 128]):
    12.             pi_h = activ(tf.layers.dense(pi_h, layer_size, name='pi_fc' + str(i)))
    13.         pi_latent = pi_h
    15.         vf_h = extracted_features
    16.         for i, layer_size in enumerate([32, 32]):
    17.             vf_h = activ(tf.layers.dense(vf_h, layer_size, name='vf_fc' + str(i)))
    18.         value_fn = tf.layers.dense(vf_h, 1, name='vf')
    19.         vf_latent = vf_h
    21.         self._proba_distribution, self._policy, self.q_value = \
    22.             self.pdtype.proba_distribution_from_latent(pi_latent, vf_latent, init_scale=0.01)
    24.     self._value_fn = value_fn
    25.     self._setup_init()
    27. def step(self, obs, state=None, mask=None, deterministic=False):
    28.     if deterministic:
    29.         action, value, neglogp = self.sess.run([self.deterministic_action, self.value_flat, self.neglogp],
    30.                                                {self.obs_ph: obs})
    31.     else:
    32.         action, value, neglogp = self.sess.run([self.action, self.value_flat, self.neglogp],
    33.                                                {self.obs_ph: obs})
    34.     return action, value, self.initial_state, neglogp
    36. def proba_step(self, obs, state=None, mask=None):
    37.     return self.sess.run(self.policy_proba, {self.obs_ph: obs})
    39. def value(self, obs, state=None, mask=None):
    40.     return self.sess.run(self.value_flat, {self.obs_ph: obs})

Create and wrap the environment

env = DummyVecEnv([lambda: gym.make(‘Breakout-v0’)])

model = A2C(CustomPolicy, env, verbose=1)

Train the agent

model.learn(total_timesteps=100000) ```