envs.cartpole_wrapper API documentation

# envs.cartpole_wrapper

Author: John Mansfield BSD 3-Clause License

View Source

     1"""
     2Author: John Mansfield
     3BSD 3-Clause License
     4"""
     5
     6import gymnasium as gym
     7import numpy as np
     8from bettermdptools.envs.cartpole_model import DiscretizedCartPole
     9
    10
    11class CustomTransformObservation(gym.ObservationWrapper):
    12    def __init__(self, env, func, observation_space):
    13        """
    14        Helper class that modifies the observation space. The v26 gymnasium TransformObservation wrapper does not
    15        accept an observation_space parameter, which is needed in order to match the lambda conversion (tuple->int).
    16        Instead, we subclass gym.ObservationWrapper (parent class of gym.TransformObservation)
    17        to set both the conversion function and new observation space.
    18
    19        Parameters
    20        ----------------------------
    21        env {gymnasium.Env}:
    22            Base environment to be wrapped
    23
    24        func {lambda}:
    25            Function that converts the observation
    26
    27        observation_space {gymnasium.spaces.Space}:
    28            New observation space
    29        """
    30        super().__init__(env)
    31        if observation_space is not None:
    32            self.observation_space = observation_space
    33        self.func = func
    34
    35    def observation(self, observation):
    36        """
    37        Applies a function to the observation received from the environment's step function,
    38        which is passed back to the user.
    39
    40        Parameters
    41        ----------------------------
    42        observation {Tuple}:
    43            Base environment observation tuple
    44
    45        Returns
    46        ----------------------------
    47        func(observation) {int}:
    48            The converted observation (int).
    49        """
    50        return self.func(observation)
    51
    52
    53class CartpoleWrapper(gym.Wrapper):
    54    def __init__(self,
    55                 env,
    56                 position_bins=10,
    57                 velocity_bins=10,
    58                 angular_velocity_bins=10,
    59                 angular_center_resolution=.1,
    60                 angular_outer_resolution=.5):
    61        """
    62        Cartpole wrapper that modifies the observation space and creates a transition/reward matrix P.
    63
    64        Parameters
    65        ----------------------------
    66        env {gymnasium.Env}: Base environment
    67        position_bins (int): Number of discrete bins for the cart's position.
    68        velocity_bins (int): Number of discrete bins for the cart's velocity.
    69        angular_velocity_bins (int): Number of discrete bins for the pole's angular velocity.
    70        angular_center_resolution (float): The resolution of angle bins near the center (around zero).
    71        angular_outer_resolution (float): The resolution of angle bins away from the center.
    72        """
    73        dpole = DiscretizedCartPole(position_bins=position_bins,
    74                                    velocity_bins=velocity_bins,
    75                                    angular_velocity_bins=angular_velocity_bins,
    76                                    angular_center_resolution=angular_center_resolution,
    77                                    angular_outer_resolution=angular_outer_resolution)
    78        self._P = dpole.P
    79        self._transform_obs = dpole.transform_obs
    80        env = CustomTransformObservation(env, self._transform_obs, gym.spaces.Discrete(dpole.n_states))
    81        super().__init__(env)
    82
    83    @property
    84    def P(self):
    85        """
    86        Returns
    87        ----------------------------
    88        _P {dict}
    89        """
    90        return self._P
    91
    92    @property
    93    def transform_obs(self):
    94        """
    95        Returns
    96        ----------------------------
    97        _transform_obs {lambda}
    98        """
    99        return self._transform_obs

class CustomTransformObservation(gymnasium.core.Wrapper[~WrapperObsType, ~ActType]): View Source

    12class CustomTransformObservation(gym.ObservationWrapper):
    13    def __init__(self, env, func, observation_space):
    14        """
    15        Helper class that modifies the observation space. The v26 gymnasium TransformObservation wrapper does not
    16        accept an observation_space parameter, which is needed in order to match the lambda conversion (tuple->int).
    17        Instead, we subclass gym.ObservationWrapper (parent class of gym.TransformObservation)
    18        to set both the conversion function and new observation space.
    19
    20        Parameters
    21        ----------------------------
    22        env {gymnasium.Env}:
    23            Base environment to be wrapped
    24
    25        func {lambda}:
    26            Function that converts the observation
    27
    28        observation_space {gymnasium.spaces.Space}:
    29            New observation space
    30        """
    31        super().__init__(env)
    32        if observation_space is not None:
    33            self.observation_space = observation_space
    34        self.func = func
    35
    36    def observation(self, observation):
    37        """
    38        Applies a function to the observation received from the environment's step function,
    39        which is passed back to the user.
    40
    41        Parameters
    42        ----------------------------
    43        observation {Tuple}:
    44            Base environment observation tuple
    45
    46        Returns
    47        ----------------------------
    48        func(observation) {int}:
    49            The converted observation (int).
    50        """
    51        return self.func(observation)

Superclass of wrappers that can modify observations using `observation()` for `reset()` and `step()`.

If you would like to apply a function to only the observation before passing it to the learning code, you can simply inherit from `ObservationWrapper` and overwrite the method `observation()` to implement that transformation. The transformation defined in that method must be reflected by the `env` observation space. Otherwise, you need to specify the new observation space of the wrapper by setting `self.observation_space` in the `__init__()` method of your wrapper.

Among others, Gymnasium provides the observation wrapper `TimeAwareObservation`, which adds information about the index of the timestep to the observation.

CustomTransformObservation(env, func, observation_space) View Source

    13    def __init__(self, env, func, observation_space):
    14        """
    15        Helper class that modifies the observation space. The v26 gymnasium TransformObservation wrapper does not
    16        accept an observation_space parameter, which is needed in order to match the lambda conversion (tuple->int).
    17        Instead, we subclass gym.ObservationWrapper (parent class of gym.TransformObservation)
    18        to set both the conversion function and new observation space.
    19
    20        Parameters
    21        ----------------------------
    22        env {gymnasium.Env}:
    23            Base environment to be wrapped
    24
    25        func {lambda}:
    26            Function that converts the observation
    27
    28        observation_space {gymnasium.spaces.Space}:
    29            New observation space
    30        """
    31        super().__init__(env)
    32        if observation_space is not None:
    33            self.observation_space = observation_space
    34        self.func = func

Helper class that modifies the observation space. The v26 gymnasium TransformObservation wrapper does not accept an observation_space parameter, which is needed in order to match the lambda conversion (tuple->int). Instead, we subclass gym.ObservationWrapper (parent class of gym.TransformObservation) to set both the conversion function and new observation space.

###### Parameters

- env {gymnasium.Env}:: Base environment to be wrapped
- func {lambda}:: Function that converts the observation
- observation_space {gymnasium.spaces.Space}:: New observation space

func
def observation(self, observation): View Source

    36    def observation(self, observation):
    37        """
    38        Applies a function to the observation received from the environment's step function,
    39        which is passed back to the user.
    40
    41        Parameters
    42        ----------------------------
    43        observation {Tuple}:
    44            Base environment observation tuple
    45
    46        Returns
    47        ----------------------------
    48        func(observation) {int}:
    49            The converted observation (int).
    50        """
    51        return self.func(observation)

Applies a function to the observation received from the environment's step function, which is passed back to the user.

###### Parameters

- observation {Tuple}:: Base environment observation tuple

###### Returns

- func(observation) {int}:: The converted observation (int).

class CartpoleWrapper(gymnasium.core.Env[~WrapperObsType, ~WrapperActType]): View Source

     54class CartpoleWrapper(gym.Wrapper):
     55    def __init__(self,
     56                 env,
     57                 position_bins=10,
     58                 velocity_bins=10,
     59                 angular_velocity_bins=10,
     60                 angular_center_resolution=.1,
     61                 angular_outer_resolution=.5):
     62        """
     63        Cartpole wrapper that modifies the observation space and creates a transition/reward matrix P.
     64
     65        Parameters
     66        ----------------------------
     67        env {gymnasium.Env}: Base environment
     68        position_bins (int): Number of discrete bins for the cart's position.
     69        velocity_bins (int): Number of discrete bins for the cart's velocity.
     70        angular_velocity_bins (int): Number of discrete bins for the pole's angular velocity.
     71        angular_center_resolution (float): The resolution of angle bins near the center (around zero).
     72        angular_outer_resolution (float): The resolution of angle bins away from the center.
     73        """
     74        dpole = DiscretizedCartPole(position_bins=position_bins,
     75                                    velocity_bins=velocity_bins,
     76                                    angular_velocity_bins=angular_velocity_bins,
     77                                    angular_center_resolution=angular_center_resolution,
     78                                    angular_outer_resolution=angular_outer_resolution)
     79        self._P = dpole.P
     80        self._transform_obs = dpole.transform_obs
     81        env = CustomTransformObservation(env, self._transform_obs, gym.spaces.Discrete(dpole.n_states))
     82        super().__init__(env)
     83
     84    @property
     85    def P(self):
     86        """
     87        Returns
     88        ----------------------------
     89        _P {dict}
     90        """
     91        return self._P
     92
     93    @property
     94    def transform_obs(self):
     95        """
     96        Returns
     97        ----------------------------
     98        _transform_obs {lambda}
     99        """
    100        return self._transform_obs

Wraps a `gymnasium.Env` to allow a modular transformation of the `step()` and `reset()` methods.

This class is the base class of all wrappers to change the behavior of the underlying environment. Wrappers that inherit from this class can modify the `action_space`, `observation_space`, `reward_range` and `metadata` attributes, without changing the underlying environment's attributes. Moreover, the behavior of the `step()` and `reset()` methods can be changed by these wrappers.

Some attributes (`spec`, `render_mode`, `np_random`) will point back to the wrapper's environment (i.e. to the corresponding attributes of `env`).

Note: If you inherit from `Wrapper`, don't forget to call `super().__init__(env)`

CartpoleWrapper( env, position_bins=10, velocity_bins=10, angular_velocity_bins=10, angular_center_resolution=0.1, angular_outer_resolution=0.5) View Source

    55    def __init__(self,
    56                 env,
    57                 position_bins=10,
    58                 velocity_bins=10,
    59                 angular_velocity_bins=10,
    60                 angular_center_resolution=.1,
    61                 angular_outer_resolution=.5):
    62        """
    63        Cartpole wrapper that modifies the observation space and creates a transition/reward matrix P.
    64
    65        Parameters
    66        ----------------------------
    67        env {gymnasium.Env}: Base environment
    68        position_bins (int): Number of discrete bins for the cart's position.
    69        velocity_bins (int): Number of discrete bins for the cart's velocity.
    70        angular_velocity_bins (int): Number of discrete bins for the pole's angular velocity.
    71        angular_center_resolution (float): The resolution of angle bins near the center (around zero).
    72        angular_outer_resolution (float): The resolution of angle bins away from the center.
    73        """
    74        dpole = DiscretizedCartPole(position_bins=position_bins,
    75                                    velocity_bins=velocity_bins,
    76                                    angular_velocity_bins=angular_velocity_bins,
    77                                    angular_center_resolution=angular_center_resolution,
    78                                    angular_outer_resolution=angular_outer_resolution)
    79        self._P = dpole.P
    80        self._transform_obs = dpole.transform_obs
    81        env = CustomTransformObservation(env, self._transform_obs, gym.spaces.Discrete(dpole.n_states))
    82        super().__init__(env)

Cartpole wrapper that modifies the observation space and creates a transition/reward matrix P.

###### Parameters

- env {gymnasium.Env} (Base environment):

- position_bins (int) (Number of discrete bins for the cart's position.):

- velocity_bins (int) (Number of discrete bins for the cart's velocity.):

- angular_velocity_bins (int) (Number of discrete bins for the pole's angular velocity.):

- angular_center_resolution (float) (The resolution of angle bins near the center (around zero).):

- angular_outer_resolution (float) (The resolution of angle bins away from the center.):

P View Source

    84    @property
    85    def P(self):
    86        """
    87        Returns
    88        ----------------------------
    89        _P {dict}
    90        """
    91        return self._P

###### Returns

- _P {dict}

transform_obs View Source

     93    @property
     94    def transform_obs(self):
     95        """
     96        Returns
     97        ----------------------------
     98        _transform_obs {lambda}
     99        """
    100        return self._transform_obs

###### Returns

- _transform_obs {lambda}