Examples

Examples of the PCT library in use.

import gym, warnings
render=False
runs=1

render=True
runs=2000

Cartpole

Cartpole is an Open AI gym environment for the inverted pendulum problem. The goal is to keep the pole balanced, by moving the cart left or right.

The environment provides observations (perceptions) for the state of the cart and pole.

0 - Cart Position
1 - Cart Velocity
2 - Pole Angle
3 - Pole Angular Velocity

It takes one value, of 0 or 1, for applying a force to the left or right, respectively.

The PCT solution is a four-level hierarchy for controlling the perceptions at goal values. Only one goal reference is manually set, the highest level which is the pole angle of 0.

This example shows how a perceptual control hierarchy can be implemented with this library.

import matplotlib.pyplot as plt
import numpy as np
from pct.hierarchy import PCTHierarchy
from pct.putils import FunctionsList
from pct.environments import CartPoleV1
from pct.functions import IndexedParameter
from pct.functions import Integration
from pct.functions import GreaterThan
from pct.functions import PassOn

Create a hierarchy of 4 levels each with one node.

cartpole_hierarchy = PCTHierarchy(levels=4, cols=1, name="cartpoleh", build=False)
namespace=cartpole_hierarchy.namespace
cartpole_hierarchy.get_node(0, 0).name = 'cart_velocity_node'
cartpole_hierarchy.get_node(1, 0).name = 'cart_position_node'
cartpole_hierarchy.get_node(2, 0).name = 'pole_velocity_node'
cartpole_hierarchy.get_node(3, 0).name = 'pole_angle_node'
#FunctionsList.getInstance().report()
#cartpole_hierarchy.summary(build=True)

Create the Cartpole gym environment function. This will apply the “action” output from the hierarchy and provide the new observations.

cartpole = CartPoleV1(name="CartPole-v1", render=render, namespace=namespace, seed=1)

Create functions for each of the observation parameters of the Cartpole environment. Insert them into the hierarchy at the desired places.

cartpole_hierarchy.insert_function(level=0, col=0, collection="perception", function=IndexedParameter(index=1, name="cart_velocity", links=[cartpole], namespace=namespace))
cartpole_hierarchy.insert_function(level=1, col=0, collection="perception", function=IndexedParameter(index=0, name="cart_position", links=[cartpole], namespace=namespace))
cartpole_hierarchy.insert_function(level=2, col=0, collection="perception", function=IndexedParameter(index=3, name="pole_velocity", links=[cartpole], namespace=namespace))
cartpole_hierarchy.insert_function(level=3, col=0, collection="perception", function=IndexedParameter(index=2, name="pole_angle", links=[cartpole], namespace=namespace))

Link the references to the outputs of the level up.

cartpole_hierarchy.insert_function(level=0, col=0, collection="reference", function=PassOn(name="cart_velocity_reference", links=['proportional1'], namespace=namespace))
cartpole_hierarchy.insert_function(level=1, col=0, collection="reference", function=PassOn(name="cart_position_reference", links=['proportional2'], namespace=namespace))
cartpole_hierarchy.insert_function(level=2, col=0, collection="reference", function=PassOn(name="pole_velocity_reference", links=['proportional3'], namespace=namespace))

Set the highest level reference.

top = cartpole_hierarchy.get_function(level=3, col=0, collection="reference")
top.set_name("pole_angle_reference")
top.set_value(0)

Link the output of the hierarchy back to the Cartpole environment.

cartpole_hierarchy.summary(build=True)

**************************
cartpoleh PCTHierarchy [1, 1, 1, 1] f408427f-1962-11f0-befb-04d9f58727ab
--------------------------
PRE: None
Level 0 Cols 1
cart_velocity_node PCTNode f408427f-1962-11f0-befb-04d9f58727ab
----------------------------
REF: cart_velocity_reference PassOn | 0 | links  proportional1 
PER: cart_velocity IndexedParameter | index 1 | 0 | links  CartPole-v1 
COM: subtract Subtract | 0 | links  cart_velocity_reference cart_velocity 
OUT: proportional Proportional | gain 1 | 0 | links  subtract 
----------------------------
Level 1 Cols 1
cart_position_node PCTNode f408427f-1962-11f0-befb-04d9f58727ab
----------------------------
REF: cart_position_reference PassOn | 0 | links  proportional2 
PER: cart_position IndexedParameter | index 0 | 0 | links  CartPole-v1 
COM: subtract1 Subtract | 0 | links  cart_position_reference cart_position 
OUT: proportional1 Proportional | gain 1 | 0 | links  subtract1 
----------------------------
Level 2 Cols 1
pole_velocity_node PCTNode f408427f-1962-11f0-befb-04d9f58727ab
----------------------------
REF: pole_velocity_reference PassOn | 0 | links  proportional3 
PER: pole_velocity IndexedParameter | index 3 | 0 | links  CartPole-v1 
COM: subtract2 Subtract | 0 | links  pole_velocity_reference pole_velocity 
OUT: proportional2 Proportional | gain 1 | 0 | links  subtract2 
----------------------------
Level 3 Cols 1
pole_angle_node PCTNode f408427f-1962-11f0-befb-04d9f58727ab
----------------------------
REF: pole_angle_reference Constant | 0 
PER: pole_angle IndexedParameter | index 2 | 0 | links  CartPole-v1 
COM: subtract3 Subtract | 0 | links  pole_angle_reference pole_angle 
OUT: proportional3 Proportional | gain 1 | 0 | links  subtract3 
----------------------------
POST: None
**************************

cartpole_hierarchy.insert_function(level=0, col=0, collection="output", function=Integration(gain=-0.05, slow=4, name="force", links='subtract', namespace=namespace))

Set the names and gains of the output functions. This also shows another way of getting a function, by name.

FunctionsList.getInstance().get_function(namespace=namespace, name="proportional3").set_name("pole_angle_output")
FunctionsList.getInstance().get_function(namespace=namespace, name="pole_angle_output").set_property('gain', 3.5)

FunctionsList.getInstance().get_function(namespace=namespace, name="proportional2").set_name("pole_velocity_output")
FunctionsList.getInstance().get_function(namespace=namespace, name="pole_velocity_output").set_property('gain', 0.5)

FunctionsList.getInstance().get_function(namespace=namespace, name="proportional1").set_name("cart_position_output")
FunctionsList.getInstance().get_function(namespace=namespace, name="cart_position_output").set_property('gain', 2)

Add a post function to convert the output to 1 or 0 as required by the Cartpole environment.

greaterthan = GreaterThan(threshold=0, upper=1, lower=0, links='force', namespace=namespace)
cartpole_hierarchy.add_postprocessor(greaterthan)

Add the cartpole function as one that is executed before the actual hierarchy.

cartpole_hierarchy.add_preprocessor(cartpole)

Set the output of the hierachy as the action input to the Cartpole environment.

#link = cartpole_hierarchy.get_output_function()
cartpole.add_link(greaterthan)

Sit back and observe the brilliance of your efforts.

cartpole_hierarchy.set_order("Down")

cartpole_hierarchy.summary()

**************************
cartpoleh PCTHierarchy [1, 1, 1, 1] f408427f-1962-11f0-befb-04d9f58727ab
--------------------------
PRE: CartPole-v1 CartPoleV1 | 0 | links  greaterthan 
Level 3 Cols 1
pole_angle_node PCTNode f408427f-1962-11f0-befb-04d9f58727ab
----------------------------
REF: pole_angle_reference Constant | 0 
PER: pole_angle IndexedParameter | index 2 | 0 | links  CartPole-v1 
COM: subtract3 Subtract | 0 | links  pole_angle_reference pole_angle 
OUT: pole_angle_output Proportional | gain 3.5 | 0 | links  subtract3 
----------------------------
Level 2 Cols 1
pole_velocity_node PCTNode f408427f-1962-11f0-befb-04d9f58727ab
----------------------------
REF: pole_velocity_reference PassOn | 0 | links  pole_angle_output 
PER: pole_velocity IndexedParameter | index 3 | 0 | links  CartPole-v1 
COM: subtract2 Subtract | 0 | links  pole_velocity_reference pole_velocity 
OUT: pole_velocity_output Proportional | gain 0.5 | 0 | links  subtract2 
----------------------------
Level 1 Cols 1
cart_position_node PCTNode f408427f-1962-11f0-befb-04d9f58727ab
----------------------------
REF: cart_position_reference PassOn | 0 | links  pole_velocity_output 
PER: cart_position IndexedParameter | index 0 | 0 | links  CartPole-v1 
COM: subtract1 Subtract | 0 | links  cart_position_reference cart_position 
OUT: cart_position_output Proportional | gain 2 | 0 | links  subtract1 
----------------------------
Level 0 Cols 1
cart_velocity_node PCTNode f408427f-1962-11f0-befb-04d9f58727ab
----------------------------
REF: cart_velocity_reference PassOn | 0 | links  cart_position_output 
PER: cart_velocity IndexedParameter | index 1 | 0 | links  CartPole-v1 
COM: subtract Subtract | 0 | links  cart_velocity_reference cart_velocity 
OUT: force Integration | gain -0.05 slow 4  | 0 | links  subtract 
----------------------------
POST: greaterthan GreaterThan | threshold 0 upper 1 lower 0  | 0 | links  force 
**************************

cartpole_hierarchy.draw(font_size=10, figsize=(8,12), move={'CartPole-v1': [-0.075, 0]}, node_size=1000, node_color='red')

C:\Users\ruper\Versioning\python\nbdev\pct\pct\hierarchy.py:323: UserWarning:

This figure includes Axes that are not compatible with tight_layout, so results might be incorrect.

cartpole_hierarchy.save("cartpole.json")

import networkx as nx
gr = cartpole_hierarchy.graph()
with warnings.catch_warnings():
    warnings.simplefilter("ignore")
    print(nx.info(gr))
    print(gr.nodes())

DiGraph with 18 nodes and 21 edges
['greaterthan', 'force', 'CartPole-v1', 'cart_velocity_reference', 'cart_position_output', 'subtract', 'cart_velocity', 'cart_position_reference', 'pole_velocity_output', 'subtract1', 'cart_position', 'pole_velocity_reference', 'pole_angle_output', 'subtract2', 'pole_velocity', 'pole_angle_reference', 'subtract3', 'pole_angle']

Run the hierarchy for 500 steps.

cartpole_hierarchy.run(1,verbose=False)

---------------------------------------------------------------------------
TypeError                                 Traceback (most recent call last)
Cell In[21], line 1
----> 1 cartpole_hierarchy.run(1,verbose=False)

File ~\Versioning\python\nbdev\pct\pct\hierarchy.py:219, in PCTHierarchy.run(self, steps, verbose)
    216     elif ex.__str__().startswith('1001'):
    217         return False
--> 219     raise ex
    221 if self.error_collector:
    222     if verbose:

File ~\Versioning\python\nbdev\pct\pct\hierarchy.py:207, in PCTHierarchy.run(self, steps, verbose)
    205         print(f'[{i}]', end=' ')
    206     self.set_current_step(i)
--> 207     out = self(verbose)
    208 except Exception as ex:
    209     # if self.error_collector != None:
    210     #     print(f'<{i} {self.error_collector.error()}>')
    211     if ex.__str__().startswith('1000'):

File ~\Versioning\python\nbdev\pct\pct\hierarchy.py:85, in PCTHierarchy.__call__(self, verbose)
     83 for ctr in range(len(self.preCollection)):
     84     func = self.preCollection[ctr]
---> 85     func(verbose)
     86     if self.prepost_data != None:                
     87         self.prepost_data.add_data(func)

File ~\Versioning\python\nbdev\pct\pct\environments.py:630, in CartPoleV1.__call__(self, verbose)
    629 def __call__(self, verbose=False):
--> 630     super().__call__(verbose)
    632     return self.value

File ~\Versioning\python\nbdev\pct\pct\environments.py:239, in OpenAIGym.__call__(self, verbose)
    236 def __call__(self, verbose=False):
    237     out = super().__call__(verbose)
--> 239     self.render()
    241     return out

TypeError: 'bool' object is not callable

cartpole_hierarchy.run(runs,verbose=False)

1

cartpole.close()