{
"cells": [
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"#default_exp agents.navigate"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Basic MineRL Agent\n",
"> Run an agent for a navigation task"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"#export\n",
"\n",
"import time\n",
"\n",
"import gym\n",
"import minerl # noqa\n",
"\n",
"# OpenCog\n",
"from opencog.logger import log\n",
"from opencog.pln import *\n",
"from opencog.type_constructors import *\n",
"from opencog.utilities import set_default_atomspace\n",
"\n",
"# ROCCA\n",
"from rocca.envs.wrappers import MineRLWrapper\n",
"from rocca.envs.wrappers.minerl_wrapper import minerl_single_action\n",
"from rocca.agents import OpencogAgent\n",
"from rocca.agents.utils import *\n",
"\n",
"from rocca.utils import *"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Agent definition"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Below is pretty much `OpencogAgent` but with the `step` method fixed to use the `minerl_single_action` helper."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"#export\n",
"\n",
"class NavigateAgent(OpencogAgent):\n",
" def __init__(self, env, action_space, p_goal, n_goal, log_level=\"info\"):\n",
" OpencogAgent.__init__(self, env, action_space, p_goal, n_goal, log_level)\n",
"\n",
" def control_cycle(self):\n",
" \"\"\"Run one step of observation, decision and env update\"\"\"\n",
" agent_log.debug(\"atomese_obs = {}\".format(self.observation))\n",
" obs_record = [\n",
" self.record(o, self.cycle_count, tv=TRUE_TV) for o in self.observation\n",
" ]\n",
" agent_log.debug(\"obs_record = {}\".format(obs_record))\n",
"\n",
" # Make the goal for that iteration\n",
" goal = self.make_goal()\n",
" agent_log.debug(\"goal = {}\".format(goal))\n",
"\n",
" # Plan, i.e. come up with cognitive schematics as plans. Here the\n",
" # goal expiry is 2, i.e. must be fulfilled set for the next two iterations.\n",
" cogscms = self.plan(goal, self.expiry)\n",
" agent_log.debug(\"cogscms = {}\".format(cogscms))\n",
"\n",
" # Deduce the action distribution\n",
" mxmdl = self.deduce(cogscms)\n",
" agent_log.debug(\"mxmdl = {}\".format(mxmdl_to_str(mxmdl)))\n",
"\n",
" # Select the next action\n",
" action, pblty = self.decide(mxmdl)\n",
" agent_log.debug(\n",
" \"action with probability of success = {}\".format(\n",
" act_pblt_to_str((action, pblty))\n",
" )\n",
" )\n",
"\n",
" # Timestamp the action that is about to be executed\n",
" action_record = self.record(action, self.cycle_count, tv=TRUE_TV)\n",
" agent_log.debug(\"action_record = {}\".format(action_record))\n",
" agent_log.debug(\"action = {}\".format(action))\n",
"\n",
" # Increment the counter for that action and log it\n",
" self.action_counter[action] += 1\n",
" agent_log.debug(\"action_counter = {}\".format(self.action_counter))\n",
"\n",
" # Increase the step count and run the next step of the environment\n",
" self.cycle_count += 1\n",
" # TODO gather environment info.\n",
" reward, self.observation, done = self.env.step(\n",
" minerl_single_action(self.env, action)\n",
" )\n",
"\n",
" self.accumulated_reward += float(reward.out[1].name)\n",
" agent_log.debug(\"observation = {}\".format(self.observation))\n",
" agent_log.debug(\"reward = {}\".format(reward))\n",
" agent_log.debug(\"accumulated reward = {}\".format(self.accumulated_reward))\n",
"\n",
" reward_record = self.record(reward, self.cycle_count, tv=TRUE_TV)\n",
" agent_log.debug(\"reward_record = {}\".format(reward_record))\n",
"\n",
" if done:\n",
" return False\n",
"\n",
" return True"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Experiment\n",
"\n",
"For this experiment we use the basic _MineRLNavigateDense-v0_ environment of MineRL. The task of the agent is to touch a diamond block placed somewhere in the vicinity. This is a _Dense_ version of the environment, which means the agent receives rewards all the time, based on whether the distance to the target decreases or increases."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"env = gym.make(\"MineRLNavigateDense-v0\")"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"env.action_space"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"env.observation_space"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"atomspace = AtomSpace()\n",
"set_default_atomspace(atomspace)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"wrapped_env = MineRLWrapper(env)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"# Create Goal\n",
"pgoal = EvaluationLink(PredicateNode(\"Reward\"), NumberNode(\"100\"))\n",
"ngoal = EvaluationLink(PredicateNode(\"Reward\"), NumberNode(\"0\"))"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"action_space = {\n",
" ExecutionLink(SchemaNode(\"attack\"), NumberNode(\"0\")),\n",
" ExecutionLink(SchemaNode(\"attack\"), NumberNode(\"1\")),\n",
" ExecutionLink(SchemaNode(\"forward\"), NumberNode(\"0\")),\n",
" ExecutionLink(SchemaNode(\"forward\"), NumberNode(\"1\")),\n",
" ExecutionLink(SchemaNode(\"back\"), NumberNode(\"0\")),\n",
" ExecutionLink(SchemaNode(\"back\"), NumberNode(\"1\")),\n",
" ExecutionLink(SchemaNode(\"left\"), NumberNode(\"0\")),\n",
" ExecutionLink(SchemaNode(\"left\"), NumberNode(\"1\")),\n",
" ExecutionLink(SchemaNode(\"right\"), NumberNode(\"0\")),\n",
" ExecutionLink(SchemaNode(\"right\"), NumberNode(\"1\")),\n",
" ExecutionLink(SchemaNode(\"jump\"), NumberNode(\"0\")),\n",
" ExecutionLink(SchemaNode(\"jump\"), NumberNode(\"1\")),\n",
" ExecutionLink(SchemaNode(\"sprint\"), NumberNode(\"0\")),\n",
" ExecutionLink(SchemaNode(\"sprint\"), NumberNode(\"1\")),\n",
" ExecutionLink(SchemaNode(\"sneak\"), NumberNode(\"0\")),\n",
" ExecutionLink(SchemaNode(\"sneak\"), NumberNode(\"1\")),\n",
" ExecutionLink(SchemaNode(\"place\"), ConceptNode(\"dirt\")),\n",
" ExecutionLink(SchemaNode(\"place\"), ConceptNode(\"none\")),\n",
" ExecutionLink(SchemaNode(\"camera\"), ListLink(NumberNode(\"2.5\"), NumberNode(\"0.0\"))),\n",
" ExecutionLink(\n",
" SchemaNode(\"camera\"), ListLink(NumberNode(\"0.0\"), NumberNode(\"-1.5\"))\n",
" ),\n",
"}"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"agent = NavigateAgent(wrapped_env, action_space, pgoal, ngoal)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"from tensorboardX import SummaryWriter\n",
"\n",
"tb_writer = SummaryWriter(comment=\"-minerl-navigate\")\n",
"\n",
"epochs = 1 # Number of epochs (learning / interacting episodes)\n",
"epoch_len = 200\n",
"\n",
"for i in range(epochs):\n",
" wrapped_env.restart()\n",
" agent.reset_action_counter()\n",
" accreward = agent.accumulated_reward # Keep track of the reward before\n",
"\n",
" # Learning phase: discover patterns to make more informed decisions\n",
" log_msg(agent_log, f\"Learning phase started. ({i + 1}/{epochs})\")\n",
" agent.learn()\n",
"\n",
" # Run agent to accumulate percepta\n",
" log_msg(agent_log, f\"Interaction phase started. ({i + 1}/{epochs})\")\n",
" for j in range(epoch_len):\n",
" done = agent.control_cycle()\n",
" # wrapped_env.render() uncomment to see the rendered env\n",
" time.sleep(0.01)\n",
" log.info(\"cycle_count = {}\".format(agent.cycle_count))\n",
" if done:\n",
" break\n",
"\n",
" new_reward = agent.accumulated_reward - accreward\n",
" tb_writer.add_scalar(\"train/accumulated_reward\", new_reward, agent.cycle_count)\n",
" log_msg(\n",
" agent_log, \"Accumulated reward during {}th epoch = {}\".format(i + 1, new_reward)\n",
" )\n",
" log_msg(\n",
" agent_log,\n",
" \"Action counter during {}th epoch:\\n{}\".format(i + 1, agent.action_counter),\n",
" ) # TODO: make the action counter look good\n",
"\n",
"log_msg(agent_log, f\"The average total reward over {epochs} trials (training): {agent.accumulated_reward / epochs}.\")"
]
}
],
"metadata": {},
"nbformat": 4,
"nbformat_minor": 5
}