initial commit

CITATION.cff

@@ -0,0 +1,30 @@
+cff-version: 1.2.0
+message: "If you use this software, please cite it as below."
+authors:
+- family-names: "WAN"
+  given-names: "Ching Pui"
+  orcid: "https://orcid.org/0000-0002-6217-5418"
+- family-names: "LI"
+  given-names: "Tung"
+- family-names: "WANG"
+  given-names: "Jason Min"
+title: "RLOR: A Flexible Framework of Deep Reinforcement Learning for Operation Research"
+version: 1.0.0
+doi: 10.5281/zenodo.1234
+date-released: 2023-03-23
+url: "https://github.com/cpwan/RLOR"
+preferred-citation:
+  type: misc
+  authors:
+  - family-names: "WAN"
+    given-names: "Ching Pui"
+    orcid: "https://orcid.org/0000-0002-6217-5418"
+  - family-names: "LI"
+    given-names: "Tung"
+  - family-names: "WANG"
+    given-names: "Jason Min"
+  doi: 10.48550/arXiv.2303.13117
+  title: "RLOR: A Flexible Framework of Deep Reinforcement Learning for Operation Research"
+  year: 2023
+  eprint: "arXiv:2303.13117"
+  url : "http://arxiv.org/abs/2303.13117"

LICENSE

@@ -0,0 +1,339 @@
+MIT License
+
+Copyright (c) 2023 Ching Pui Wan (cpwan), Tung Li (TonyLiHK)
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
+--------------------------------------------------------------------------------
+MIT License
+
+Copyright (c) 2018 Wouter Kool
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
+--------------------------------------------------------------------------------
+MIT License
+
+Copyright (c) 2019 CleanRL developers
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
+
+--------------------------------------------------------------------------------
+
+Code in `cleanrl/ppo_procgen.py` and `cleanrl/ppg_procgen.py` are adapted from https://github.com/AIcrowd/neurips2020-procgen-starter-kit/blob/142d09586d2272a17f44481a115c4bd817cf6a94/models/impala_cnn_torch.py
+
+**NOTE: the original repo did not fill out the copyright section in their license
+so the following copyright notice is copied as is per the license requirement.
+See https://github.com/AIcrowd/neurips2020-procgen-starter-kit/blob/142d09586d2272a17f44481a115c4bd817cf6a94/LICENSE#L190
+
+
+Copyright [yyyy] [name of copyright owner]
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+--------------------------------------------------------------------------------
+Code in `cleanrl/ddpg_continuous_action.py` and `cleanrl/td3_continuous_action.py` are adapted from https://github.com/sfujim/TD3
+
+
+MIT License
+
+Copyright (c) 2020 Scott Fujimoto
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
+--------------------------------------------------------------------------------
+Code in `cleanrl/sac_continuous_action.py` is inspired and adapted from [haarnoja/sac](https://github.com/haarnoja/sac), [openai/spinningup](https://github.com/openai/spinningup), [pranz24/pytorch-soft-actor-critic](https://github.com/pranz24/pytorch-soft-actor-critic), [DLR-RM/stable-baselines3](https://github.com/DLR-RM/stable-baselines3), and [denisyarats/pytorch_sac](https://github.com/denisyarats/pytorch_sac).
+
+- [haarnoja/sac](https://github.com/haarnoja/sac/blob/8258e33633c7e37833cc39315891e77adfbe14b2/LICENSE.txt)
+
+COPYRIGHT
+
+All contributions by the University of California:
+Copyright (c) 2017, 2018 The Regents of the University of California (Regents)
+All rights reserved.
+
+All other contributions:
+Copyright (c) 2017, 2018, the respective contributors
+All rights reserved.
+
+SAC uses a shared copyright model: each contributor holds copyright over
+their contributions to the SAC codebase. The project versioning records all such
+contribution and copyright details. If a contributor wants to further mark
+their specific copyright on a particular contribution, they should indicate
+their copyright solely in the commit message of the change when it is
+committed.
+
+LICENSE
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice, this
+   list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright notice,
+   this list of conditions and the following disclaimer in the documentation
+   and/or other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
+ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+CONTRIBUTION AGREEMENT
+
+By contributing to the SAC repository through pull-request, comment,
+or otherwise, the contributor releases their content to the
+license and copyright terms herein.
+
+- [openai/spinningup](https://github.com/openai/spinningup/blob/038665d62d569055401d91856abb287263096178/LICENSE)
+
+The MIT License
+
+Copyright (c) 2018 OpenAI (http://openai.com)
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+
+- [DLR-RM/stable-baselines3](https://github.com/DLR-RM/stable-baselines3/blob/44e53ff8115e8f4bff1d5218f10c8c7d1a4cfc12/LICENSE)
+
+The MIT License
+
+Copyright (c) 2019 Antonin Raffin
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+
+- [denisyarats/pytorch_sac](https://github.com/denisyarats/pytorch_sac/blob/81c5b536d3a1c5616b2531e446450df412a064fb/LICENSE)
+
+MIT License
+
+Copyright (c) 2019 Denis Yarats
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
+- [pranz24/pytorch-soft-actor-critic](https://github.com/pranz24/pytorch-soft-actor-critic/blob/master/LICENSE)
+
+MIT License
+
+Copyright (c) 2018 Pranjal Tandon
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
+
+---------------------------------------------------------------------------------
+The CONTRIBUTING.md is adopted from https://github.com/entity-neural-network/incubator/blob/2a0c38b30828df78c47b0318c76a4905020618dd/CONTRIBUTING.md
+and https://github.com/Stable-Baselines-Team/stable-baselines3-contrib/blob/master/CONTRIBUTING.md
+
+MIT License
+
+Copyright (c) 2021 Entity Neural Network developers
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
+
+
+MIT License
+
+Copyright (c) 2020 Stable-Baselines Team
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
+
+---------------------------------------------------------------------------------
+The cleanrl/ppo_continuous_action_isaacgym.py is contributed by Nvidia
+
+SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+SPDX-License-Identifier: MIT
+
+Permission is hereby granted, free of charge, to any person obtaining a
+copy of this software and associated documentation files (the "Software"),
+to deal in the Software without restriction, including without limitation
+the rights to use, copy, modify, merge, publish, distribute, sublicense,
+and/or sell copies of the Software, and to permit persons to whom the
+Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.

demo/cvrp_search.ipynb

@@ -0,0 +1,411 @@
+{
+  "cells": [
+    {
+      "cell_type": "code",
+      "execution_count": 1,
+      "id": "3f3052cd",
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        },
+        "id": "3f3052cd",
+        "outputId": "78d129fd-0956-4f88-ae39-def9953a982e"
+      },
+      "outputs": [
+        {
+          "output_type": "stream",
+          "name": "stdout",
+          "text": [
+            "Cloning into 'RLOR'...\n",
+            "remote: Enumerating objects: 52, done.\u001b[K\n",
+            "remote: Counting objects: 100% (52/52), done.\u001b[K\n",
+            "remote: Compressing objects: 100% (35/35), done.\u001b[K\n",
+            "remote: Total 52 (delta 12), reused 52 (delta 12), pack-reused 0\u001b[K\n",
+            "Unpacking objects: 100% (52/52), 5.19 MiB | 4.39 MiB/s, done.\n",
+            "/content/RLOR\n"
+          ]
+        }
+      ],
+      "source": [
+        "!git clone https://github.com/cpwan/RLOR\n",
+        "%cd RLOR"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 2,
+      "id": "f01dfb64",
+      "metadata": {
+        "id": "f01dfb64"
+      },
+      "outputs": [],
+      "source": [
+        "import numpy as np\n",
+        "import torch\n",
+        "import gym"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "id": "985bf6e6",
+      "metadata": {
+        "id": "985bf6e6"
+      },
+      "source": [
+        "# Define our agent"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 3,
+      "id": "953a7fde",
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        },
+        "id": "953a7fde",
+        "outputId": "9b37d746-b9a0-4d53-c12a-b2445ed6bd9d"
+      },
+      "outputs": [
+        {
+          "output_type": "execute_result",
+          "data": {
+            "text/plain": [
+              "<All keys matched successfully>"
+            ]
+          },
+          "metadata": {},
+          "execution_count": 3
+        }
+      ],
+      "source": [
+        "from models.attention_model_wrapper import Agent\n",
+        "device = 'cuda'\n",
+        "ckpt_path = './runs/cvrp-v0__ppo_or__1__1678159979/ckpt/12000.pt'\n",
+        "agent = Agent(device=device, name='cvrp').to(device)\n",
+        "agent.load_state_dict(torch.load(ckpt_path))"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "id": "2cbaa255",
+      "metadata": {
+        "id": "2cbaa255"
+      },
+      "source": [
+        "# Define our environment\n",
+        "## CVRP\n",
+        "Given a depot, n nodes with their demands, and the capacity of the vehicle, \n",
+        "find the shortest path that fulfills the demand of every node.\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 4,
+      "id": "81fd7b68",
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        },
+        "id": "81fd7b68",
+        "outputId": "f4a9d9d8-29f7-413d-b462-d27f04a0153a"
+      },
+      "outputs": [
+        {
+          "output_type": "stream",
+          "name": "stderr",
+          "text": [
+            "/usr/local/lib/python3.9/dist-packages/gym/utils/passive_env_checker.py:31: UserWarning: \u001b[33mWARN: A Box observation space has an unconventional shape (neither an image, nor a 1D vector). We recommend flattening the observation to have only a 1D vector or use a custom policy to properly process the data. Actual observation shape: (50, 2)\u001b[0m\n",
+            "  logger.warn(\n",
+            "/usr/local/lib/python3.9/dist-packages/gym/core.py:317: DeprecationWarning: \u001b[33mWARN: Initializing wrapper in old step API which returns one bool instead of two. It is recommended to set `new_step_api=True` to use new step API. This will be the default behaviour in future.\u001b[0m\n",
+            "  deprecation(\n",
+            "/usr/local/lib/python3.9/dist-packages/gym/wrappers/step_api_compatibility.py:39: DeprecationWarning: \u001b[33mWARN: Initializing environment in old step API which returns one bool instead of two. It is recommended to set `new_step_api=True` to use new step API. This will be the default behaviour in future.\u001b[0m\n",
+            "  deprecation(\n",
+            "/usr/local/lib/python3.9/dist-packages/gym/vector/vector_env.py:56: DeprecationWarning: \u001b[33mWARN: Initializing vector env in old step API which returns one bool array instead of two. It is recommended to set `new_step_api=True` to use new step API. This will be the default behaviour in future.\u001b[0m\n",
+            "  deprecation(\n"
+          ]
+        }
+      ],
+      "source": [
+        "from wrappers.syncVectorEnvPomo import SyncVectorEnv\n",
+        "from wrappers.recordWrapper import RecordEpisodeStatistics\n",
+        "\n",
+        "env_id = 'cvrp-v0'\n",
+        "env_entry_point = 'envs.cvrp_vector_env:CVRPVectorEnv'\n",
+        "seed = 0\n",
+        "\n",
+        "gym.envs.register(\n",
+        "    id=env_id,\n",
+        "    entry_point=env_entry_point,\n",
+        ")\n",
+        "\n",
+        "def make_env(env_id, seed, cfg={}):\n",
+        "    def thunk():\n",
+        "        env = gym.make(env_id, **cfg)\n",
+        "        env = RecordEpisodeStatistics(env)\n",
+        "        env.seed(seed)\n",
+        "        env.action_space.seed(seed)\n",
+        "        env.observation_space.seed(seed)\n",
+        "        return env\n",
+        "    return thunk\n",
+        "\n",
+        "envs = SyncVectorEnv([make_env(env_id, seed, dict(n_traj=50))])"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "id": "c363d489",
+      "metadata": {
+        "id": "c363d489"
+      },
+      "source": [
+        "# Inference\n",
+        "We use the Multi-Greedy search strategy: running greedy sampling with different starting nodes"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 5,
+      "id": "bbee9e3c",
+      "metadata": {
+        "id": "bbee9e3c",
+        "outputId": "5632253d-9a70-433b-c35a-3d97e478da0d",
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        }
+      },
+      "outputs": [
+        {
+          "output_type": "stream",
+          "name": "stderr",
+          "text": [
+            "/usr/local/lib/python3.9/dist-packages/gym/utils/passive_env_checker.py:174: UserWarning: \u001b[33mWARN: Future gym versions will require that `Env.reset` can be passed a `seed` instead of using `Env.seed` for resetting the environment random number generator.\u001b[0m\n",
+            "  logger.warn(\n",
+            "/usr/local/lib/python3.9/dist-packages/gym/utils/passive_env_checker.py:190: UserWarning: \u001b[33mWARN: Future gym versions will require that `Env.reset` can be passed `return_info` to return information from the environment resetting.\u001b[0m\n",
+            "  logger.warn(\n",
+            "/usr/local/lib/python3.9/dist-packages/gym/utils/passive_env_checker.py:195: UserWarning: \u001b[33mWARN: Future gym versions will require that `Env.reset` can be passed `options` to allow the environment initialisation to be passed additional information.\u001b[0m\n",
+            "  logger.warn(\n",
+            "/usr/local/lib/python3.9/dist-packages/gym/utils/passive_env_checker.py:141: UserWarning: \u001b[33mWARN: The obs returned by the `reset()` method was expecting numpy array dtype to be float32, actual type: float64\u001b[0m\n",
+            "  logger.warn(\n",
+            "/usr/local/lib/python3.9/dist-packages/gym/utils/passive_env_checker.py:165: UserWarning: \u001b[33mWARN: The obs returned by the `reset()` method is not within the observation space.\u001b[0m\n",
+            "  logger.warn(f\"{pre} is not within the observation space.\")\n",
+            "/usr/local/lib/python3.9/dist-packages/gym/utils/passive_env_checker.py:227: DeprecationWarning: \u001b[33mWARN: Core environment is written in old step API which returns one bool instead of two. It is recommended to rewrite the environment with new step API. \u001b[0m\n",
+            "  logger.deprecation(\n",
+            "/usr/local/lib/python3.9/dist-packages/gym/utils/passive_env_checker.py:234: UserWarning: \u001b[33mWARN: Expects `done` signal to be a boolean, actual type: <class 'numpy.ndarray'>\u001b[0m\n",
+            "  logger.warn(\n",
+            "/usr/local/lib/python3.9/dist-packages/gym/utils/passive_env_checker.py:141: UserWarning: \u001b[33mWARN: The obs returned by the `step()` method was expecting numpy array dtype to be float32, actual type: float64\u001b[0m\n",
+            "  logger.warn(\n",
+            "/usr/local/lib/python3.9/dist-packages/gym/utils/passive_env_checker.py:165: UserWarning: \u001b[33mWARN: The obs returned by the `step()` method is not within the observation space.\u001b[0m\n",
+            "  logger.warn(f\"{pre} is not within the observation space.\")\n",
+            "/usr/local/lib/python3.9/dist-packages/gym/utils/passive_env_checker.py:260: UserWarning: \u001b[33mWARN: The reward returned by `step()` must be a float, int, np.integer or np.floating, actual type: <class 'numpy.ndarray'>\u001b[0m\n",
+            "  logger.warn(\n"
+          ]
+        }
+      ],
+      "source": [
+        "trajectories = []\n",
+        "agent.eval()\n",
+        "obs = envs.reset()\n",
+        "done = np.array([False])\n",
+        "while not done.all():\n",
+        "    # ALGO LOGIC: action logic\n",
+        "    with torch.no_grad():\n",
+        "        action, logits = agent(obs)\n",
+        "    if trajectories==[]: # Multi-greedy inference\n",
+        "        action = torch.arange(1, envs.n_traj + 1).repeat(1, 1)\n",
+        "    \n",
+        "    obs, reward, done, info = envs.step(action.cpu().numpy())\n",
+        "    trajectories.append(action.cpu().numpy())"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 6,
+      "id": "f0fbf6fd",
+      "metadata": {
+        "id": "f0fbf6fd"
+      },
+      "outputs": [],
+      "source": [
+        "nodes_coordinates = np.vstack([obs['depot'],obs['observations'][0]])\n",
+        "final_return = info[0]['episode']['r']\n",
+        "best_traj = np.argmax(final_return)\n",
+        "resulting_traj = np.array(trajectories)[:,0,best_traj]\n",
+        "resulting_traj_with_depot = np.hstack([np.zeros(1,dtype = int),resulting_traj])"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "source": [
+        "## Results"
+      ],
+      "metadata": {
+        "id": "ViNGfd1PQwlw"
+      },
+      "id": "ViNGfd1PQwlw"
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 7,
+      "id": "dff29ef4",
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        },
+        "id": "dff29ef4",
+        "outputId": "8a57a330-b340-4d60-dc83-2a1ea548c7d0"
+      },
+      "outputs": [
+        {
+          "output_type": "stream",
+          "name": "stdout",
+          "text": [
+            "A route of length -11.283475875854492\n",
+            "The route is:\n",
+            " [ 0 16 32 27  7 30 23 38 40  2 11  0  9 10 19 36 35  4  5 18  0 25 42 43\n",
+            " 48 21 37 31 50  0 13  8 41 17 12 46  0  3  6 44  0 22 26 49 34 33 28  0\n",
+            "  1 14 29 39 47  0 20 24 45 15  0  0]\n"
+          ]
+        }
+      ],
+      "source": [
+        "print(f'A route of length {final_return[best_traj]}')\n",
+        "print('The route is:\\n', resulting_traj_with_depot)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "id": "1b78c529",
+      "metadata": {
+        "id": "1b78c529"
+      },
+      "source": [
+        "### Display it in a 2d-grid\n",
+        "- Darker color means later steps in the route.\n",
+        "- We abuse the errorbar to show the relative size of demand at each customer."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 8,
+      "id": "dc681a06",
+      "metadata": {
+        "tags": [
+          "\"hide-cell\""
+        ],
+        "cellView": "form",
+        "id": "dc681a06"
+      },
+      "outputs": [],
+      "source": [
+        "#@title Helper function for plotting\n",
+        "# colorline taken from https://nbviewer.org/github/dpsanders/matplotlib-examples/blob/master/colorline.ipynb\n",
+        "import matplotlib.pyplot as plt\n",
+        "from matplotlib.collections import LineCollection\n",
+        "from matplotlib.colors import ListedColormap, BoundaryNorm\n",
+        "\n",
+        "def make_segments(x, y):\n",
+        "    '''\n",
+        "    Create list of line segments from x and y coordinates, in the correct format for LineCollection:\n",
+        "    an array of the form   numlines x (points per line) x 2 (x and y) array\n",
+        "    '''\n",
+        "\n",
+        "    points = np.array([x, y]).T.reshape(-1, 1, 2)\n",
+        "    segments = np.concatenate([points[:-1], points[1:]], axis=1)\n",
+        "    \n",
+        "    return segments\n",
+        "\n",
+        "def colorline(x, y, z=None, cmap=plt.get_cmap('copper'), norm=plt.Normalize(0.0, 1.0), linewidth=1, alpha=1.0):\n",
+        "    '''\n",
+        "    Plot a colored line with coordinates x and y\n",
+        "    Optionally specify colors in the array z\n",
+        "    Optionally specify a colormap, a norm function and a line width\n",
+        "    '''\n",
+        "    \n",
+        "    # Default colors equally spaced on [0,1]:\n",
+        "    if z is None:\n",
+        "        z = np.linspace(0.3, 1.0, len(x))\n",
+        "           \n",
+        "    # Special case if a single number:\n",
+        "    if not hasattr(z, \"__iter__\"):  # to check for numerical input -- this is a hack\n",
+        "        z = np.array([z])\n",
+        "        \n",
+        "    z = np.asarray(z)\n",
+        "    \n",
+        "    segments = make_segments(x, y)\n",
+        "    lc = LineCollection(segments, array=z, cmap=cmap, norm=norm, linewidth=linewidth, alpha=alpha)\n",
+        "    \n",
+        "    ax = plt.gca()\n",
+        "    ax.add_collection(lc)\n",
+        "    \n",
+        "    return lc\n",
+        "\n",
+        "def plot(coords, demand):\n",
+        "    x,y = coords.T\n",
+        "    lc = colorline(x,y,cmap='Reds')\n",
+        "    plt.axis('square')\n",
+        "    x, y =obs['observations'][0].T\n",
+        "    h = obs['demand']/4\n",
+        "    h = np.vstack([h*0,h])\n",
+        "    plt.errorbar(x,y,h,fmt='None',elinewidth=2)\n",
+        "    return lc"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 9,
+      "id": "aa5e32f2",
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/",
+          "height": 282
+        },
+        "id": "aa5e32f2",
+        "outputId": "8da68d19-138b-4e3e-c481-02e06416c5e7"
+      },
+      "outputs": [
+        {
+          "output_type": "execute_result",
+          "data": {
+            "text/plain": [
+              "<matplotlib.collections.LineCollection at 0x7f6c20062b20>"
+            ]
+          },
+          "metadata": {},
+          "execution_count": 9
+        },
+        {
+          "output_type": "display_data",
+          "data": {
+            "text/plain": [
+              "<Figure size 432x288 with 1 Axes>"
+            ],
+            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAP8AAAD4CAYAAAAjDTByAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAAsTAAALEwEAmpwYAABXs0lEQVR4nO2dd3RURRfAf/N203ujJ4TepCdIlSbSFLBRbCgqYkUU6U1QBAEFu4gVK/ipgIANQRSlhN4hCSUJJUACIT27b74/NoEkpOwmb7MbeL9zOGfz3rw7d0Pum5k7d+4VUkp0dHRuPBRHK6Cjo+MYdOPX0blB0Y1fR+cGRTd+HZ0bFN34dXRuUIyO6jg4OFiGh4c7qnsdnRuC7du3n5dShhR1z2HGHx4eTlRUlKO619G5IRBCnCjuXqnTfiHEJ0KIRCHEvmLuCyHEW0KIaCHEHiFEm/Ioq6OjUzFYs+b/DOhTwv2+QIPcfyOB98uvlo6Ojr0p1fillBuBpBKaDAS+kBY2A/5CiOpaKaijo2MftPD21wTi8v0cn3vtGoQQI4UQUUKIqHPnzmnQtY6OTlmp0K0+KeViKWWElDIiJKRIB6SOjk4FoYXxJwCh+X6ulXtNR0fHidHC+FcCD+V6/dsDl6SUpzWQq6OjY0dK3ecXQnwDdAOChRDxwHTABUBK+QGwBugHRAPpwCP2UlanchE+YTUAx+f0d7AmOkVRqvFLKYeVcl8CT2umkY6OToVww8f2h09YfWWE0tG5kbjhjV9H50ZFN34dnRsU3fh1dG5QdOPXsTu6X8U50Y0/l8xfV3MjZDLWDVEnjxva+NOWf82W3a9zoEMWGV9/zsXH7idn725Hq6WjUyE4LJmHI5FSkrr4HTLW/kzQR19irBWKxx2DyFq7kpSJY3Bp3Ravp8dgqFbD0arq6NiNG27kl2YzKa+9TObG9QR9/BXGWpZjCUJRcO8/iMDlP2MIrU3yg/eQ9sEiZHqagzXW0bEPN9TIL7OyuDjlJdTUywR9+AWKt/c1bYSHJ14jn8F94D2kvb+IpMG34znyWdz7D0QYDA7QuvIgpSQrJobL//zL5U2b+HnvPpQaNUjdsYvIY0ccrZ5OIW4Y41dTL5P84jMo/gEELvoQ4epaYntD1Wr4zniNnAN7SX1zLpnLv8Lr+fG4tm1XQRpXDrLj47m86V8u//Mvqf/+CwYjPp074t+3D6GzZpJ9/jzRz4xGCOFoVXUKcUMYv/n8OZKeewLXFi3xfWmKTSO4S9Pm+C9eSvafv3F51hSMDRvj/eyLGEJr21Fj5yXn/HlSN/3H5U2buLzpX9TUNLw7dcSnU0eqj30B17DQAoZ+eedO3MNCS5Co4yiue+M3xZ8k6enH8Og/EO/HnyrTCCSEwK1nb1w7dyPju6UkP3of7v0G4DliFIqvnx20dh7MKSmkbt5yZSqfc+o03u1vxrtzJ0JGPIJ7o4Yl/k6zTsbhFqobvzNyXRt/zqEDJD0/Cu/HnsLrnqHllifc3PB86DHcb7+TtMXvkDT4DrwefQL3OwcjjC4aaOx41IwMUrdFkbrpXy5v+pfMI0fxatsG704dCZv/Op7Nb0IYrf+zyYqLxy20lh011ikr163xZ0Vt4eLEF/AdPw2PW3trKlsJDMJnwnQ87hlG6qJ5ZHz/LV7PvYRrxy6Vbm0rc3JI27XLsmbf9C/pu/fg0awpPp06UmPyRLzatEZxcyuz/Ky4OHw7ttdQY/twI+YeuC6NP2Pdb6TMeRn/197ALeJmu/VjrN8Qv7cWk71pI2mLXifju6V4jx6HsV4Du/WpFWc/+JDL/2wibdt23MJr49O5E1WeGoX3ze0weHlp1o8+7XderjvjT/v+W1KXvEfg2x/h0rip3fsTQuDWuSuu7TuS+cMyLj49Ardut+I18hmUwCC7928rx2b35dCtt5GZ3YrgYUMJf2sRxsAAu/QlpbRM+3WHn1Pi1EE+tsShSym5vPhd0pZ+QtBHSyvE8PMjjC54DL6fwGU/I9zcSBo6gPSlHyOzsytUj9IQioL/rT0RZhP+/fvZzfABTMkXQVEw+l3fTtHKilMbv7VIs5mU12eRuWGdJWrPgdtwiq8f3mMm4L/kK3J27yRpyB1krfvVqQ4NVX1+NOnbt3P5r7/s2k9WXJw+6jsxld74ZXY2FyePxRQbQ9DizzEEO0c9AGNYOH7z38Fn8sukffohl0YNJ+dgkeUOKxzF05Oar8wiYcpU1IwMu/VjWe/rnn5npVIbv5qaStLoJ8BsJvCtxSjePo5W6RpcI9oT8Ply3PoPJGXsM6S8PBFz4llHq4VPt254tmrF2UWL7NZHZRn5ZWb61c/JN04lqUpr/OYL57kwajjG0Nr4z3kTUY7tKHsjDAY8BtxNwLLVGKpUI/mBu0hb8h4yI730h+1I9WlTSVq2nIwDB+wi37LH79zGL6P3or4/lZjIeGI6JmL+YDqmj2ahbvkDmXbZ0erZlUpp/Kb4OC48dj/uXbrhO3F6pTlwo3h54fXkaAI+X475eCxJg+8gc80KpKo6RB+XkBCqvfQS8RMnIs1mzeU787RfZmWgrvoMddVnKANGYLj7CQwDHsHw0lsot9yOPHEE88KxmJfOR939LzI709Eqa06lM/6cI4e48PgDeN03HJ8nnq10QTUAhuo18H1lPr6zF5Dx/bdcHDGUnF3bHaJL4JDBKK5uXFj6peays+LinTKuXx47iPrBNFDNKE/OQtRrduWeMBpRGrXGMPgpDGMXIVp0QO7ehHneaMzL30M9vAtpNjlQe+2oVPv8Wdu3cnHCGHzHTcGjV19Hq1NuXJq3wv/jr8n6bQ0p08djbNrccmioRsWNlkJRqDl7NjGDB+Pb+zZcq2tTXd2yx+9cAT4yOwv5x3Lkoe0otw9HNGxVYnvh5o5o2QladkKmpSD3bUHduBJ+WIy4qR1Ki44QWh+hVLoxFKhExp+5/ncuzZ6B/6vzcWvXwdHqaIYQAvfe/XHr2pP0bz4n+eEhuA+4C8+HR1aYA9O9QX2CHnqQU9OnE754sSYyTecvoHi4YygiZ4IjkCePoq5YgqhZ1zLae9iml/DyRdzcC+XmXsjkROSezZhXfAI5WYgWHVBadEBUdZ4XnTVUildW+o/LuDR3FoFvLb6uDD8/wt0dr0eeIODrn5AXL5J0b38yfvgOaaqYKWaVp54iKzqaS7/+qom8zLg43Go5fr0vc7JRf/sWdfm7KLcORrnrCZsNvzAioApK1wEYnn0Nw33Pg2rG/MV8TO9MQt34M/LieW2UtzOVYuRP/ewjghZ/gTEs3NGq2B1DcAg+U2bhfvggaQvnkvH9N3g/9xKu7TvZtV/FzY2as2cT9/wYvDt2xOBTvllH1sk43MLCNNKubMiEWNSflkCVmiijZiK8fDWVL4SA6rUxVK+N7DUEThxG3fMf6vtTIaQmSsuOiGaRCE/n24IGEI6KPIuIiJBRUVEltjFduEDSmKcIeuNtDMFVKkgz50FKSfbGP0l7az6GsHC8nhuLsU49u/YZN248iocHNV+eUS45CW+/iykpmdrTp2ijmA1IUw7yr5XInRsRfe5DNGtXoY5haTIhj+5B7vkXGb0XUbsRokUHROM2CNeK3ZIWQmyXUkYUdc+pR36Dnx+mkyeQJu23oSqSsh4XFULg1rUnrh27kPH9N1wcNRy3Xn3xevxpFD9/O2gK1SdN5MitvQgYNBDP1q3LLCcrLh7Pxo001Mw65OkTltE+INgy2ntX/LkCYTQimrSBJm2QWRnIg9uRO/9GXfUZolFrRMuOiLrNHL5F7dRrfmE04t6lG5kb1jlaFYciXFzxHDacwO9WAZA05A7Sv/kcmaP9oSGjvz/Vp0yx7P3n5JRZjiW6r+Km/dJsQt3wE+qXCxAd+6AMec4hhl8Y4eaB0qozhuHjMIx+HVGrLur6HzHPew7zz18gTx61+tyH1gVXnNr4ATy630rG+hvb+PNQ/APwGTsZ//c/I2frZpKHDSLrrz81PzTkP3AAxpAQzn38cZllVGSAj0yMR/34FWR8DMoTM1BadnLK+A/h7YfS/jaMI6djGDkN4e2L+cclmN98EfMfy5GJCRWqj1XGL4ToI4Q4LISIFkJMKOJ+mBBivRBipxBijxCin1YKunXoRPb+vaiXLmolstJjrFMPvzffx3vsZNI+WMSlp0dgOnJQM/lCCGq++irnPviQrJMnbX5eqipZCQl23+OXqhn1759RP5+LiOiOcv8LCN9Au/apFSKwKkq3QRiem4Nh6HNgysH82VxM705B/Wc18lKS3XUo1fiFEAbgXaAv0BQYJoQofFh+CrBMStkaGAq8p5mCHh64RbYn42/7Hj+tjLi270TA0v/hdmsfLo5+gsuvTMV8XpuDKW5hYYSMeoKEyZNtnlnkJCZi9PbB4OmhiS5FIc+fRv1kNvLYAZTHp6O06eqUo31pCCEQNcIx9LkPw9iFKH3vQ54/g/ndyZg/mY0atR6ZnmqXvq0Z+dsB0VLKWCllNvAtMLBQGwnk7aP4Aae0UxE8uvckc/0fWoq8bhBGIx53DSFw+WqEnx/J9w0i7dMPkZnlj0UPefRRTOfOcXHFSpues2f2HqmqqP/9ivrJbESLjigPjkX4B9ulr4pGKApK3aYYBj2KYdxbiPa3IaP3Yn7jBcxfval5f9YYf00gLt/P8bnX8jMDeEAIEQ+sAZ4tSpAQYqQQIkoIEXXunPUjlPst3cjc8p8mf9DXK4q3D97PjiXg028xHT5I0pDby115WLi4UGvOHE6/8gqmixetfs5e632ZdBb18zmW8NzHpqC064kQTu+2KhPC6ILSNALD0OcwjF0EjVsTXeVPjr1ym2Z9aPWbGwZ8JqWsBfQDlooi/leklIullBFSyoiQEOuTbhj8A3Bt3JTMzf9qpO71i6FmKH5zFuIzY64mlYc9W7XCr39/Tr862+pntPb0S6mibv0DdckriMZtUYZPQARW1Uy+syPcPRD+IeDmoWmKeGuMPwHIP4erlXstP48CywCklP8B7oCmczH3HreSoU/9rca1dVv8P/0Wj7uHkDJxDClTX8J8pmyrsWpjXyT1779J3bzZqvZa5uqXF8+jLp2P3PMfyohJKB16V9qDNOVBHtqB0rjscRdFYc1vcRvQQAhRRwjhisWhV3gReBLoCSCEaILF+DVNieLRrSeZG9dXWKz79YBQFNz7DbRUHg4Lt1Qefn8RalrBysOl7R8bfHyo8fIMEiZNQrVi6ZWpwbRfSom6/S/Uj15G1G2GMmISIlibE4eVEXloJ6JxG01llmr8UkoT8AzwK3AQi1d/vxBiphBiQG6zF4HHhRC7gW+Ah6XGm8/GGjUxVK1O9u6dWoq9IRAenng9/jQBX/6AOfEMyYP7k7Hyf1cSeMTc5UpMtT9RV32GzMkqUoZf79641a9P4nulb+SUd9ovU5JQv34TuX09yvDxKJ37I5TKkbDFHshzpyE7E2qEayrXqWP7C5Py4buoKSn4vzTRTlrZh7MHDrGoQw88AgIIjWhNWGRbQiPaUCuiFR4OSGudc3AfqW/OhYx0vEaPwxCzBXFTe4jeizx7EuXOJxAhNa55Lvv0aY7260+9Zctwb1C/SNnSbGZrnYZEHtmP4u5uk15SSks8/G/fIdr1RHTujzA4dQR6haD+swaZmIDhrsdtfrbSxvYXxqPHrZwf/RR+YydUqj3doHp18A0MoN+8VzGbTMRt28FvM+eQsGsPfjWrExrRhtCI1oRGtqVmq+a4eNhvfxzApclN+H/4haXy8CtTMSjZeLftjWHACNi9CfXLeYie91qSVeTDtXp1qo4eTcKkSdT97tsi197ZZ85iDAyw3fBTL6H+/Bkkn0d54EVE9RuzCnJRyEM7EB37aC63Uhm/sX5DUBRyjhzGtVFjR6tjNUY3N7pPHMv2T77gkTU/0HroPQCYTSbOHjhEXNQO4qJ2EvXltyQePEJIo/q5L4Q2hEW2oWqzJhhsKI5pDXmVh11qViH9w4VcfGoE7n3vwPPRJ1Fq1EH94QPUE4cRve8rcBIt6MEHuPjTjyR9t4ygYdcWP806edKmyD4pJXL/VuQvXyPa3IK492l9tM+HzExHxsei1L9Jc9mVatoPcHHBHBQvb3xHPWMHreyHKTubN5q2ZcjnH1G7U/GFK3MyMzm1a2/uC8HyUrgYl0CNFjcRGtna8lKIbEtw/bqazH7UP5aBpw80bkfa4nfI2rAOrxFP4Hb7APhjOfL0cUsCjJCroR0ZBw8Se/8DNPxlLS5VCh61Prfsey79tZH6775Vat8yLQV1zVJIPIUy6FFEzbrl/j7XG+rezciovzA8Mr5Mz5c07a90xp+1I4qLc16h6rKfytW/I6qybvv4C/Ys+x+P/rrCpucyU1KI376LuKidnNy2nbioHWSlXKZW27yXgcWP4Ffz2nV6aZgXT0cZ8CiimsVBZ4o+QuqieaiJZ/B6dixGbwOs/x+ixz2IFh2vvHBOz5lDdkICtd9+u4C8+PlvIk05hE4YV2K/8uB21DVLEc07IHrcdd2UONca87L3EGENUNr3KtPz182aH8C1ZWvM5xIxJcRjrOn4NFG20OahYWyYu4Bjf2+iThfrM/O4+/pSv/st1O9+y5VrlxPPEbfNMjvY8vFSvh/1PAYXF4vvIHfJEBrZBs8SavHJi+chMx2qXv09Fqg8/NY8lKrV8XpoOMrmX+HEIehzP8LVnaqjR3Pktt5c3rABn27drjyfFReHT7vI4vvMSEWu/QqZEIty79OIMOevaOwopKoij+xCuW2wXeRXOuMXBgPu3XqQsX4dPg8Md7Q6NmFwcaHHpJf44+XXePyPn8sly6dKCE3796Zp/96AZe2cfOIkcVE7idu2gz9ff5P4HbvxDgkiNLLtlZdCzTYtccstwS2PHUDUaXpNiGyBysM/LuPS1Em43dIND79MxCevotw1CqVKTWq++grxEyfR6PffUDw9AUuAT/A9dxWpszyyG/XnzxFN2loSbbg4b6EVpyAuGnwC7HZ2odIZP1gO+lz+/JNKZ/wArR4Yyvo5C4j962/qdu2imVwhBIHhtQkMr03LewYBoJrNJB4+esV3sGvZj5zZf5CguuGERbahpiGV0J63UiM7G6Or67UyjS543Hs/br1vJ/2TD7j46Td49OyB2xevo9x6D95duuDVti1n3lxIjcmTgKIDfGRmOvLXb5DHD6HcNRIRXnmctY5EPbRD88Ce/FS6NT+AzMriVM/OVFv5K4bAsp3fdsSaP48dS78h6tOlPL5udYVvWZqyszmz9wAntmwlbulHxKcpXDgeR7VmTQiNtOwuhEa0IaRRA5RCW3mmkydIe3s+psMH8GwWjuvNHVBv7svR2wdS54vPcW/UiK11GxEZfRAl92UiY/ahrvwU0aAFotdghJt9tzGvJ0yLxmEY9BiidsMyy7iu1vwAws0N9/Ydydy4Hq9BdztaHZtpOexe1r82n5j1G6nfo2uF9m10daVW21bUDPFE9bqAYfhEslJTSdi5h7ioHRz65Q9+nzWXtPNJ1GzdwuI7aGdZNgSEheE3722yozaT+uZcMo8ux3PfLqo++RjxEyZS6623cAkJRnF1teSu+30Z8ugelAGPIOppv1V1PSOTz8HlixBadDCVFlRK4wdLeq/039dWSuM3GI30nDKedS+/Rr3utzgkYEnG7kfUsZSpcvP2pm6XjtTtcjWoJ+1CEvFRO4mL2sGOL7/lp9HjUM3mqwFJw56gSvIZLi9djNH/ICJLcG7Jx7iFhiKPH0Jd8QkivJGlQIa7Z4V/v8qOPLQT0bCVXQ8xVVrjd+9yC8mzZ6BmpKN4lP2P6+KPP0J+4xPiqjEKcfVe/s/k5mwvpV0BOYXu1akWzLr4ePa++RZ1I/Ot6/La5JdRjA5W6SoEIl/7vH/qpo2IDr0RuRV6RSG5BiGoHV6L2nVCYfAgAFLOJhK/7wAJew/w1+tvkrDvAO7eXlTz9SA45QJuew5Ru0Vj1B8+tKoclk7xyEM7ERH2nRVWWuNXfP1wvaklWf/+g0dP2xMcbA3YS9r2HaRm1YQ8v4eUBZNf5Lt+5R8WzzoltcvfJv+9fO2llLSsVZU/X1+Iz625CScLyZWFny/0WeaTV1w7Wbh/KcFsRp4/Df8eRiCu7aeQ3PwyfKSkMdDYKJEt6nEpI4tzqekkqkaiLySRtGU3xkbNqHYxBx8pK1UYtrMgszORJw6jDCsyJ45mVFrjB0usf5Pfc+D31TY77jJ37aLK6NF4dbJvJZySCDOb2demE9kPDKdRH0sQR0U4ItW9/yGj92K4c6RmMre98grnZ7+JEXBzN3D0hZeQZjNVhwymyuC7cathewDSjYqM3oeoVdfuy6VKnRXBvVuPMj0npSTryGHcGlV8UYn8KAYDPXLX/hW66xK7H1G3WentrCTlxHH+mfc2TbrdjKIoGGN20XrtChq+vZCs+Hh2du/FviH3c+7HFVblA7jRscfZ/aKo1MZvrFqtTM+ZzpxBuLhgDHZ84seb7h5ITkYmh9doUyCzNKSqIo8fRNQpnIC5jPKkZPWwh1CEoOuUcbi7GTkedwn12w/xjWhL/flzidwVRZXB93D262/Z2iqC6HETubxjZ8W+8CoJUkrLKT6Ns/YURaU2/rLiDKN+Hoqi0HPaeNbNnFMxxnDmBHj7IXyLD/u1hV1vvUVS7AlaPPIQnmG1CQzx41zcWcyH96Du3QaAwcODKnffyU3Lv6HVH7/iWq0ah598hp1dexL/7vtkJyZa1ZfWFWucklPHLbn6KiBrUaU3/q2pq9jT6qJNz2QdOoxbQ+cwfoCmg+5ANZk49PNau/clYw9c2eIrL8nRMfzzynxQVSLHjUH4+FGtVgDmbBPp7fth+uItZGZGgWfca9Uk7IXRtN38D/Vef430I9Fs79yd/Q88zPmfV6Nma1+CrDIhD+2w1PmrACq18adv3EDO8WP43HWvTc9ZRn7nCTG1jP4T+OPl1+zelzymzXpfNZtZO3wEVeuF0uS+oXhXrw5evgT6GBCKQvz6TSgNm2P+4dMinxdC4Nf+ZhouWkC7nVsJvqM/pz7+lK2tIoiZPI3UvfvKrWNlpO7v7tTbqM2srDQqrfGrmZkkz55F4KSpiCLi0ksi+8hh3BqVPWTSHjQZ0N/u22IyIw3On4Ja5S/zvf2NhaiXL5EUd4aIF0cDuQVEAvwxursT/9s6DMNGoW79CzXmQImyDF5eVB1yLy1+/J6Wa1Zi9PXh4MOPsrPHbSQsXkLO+Qvl1reyEF13O7FPVExeg0pr/ClLPsS1aTM8Ona26TmZk0P2sWO41neuo6RCCHpOt29uQnn8INRqUO6z8+cOHGLz/Leo3rYV9QcNwDffQR7h40dw25ZkJF8iLekihmGjMH9qOeNvDR7h4dQe/xIR2/6jzoxppO7aTVSHLhx85LFy6VxZUBq3Qf73e8X0VSG9lJHiHDw5J45z+buvCRg3yWZ5dab+hrFGDRQ758krC437a5+nrQCxB8o95Tfn5LB2+KPcfM+tRP+xicixzxds4O1LcNtWuLu5EP/5FyjtukFwNdTV39rUj1AU/G/pTKP33iZy+2YCenTnx13v8Yf7jnLp7+yIyO7Iwzudo1CnsyGlJOm1WfiOeBxjtbJt9TmTsy8/QgiWBh9liav2f+BSSk3W+1vmzsfdYEYNqkWdPr3wqxNe4L7w8SOwfjjS1Y2ElasQQmB86DnMf6xAJpwoU59GX1+qPXg/LX78nnMrVnFx85ZyfQdnRnh4IVp1Qt1i/wI1lc740//4DfOZM/iW4yy/s2zzFYXB6EJOekbpDW3lXAIYXREB1pdJK8zZXXvY8e5iekx4hj2ff0tE4VEfwNsP36qB5GRmk3Iinqzz5xGBIRjuHI7p0zeQqrnM/Rs8PKg7eQIx02YiVbXMcpwdpWMf5ObfrV4qlbkfu0rXGDU9jeTXXyNw8nSES9nXrc5s/FJV7XKSS8buR9Qte2CPKSuLNQ8/TtcH+3I8OpHQbl0ILMJpKnx8UTLTCWjZAm83Awk/WvIVKt36g6Kg/rmqzDoAVLlrECiCs9//UC45zoyoWgtRvTZyz3927adSGf+lD97DPSIS98h25ZLjzMavqmYUg7b/LeETVlP3J1O5pvybZs7Gz9+dho8/yc53PyLypReKbCd8/JCplwhq3w634GDiv/vOcl1RMD4yBvOKpcjzZwvoZkvgjlAU6s+cwbHZczGnpZf5+zg7olNf1H/W2jXwq9IYf3ZMNKk//Y+AF0vOCmsNLqHaVZDVGqmqCIO2pamOTe1CTJsYCCvb9mbCf1vZ9/mX9HppJAd++5fq7SMJvqmYWYS3L1xOIbBdJKqHF+f3HiDncioAonoYhtvuwvTFonL9Ufu1i8C3XSRx731QZhnOjmjcGjJS4eRRu/VRKYxfSknSqy/j98TTGIJtW7NKk4msA/u59PVXV645c5VXadZ+2i93bkQEVEG42lZFByA7LY21jz1Jz+H9cO/Ymx0L36bduBeLbS98/JCXLxEY2ZaUhNP4ebhw+verziul7xC4eAH1v3Vl+i551J0ykfgln5B1+nS55DgrQlFQOvRG3WS/qE/ntYJ8pK1ehXr5Mj5DhpXa1pycTNqG9VxY+AYJDz9IbPsIzox7kcz9+9jdNo2YccUXzHAGVLMZRcORX2ZnIXf/g4go2wnIjVNnUbV2FRo+8hgHv19JcIvmVGndstj2edN+95AQ3EJC8A8JIOGbq9t8wmjE8MgLmL/7EJlysUw6AXiEhVLjwfuJnf16mWU4O5Ztv1122/arFOf5L77xOiFvvI0oVLJKms1kx0STuWun5d/OnZjOJeLevCXurVvjP+Ix3Fu0xODv7xjFy4DW0365bzPUqocIqFJ640KcWP8XR3/4kYcWjUcNbUTUgofo+8WSkh/y9oPUFACC2kXglpTIyY3/Yc7KwuBmSdWt1GmE0uFWzN+8D7SwWa88wkY/w9YOXbm8ew8+Lcsux1nJv+1nsEPu/kph/B5duuLWqjXmlBQy9+zONfQdZO3dgyEwCPdWrXFv1Qr/4Y/gWr+B5mvmikSazSga1eWTqhm5/U+U/g/b/GzWpRTWjnyWXiP649n1DvZ/+z3+9etRvYSCHHB12g8QGBnBhV9/w9vDlcSN/1C9V88r7Qx3Didn6khQym60Rh8fwse/SPTUl2m14vvrMmuQ0rEP5g9fRva4U/OqRpVi2p+TkcXJO/pxvPstJC/+AHJy8H/gQWr/8ju1f/mdqnNex2/ofbg1alypDR/ypv3a/LfII7stx3dr1LH52fXjp1CnVUPq3nk30juAqPkLuXnC2NIf9PKBtMtI1UxQu0iSY44R4mUk/n8Ft+aEmzvGh5+3Wa/CVL9vKKaUFM7/vKbcspwRUbUWokY4crf2235ObfzRY2/m3/Pf496yJVXmzKPulihqffEVQWNexKt7TwyBQY5WUXO0mvZLKZHb/kCJvNXmZ2PW/MqJdeu5ZVhPRNObObL8B7xrVKdm546lPisMBvDwgrRUfBs3IjPxHMHtIjm19lekuWCAj9K0DdGdUziiLEf951ekyWSzrsJgoN7L04iZORs1K8vm5ysDomMf6i6/rHkuA6uMXwjRRwhxWAgRLYSYUEybwUKIA0KI/UKIr7VQzhgcTI0ln+J//4O4N2t2zZr/ekSa1WvKZ5WJuKOQkwU25svPuJDEb8+8QO+Rd+DeqT8SwdZ5b9BufPEe/sLkOf2EwUBg2zaotevh5mLgwtZri7QYbx+G8eEXUbdtwDR1BOa/ViNzbDvTH9i1C16NGhC/pOjjw5Ude2X1KfWvTAhhAN4F+gJNgWFCiKaF2jQAJgKdpJTNgOe1V/XGQCtvvxq1DhHRw+YXyR/Pj6Nh10jCunVFVA0j+qdVuPn5EdrdhjTSuXv9AIHtIkhJzybYTRC/quj6hErD5hjHvIbhsQnI3ZsxTRmB+c8VyGzrR/J6M6YS9857ZF+Hx3/ttTVtjdR2QLSUMlZKmQ18Cwws1OZx4F0pZTKAlNK6vEw616DFtF+ePw1nTiKa2hYJeWj5jyTu3EWn2yMRLW5Bqipb5y6g3fgXbXKmCW9fZKrF6RcUGUHSvgNUa1iHhJ9Wlhjco9RrivG5WRienIY8uBPT5Ecw//4/ZFbpST8969ejyl2DOP76Aqv1rEzEPlab6Jr/airTGuOvCcTl+zk+91p+GgINhRCbhBCbhRBFnk0VQowUQkQJIaLOnTtXNo2vc6TZXO43vYxah2jd1SbvcOqZs6x7cSK9n7oL14juCDcPYlevxeDiQnhv22rDWzz+lpE/IKINybt243drL2RmBpf2lZzYA0AJb4jx6RkYn5uFjDloeQn8sgyZWXI4b/jYMZz7eTVphw7bpG9lwB47GVrNJ4xAA6AbMAz4SAjhX7iRlHKxlDJCShkRElL202XXM+X19svUi8joPYhW1ic5kVLy29NjaD6oF9VvaoQIa4KUkq1zFhA5/gXb//B8/CB3u8/Vzw+vsDAyq9SkSoAXCT/bEMcfWg/jqCkYx7yGjIuxvARWf41MTyuyvUtAALWff5bo6TNt07eyoPELwJq/sgQgNN/PtXKv5SceWCmlzJFSHgOOYHkZ6NhIeaf9csdfiCaRCA9vq5/Zt/RrUk6coH23RihteiKE4MRv6zBnZ1Pv9n4265B/rx8gsF0kly6kEOKuEP/TStvl1QzH+PhEjGPnI88mWF4CK5ci0y5f07bGww+ReeIkF/5cb3M/To0dzvdYY/zbgAZCiDpCCFdgKFD4f/AnLKM+QohgLMuAWO3UvHEoz5FemZ2J3Psvom13q5+5dCKOvya/TJ/nhmJs2g7h7W8Z9efOt6z1y6JLvig/sET6JUVtJ7hnDzLPnCH12HHbZQKieijGES9hnLgQmXzO4hj88dMCLxrF1ZV6M6YSM30Wahm2Dm8kSv2flVKagGeAX4GDwDIp5X4hxEwhxIDcZr8CF4QQB4D1wEtSyuvP7VoBlMfbL/f8iwhrhPC3rhiJVFV+GfUsbR+6l5Bq/oiGbQGI27CRzORk6t85oBQJRZPf4Qe5kX5bt+HaqStVaoSQUM6AHFGlBsbhL2Cc/DakXcY09VHM33+ETEkGIKh3L1yrhHD6i69KkVR5kHYY+q16rUsp10gpG0op60kpX829Nk1KuTL3s5RSviClbCqlbC6ltC1hm84VpKqWyfil2Yzcvh4R2bP0xrnsXPwJ41s+zjBzR5S2PRGKpd+tcxcQOXZMmV9Chaf93vXqYk7PwBRWj2BzGgmrtAlWEcHVMDzwHMbp74MpB9O0xzF/9wFcvED9l6dzfP6b5Fy6VLqgyoCUDlnz61QgFm+/7f/J8vAO8A9GVKttVfvk6Bj+nTWHbfX3EzuiGiLIUkgzYdN/pMYn0GjIPTbrcIV8Dj+weKoD20WQfPgowc2bkXLwIJlntdsNFgEhGIY+hXHGh6AomF4ehceePwnq1oUTb7ylWT+OJl+xdU3Qjd/JUM1mmx1+tobyqmYzax4awc392xPUvQ+iwdUKMVvnLiBi7PPlOlxkifBLKXAtqF0kSdu24965G8HhtWj85jbNw1WFfxCGe0dinLkE3DwINZ/gzOdfkL7j2shCHd34nQ6pStudbCcPg2oGK4pvStXM1nEvoGSn03b26yj1WlzZyjsTtZ3kI0dpct+Qsqh+FS9vyEgrkKwzqJ1l3e/SsQshBvs64oSvP4a7H8Vz/lJq3taJmMdHYPpsAfJs4U2qSobGW/268TsZsgwOP3XbH4iInqXux8v0yyR++iZRX/1E36+/QgmuUeD+1jkLaPvCcxhsrIBUGKFcPdyTh3+rlqQcPAi1ahPoWTFnNISPH2GL3idVdefShXRMc1/AtGQu8lTJKcSdsiCoHXL56cbvZNi6zy8T4+H8aUSTiJLbnTlBzprP+OXtZXSZPQv/hgXz+SXu2kPirt00e+j+MuldGOHtW8DpZ/T0xKdRQy7u3YdHp1s06cMaDB4e1J02mWNr/8UwawmiZm1MC8ZhWjwbGX+swvTQBN3hd32jms02HcaxhPJ2KzaUV0qJun8z6uY1bN12Cq/wOrR45MFr2m19fQFtRj+D0d32PH9FUsjpBxAUGUnS1m0YIzvy9rej2TrIui3J8lJl0AAUFxfOrlqLoe9QjK9+hghviGnhJEzvzUSesF+STO3QR/7rHmlDeK9MSbbk42/Zqej7WRmoG39Anj5GYkgLdn/zA33eW3jN8uD8/oOc/m8rzR8teyGUwhTt9IvgwrbtxO4+iJuaTXDL5pr1V6IuQlBv1vQr6b6FuweG2+7B+OqniEYtML07A9Pb01BjD1WIPmVCSrRe9OvG72TYMu2XO9Yjmt2McPe89l7SGdTfliJ8A1E7DmDtM2PpMW823jWqX9N227w3af3MKFw8r5VTVoR3wb1+sIT5nv9vMwff/4jm3dtj2rFVs/5Kwy+iLf4d23Pynfeu6ujmjqHnIMtLoHkk5g9fxbTQtvqPFYru8Lu+Ua1M3S2zMpD7Nl8TyiulRI3ejfrX/1BadUNp3Z1Ns14nqEljGg++6xo5yUejiVv/Fy1GjtDsOwCWM/2pBY3fo2YNMlPTqDP0Xvxv60POv39r22cp1J0yiYRPPiPz1KkC14WLK4Zud2B89ROUtl2IDt1KbJdLdi2Y4Qzoxu9kWOvtl7v/QdRphvANvHrNlI3cshZ5dCdKz2GI0IbE//MfB75dTq9F84rcDdg2701aPvk4rj4+mn6PwlF+APGrViOMRgKbNsGlwy3k/Pd3hRqYe62a1Hj4IY69MqfI+8LogtKlL4YHRqPuj0L9dJ7NWYXsiz7tv66RsvRpvzSbkDs2FAjllSlJqL9bsqcpve5D+AaSnZrKmpHP0GvRPDxDrnWuXTp2nGNrf6XVqMe1/RJwjcMv53IqOyZOoe6wISTv3IWhTj2QEvV4xZ7/Cnv2KZL/3kTKjp3FthF+gRiefw1UFfPCiVfODDgUfavv+ke1IpmHPBgFQdURVWpZfo47jLruG0SD1oib+yKMln36vybPoFbHm2kwoH+RcqIWLKL5Y4/g5u+n7ZfAUrAzv8Nvz6tzqNr1FsKH3kvStu0IIXDp2IXs/zZq3ndJGL29qTNhLNHTZpY46xCubigjxiEat8I870XkqeMVp2RR6LH91z+lefstobzrUCJ7IlUz6o71qLs2onS9G6V+yytT+2N/rCdmzW/0mP9akXJS4uI5+tNKWj89yi7fI7/DL2nXHk787wdaz5qOf/ObSD12jJyUy7h0vIWcfyvW+AGqDR2Mmp7OuVIOGAlFwXDHgyi3P4B54UTU/Y4LE677+WnqHWurqUzd+J2MUr39xw6AoiBDaqL++R0yNRnltgcQgdWuNMm8eIlfnxxN7/cX4V7MqL5j4dvcNPxBPILtlP7cxxdSU1DNZraNGUur6VNxCwpCcXXFv0ULknfsxKVNO0yH9qOmXpuUw54Ig4F6M6cRO/NVzJml5wdUbu6B4YmpqEsXom4oX4lxZ0I3fiejNG+/um0dokFL5O9fIarXRelyJ8LNo0CbP8dOpF6/26hza9FJPdJOn+HQd9/T+rmnNNU9P3kjf/SSTzF4eFDn/qFX7gVFtuXC1m0ID09cmrfGtG2z3fQojoDOnfBq2pSExR9b1V7Ua4ph7DzUv9dg/u79a2oQVEZ043cySvL2q2dOwLl45NnjKO37oTRrf40H/+jK1ST8t5Wur84oto/ti96lybAheFW1vX6f1Xh5k5F0kb1z5xH5ZsGdhsB2kVdy+Lt0qPh1fx71pk/m5HsfkJ1oXTJZEVwdw9j5kHgK9f2XkRlF5xKsLOjG72QUN+2XWRnINZ9DYBWU3g8WeW4//dx5fh/9En0Xv4Ord9E5/NLPnefAl1/TdswzmuueH6EY2HMwkfr3D8WvUcFzBIGRbUnevgOpqpZ1fwVv+eXhWa8u1e69m2M2pPsWHl4oT82AoKqY57+EPH/GfgraGd34nYyivP0y6Qzq6o8h5SLizicRntfuyUsp+f25sTQdei+1OhVfhnzn2+/R8O5BeNeoUWwbLTj12x9cupRBk4fvu+aee0gIrkGBXD58BEOtMISXN+Yjjgmtrf3i85xfs5bUAwetfkYYDChDn0Lp3AfzgpeQsdY/60zoxu9k5Pf254/Ww+CGaH0LdV/eUORx04Pf/Y8Lhw7TefrEYmVnJiWz79MviHjxeXupD4ApLY2osRNo1SsSo6noqjuW8/1Xp/6O8PoDuPj7U/uF0cRMn2XT7EMIgdJ9AMr9z2H+YBbqtg32U9JO6MbvZOQl85CmnCvReqLTQIg7imjbrchnUk+dZv24yfRb8l6Jp/J2vb+Yurf3wzcstNg2WrDv9QUERbal2k2Nr4nyy8Oy7t8GgGuHLuQ4aN0PUGP4g2QmJJD0x582P6vcFIlh9GzUFZ9j/vmrShUSrBu/k6GazZCRivq7JfOs0us+iN2HqNcc4e1/TXspJb889TwtH3+Eam2LL+iYlZLCnsUfEzn2eTtpbuHi/gPEfvkNbWbPLDLEN4+gyAiStllGfmOrCEyx0agXHRNJp7i4UG/GVKKnz0TNybH5eVEzHMNLC5AHtjthSHDx6MbvZMjsTNiy5kq0Hgjkzo2IYvLz7f3sS9LOJtJhQslVdHd/uITat/bAv15dO2htQaoq28aMpfmkcXhUrXplr78ofBs3IvNsIlkXLiBcXXFp046crdrWorOFoF49ca9Rg1NffFmm56+EBEtpl5Dg2N4mYiLiNZWpG7+TIFUz6s71qBnpGNr3vRKtJw9sgyq1EMHXHsW9ePwEG6fNot+S9zC4FF+XLzs1lV3vLibypTH2/ArEfPElUpXUf8SSF6BwNp/85JXvToraDuCwaL8r+ghBvZnTOLFgETkXL5ZNhqsbyiMvIZq01jwkWCYmIEIKl8gsH7rxOwEy/TLqn8uQl5ORbh4oudF6UqrIqHVFZuWVqsovI58l8vlnCGnWpET5e5d8Rq0uHQls3EgznQvnuctITGTPK3OIXDj/6m5FvoKdRRFYwOnXmZwtmxwaPOPdtAnBfXtzYsGiMssQioLh9gdQ7ngQ88JJ2oUEnzsFVXTjv66QZ0+i/vYlonodlC53IqW8Gtsfsw9cXCH02rKHO977CHN2NpHPP12ifFNGBjveeo/IcSUvC8rLzinTqTNsMAE3NbtyTXj7XXOmPz9B7SxpvQAMVaujBIdgOrDXrnqWRp3xYzmz/HvSY8t32lBp1x3DE1MsIcHrba9PWBiZeApRRdvtWd34HcSV3Hr/rS4QrSfzhfeq29YhIm8t8hz+f3Pm02/Ju6We/d/32VKqRbYhpHmzEtuVhzMb/uL8f1tpPuGlAtfrLUum4bmiU4wBBOaW785zsuWd8XckrlVCCH3qCWJffrXcsiwhwfNR/1lbrpBgmZUJaSkQoG1la934HYDMykD9+0fk6VjLoZx80Xp5RTvkqWNw+SKiYasiZXScMp6A+vVK7MeUlcX2N9+m3YSxWqpfAHNmJlEvjqft67MxennZ9KyLry9eYWFc2n/A8nOHLg43foBaIx8jdf8Bkv/ZVG5ZIrhavpDgGWULCT53CoKrl7mAa3Hoxl/BXMmt5xOA0mPINdF6793/Js0XRllG/YjuV+rn5fF9+GnmH/+O1lak3Tr45TcENWtK1dattPwKBTjwxiL8mjSmZt/e19yLHd2SQ5klF8sMzDf1NzZviXoqHvW8dbH29sLg7k7dqZOImTZTEx/ElZDg4OqY54+1KSQ4fMJq6rxzEKHxeh90468wCuTWa9kVpfW1hl2A+GjETR0KXEpLPMffr8yl67SJpY4C5pwcohYsot14+671jyz5lLZzZxejhAlKWZbkj/QTRhdc2nUgZ7PjR/+QAbejeHpy5rvlmsgTBgOGoU+hdO5reQHEHLBNgMbrfdCNv0K4Gq23w5JbL6x0r7to2Qnh6lbg2p+TZ9D8gaGENC3Zuw9w6Jtl+IbXpkb7dmXW2xqajR2DZ81i/jDNJiimnkAeQfki/cCy7s+u4MSeRSGEoP7MaRx7bR6m1NTSH7ASpfsAlAdGY/7wFZtCgvWRvxJiya2XF613f4GEmyUhWnct8HPcv5s59ucGukweV+qzqsnEtvkLudmOa/08Go58tPibphwwlFyay6tuHcwZGWScOg2AS/tOmKI2I022R9ppjW+b1vh37kjcO+9rKrcsIcEOM34hRB8hxGEhRLQQYkIJ7e4WQkghRMm1o24QZNyRInPrFUf+fXPh5Xvls2oy8cvz4+j52kzcrMiye+R/P+FVtQo1uxTvaS8P5qyrh3VKquYrzWYwljztF0IQmC/UVwkMRqkVhmlP8Qk2K5K6UyaS8OnnZMZrW+RT1AzHMO6N3JDg15HZRR+AuoIjpv1CCAPwLtAXaAoME0JcUw5WCOEDjAa2aK1kZeNKtN6uDdfk1rOGI0MKpt7avvgTPAICaHrvtXn3r+1bZdu8N2g3fqxNfdrC6Q0beX/n+xyfU3Ri0CuYTAhDydN+yJv6Xw2GcenoHF5/APeaNaj5yHBiXyk6F2J5EL4BlpBgwLxoUokhwfkHA62wZuRvB0RLKWOllNnAt8DAItrNAuYCpSdFu465Eq2XkoRy24MFcutZi+J/Nc126tlE/pk9j94L51plzNErfsbFy4uwnt1s7tdaTqz8mdoD7yi9oSkHSpgZ5JFXvjsP145dK7ygR0mEPvMUF//bTMr2HZrLtoQEj7NLSHBpWGP8NYG4fD/H5167ghCiDRAqpXSyusYVS4FovVvuuia3nrUIv6t+gfWTX6bFQ/cR0qRx6f1Lyda5C2g3/kW7jfpqTg5xq38h7I5SRn0As7lUbz9cLd9tzsgAwNC4GWpyEubTp0p5smIwentRZ+I4oqe+bJcju0II+4QEl0K5HX7CUlL2DaDUPSUhxEghRJQQIurcOcfu5WpJcdF6ZSXPKRi3aTPH1v9F54nWOe6OrfkFIQR1ithz14ozf2/Ct24dvENrld7YbCrV4QeW8t2+jRuRvHsPYImPd2nfyaFn/AtTbfA9qFlZnFthv+y9WocEl9qfFW0SgPzZH2rlXsvDB7gJ2CCEOA60B1YW5fSTUi6WUkZIKSNCQrQNVXQUMjuz2Gi9sqIYjblOvpe4de4sq5x8eaN+5PgX7DbqA5xYsYqwYoqAXIMpp9StvjwCIyJIyr/ud5JovzyEolB/1nRiZs22Kt13mfspFBJsT6wx/m1AAyFEHSGEKzAUuPJaklJeklIGSynDpZThwGZggJTScRUO7EThk2wy6Qzqr0sR3kVH65WH7R9+jGdwME3uHmRV+5N/rMeUnkH9AbdrpkNhVLOZuNVrqW1lH9JssrrisKV8dz7jv7kTOTujkFmleMErEP+OHfBpfhPxHy6xaz9XQoLP2XfZU6rxSylNwDPAr8BBYJmUcr8QYqYQYoBdtXNSCkbr3YLSppRoPRtJPXOWf16bT+8351g1iksp2TJ3PpHjXtA8/js/if9uxqNaNXzq1rHuAZPZKocfXA3zzVtTK75+GBs0Imenc40hdadNJu79D8k6m2jXfoSHF8qTM4h9viXHZve1Sx9W/aVIKddIKRtKKetJKV/NvTZNSnnNwkRK2e16HPXzY2u0nq38OWkGLYffT7CV5+/jN/5DxrnzNLByllBWrPby52FFhF8enrVqori4knb8xJVrlsSef9mqpl3xrFuHakMGc3zuPLv3JQwGlGq17PZC1yP8yoK0LVrPVo7/9bfVTj6ArXPmEzn2eatKe5cVqaqcXLWa2oNsMH5TjlXe/jwC20VcOeQDV7P6OltSzNovPMf5X34ndZ+N8flOhm78NnDs+WbEdD+PaF96tF556PX6K8UW3SjMqf+2kHIyjkZD77WbPgDntkXh5u+PX4P61j9kNlvl7c8jKLLgut9QvxEyOxs17kQJT1U8Ln5+hL/4PNHTS6706+zoxm8LRldQVbt506Mb72LnfdVofFdRMVRFs3XuAiJeHF1iDj8tOLFiFWEDbXMmWhx+1hu/ZeS/avxCiNy03s7j9c+j+vAHyD57lgu//eFoVcqMbvy24OoGOfbxPkspEamX8G/czOqXy5ntO7hw8BBN8hXBtJduJ1euttrLfwUbtvqAAuW783Dp2IVsByb2LA7FaKTey1OJmTELNbtypOoujG78tuDibjfjJzMNDEaEa/FFNwqzbe4CIsY8i9HNrfTG5eDCjl0Y3Nzwt+IocQFMpR/syU/+8t15uER0wLR/NzI93ba+K4Cgnj1wDwvl1GdfOFqVMqEbvy24ukFpp6/KSkoy+Phb3fzcnn2c3b6TZsMfsI8++Tix8mfCBtxu+3LHygi//BSO8xdeXhibtiAnquLLeFtDvRlTObHwbXKSHVNwpDzoxm8DeetXaTZpL/xyMtiwe7D19QW0ee5pjB5lOz9gLVJKTq5YZdsWXx42bPXlkf94bx7OFu2XH+8mjQnp34/jCxY6WhWb0Y3fVlzsM/rLlCSEb4BVbS8cPETCP//R/LGHNdejMMn79iPNKoEtm9v8rDSVnsarMIGRbUmKspTvziOvlp+zetbDx48l8X8/kh4d42hVbEI3fltxdYMcO8R2X04GH+uMf9u8N2n99BO42JgttyycWPEzYQPLMOUHsNHbDwXLd+eh1K4DRhfMsUdt16ECcA0OIvTpJ4l5+RVHq2ITuvHbiot9PP7ycjLCp/Rpf3J0DCfXrafFEyWkz9KQkyt/praNW3xXKMO0Hwom9QTLlp+lnJdzTv0Baj0+grSDh0ne+I+jVbEa3fhtxdXdPk6/lGSwYtofNX8hLZ54DDdf7TO7FObiocPkpKYS3LZN2QTYGOGXR2ChpJ5wNdrPWVHc3Kg7fRLR0152aMkxW9CN31bsNPJbM+1POXGS2NVrafXkSO37L4KTK38m7I7+ZY8tt+FgT36CIiOuFPDMw6VNJKajB1FTii//5WhCbu+P0deX099852hVrEI3fhsRLm6lJ1u0EZmTDdmZ4FlySO+2BYu4acRw3AP8Ne2/OE6U1cufRxm2+iC3fPeZs2RduHDlmnBzx6VlW3K2/Vd2fexMXqXf43MXaJru217oxm8r9ojyu2zZ47ckRSqmSUICR3/4idbPPKlt38WQEhNLRuI5Qm6OLLMMacpBlGHNX7h8dx4uHR1fy680fFu1JKBrF04uesfRqpSKbvy24mIHb//lZCjF2bfjzXdo9uD9eIYEl9hOK06ssEz5y3VS0MocfkURWMjpB1f3+/NvAzojdSeP59QXX5JxMq70xg5EN35bsUOUn0xJLnGPP+3MWQ5+u4w2o0sux60lJ1f+bHssf2HKOO2HguW78zDUqIXw86fOpLUFMio5G27Vq1Pz0Uc49uocR6tSIrrx24o94vsvJ5Xo7Nvx1ns0HnovXtWqattvPvKnKEs9cZLUk3FU7dyxfEJNZdvqA0v57ou791wp352Ha4cu5dOpggh7+kkubt7CpW3bS2/sIHTjtxF7OPxK8vRnnL/A/i++pO3zz2rbZwmcXLWa0P59S6zGYw2N0npR78vTZXrWxdcXz9BQ6k79rcAo79LhlnLpVFEYvDypO2kC0dNmOO0yRTd+W7GDw88y7S96zb/znfdpcOdAfGppX6utOCxefvslArWWwHbXOhuNLcsYc+AAqt57F9JkJvHHFY5WpUh047cVe+zzFzPyZyZfZO/HnxHx4mht+yuB9FOnuXQ0hmq3dK6wPosjqN21JR+FnZOWaEleuu/YV17DnJ7haHWuQTd+W3F1s+zJa4RUVUi7BN7+19zb/cFH1OnXB7/w8tcCsJYTq1ZTq08vDK72S1NmLUFFjPyVDf/2N+PTuhXxH37kaFWuQTd+W9Ha4Zd2Cdy9EIXW11kpKez+4CMiXxqjXV9WcNLWDL12xKuYFOFHB/my31h50mfVmzaZuA8/IuvsWUerUgDd+G0l90ivZsdLi5ny71n8CWE9uhNQv542/VhJ0t591OjetdxyZE42B85/SOyM7mWWUdxJQvV0AoqfdScgnQGP8NpUHzaEY6+97mhVCqAbv40IgwEUxbKHrQFFOfty0tLY+c4HFT7qA9TsdSsGd+tTiRWHevQgSmAIirunBloVkn08FiW8ruZy7UntMc9x4fc/ubx3n6NVuYJu/GVBS6dfyrV7/Hs//pyandoT1LT0yrxaU9vaOnyloO7fiXKTdp759HzRcubjMSjhFTsjKi9GX1/Cx44hZpp9Kv2WBd34y4KWUX6XCx7lNWVmsmPRO7QbV2rRY7tQs1dPTeSY9+/A0Kx1ueUcn9OfP+uc5p/+g7h86DAA6olYDLWtLBnmRFR/8D6yLyRx4ZffHK0KoBt/2dAwvr9wEo/9n39JlTatCSlD2qyykpZwire3v8uK4CMYPcs/TZfZWajRB1GatNRAO6j7xGM0mTKBf++8l6S//0FmZiJC7BftaC8Uo5H6M5wn3bdu/GVByzx++Rx+5uxsot54i3YTKm7UP7n2N1Z0vZXQ3r1oPuY5TWSqRw+g1ApHeGqXZix0yL20fGMeW4c/SopngF3LkNuTwB7d8Khbh4RPPne0KrrxlwlXd6QGa34pZYEMPge/+pagxo2oVtbMOTZgzs5m88Sp/Dt2PD2WfkKrsWM0q/Vn3rdD0/V+HtX69qblqOHs23qQ02t+0Vx+RVFvxlROLnqbnCTHpvvWjb8MCA0cfuETVlNn4hoQAuHmgTknh23zFxI53v6jfkrsMX7u1Z/Lx45z599/Uq1De03lq/t3arLeLwp/DyNtnxnBnhfHEfftMrv0YW+8GjUkZMAdHJ+3wKF66MZfFrR0+OWO+oe/+x7fsFBqdtTWEAsT+78fWdWzL/WG3MOt33yBW6C2++UyKws19jBK4xaays1DPRFLQNeudPzpew7NnkusE0bOWUP4uBdJ/GklaUejHaZD+Y5t3ahomdDDJxDVbGbb/DfpsXC+NjKLwJSRweYJUzi1YSO9f/iO4NbaOOMKox7ZhxJWD+Gh/f4+gPl4LIbadfGpUYtOq3/iv7uHkp18kUbjx1YqP4BrUCBhzz5NzIxZtPjKMet/q0Z+IUQfIcRhIUS0EGJCEfdfEEIcEELsEUKsE0JUXDC6I9DQ4Sd8Ajj6wwo8goKo1dU+Z9WTDx1mZffe5FxOZdDff9rN8CFvvW+fKb9MS0WmXERUqwGAZ2gonX7+iTO//Ma+iVOd9uhscdR89GHSj0aTtMExWYlLNX4hhAF4F+gLNAWGCSGaFmq2E4iQUrYAvgecK45RazQ81iu9/dg2dwHt7DBySSk5svQr1vQdSLOnRtLt4w9w9fXRtI/CqPt3YGhmH4el+cQxlNDwAtmE3auE0GnF91zau5edTz13TfIPZ0Zxc6Pe9CnETH8Z1WSHEnCl9W9Fm3ZAtJQyVkqZDXwLFCggL6VcL6XMK6O6GailrZrOhXBxp+5KRZNUUjFRBzB4uFO7Vw8NNLtK9uVU/nrsSfa+8wH91vxEo4cesPu0WGZmoB6PRmlsnxgF9UQshiIi+1z8/Gi//Buyk5PZ9vBjmDOc7/hscQT364MxIIAzX39b4X1bY/w1gfyZCONzrxXHo8Daom4IIUYKIaKEEFHnzp2zXktnw0W7kthRn35Hu/EvamqY53ftZkWXHhi9PBm4/lcCmlRMmLB6eB9KeAOEW/nPBhSF+XispXRXERg9PWm39FMMHh5sHnI/OSmX7aKD1gghqD9zOsdffwPT5YrVWVNvvxDiASACmFfUfSnlYillhJQyIiQkRMuuKxZX7YxfIqjbv682sqRk//uL+fXOIbSZMpHOb72hScSetdhrfz8P9URMkSN/HoqrK20/fBfvBg34d9A9ZJ0/bzddtMSnRXMCe3TjxMK3K7Rfa4w/AQjN93Ot3GsFEELcCkwGBkgp7VTE3knQcOSPnKDNWj8rKZk/7htO9LfLuOOPNdS7504NtLMNVaN4/sLkJRe15jSfMBhoMX8OVXp0Y9Ptd5KRcM2fqlNSZ9I4Tn/5NRknTlZYn9YY/zaggRCijhDCFRgKrMzfQAjRGvgQi+Enaq+mk6HhyN/gzoGlNyqFs1u28mOXHvjUDuP231bjW6/ij7vKjHTUk7EoDW+yWx/q+XMoNUJLbSeEoMmUiYQ9cB//9B9EaiUone1WrRq1Rj5G7KzZFdZnqcYvpTQBzwC/AgeBZVLK/UKImUKIAbnN5gHewHIhxC4hxMpixF0faDDyr298mnc2vlL2OnhYUoDtXrCQP+57mI7z59B+zisY3LR7MdmCemgPSt1GCDv2r9SsdU3Go5Ko/8yTNBw7hk0D7ubSnr1200srQp98gpSo7Vzasq30xhpg1W9SSrkGWFPo2rR8n2/VWC+nRijli4G37BJU5/XL6aW2LY6MxEQ2PP405swMBv71O94VmN23KMz7dqDYKaQ3j5LW+8VR+4H7cPH15b97hxH52RKCNA5l1hKDpwd1JlvSfbdZu6pcA4M16OG9DiQtOaVAsQxrSVj/Fz916UmViDb0W/2Tww0fcuP57ejsA1Bql205U2PA7bT54F22DX+Us7+v01grbal6t8VXc/Z/P9q9L934C1EWYywr2am27UerJhNRM19l46hn6Prhu7SdOrHchTW0QKalosYfR2nYzK79lNX4Aap070q7r75g57PPE18BhlVWhKJQf+YMjr06x+7pvh3/l1PJkKkXkbv+IqbzOZTeD5ZLlkdwycU585Man8CGESMxenoy6O91eFSpUq6+tUQ9tAelflOEi33TfRvKmbcvMLItHX9YxubB95NzKYU6I4ZrpJm2+N0ciW9kBHHvfUD4WPvlcdRHfiuRmemo235D/XkJePmh3Pag5WhvOfCqVs2qdifW/MKKrr0I7dub3j9851SGD5b1vr2O8OZHCQ0vtwzfpk3o9POPxLz7PkfeWOQ0+fQKU3fKROI/+pisM2fs1odu/KUgc7JR9/yD+tN7YDahDByF0robQoPtPq9S1urmrCw2j5/M5nGTuPXrz2g55jm7O4HKgnm//Q7z5EdoVEjEK7w2nX/+iYQffuLA9JlWvwAqcknoUTuM6g/cx7HZ9jsm43x/SU6CVM2oh7ej/vgeJJ9F6fcISvt+CA9vzfrwqlm88afExLKqVz9S4+IZtHEdVW9up1m/WiLTLiNPxaHUL3zWy7lxr16NTqt+IGnzVnaPftEhB2tKo/bzz5L05wYu22mbUjf+YlBXfIA8fgClx2CUrncjfIMK3NdiFPCqXvS0P2b5D6y6tR8N7x9Gz68+0zzhhpaYD+xGaXhTpaqhl4drQAAdflhGenwC2x99AnOWcwWmGn18CB/3AtFTZ9hleaIbfz7kmeNXPivt+qDc9gAiuIbd+vOqUb3Az6b0dP5+5nl2zJ5L75+W0fSJx5w2QcWVkNsKWO9HP1ybPTGLUM9pX+7K6O3Fzd98AUKwZdhDmFLTrHquopYA1e8fhuniJc6vLvKsXLnQjR+QSWcx//EN6qZVV66JmvWKNTyZdunKZ3XXRmTMXmRiPDIjzaY3tFf1q8afdOAgK7rdhjkrm4Eb1xHc0j5psLTGst637/6+sfFNuN01jPQpY5B2OK5rcHOj7ZIP8KxVk//uGkx2smMTa+ZHGAzUe3kaMTNfRc2dmWj14rmht/pk6kXkzg3IU7GIFp0R3QfDtqKzwsrsLOSx/cijO+H8KWL6NEV4+UFKEmpCjCUF9+VkUM2Wirs+AQifAEtabp8AhG/uZ0/fKzIt037LMc61/e8k8pXpNLhvqNOO9kUhz55CqWf/I8OuQx9GPXmMjDlT8Jg+T3PHp2I00nLRAg5Mn8mmO+6iw/ff4l7NOWoDBHa7Bc8G9Un4+DNCn3pCM7k3pPHLzHTknn+QsXsQjSNQ7ny6SO+9NJshIRp5ZCcy7ghUr4PStB2ENUYYi17jyuxMSzruy8nIvBfChdOoudfITANuAfKm/UcB6P/LSvwbNbTXV7YbSqPmNsXblxUhBO5jppD+0iiyPn0f90eftksfTV+ehou/P//0H0iH77/Fq0645v2UhXozprJrwF1UHXyPZjJvKOOXOdnIg1uQB7YgwpuhDBxVrPde3bQKGbMHfIMQDVqhdLoD4VF6EQrh6g7B1SG4OkWN39Js4lhmOsLLMgPY+3RLPIKDMHp4lOerOYyK2N/PQ7i64vHyfNKefgglLBzXXtrUFSzQhxA0fGE0Lv7+bLr9Ttov/xrfpk0078dWvBrUp8qdAzk+7w0QHTWReUMYv1TNyKO7kLv/RlQNRek3okBlXJmdCaePIU/FXn3IzdPycvALKkJi2REGI3hdnfr7hFbujGf2Xu9f059/IJ6vLCL9xcdRatTC2Mw+yUjrjBiOq78f/941mHZLPyMwsq1d+rGF8LEvsLVTVxioG3+pSCnhxEHUnestUXk9hyCCqiPNJuTpY8iEGIvBX0yEkFBEzbrEvlQfEVitUq27HYlSt+KXKoY69XAfP5OMGWPxevtzlGr22ZGpedcgjD4+bL3/Idp8+J5d+rAFl8AAwkY/C8e1kXfdGH+e9/P4HMtUUJ45jrp9Hagqol0fhJsnMiEGddtvkBgP/sGIGvVQIm6FKqHFruF1SkYYHPMn5HJzZ9QhD5M+5Xm83vpU07qA+anaqyeRn3/MtocfgzvnArm+IAdgTksn8+xZlu9cQcTa8qfMuG6MPw+ZdAZ1x3pISoQa4ZCViVy/HOnuiahRF6VxO+g+GOFWOdfYOldxvfs+1BOx1Jm5Abj64teaoA7t6bD8G/739rtcWLeetZ+MhBGL7dJXcZz75TeOTJqKX7tIWnzxiSa7Hded8aurloDBAIoBYTZBWENE+z6WbTkdzTh6uzs5yz5Fpnez26hbGkII3EdPgCn2r3fv16I5bT/6ALBkUFox6mmCu3eze78ZJ+M4Mmkq6TGxNFm0gMAunTWTfd0Yf+xdRuTJw4iG3RChDcE/RF+32xFjpx6o+3aQNX8qbpNfd9j0P/9yLXvNj7j2s3/iUqEo5CQl4VbVfnEAanY2J9/7kJPvf0jYqJE0//hDFI1TpF03EX5Ku94Y7nkOpUVnREAV3fDtjBAC18fHgJRkf/SmUxyNzfr4HUy7o4q8p3U4buaZs7hVtc/R6qS//2FLt15c2hZF5G9rCB/znOaGD9eR8etUPMJgxO2lVzAf3INp1XeOVgePSa+SMXMCakJc6Y3LSdbZRNw0jgDMOpvI/ief5eBzL1B/6iRafvU5HrXDNO0jP7rx65QL4emF+5R55Pz0NaYtjik4mYexbXvcHhpJ+uTRyFT7Vb8xZ2ZiTk/HJdD6TEwlIc1m4pZ8ypauPXGrUYP2/2wgpG9vTWSXxHWz5tdxHEpINdwmzSVz5ouIoCoY6ldMebCicB04GPOJWNJnTcBz9iK7+CKyE8/hFhKsydIyZecuDo2dgMHbmzYr/od3BYZ46yO/jiYY6jfB7anxZL023i5Hb4tD5iuVriZaUl65Pz0WpCTrvQV26TPzzJlyO/tyLl7k0LiJ7H5wBKGjHqfNT8sr1PBBN34dDTG274rLHUPIfGUsMt26c/FlQaoq5r07yFz0KmkP3s5BZQ2HQveR9uR9ZL4zF5l0Ac9pczFt30L2imWa9h0+YTWtfyj7el9KyenvlrO5U3dA0H7Teqrfe7dDHNT6tF9HU4wDh6GejrfLFqB6Mpac9b9gWv8rwssbY48+eL77JUqwxRBd7x9J9vdfkDZqGC49+uA+dhoZ08ei1NLeaVaWkT/10GEOj5uEOT2dll99hm8r+5xLsBZ95NfRFCEEriNfsGwBLllY7i1ANekC2T9+Q/pzD5Ex5TkwmXCfPh/Pd7/C9e4Hrxg+gOIfgPtjo/FashyMLmRMfwFDyzZkzJpY3q91DbZs85nT0ome+So7Bt1D1UF3EPnrzw43fNBHfh07kLcFmDHhCUyrluEyYIhNz8vMDEz/bsC0/hfMh/djbH8Lro88g6FFW4Sh9FJpSkAQ7k+MwfWeB8le9jkmU3ZZv0qxuFmRPl1Kyfm1v3Jk8jT8O7bn5o1/4lbFeUrT68avYxfytgAzJzyBqFod4823lNhemk2Yd0dh+nMtpq3/YGjSAmPPfrhPnotwdy+TDkpQMO5PvojrvQ9ycPlS3Ee9UCY5RVHatD/j+AkOT5pKxomTNH1nIQGdtDmGqyW68evYDaVKddwmziFz1tgitwCllKixRywG/9dviJCqGLv3wfOx0Sj+2uyhAyjBVfB48kXN5AHFOvzUrCxOvPM+cYuXEPb0k7T4bAmKRvUGtEY3fh27YmjQ9MoWoPucxSghVVETz2Da8Cum9WuRWZkYu/fB47X3NKnIU1EUNfInbfybw+Mm4dmwAZF//IKHkydq0Y1fx+4Y23dFnkmg7gJL3P2+4wsxdu6J2zMTUJq0cMoqRKXhGnw1w1PWmbMcnT6TS1HbaTh7JiG9b3OgZtajG79OhWAcOIzo2lshPR1Du9V2L+ppb4TBgGoykfDJ5xx7YyE1H7yfJm/Ox+BZefJEWGX8Qog+wCLAACyRUs4pdN8N+AJoC1wAhkgpj2urqk5lRgiBsfXNjlZDMy5t38HhlyZi9POl7aof8WpQ39Eq2Uyp8y0hhAF4F+gLNAWGCSEKF2Z7FEiWUtYH3gTmaq2ojo4zsWf4Y4Q9PYrWPyyrlIYP1gX5tAOipZSxUsps4FtgYKE2A4HPcz9/D/QU+oF6neuQI+M6sC7gIO03rafa3XdW6rwR1hh/TSD/Aen43GtFtpFSmoBLwDU5r4UQI4UQUUKIqHPnzpVNYx0dB+IaGEi98WNx8av8aeEq1M0qpVwspYyQUkaEhDhPpJOOzo2INcafAITm+7lW7rUi2wghjIAfFsefjo6Ok2KN8W8DGggh6gghXIGhQOGk4SuB4bmf7wH+lM6Q1E1HR6dYSt3qk1KahBDPAL9i2er7REq5XwgxE4iSUq4EPgaWCiGigSQsLwgdHR0nxqp9finlGmBNoWvT8n3OBO7VVjUdHR17UvniKnV0dDRBN34dnRsU3fh1dG5QhKOc8kKIc8CJIm4FA+crWB1b0XUsP86uH1wfOtaWUhYZVOMw4y8OIUSUlDLC0XqUhK5j+XF2/eD611Gf9uvo3KDoxq+jc4PijMa/2NEKWIGuY/lxdv3gOtfR6db8Ojo6FYMzjvw6OjoVgG78Ojo3KA4zfiFEHyHEYSFEtBBiQhH33YQQ3+Xe3yKECHdCHV8QQhwQQuwRQqwTQtR2Jv3ytbtbCCGFEBW+bWWNjkKIwbm/x/1CiK+dTUchRJgQYr0QYmfu/3W/CtbvEyFEohBiXzH3hRDirVz99wgh2lglWEpZ4f+wnA6MAeoCrsBuoGmhNk8BH+R+Hgp854Q6dgc8cz8/WZE6WqNfbjsfYCOwGYhwwt9hA2AnEJD7cxUn1HEx8GTu56bA8QrW8RagDbCvmPv9gLWAANoDW6yR66iRvzLkBSxVRynleilleu6Pm7EkOnEa/XKZhSWhamYF6paHNTo+DrwrpUwGkFImOqGOEvDN/ewHnKpA/ZBSbsRyVL44BgJfSAubAX8hRPXS5DrK+DXLC2hHrNExP49ieftWFKXqlzv9C5VSrq5AvfJjze+wIdBQCLFJCLE5N018RWKNjjOAB4QQ8ViOtj9bMapZja1/q4BetEMThBAPABFAV0frkocQQgHeAB52sCqlYcQy9e+GZea0UQjRXEp50ZFKFWIY8JmUcoEQogOWxDU3SSlVRytWHhw18leGvIDW6IgQ4lZgMjBASplVQbpB6fr5ADcBG4QQx7GsBVdWsNPPmt9hPLBSSpkjpTwGHMHyMqgorNHxUWAZgJTyP8Ady4EaZ8Gqv9VrqEjHRT4HhRGIBepw1cnSrFCbpyno8FvmhDq2xuIsauCMv8NC7TdQ8Q4/a36HfYDPcz8HY5m+BjmZjmuBh3M/N8Gy5hcV/LsMp3iHX38KOvy2WiWzIr9AIYX7YXnLxwCTc6/NxDKCguXtuhyIBrYCdZ1Qxz+As8Cu3H8rnUm/Qm0r3Pit/B0KLMuTA8BeYKgT6tgU2JT7YtgF3FbB+n0DnAZysMyUHgVGAaPy/Q7fzdV/r7X/z3p4r47ODYoe4aejc4OiG7+Ozg2Kbvw6OjcouvHr6Nyg6Mavo3ODohu/js4Nim78Ojo3KP8HWNDOFqce/gUAAAAASUVORK5CYII=\n"
+          },
+          "metadata": {
+            "needs_background": "light"
+          }
+        }
+      ],
+      "source": [
+        "plot(nodes_coordinates[resulting_traj_with_depot], obs['demand'])"
+      ]
+    }
+  ],
+  "metadata": {
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    },
+    "language_info": {
+      "name": "python"
+    },
+    "colab": {
+      "provenance": []
+    },
+    "accelerator": "GPU",
+    "gpuClass": "standard"
+  },
+  "nbformat": 4,
+  "nbformat_minor": 5
+}

demo/tsp_search.ipynb

@@ -0,0 +1,397 @@
+{
+  "cells": [
+    {
+      "cell_type": "code",
+      "source": [
+        "!git clone https://github.com/cpwan/RLOR\n",
+        "%cd RLOR"
+      ],
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        },
+        "id": "5a69iB04JzY2",
+        "outputId": "13a3a63e-34bf-4d8b-a853-9d2597cd03d5"
+      },
+      "id": "5a69iB04JzY2",
+      "execution_count": 1,
+      "outputs": [
+        {
+          "output_type": "stream",
+          "name": "stdout",
+          "text": [
+            "Cloning into 'RLOR'...\n",
+            "remote: Enumerating objects: 52, done.\u001b[K\n",
+            "remote: Counting objects: 100% (52/52), done.\u001b[K\n",
+            "remote: Compressing objects: 100% (35/35), done.\u001b[K\n",
+            "remote: Total 52 (delta 12), reused 52 (delta 12), pack-reused 0\u001b[K\n",
+            "Unpacking objects: 100% (52/52), 5.19 MiB | 7.89 MiB/s, done.\n",
+            "/content/RLOR\n"
+          ]
+        }
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 2,
+      "id": "dbe3c5ed",
+      "metadata": {
+        "id": "dbe3c5ed"
+      },
+      "outputs": [],
+      "source": [
+        "import numpy as np\n",
+        "import torch\n",
+        "import gym\n",
+        "from models.attention_model_wrapper import Agent"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "id": "985bf6e6",
+      "metadata": {
+        "id": "985bf6e6"
+      },
+      "source": [
+        "# Define our agent"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 3,
+      "id": "953a7fde",
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        },
+        "id": "953a7fde",
+        "outputId": "06f10aaf-57ca-4870-d22b-f71a14ea4ec4"
+      },
+      "outputs": [
+        {
+          "output_type": "execute_result",
+          "data": {
+            "text/plain": [
+              "<All keys matched successfully>"
+            ]
+          },
+          "metadata": {},
+          "execution_count": 3
+        }
+      ],
+      "source": [
+        "device = 'cuda'\n",
+        "ckpt_path = './runs/tsp-v0__ppo_or__1__1678160003/ckpt/12000.pt'\n",
+        "agent = Agent(device=device, name='tsp').to(device)\n",
+        "agent.load_state_dict(torch.load(ckpt_path))"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "id": "2cbaa255",
+      "metadata": {
+        "id": "2cbaa255"
+      },
+      "source": [
+        "# Define our environment"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 4,
+      "id": "c2bd466f",
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        },
+        "id": "c2bd466f",
+        "outputId": "92450c8d-f5db-444d-f465-da4a98667799"
+      },
+      "outputs": [
+        {
+          "output_type": "stream",
+          "name": "stderr",
+          "text": [
+            "/usr/local/lib/python3.9/dist-packages/gym/utils/passive_env_checker.py:31: UserWarning: \u001b[33mWARN: A Box observation space has an unconventional shape (neither an image, nor a 1D vector). We recommend flattening the observation to have only a 1D vector or use a custom policy to properly process the data. Actual observation shape: (50, 2)\u001b[0m\n",
+            "  logger.warn(\n",
+            "/usr/local/lib/python3.9/dist-packages/gym/core.py:317: DeprecationWarning: \u001b[33mWARN: Initializing wrapper in old step API which returns one bool instead of two. It is recommended to set `new_step_api=True` to use new step API. This will be the default behaviour in future.\u001b[0m\n",
+            "  deprecation(\n",
+            "/usr/local/lib/python3.9/dist-packages/gym/wrappers/step_api_compatibility.py:39: DeprecationWarning: \u001b[33mWARN: Initializing environment in old step API which returns one bool instead of two. It is recommended to set `new_step_api=True` to use new step API. This will be the default behaviour in future.\u001b[0m\n",
+            "  deprecation(\n",
+            "/usr/local/lib/python3.9/dist-packages/gym/vector/vector_env.py:56: DeprecationWarning: \u001b[33mWARN: Initializing vector env in old step API which returns one bool array instead of two. It is recommended to set `new_step_api=True` to use new step API. This will be the default behaviour in future.\u001b[0m\n",
+            "  deprecation(\n"
+          ]
+        }
+      ],
+      "source": [
+        "from wrappers.syncVectorEnvPomo import SyncVectorEnv\n",
+        "from wrappers.recordWrapper import RecordEpisodeStatistics\n",
+        "\n",
+        "env_id = 'tsp-v0'\n",
+        "env_entry_point = 'envs.tsp_vector_env:TSPVectorEnv'\n",
+        "seed = 0\n",
+        "\n",
+        "gym.envs.register(\n",
+        "    id=env_id,\n",
+        "    entry_point=env_entry_point,\n",
+        ")\n",
+        "\n",
+        "def make_env(env_id, seed, cfg={}):\n",
+        "    def thunk():\n",
+        "        env = gym.make(env_id, **cfg)\n",
+        "        env = RecordEpisodeStatistics(env)\n",
+        "        env.seed(seed)\n",
+        "        env.action_space.seed(seed)\n",
+        "        env.observation_space.seed(seed)\n",
+        "        return env\n",
+        "    return thunk\n",
+        "\n",
+        "envs = SyncVectorEnv([make_env(env_id, seed, dict(n_traj=1))])"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "id": "c363d489",
+      "metadata": {
+        "id": "c363d489"
+      },
+      "source": [
+        "# Inference"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 5,
+      "id": "bbee9e3c",
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        },
+        "id": "bbee9e3c",
+        "outputId": "11750d60-eb7c-4d9c-8b40-3a8a92021ffe"
+      },
+      "outputs": [
+        {
+          "output_type": "stream",
+          "name": "stderr",
+          "text": [
+            "/usr/local/lib/python3.9/dist-packages/gym/utils/passive_env_checker.py:174: UserWarning: \u001b[33mWARN: Future gym versions will require that `Env.reset` can be passed a `seed` instead of using `Env.seed` for resetting the environment random number generator.\u001b[0m\n",
+            "  logger.warn(\n",
+            "/usr/local/lib/python3.9/dist-packages/gym/utils/passive_env_checker.py:190: UserWarning: \u001b[33mWARN: Future gym versions will require that `Env.reset` can be passed `return_info` to return information from the environment resetting.\u001b[0m\n",
+            "  logger.warn(\n",
+            "/usr/local/lib/python3.9/dist-packages/gym/utils/passive_env_checker.py:195: UserWarning: \u001b[33mWARN: Future gym versions will require that `Env.reset` can be passed `options` to allow the environment initialisation to be passed additional information.\u001b[0m\n",
+            "  logger.warn(\n",
+            "/usr/local/lib/python3.9/dist-packages/gym/utils/passive_env_checker.py:165: UserWarning: \u001b[33mWARN: The obs returned by the `reset()` method is not within the observation space.\u001b[0m\n",
+            "  logger.warn(f\"{pre} is not within the observation space.\")\n",
+            "/usr/local/lib/python3.9/dist-packages/gym/utils/passive_env_checker.py:141: UserWarning: \u001b[33mWARN: The obs returned by the `reset()` method was expecting numpy array dtype to be float32, actual type: float64\u001b[0m\n",
+            "  logger.warn(\n",
+            "/usr/local/lib/python3.9/dist-packages/gym/utils/passive_env_checker.py:227: DeprecationWarning: \u001b[33mWARN: Core environment is written in old step API which returns one bool instead of two. It is recommended to rewrite the environment with new step API. \u001b[0m\n",
+            "  logger.deprecation(\n",
+            "/usr/local/lib/python3.9/dist-packages/gym/utils/passive_env_checker.py:234: UserWarning: \u001b[33mWARN: Expects `done` signal to be a boolean, actual type: <class 'numpy.ndarray'>\u001b[0m\n",
+            "  logger.warn(\n",
+            "/usr/local/lib/python3.9/dist-packages/gym/utils/passive_env_checker.py:141: UserWarning: \u001b[33mWARN: The obs returned by the `step()` method was expecting numpy array dtype to be float32, actual type: float64\u001b[0m\n",
+            "  logger.warn(\n",
+            "/usr/local/lib/python3.9/dist-packages/gym/utils/passive_env_checker.py:165: UserWarning: \u001b[33mWARN: The obs returned by the `step()` method is not within the observation space.\u001b[0m\n",
+            "  logger.warn(f\"{pre} is not within the observation space.\")\n",
+            "/usr/local/lib/python3.9/dist-packages/gym/utils/passive_env_checker.py:260: UserWarning: \u001b[33mWARN: The reward returned by `step()` must be a float, int, np.integer or np.floating, actual type: <class 'numpy.ndarray'>\u001b[0m\n",
+            "  logger.warn(\n"
+          ]
+        }
+      ],
+      "source": [
+        "num_steps = 51\n",
+        "trajectories = []\n",
+        "agent.eval()\n",
+        "obs = envs.reset()\n",
+        "for step in range(0, num_steps):\n",
+        "    # ALGO LOGIC: action logic\n",
+        "    with torch.no_grad():\n",
+        "        action, logits = agent(obs)\n",
+        "    obs, reward, done, info = envs.step(action.cpu().numpy())\n",
+        "    trajectories.append(action.cpu().numpy())"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 6,
+      "id": "f0fbf6fd",
+      "metadata": {
+        "id": "f0fbf6fd"
+      },
+      "outputs": [],
+      "source": [
+        "nodes_coordinates = obs['observations'][0]\n",
+        "final_return = info[0]['episode']['r']\n",
+        "resulting_traj = np.array(trajectories)[:,0,0]"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "source": [
+        "## Results"
+      ],
+      "metadata": {
+        "id": "5n9rBoH5Q8gn"
+      },
+      "id": "5n9rBoH5Q8gn"
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 7,
+      "id": "dff29ef4",
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        },
+        "id": "dff29ef4",
+        "outputId": "dcffcda5-5728-464c-ee3e-0fcc702443ff"
+      },
+      "outputs": [
+        {
+          "output_type": "stream",
+          "name": "stdout",
+          "text": [
+            "A route of length [-5.908508]\n",
+            "The route is:\n",
+            " [26 34 33 49 37 21 48 43 31 28 42 29 47 39 38 23 27 30  7 32 24 40 20 14\n",
+            " 25  1 18 22  0 11  2 16 45 15 46 12 17 41  8 13  3  6 44  9 10 19 36  5\n",
+            " 35  4 26]\n"
+          ]
+        }
+      ],
+      "source": [
+        "print(f'A route of length {final_return}')\n",
+        "print('The route is:\\n', resulting_traj)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "id": "b009802e",
+      "metadata": {
+        "id": "b009802e"
+      },
+      "source": [
+        "## Display it in a 2d-grid\n",
+        "- Darker color means later steps in the route."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 8,
+      "id": "dc681a06",
+      "metadata": {
+        "tags": [
+          "\"hide-cell\""
+        ],
+        "cellView": "form",
+        "id": "dc681a06"
+      },
+      "outputs": [],
+      "source": [
+        "#@title Helper function for plotting\n",
+        "# colorline taken from https://nbviewer.org/github/dpsanders/matplotlib-examples/blob/master/colorline.ipynb\n",
+        "import matplotlib.pyplot as plt\n",
+        "from matplotlib.collections import LineCollection\n",
+        "from matplotlib.colors import ListedColormap, BoundaryNorm\n",
+        "\n",
+        "def make_segments(x, y):\n",
+        "    '''\n",
+        "    Create list of line segments from x and y coordinates, in the correct format for LineCollection:\n",
+        "    an array of the form   numlines x (points per line) x 2 (x and y) array\n",
+        "    '''\n",
+        "\n",
+        "    points = np.array([x, y]).T.reshape(-1, 1, 2)\n",
+        "    segments = np.concatenate([points[:-1], points[1:]], axis=1)\n",
+        "    \n",
+        "    return segments\n",
+        "\n",
+        "def colorline(x, y, z=None, cmap=plt.get_cmap('copper'), norm=plt.Normalize(0.0, 1.0), linewidth=1, alpha=1.0):\n",
+        "    '''\n",
+        "    Plot a colored line with coordinates x and y\n",
+        "    Optionally specify colors in the array z\n",
+        "    Optionally specify a colormap, a norm function and a line width\n",
+        "    '''\n",
+        "    \n",
+        "    # Default colors equally spaced on [0,1]:\n",
+        "    if z is None:\n",
+        "        z = np.linspace(0.3, 1.0, len(x))\n",
+        "           \n",
+        "    # Special case if a single number:\n",
+        "    if not hasattr(z, \"__iter__\"):  # to check for numerical input -- this is a hack\n",
+        "        z = np.array([z])\n",
+        "        \n",
+        "    z = np.asarray(z)\n",
+        "    \n",
+        "    segments = make_segments(x, y)\n",
+        "    lc = LineCollection(segments, array=z, cmap=cmap, norm=norm, linewidth=linewidth, alpha=alpha)\n",
+        "    \n",
+        "    ax = plt.gca()\n",
+        "    ax.add_collection(lc)\n",
+        "    \n",
+        "    return lc\n",
+        "\n",
+        "def plot(coords):\n",
+        "    x,y = coords.T\n",
+        "    lc = colorline(x,y,cmap='Reds')\n",
+        "    plt.axis('square')\n",
+        "    return lc"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 9,
+      "id": "bb0548fb",
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/",
+          "height": 282
+        },
+        "id": "bb0548fb",
+        "outputId": "3e967b86-32e9-4be3-e5b7-c8a457aa12a7"
+      },
+      "outputs": [
+        {
+          "output_type": "execute_result",
+          "data": {
+            "text/plain": [
+              "<matplotlib.collections.LineCollection at 0x7f15aabac8b0>"
+            ]
+          },
+          "metadata": {},
+          "execution_count": 9
+        },
+        {
+          "output_type": "display_data",
+          "data": {
+            "text/plain": [
+              "<Figure size 432x288 with 1 Axes>"
+            ],
+            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAP8AAAD4CAYAAAAjDTByAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAAsTAAALEwEAmpwYAAA4QElEQVR4nO2dd3iUVdqH72dm0kklBUKAUBIhdAgqIk1aAAUBRWy7Kuradm27a1td1911dXVddXU/xYJ1VURUFKSIUpTeqyGhJgTSgCQkpMzM+f6YgBECTJJ3Wubc15UrM/OeOe8vk/mdfp4jSik0Go3/YfK0AI1G4xm0+TUaP0WbX6PxU7T5NRo/RZtfo/FTLJ66cWxsrEpOTvbU7TUav2D9+vVFSqm4+q55zPzJycmsW7fOU7fXaPwCEdl/tmvnbfaLyNsiUiAi285yXUTkZRHJFpEtItK3KWI1Go17cKbP/w6QcY7rY4CU2p/bgf9ruiyNRuNqzmt+pdQy4Mg5kkwA3lMOVgFRItLaKIEajcY1GDHa3wbIqfM8t/a1MxCR20VknYisKywsNODWGo2msbh1qk8pNV0pla6USo+Lq3cAUqPRuAkjzH8QaFvneVLtaxqNxosxwvxzgF/VjvpfDJQopQ4ZkK9Go3Eh553nF5GPgKFArIjkAn8GAgCUUq8B84CxQDZQAdzsKrEajcY4zmt+pdS157mugLsNU6TRaNyCXtuv0fgp2vwajZ+iza/R+Cna/BqNn6LNr9H4KR7b0ust5H35FftnzsJWWUVkt65EdksjolsaEakpmAIDPS1Po3EZfm3+fW+9Q9YLL9HrlX9jq6mhdPtODi9aTOaL/6F8/wFadOxAZLc0Irt1JSItjchuaQS3SkBEPC1do2kyfml+pRS7nn2e3FmzuWTuF4Qltweg9aiRp9LYKisp25VFybbtlGzfSf73/0fJtu2gqFMYOFoK4RekYgkN9dSfo9E0Cr8zv7LZ2PrHRzi6YSOXzptDUHz9G4zMwcFE9exBVM8eP79XKaoKCinZvoOS7Tso+nEFu6e/RVn2bkLbJhGZ9nO3IbJbV0LbtvXrVoKtpoacL7+icP1GJCiQpNEjie+fjsnid187r0Q8dWJPenq6cncYL1tlJRt+czfWklLS33ubgIhwQ/K119RQlpVN6Y6dpwqGku07sR4/TmRaVyJqWwiRaWlEdO1i2H29DaUUZVnZFCz7gfxlyzm6eQshycm0u2oSpXv3kbNwEeU5B2kzfBhtR4+kzYjLCG4Z42nZzRoRWa+USq/3mr+Yv6a0lLU33kxgTAx9XnsFc1CQy+9ZdeRIbYHgKBRKd+yg9KdMguLiaguDn1sKLTokI2azyzUZTfmBHAqW/0DBsh8oWP4DYrYQP+RSEgYPIn7QpYS0bvXL9HmHyF34LTkLFnFo+Q9Ed+1C24xRtB09kuhuaX7dUnIFfm/+yvwCVl9zPdHp/ejx7N89ajJls3F87z5KT7YQalsLVYWFRHS54NTA4snxhMDoaI9prY/KwkIKlv9IwbLlFCz7gZrjx4kfdCnxgy8lYcggwtq3d9rA1spKDv+wgtwFi8hZ8C32mmqSRo+k7eiRtB58KQFhYS7+a5o/fm3+8r37WHXVVJKuuZrUPzzgtTVLTWkZpTt/omSHo1AorW0tWMLDTxUEkd3SiEjrSnhKZ0wBAW7RVV1SSuGPKxy1+9LlVBzMI27gAOKHDCJh0KVEdO1iyGeqlKIkK5ucBYvIXbCIoo2biL/oQtrWFgbhtYOymobht+Yv2bKV1VNvJPUP95N8869dei9XoJSiIieHku07f24pbN9BRe5Bwjt3+sUUZGS3NIIT4pt8T2tFBUWr1zpq9uU/UJq5i5b900/V7tG9erplwK66pJSD3y9xtAoWLiY4JvpUqyDh4gvdVvj5On5p/qLlP7L+1t/Q47lnSBx/ucvu4wmsFRWU/pR5qnVQssPRUhCL2TGoWKelEJ6agjk4+Kx52WtqOLJ+I/m1zfijmzYR1b078YMvJX7IIFqm93PL+Mi5UHY7RRs3n2oVlO7dS+KwIbQdNZKkUcMJ0SHhzorfmT9vztds/cPD9HvzdWIHDXTJPbwNpRSVhw7/ojAo2b6D43v3Eda+3c/dhm5dCYiIoGjtegqWLqdozRpadOjgGKAbfCmxAy4moIV397Ur8vPJXbSY3AWLyFuyjIjOnU91D1r26oGY9Kr1k/iV+ffNeJddz/2biz5+n8g6c/T+iq2qyjENWWcK0hwVSWB0NAmDLyXu0ksIivHd6TZbdTX5K1efahVUl5WRNHK4Yypx2FACwlt4WqJH8QvzK6XY9dwL5H7yKRd/+hFhHTsYlrfGdyjdvYec2tmDgrVrie+fTttRI0gaPZLIzp08Lc/tNHvzK5uNbQ//iSNr13HRJx8aMvCl8X1qjh8nb8kyR6tg4bdYQkNpO3okSaNH0mrgAMx+sHGrWZvfVlXFxjt/S3VxMf3fn0FARIQB6jTNDaUUR7ZsPdUqKNm1i9aDB9F29AiSRo0gtFWr82figzRb89eUlrHuV7dgiYyg7+uvnnNUW6Opy4miIg4u+o6cBYs4+N0SYtP7kHDJJXS+ehLh7dt5Wp5hNEvzVxUUsnrq9UT16U2Pf/7DJ5fGarwDu9XK4VVr2D1rNnu++IqolM50njKZjhMnEBLb0tPymkSzM3/5vv2svvpa2kyeSOpDv/faVXsa38NWXU3u4u/J/vQzDiz8llYXXUjna64iedwYn1xu3KzMX7JtO2uuuYGUB+4ledpNxgvTaGqpOX6cfXO/IWvmZ+SvXkO70SPpfPVkkoYPw+wjKwybjfmLflzB+ltup8ezT5N45XgXKdNozuREYRG7P/+S7JmfUZK9m44Tx5My5SoSLurv1YuKmoX5D309jy0P/pG+0/+PuCGDXKhMozk3pfv2k/3pbLJnzqKmooLOV08mZcpkYtK6elraGfi8+fe/9wGZzzzPhf97l6jevVysTKNxDqUUR7ZtJ+uTWWTPmk1QdDQpV0+m09WTCG+b5Gl5gA+bXylF1r9eJOd/H3PRpx/RolNHN6nTaBqGsts5tGIl2TM/Y++XXxPVJZWUa66i44TxHo1W5LPmryosZO0NN5H+7tsEt0pwkzKNpmnYqqvJWbSY7E8/I+fb72h9yQA6T5lM+7EZBLg50Ou5zO/VkRQDoqOp3LsPZbN6WopG4zTmwECSx40hedwYqsvK2Pf1N2R++DHL7/s97ceMpvOUq0gaNsTjgUy9d5gSMFksxI8aScE3CzwtRaNpFIHh4aReO4Vxn8/kmg2riO/Xl/X/eI4PUnvww+8f5vDqtXiq9e3V5gdIGDua/HnzPS1Do2kyofHxdL/jNiZ+N58J384jND6OpXf9jo96prPmqac5+tMut+pxyvwikiEimSKSLSIP13O9nYh8LyIbRWSLiIw1SmDs0CGUbNxM9dGjRmWp0XicyI4d6PvHB5mybgWjPpiBrbKSr8dPYtbAYWx+6RWOH8xzuYbzml9EzMCrwBggDbhWRNJOS/YnYKZSqg8wFfivUQLNoaG0HDSQwkWLjcpSo/EaRITYXj0Z8PRTXL9zMwP+8RTHsrKZNWAIX42dwM4Z71F5xDUVnzM1/4VAtlJqj1KqGvgYmHBaGgWc3EsbCRhabMWPzdBNf02zx2Q202bwIIa88iI3Zm2j+52/Ife7JXzUoy8Lpt7IgQWLjL2fE2naADl1nufWvlaXJ4EbRCQXmAf8tr6MROR2EVknIusKCwudFhk/agTFS5djO3HC6fdoNL6MOSiIDleMZeT7b3P9zi20G5PBkof/hLWqyrB7GDXgdy3wjlIqCRgLvC8iZ+StlJqulEpXSqXHNSDiamBMDBE9e1C0ZJlBcjUa3yEwIpwWye0IDA/HYmAkZWfMfxBoW+d5Uu1rdZkGzARQSq0EgoFYIwSeJGFcBvnf6Ka/xj/Z+81COowZbWiezph/LZAiIh1EJBDHgN6c09IcAIYDiEhXHOZ3vl3vBAljMihYsAi7VS/4MRpltVJ9YL+nZWjOwd75C+iQMcrQPM9rfqWUFbgHWADsxDGqv11EnhKRk/tqHwRuE5HNwEfATcrglQshbZMISUzk2Br3nuxrBPbqak9LOCdH33uXnGumkP/Yw9j1uIrXcTQrm5rj5cT17mlovk71+ZVS85RSqUqpTkqpv9e+9oRSak7t4x1KqYFKqV5Kqd5KqYWGqqwlfmwG+XO/cUXWLqEq9yD7n/o7G3r2Y88993pazlkp/2E5cY88CjVWcqdMpjo7y9OSNHXYO38RHTJGGR6xyutX+NWl1bgx5H8z32PLIZ1BKUXZmrXsuu1OtowYjbLWkPbVbI6vXM3x9Rs8Le8M7JWVnNiwgbBhlxH/7HNE3XQzub+6ntIvZntamqaWvd8soMMYY5v84GPmb9G1C5hMlG3f4WkpZ2Cvrqbos8/ZlnE52b+7n/CL+tNn7UqSn3qS0JQUWt97D3nPveBpmWdwYs1qgrumYQ4PR0SImHw1bd75gKNvvE7+ow9hr6jwtES/pqqklIING2k7dLDhefuU+UWEhDHeteCnpqiY3H+/xMb+Ayj46BOSHryP3j8upfWtt2AJDz+VruXUKVTu3kPZmrUeVHsm5cuWETb4l1+soNRU2s6cDXZF7pTJVGXpboCnOLD4e1oPuNglwUN9yvwACWMzyJ/n+X5/+Y6d7L7/92waOISqnFy6fPQBabM+JnrUyHrDiJsCA2l932/Je967av/y5WeaH8AUFkbCM/8katqtHPz19ZTOnuXV3a3myt5vFtBxrLFTfCfxOfNHX5hO1eF8KvYfcPu9lc3GkQUL2TH5Gn669gaC2rWl949L6fTCc4Q5Eb+t5ZSrqNp/gLKVq92g9vzU5OZiKykhqOvpWzV+JmLiZNq8+yHH3n6Lgof/iL283I0K/Ru7zca+hd+SbPAU30l8zvxiNhM/ehT5btzjby0r49Abb7Fp4BAOvvAS8ddNpc/alSTdfy8BDTjUwRQQQOL993pN7V++fBlhgwadN/psUEoKSTM/A4uZnCmTqNrl3q2n/kr+ug2EtWpFhIviAfqc+cHR9C9wQ7+/cv9+9j3xJBv7D6BszTo6/+dFus//mtjJEzE18pDHlldNojovj9IVKw1W23Ac5h/iVFpTaCgJf3+G6Nt+w8GbbqB01qe6G+Bi9syb75JR/pP4pPlbDhlE6bbtVBUVG563UoqSH1eSedM0to25AgkIpOe3C0h94/8I75/e5LlWsVhoff+95D33gkfNo6qrqVi1itCBAxv0vogrJ5H03occe3cGBQ/9QXcDXIgrlvTWxSfNbw4OJnboYAoXGrfF0V5ZScFHn7B1+Gj2PvwoUcOG0mftKto//ihBSadvYmwaLSddSU1BAWU//Ghovg3hxMaNBHbogKVlw8+iC+zs6AZIYCA5V0+ialemCxT6N6UHcqjIzychva/L7uGT5geIN2jKrzo/n5xnn2dj+gCK53xNuz89Qq+li0n49Y2Yw1wTaVUsFhIfuM+jtb+jyd/4uWNTSAjxf3uamDvu4uBNN1Ly6UzdDTCQvfMXkjxqBCYXHkDru+YfeRnFP6zAWt64RSjWsjKy77mPzYOHU1NcTNrsmXT96H2iLhvmluOXYq4cj/XoMUqXLnf5veqjfNkyQgc1/eSj8PETSPrgI0ref5f8Pz6IraTEAHUaVzf5wYfNHxAVRVS/PhR9v6RR78/590sos4k+q5bT8Z//ICQ1xViB50HMZhIfvI+8591f+1sLCqg5mEtIr96G5BfYsRNJMz/DEp9A3vgxFP/5MSo3rtctgUZSU17OoZWraDd8mEvv47PmB0gYN6bRTf+y9RuJmzQRS3S0waqcJ/qKcdjKyij9bolb71v+w3JCLxmIGBg33hQcTOwfHqL1p59jaZ9M8Z//RN4VGZS88RrWw4cNu48/kLNkGfF9+xAUGXH+xE3Ap80fnzGKwkWLsdfUNOh9SinKd/5EaNcuLlLmHCdr/4Nurv3rW9JrFJb4BCJvuY3EL+cR+/dnseblcWjyePJ/M43yb+aiDAxD1VxxR5MffNz8IYmJhCa352gDV8xVHzqEKSCAwHjnQ4m5iujLx6EqKyn51j3RiZXNRsWKHwm71LUnHYsIQb160/LPT9Fm8TJajL+S47NnkTt8MMV//TNVW7fobkE9KKVctovvdHza/ACxw4ZQsPi7Br3HG2r9k4jJROKD95P3/L/dYobKrVsxx8UR0Lq1y+91ElNwMGHjriDhjRm0/vRzzPEJFD30IIcmXk7JjDexFRka9MmnKdy0hYDwFkR37uTye/m8+Su276BFu7bnT1j3PTt2EuYl5geIGpuBstkoMXDdwtkoX768SVN8TcXSOpGo39xF4tyFxDz+F2r27CZv/FgK7rmD8kULUDXeHfXI1eydv5COYzLcci+fNn/Rwm+pyN5Nmxuua9D7ynf+5NRGHHdxsvY/6Ibav3z5Upf19xuCiBDcL53Yv/6DNt8uIXTkaMr+9wG5wwdz5Jm/Ub3T+2I2uIM98+aTnDHSLffyWfPbTpwg89HHueCZv2NqYDhjb2r2nyQqYxSCcMyFG5Zsx45RnZVFSL96T2z2GKbQMFpMmEirGe/T6sOZmFqEU3Dv3eRdNYHSD97FduSIpyW6DWkRDm46vddnzb/vpVcI79mDlsOc25hyEntNDZV79hCamuoiZY1DREj8w/2OeX+73SX3KF/xIyH9L2xwYelOAtq2I+qee2kzfzHRv3+Y6m1bOXj5KAruu4eK7xejGjiz42t0GjOKTW/McMu9fNL8Fbv3kDvjXVL/9mSD33siezdBbdpgDg0xXlgTiRw5AgkMpHjmLJfk78opPqMRk4mQiwcQ+8zzJC34npABl3DkqSeo3rnd09JcSo9fXc+ehd9SlnfI5ffyOfMrpch85HHa//ZughMTG/z+Ci9s8p9EROjw8gscfv558l/+D8pmMyxvpVST1/N7CgkKonrFckJ69ybwHIFHmgPBUZF0vXoSm956x+X38jnzF3w9j8q8PNr95tZGvb98509eNdJ/OiGpqVzw9RyOr1jB7qnXUp1nzJmnVZmZmIKDCWzf3pD83IWqqqL4wd9CQAAx/3wRCWhcHAVfou8dt7L5rXcNPZevPnzK/Nbj5ex6/Em6PPs0poCARuVRvvMnQr1opL8+Alq1ouOHHxA+dChZl4/n2Nx5Tc6zfNlSwgY3bHzE06jKSorvvwtTaBgx//gX0sj/ua8R27ULcd3TyJz9pUvv41Pm3/vCi0RfcjHRAwc0Oo+Knd41x382xGwm4e676PD2mxx65lly/vgwtiaE0a7wsSa//cQJiu+9A1NUNNF//6ffGP8k/e76Deteed2lU78+Y/7jmbvI+9/HpPz58UbnYS0tpebIUYJ9qOkb2rs3qd/MRdVUkzX2ciq2bmtUPnaThUAvm+E4G/aKcop/ezvmhFZE//VZQzcg+QqdMkZSeewYeS48ns4nzK+UIvOhR+nw4P0EJcQ36L12q5Wyrds4+PY77LznPrft1zcSc4sWtPv3CyTcdy97bvwVBdPfaPB0oDkujpKvv3aRQuOwlx+n+J7bsbRtT9STT9cbBt0fEJOJvr+Zxvr/TnfdPTy1uSI9PV2tW+dcqXZo1mz2v/oaFy6ah+k8tUDNkSOUrN9A6dp1lKxbT9mmLQQltiYyvR8R6f2IzRhNYAMi7nobVQdyOHDvvZhCw2j3wr8IcLIwPLFlCwfuuovUpUu91lD2sjKK77kNS8oFRD36Z58rpI2m8lgJr3XtzbT1KwhPbNxeDBFZr5Sqd1WX17enrKWlZD35N3rOmH6G8ZXNRnnmLkrXradk3XpK1q6jOr+AiD69ieifTru77ySibx8CPLhn32iC2rWl86czyX/pZXaNHUfbZ58hYsTw874vpGdPLLGxlH3/PREjRrhBacOwl5VSdNetBHbrTuRDjxt+KKUvEhwVSdo1V7HprXcY9Pgjhufv9TV/5qNPYKuoIO3F56kpKaF0/UaH2deuo2zjJgJiW56q1SP7pxPW5QKvrdmM5vjqNRy4734iR46k9aMPYwoOPmf6o7NnU/LFFyS/956bFDqHveQYRXfeQmCfdCJ//4g2fh2Kfsrk44wJ3JG5GUsjVmaeq+b3avMfW7eejVOuIz5jFOXbtlOZk0t4r55E9O/nMHy/fj7dhDcC67ESch95lKrdu2n/n5cJvuDsg3r2qioyBw6k48yZBHXs6EaVZ8d29CjFd95C0EUDiLjvD9r49fDJFZPpdu0Uul93TYPfey7zo5TyyE+/fv3U+ajML1Abrp6qct58W5Vu3qJsNTXnfY8/YrfbVfEnn6itvfqownffU3a7/axpDz37rMr7y1/cqO7clM14Qx175q/n1OzvZM2dr9655LJGfUbAOnUWDzo1oiIiGSKSKSLZIvLwWdJMEZEdIrJdRP7X4CKqHoLi4+gz8yOSpt1MeM8e5x3s81dEhJgpU0iZPYsjH3/Cvltvw3qWnXAx113Hsc8/95qjt2vWrCB40BBd458DV037ndf8ImIGXgXGAGnAtSKSdlqaFOARYKBSqhtwn6EqNU4R1LEjnb+YTVDHTuzKGFvvoSCBSUmE9u/PsS++cL/AerDlHcTcpmHBWPwNW3U1Cen92Dnrc0PzdabmvxDIVkrtUUpVAx8DE05LcxvwqlLqKIBSqsBQlRqnMQUGkvjYI7R9/jkO3P8AeU//A3v1L6PjtPzVryh+7z2Px9BTNhu2gsOYWzd8g5Y/cGT3HhY9/Dgvde5BxZGj9Lz5RkPzd8b8bYCcOs9za1+rSyqQKiI/isgqEak3DpGI3C4i60RkXWGhjtvmSsIHD+KC+fOoysome+JkqvbsOXUtbOBAVE0NFWvXelAh2AsOY4qOQRp56GlzxG618tOXX/PhuEnMGJqBmM3csmwR1331KfEG70kxqhNtAVKAoUASsExEeiiljtVNpJSaDkwHx2i/QffWnAVLy5Ykv/0mxe+9T9akq0h89GGir74aEaHlDTdw5P33CbvwQo/ps+UdxJxo7DmIvkpZ3iE2znifjW+/R2S7tvS7/RaumTS+UdN7zuKM+Q8CdTtlSbWv1SUXWK2UqgH2isguHIWBZ6sWDSJC7K9/RdjFF3Hgnt9RumQpbf/xNFGTJ5P/4ovUFBQQEN+wJdNGYTuYiznRNWfP+wJKKfZ9v4x1b7zNviXL6XbVRKZ+8QkJPbq55f7ONPvXAiki0kFEAoGpwJzT0nyBo9ZHRGJxdAP2oPEaQi64gJSvvsQSG0vmmLGc+CmTqCuu4OhHH3lMk7/W/CeOHmPVy//lvz0vYuEfH6PDsCH8btdmxv7nX24zPjhR8yulrCJyD7AAMANvK6W2i8hTOOYQ59ReGyUiOwAb8AelVLErhWsajik4mKSn/kLp4MHsv/NOIkePpvjDD4m76y6PbJm1Hcwh8KJL3H5fT5G3bgPrXn+LzK/m0TljFONff5mkARd5bJrTqT6/UmoeMO+0156o81gBD9T+aLyciBHDSZ03jwP3P4C1vIIjH39MyxuNHUl2BkfN37yb/dXl5WyfOZv1b8zgxJGj9LvtZoY//RfC4mI9Lc37N/ZoXENAQjwdP3iPQ8/+k5A+fTyiwZaX22yb/YU/ZbLhjRls/ehT2l5yMUOffJROIy7zqp2K2vx+jJhMJD5S74JNl6MqK7GXlmCK88xgoyuwVVfz05y5rJ8+g+JdWfS+6QZuW7WUyHbe2brR5td4BNvhPMwJrZrFDsySA7lseOtdNr37AS1TU+h3+810GT8Os5evX9Dm13gEX5/mU3Y7uxctZt30GeSuXE2Pa6/mhvlfENflAk9Lcxptfo1HsObm+GR/v7ywiM3v/Y/1b8wgJCaafrfdzKT33iAwLMzT0hqMNr/GI9h3biXgwsZHYXYnSilyV65m3fQZZM9fyAXjxzH5g7dITO/raWlNQptf43aU1UrNyuWE3nWfp6Wck6qyMrZ+9Cnrp7+Ntaqa9NtvJuPfzxISHeVpaYagza9xO9YtGzEltsEc38rTUurl0JatbJg+gx2ffUHy0EGMeu5pkocOanYxB7T5NW6nevn3BF46zNMyzsoP//oPmd8soseUifT91XUk9e/X7IwPPhK3X9N8UEo5zD/Ye81/1Tuvc8vCOYS2bMln0+7muU7dmfvgI+z7YQV2Aw9P9TS65te4Fdve3WCzYe7svVNiIkJi7x4k9u7ByL88RsGOn9g2ew5f3ftHjhcWkTZhHN0nTSB50CWYfTi0nFdH79U0P068+wb24kLCHnjU01IaRVHWbrZ/Podts+dw7EAOaePH0W3iFXQcNhiLFy7q8dnQ3ZrmR8mt1xFy+28J9JFpvnNxdN8Btn8+h+2fz6FwVzZdxo6m26TxdB4xjIDznKHgLrT5NV6BvbiIY9eOJ3ru0mZ36m5J7kF2fDmX7Z/P4dDmraRmjKTbxPGkZowgMDTUY7q0+TVeQeWXs6hZv5rwp57ztBSXcjy/gB1z5rJt9hxy166n8/BhdJ80ngvGjiIoPNytWrT5NV5B6e/vJmjUOIJGjfW0FLdRUXyEHV/NY/vnc9j/wyo6DLmU7hPH0+WKMYRERbr8/tr8Go+jTlRw9PJhRH2xCFN4hKfleIQTx0rInDufbbPnsGfpctoNuJDukybQ9YqxhLno2DmfPqVX0zyoWbMSS1oPvzU+QEhUJL2vv4be119DVVkZmd8sYvvnXzHvD4+RlN6XbhOvIG3C5YS3SnCLHm1+jVuoXv49AYOGelqG1xAUHk7PKZPoOWUS1RUVZC38ju2zv2Th43+lVY9udJs4nm5XXk5kkut2Pupmv8blKJuNo1cMI/KtjzC39r1tvO7EWlVF9rffs/3zr9j59TfEpnSm28Qr6DZxPDEd2jc4P93s13gU67bNmFrGauM7gSUoiC7jMugyLgNbTQ17lixn2+w5vHrxMEb9/QkuuvUm4+5lWE4azVmoXv49gYMu87QMn8McEEDKyMtIGXkZMamdyduy3dD89cYejcupXr6EQN3fbxLF2XuIT+1saJ7a/BqXYtu/F1VRjrmL+06iaY4U7comLjXF0Dy1+TUuxdHkH9os98O7k8Ks3cSmdjI0T21+jUtxmN979+77AtXl5VQUHyGqXdvzJ24AesDPx1BlpVh3bEFEkMQkTK0SEYv3bZJRVivV8+egqioJ6HeRp+X4NEVZu2nZqQMmg0/70eb3EVR1NTVfzaT6k3cx9R+IKsjHnpeDKipAYuMxJSY5flq3xdSmbe3ztkhIiHt1WmuoXvA1Ve9Nx5TQmhaPPIl44T53X6JgVzZxBg/2gTa/16PsdqxLF1I147+YkjsR8s/XMCf/3PdT1hpU/iHsB3Ow5+Viz8uhZttGx/PDeUhoWJ3CoLZAqH0uEVGG9cVVTQ3V8+dQ9f4bmFonEfrIX7H0rndtiaaBFO3KJjZFm9+vsG5eR9WbLwMQ/OATWHqdaSaxBCBt2mFq0+6Ma8puRxUXYc87UFsw5FKzcumpggKlMLVO+mXh0Kado2CIjXfqUElVU0P1vM+p/OAtzG3bE/qnp7H09O149t5G4a5sUkYMNTxfbX4vxLZvN1Vvv4J9/x6Cbr4Ly+CRjTrdVUwmJC7ecRhmPQWHKi2pLQgchYFt22ZqFn7t6E6UlWFqlYgp8eeCQU4WDq0SQSmq5852mD65E2F/fhZL994G/PWa0ynalc0ld95qeL5OmV9EMoCXADPwplLqmbOkmwzMAvorpfTC/UZg3bKByr89RODUmwn407Mu7S9LRCTmiEjMXbufcU2dOIH9UO6proRt/x7sK5dhz8tFFeVj6toTAgIIe+p5LN16uUyjv6OUonBXNrGe6POLiBl4FRgJ5AJrRWSOUmrHaenCgXuB1Yar9COsP3xLwPXTCJww1aM6JCQEc8cUzB3PXFiirDWoykpMLdwblcYfOV5QiCnAQljLGMPzdqYteSGQrZTao5SqBj4GJtST7q/As0Clgfr8jxMVXr/nXSwB2vhuojAzizgXDPaBc+ZvA+TUeZ5b+9opRKQv0FYpNddAbX6JOlGBhHgu4KPGuyjM2k3cBZ4z/zkRERPwAvCgE2lvF5F1IrKusLCwqbdunpyoAG1+TS2umuYD58x/EKi7rjCp9rWThAPdgSUisg+4GJgjImcMLyulpiul0pVS6XFxcY1X3YzRNb+mLoW7sjxa868FUkSkg4gEAlOBOScvKqVKlFKxSqlkpVQysAoYr0f7G4c6UY6EhHlahsZLKN6zj9jOxm7oOcl5za+UsgL3AAuAncBMpdR2EXlKRMa7RJU/o5v9mjqUn6jE7KLTf5ya51dKzQPmnfbaE2dJO7TpsvwX3ezX1CWsZQzlRcWQYnztr1f4NQBVWYH1x8VQXg7BwUhgsON3UEjt72AICobgkNrHIWCxOL1+XikFlSfAzZtxNN5LeHwcZfkFLslbm78B1Hz8BvbDeZgS2kBVJfaqE1BZiaqqhKpKVO1zqk6gqqqg6gTYVZ2CIQSCgpFgRyEhtc9PXlfmADBbELP+t2gchMfHUVbgmpkx/S1zElVehm3ZQoKfm4G0jHf+fVarozCoLRQchUQlVJ6o/f1zwSFVlQT/7SUX/hUaX8NR82vzexTr4q8x97m4QcYHEIsFLOFImF4Rp2k44QnxFGbtdkneOoyXEyirFdv8WVjGTfG0FI2f0SI+ltIC1/T5tfmdwLZ6CZLQBlPHCzwtReNnhCfEc7ygyCV5a/OfB6UU1q8/wXL5NZ6WovFDXDnar81/Huw7N8OJckx9BnhaisYPCU+Id9lovzb/ebDOnYll7JRGRdLRaJpKaEw0laVl2GpqDM9bf6PPgf1QDvbMrZgHj/a0FI2fYjKZaBHbkuOFxvf7tfnPgfWbWVhGjEeC9Yo7jedoER9HqQv6/dr8Z0EdL8X2wyIsoyZ6WorGzwlPiOO4C/r92vxnwfrtHMz9BiIxsZ6WovFzXLXKT5u/HpS1Buv82VjG6ek9jecJj3fNiL82fz3YVnyHqU17TMmuiaCi0TSEFvGxLpnr1+Y/DaWUY3rvcr2UV+MdRCTE69F+d2DfvhGqqzD10ifLarwDPdrvJqxzZ2IZpxf1aLwHPdrvBux5B7Bn79CLejRehR7tdwPWeZ9iGTkBCQzytBSN5hQt4uM4XljkCPNmINr8tajSY9hWLNaLejReR0BQEIGhoVQcPWpovjqSTy3WRV9ivnAwEmX8gYj+hrJaIf8AqqTYMXYSHQ/R8bpF1QQc/f4iwmKM+35q8wOqphrrws8JeuwFT0vxOZTdDsWHUAf3onL3oA7ugYJciElAkrugyo7CkXw4VgTBoY7XY+IhOs7xODoeYuKRYB2u/FycHPFP6JJqWJ7a/IDtx8WY2nXC1K6jp6V4NUopKD3ys8kP7kHl7YPQcCSpI5LYAVOPi6B18hm1vFJ2KD0GR/JRRwvgSAFqx1rUEcdjLAGOQiDG0UpwFAxxEBPvyN/J8OfNFVeM+Pu9+R2Lej4h4Pq7PC3F61AVx382+cG9jsdKIW06Ikkd4dJxmBI7OBWcVMQEkTEQGYN06PrL+ygF5aWOAuFogaOlkLUZ+9ECKM6HvkMwj/TvRVfhccaP+Pu9+e1b14PdjqlXf09L8SjKakXl7YHaWl0d3APlpUhiB2jTEel9KaZxN0JkS8NrYRGBFpHQIhJpl/JLXYcPYJ/7rqH380VcEdHH781vnfuJY1GPHzYrld2O2rsDtXklavsa6JiGRLZEOvfANHQCtGzt+cVOZgvYbJ7V4AW0iI/l4Kathubp1+a35+7DvjeLwAf/5mkpbkMpBTnZ2DevQG1dBeFRmHpdgumepx19bG/DbAGb8SGsfI2IhHgyDV7f79fmt86diWXUlc1+CkopBYcPOAy/ZSVYAjD1HIDptseRuERPyzs3FgvYrJ5W4XFcsb7fb82vSo5iW72E4Bf/52kpLkMVHUZtXoF9ywqorkJ6DsB8w/2O0Xhf6eZYAsCqa35H/H7d5zcE68IvMF88DImI8rQUQ1ElxagtK7FvXgElR5AeF2GedDu07ez5/ntjMOuaHyA8PpYygw/v8Evzq+oqrIu+IOiJ5nEopjpeitq22mH4/BykW39MGdciHdIQs9nT8pqG2QJWbf7giAhs1dVUnzhBoEFHuDtlfhHJAF4CzMCbSqlnTrv+AHArYAUKgVuUUvsNUegCbD8swtQhFVNSsqelNBpVWYHavha1eQXqQBZyQW9Mg8Yhqb0QS4Cn5RmH2QJ2G0rZHWsF/BQRITzesdAnpn07Q/I8r/lFxAy8CowEcoG1IjJHKbWjTrKNQLpSqkJE7gT+CXhlALyTkXoCfv07T0tpMKq6CpW50WH47G1IxzSk3xBMN9yPBAZ7Wp5LEJGfp/ss/mt+OBnOy43mBy4EspVSewBE5GNgAnDK/Eqp7+ukXwXcYIg6F2DfvAZMZkw9+nlailMoqxWVtQW1ZQXqp41IUiek1wBMk29HQlp4Wp57ONn0b04tmkZg9EIfZ8zfBsip8zwXOFeMq2nAN/VdEJHbgdsB2rUzpvRqKNa5n2C53LsX9Zyx+CaujWNqbuwNSHiUp+W5n1Nz/f59eIrRh3YaOuAnIjcA6cCQ+q4rpaYD0wHS09ONjUzgBPYDu7Hn7CXwkuHuvvV58bnFN+7EEqAH/ag1v5tr/oNA2zrPk2pf+wUiMgJ4DBiilKoyRp6xWOd+imXURCQg0NNSAB9ffONO9Co/wLHQ5+iBXMPyc8b8a4EUEemAw/RTgevqJhCRPsDrQIZSyjWHiTcRdawY29plXrOox36iHPv0v0JluW8uvnEnepUf4Ojz56zfaFh+5zW/UsoqIvcAC3BM9b2tlNouIk8B65RSc4DngBbAp7Vf3gNKqfGGqTQA68IvMF8y3HsW9ezZCUHBmH/7tG8uvnEneq4fqF3o4+4+v1JqHjDvtNeeqPN4hGGKXIBjUc+XBP3lFU9LOYVERENlhTa+M+hVfsDJ0X7jVvn5xTfPtnQ+ppQ0TImemWGol8gYKDniaRW+gSVAmx/jR/ubvfmV3e4IyW3woZtNDqPcIhIqyx3BLjXnxmzRm3uAsNiWVBw9ht2g+AbN3vz2TashMAhTWm/j8szcQs3vr8eeu7fReYjJBC2ioMzYcMzNEl3zA2C2WAiJiuR4UbEh+TV78zsW9Vxj2Ci6PWsb1leeQiKjoexY0zKLiNZNfycQswWla36g9tBOg+b6m7X57fuysB88gHnAMGPy270D63/+guX2h5DYBKg60aT8JDIGVapr/vOiB/xOYeRx3c3a/Na5M7FkTDJkl5t9bybWl/6MZdrvMfXoD0HBUFnZtEwjYqBU1/znRc/zn8LIEf9ma351pAjb+hVYRjR9uYF9fzbWF/+E5eb7Tx3dLcEhqKbW/BHRKN3sPyfqRDnqeIme56/FEcLbmJq/2QbzsC6YjfnSkUiLiCblY8/Zi/WFR7Hc+DtMfS75+UJQcJOb/UTGwOGc86fzM9SJ46jsLajMjZC3F9p3QfoO9rQsryA8wbj1/c3S/KryBNbFXxH0t/9rWj4H92P918OYr7sLU/qgX14Mj0btWI894ORJM404XSYiBnSfH6g1fNYW1K4NkLcPkrsi3Qcg429t9gFWG0KL+DgKsxs/y1SXZml+29L5mLr0xNQqqdF5qEMHqHn+IcxTbsN80dAzrptGTkJZTKjiw5C1FfvJY6dsNoiJO3XUlMTEO56fPIYqMhoxOUJrSUQ0dj/u89dv+EuQ8bdpw5+F8IR4jhfqmr9eHIt6ZhJwx8ONz+NwLjXPPYR58s2YL6l/5bKYTMjwSWe+90QFHC2oPYOu0PH74N7awqEQKkohuQuYTEh4JJQYM2frK6iK46jszY4m/aF9SHJXTD0uAW14pzBylV+zM799wwoIDcfUpWej3q8KDjlq/Ak3Yr50dIPfLyGhEJKMJCbXn39NDepoEZQWw9EiZOiERun0Jeo1fM+BMEEbvqE4+vzGjPY3O/Nb585scKQeVXEc+87N2LdvQOXnYh47FfOQsS7RJwEBSHxriG/tkvy9BVVRhsrajNq1EQ4f+NnwV97uNfEUfJGTNb9SqskL15qV+e17MlH5efX20euiaqqxZ+/Avm099h0bUbl7MXVOw9StH6bLrsDsw1F9PUm9hu91KVzZTRu+iShlB5uVABOYAyxUlpUREtG0maxmZX7r159gHjMZsfzyz1J2O+rAbuzb12PfvgF71g4ksR2m7v2wXD0NU+duSKD+cjaGU4bP3Aj5Jw0/CK5Ma3aGVzVVoBRYAn+xFVvZbY6NRzar4+e0x8pmdUQistb+Pj2NrcaxwetUunrystvAHAAWC1P/dBdmS9Ot22zMby/Kx7ZpNcHT7nc8L8jDvm2Dw/A7NiIRUZjS+mC+bDwBdz+BhPlJ5FsXYS8+hFr0CRQeRDqkYeo9CDo0P8PXxf7tx3D8qMOggcFgMteuPFSnjIk5wLEc2WxxbEgyBzgqI3Oda5YAx/tPpbFgsgSclua09CbzqWZ+vyuM+XuajfmtX32MKbUb1o9fx75tA8pag6lbX8y9Lybg+rsc8/Aa48jPgeNHkVuewBQW7mk17qH8GKZx0yCkBaqmGhHAbDk1detrNBvzi8XiONWlbUcCMq5CEtvreHguRLr2h0P7UAs/RE24zWcN0CCCwxzRl0LDm8UsRbMxf8CNd3tagl8hIjBsMurLN1BLZsOwq5p/YRsUCpXlnlZhGM12Y4/G9YjJjGnsTai8PaiNSz0tx+VIcBiqssLTMgxDm1/TJCQoGNP421Hrv0ft3uppOa4lWNf8Gs0vkIhoTOOnYf/2Y1R+M96lWNvnby5o82sMQRLaYRp+Dfav3kS5MS6hstagsrdg37EW1dSwauejmdX8zWbAT+N5pHNPpKQY+5fTMV19LxLkmmPDlbJD3l5U5gbUnq0Q2wZatkKtmY906on0GYq0iDT8vhIchr0Z1fza/BpDkb5D4Vgh9nnvYDJ4ClAdyXcYftcGCApGUvthmvoA0iLKcT19OGrzcuyzXkY693IUAmFNWwL7C3TNr9GcnV9OAX4OwyY3aQpQlZeisjY5DF9RhqT0wTTuZiT2zINMJaQFcvEYVM9LHYXApy8hKX2QPkOQUAMWIjWzPr82v8ZwTk4B2me+BBuXOloDDUDVVKP2bHUYPj/HsXx4wFho09mp480kNBwZMNZRCGxahn3mi5DaF+k5EFNtK6FR1Nb8Ruyo8wa0+TUuQYKCMU24DfsnL6EiWyKdepwzvbLbIDfb0azft8NxYvEF6UjGrxu9X0DCIpCBl6N6D0L9tA71xWvY4pIwpfaBdhcg5oZ9/SUgEEQcm22awR4GbX6Ny5CIGExXTMP+5euYWkQhCW1/cV0pBUV5qMz1qKxN0CIKSe2LaeAVSKhxG68kLBLpNxzVaxBq307sP62FlXORjj2Q1L5ITILzmQWHOfr92vwazbmRVj9PAZquuQ8Jj0aVHUXt2ojK3ADWauSCvpiuvAOJjnetFksg0rkXdO6FKj2CytqEfdGHENICSe2DdOiOBIWcO5OTg37h0S7V6g60+TUuxzEFWIR99msQEQXFh5HOPTENnQyt2yPi/uUmEhGD9LsM1Wco5O1G7dqIfd23SNtUJLUvtDqLrmY06KfNr3EL0ncYxLZBrNWOKL0N7G+7CjGZICkFSUpBVZajdm/FvuobsNYgKb2Rzr1/sWZAgkNRleX4/nCfk+YXkQzgJcAMvKmUeua060HAe0A/oBi4Rim1z1ipGl9GRJD2F3haxjmR4DCk28WotIugOM/RGvjyNYhtc2qQ0K9qfhExA68CI4FcYK2IzFFK7aiTbBpwVCnVWUSmAs8C17hCsEbjakTE0UqJbYO6cBRq/0+nBgmJiofo5hEYxpma/0IgWym1B0BEPgYmAHXNPwF4svbxLOAVERGllDJQq0bjdsQSiHTqCZ16okqPYN+5Bknp42lZhuDMSEsboO5Wrdza1+pNo5SyAiVAy9MzEpHbRWSdiKwrNOjUEY3GXUhEDOaLMjA1ZGrQi3HrMKtSarpSKl0plR4X1zyaThqNr+KM+Q8CdVdnJNW+Vm8aEbEAkTgG/jQajZfijPnXAiki0kFEAoGpwJzT0swBfl37+CrgO93f12i8m/MO+CmlrCJyD7AAx1Tf20qp7SLyFLBOKTUHeAt4X0SygSM4CgiNRuPFODXPr5SaB8w77bUn6jyuBK42VppGo3ElOoyXRuOnaPNrNH6KNr9G46eIpwblRaQQ2F/PpVigyM1yGorW2HS8XR80D43tlVL1LqrxmPnPhoisU0qle1rHudAam46364Pmr1E3+zUaP0WbX6PxU7zR/NM9LcAJtMam4+36oJlr9Lo+v0ajcQ/eWPNrNBo3oM2v0fgpHjO/iGSISKaIZIvIw/VcDxKRT2qvrxaRZC/U+ICI7BCRLSKyWETae5O+Oukmi4gSEbdPWzmjUUSm1H6O20Xkf96mUUTaicj3IrKx9n891s363haRAhHZdpbrIiIv1+rfIiJ9ncpYKeX2Hxy7A3cDHYFAYDOQdlqau4DXah9PBT7xQo3DgNDax3e6U6Mz+mrThQPLgFVAuhd+hinARiC69nm8F2qcDtxZ+zgN2OdmjYOBvsC2s1wfC3wDCHAxsNqZfD1V85+KC6iUqgZOxgWsywTg3drHs4Dh4t4D0s6rUSn1vVLqZCjXVTgCnXiNvlr+iiOgaqUbtZ3EGY23Aa8qpY4CKKUKvFCjAk4e9xsJ5LlRH0qpZTi2yp+NCcB7ysEqIEpEWp8vX0+Z37C4gC7EGY11mYaj9HUX59VX2/xrq5Sa60ZddXHmM0wFUkXkRxFZVRsm3p04o/FJ4AYRycWxtf237pHmNA39rgL60A5DEJEbgHRgiKe1nEQcx828ANzkYSnnw4Kj6T8UR8tpmYj0UEod86So07gWeEcp9S8RGYAjcE13pZTd08Kagqdqfl+IC+iMRkRkBPAYMF4pVeUmbXB+feFAd2CJiOzD0Rec4+ZBP2c+w1xgjlKqRim1F9iFozBwF85onAbMBFBKrQSCcWyo8Rac+q6egTsHLuoMUFiAPUAHfh5k6XZamrv55YDfTC/U2AfHYFGKN36Gp6VfgvsH/Jz5DDOAd2sfx+Jovrb0Mo3fADfVPu6Ko88vbv4skzn7gN84fjngt8apPN35B5wmeCyOUn438Fjta0/hqEHBUbp+CmQDa4COXqjxWyAf2FT7M8eb9J2W1u3md/IzFBzdkx3AVmCqF2pMA36sLRg2AaPcrO8j4BBQg6OlNA24A7ijzmf4aq3+rc7+n/XyXo3GT9Er/DQaP0WbX6PxU7T5NRo/RZtfo/FTtPk1Gj9Fm1+j8VO0+TUaP+X/Abem6TLSC0KwAAAAAElFTkSuQmCC\n"
+          },
+          "metadata": {
+            "needs_background": "light"
+          }
+        }
+      ],
+      "source": [
+        "plot(nodes_coordinates[resulting_traj])"
+      ]
+    }
+  ],
+  "metadata": {
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    },
+    "language_info": {
+      "name": "python"
+    },
+    "colab": {
+      "provenance": []
+    },
+    "accelerator": "GPU",
+    "gpuClass": "standard"
+  },
+  "nbformat": 4,
+  "nbformat_minor": 5
+}

environment.yml

@@ -0,0 +1,113 @@
+name: cleanrl
+channels:
+  - conda-forge
+dependencies:
+  - _libgcc_mutex=0.1
+  - _openmp_mutex=4.5
+  - asttokens=2.1.0
+  - backcall=0.2.0
+  - backports=1.0
+  - backports.functools_lru_cache=1.6.4
+  - bzip2=1.0.8
+  - ca-certificates=2022.9.24
+  - debugpy=1.6.3
+  - decorator=5.1.1
+  - entrypoints=0.4
+  - executing=1.2.0
+  - ipykernel=6.17.0
+  - ipython=8.6.0
+  - jedi=0.18.1
+  - jupyter_client=7.4.4
+  - jupyter_core=4.11.2
+  - ld_impl_linux-64=2.39
+  - libffi=3.4.2
+  - libgcc-ng=12.2.0
+  - libgomp=12.2.0
+  - libnsl=2.0.0
+  - libsodium=1.0.18
+  - libsqlite=3.39.4
+  - libstdcxx-ng=12.2.0
+  - libuuid=2.32.1
+  - libzlib=1.2.13
+  - matplotlib-inline=0.1.6
+  - ncurses=6.3
+  - nest-asyncio=1.5.6
+  - openssl=3.0.7
+  - packaging=21.3
+  - parso=0.8.3
+  - pexpect=4.8.0
+  - pickleshare=0.7.5
+  - pip=22.3.1
+  - prompt-toolkit=3.0.32
+  - psutil=5.9.4
+  - ptyprocess=0.7.0
+  - pure_eval=0.2.2
+  - pygments=2.13.0
+  - pyparsing=3.0.9
+  - python=3.8.13
+  - python-dateutil=2.8.2
+  - python_abi=3.8
+  - pyzmq=24.0.1
+  - readline=8.1.2
+  - setuptools=65.5.1
+  - six=1.16.0
+  - sqlite=3.39.4
+  - stack_data=0.6.0
+  - tk=8.6.12
+  - tornado=6.2
+  - traitlets=5.5.0
+  - wcwidth=0.2.5
+  - wheel=0.38.3
+  - xz=5.2.6
+  - zeromq=4.3.4
+  - pip:
+    - absl-py==1.3.0
+    - cachetools==5.2.0
+    - certifi==2022.9.24
+    - cfgv==3.3.1
+    - charset-normalizer==2.1.1
+    - cloudpickle==2.2.0
+    - distlib==0.3.6
+    - filelock==3.8.0
+    - google-auth==2.14.1
+    - google-auth-oauthlib==0.4.6
+    - grpcio==1.50.0
+    - gym==0.23.1
+    - gym-notices==0.0.8
+    - identify==2.5.8
+    - idna==3.4
+    - importlib-metadata==5.0.0
+    - llvmlite==0.39.1
+    - markdown==3.4.1
+    - markupsafe==2.1.1
+    - nodeenv==1.7.0
+    - numba==0.56.4
+    - numpy==1.23.4
+    - nvidia-cublas-cu11==11.10.3.66
+    - nvidia-cuda-nvrtc-cu11==11.7.99
+    - nvidia-cuda-runtime-cu11==11.7.99
+    - nvidia-cudnn-cu11==8.5.0.96
+    - oauthlib==3.2.2
+    - pillow==9.3.0
+    - platformdirs==2.5.3
+    - pre-commit==2.20.0
+    - protobuf==3.20.3
+    - pyasn1==0.4.8
+    - pyasn1-modules==0.2.8
+    - pygame==2.1.0
+    - pyyaml==6.0
+    - requests==2.28.1
+    - requests-oauthlib==1.3.1
+    - rsa==4.9
+    - tensorboard==2.11.0
+    - tensorboard-data-server==0.6.1
+    - tensorboard-plugin-wit==1.8.1
+    - toml==0.10.2
+    - torch==1.13.0
+    - torchvision==0.14.0
+    - typing-extensions==4.4.0
+    - urllib3==1.26.12
+    - virtualenv==20.16.6
+    - werkzeug==2.2.2
+    - zipp==3.10.0
+prefix: /opt/conda/envs/cleanrl

envs/cvrp_vector_env.py

@@ -0,0 +1,144 @@
+import gym
+import numpy as np
+from gym import spaces
+
+from .vrp_data import VRPDataset
+
+
+def assign_env_config(self, kwargs):
+    """
+    Set self.key = value, for each key in kwargs
+    """
+    for key, value in kwargs.items():
+        setattr(self, key, value)
+
+
+def dist(loc1, loc2):
+    return ((loc1[:, 0] - loc2[:, 0]) ** 2 + (loc1[:, 1] - loc2[:, 1]) ** 2) ** 0.5
+
+
+class CVRPVectorEnv(gym.Env):
+    def __init__(self, *args, **kwargs):
+        self.max_nodes = 50
+        self.capacity_limit = 40
+        self.n_traj = 50
+        # if eval_data==True, load from 'test' set, the '0'th data
+        self.eval_data = False
+        self.eval_partition = "test"
+        self.eval_data_idx = 0
+        self.demand_limit = 10
+        assign_env_config(self, kwargs)
+
+        obs_dict = {"observations": spaces.Box(low=0, high=1, shape=(self.max_nodes, 2))}
+        obs_dict["depot"] = spaces.Box(low=0, high=1, shape=(2,))
+        obs_dict["demand"] = spaces.Box(low=0, high=1, shape=(self.max_nodes,))
+        obs_dict["action_mask"] = spaces.MultiBinary(
+            [self.n_traj, self.max_nodes + 1]
+        )  # 1: OK, 0: cannot go
+        obs_dict["last_node_idx"] = spaces.MultiDiscrete([self.max_nodes + 1] * self.n_traj)
+        obs_dict["current_load"] = spaces.Box(low=0, high=1, shape=(self.n_traj,))
+
+        self.observation_space = spaces.Dict(obs_dict)
+        self.action_space = spaces.MultiDiscrete([self.max_nodes + 1] * self.n_traj)
+        self.reward_space = None
+
+        self.reset()
+
+    def seed(self, seed):
+        np.random.seed(seed)
+
+    def _STEP(self, action):
+
+        self._go_to(action)  # Go to node 'action', modify the reward
+        self.num_steps += 1
+        self.state = self._update_state()
+
+        # need to revisit the first node after visited all other nodes
+        self.done = (action == 0) & self.is_all_visited()
+
+        return self.state, self.reward, self.done, self.info
+
+    # Euclidean cost function
+    def cost(self, loc1, loc2):
+        return dist(loc1, loc2)
+
+    def is_all_visited(self):
+        # assumes no repetition in the first `max_nodes` steps
+        return self.visited[:, 1:].all(axis=1)
+
+    def _update_state(self):
+        obs = {"observations": self.nodes[1:]}  # n x 2 array
+        obs["depot"] = self.nodes[0]
+        obs["action_mask"] = self._update_mask()
+        obs["demand"] = self.demands
+        obs["last_node_idx"] = self.last
+        obs["current_load"] = self.load
+        return obs
+
+    def _update_mask(self):
+        # Only allow to visit unvisited nodes
+        action_mask = ~self.visited
+
+        # can only visit depot when last node is not depot or all visited
+        action_mask[:, 0] |= self.last != 0
+        action_mask[:, 0] |= self.is_all_visited()
+
+        # not allow visit nodes with higher demand than capacity
+        action_mask[:, 1:] &= self.demands <= (
+            self.load.reshape(-1, 1) + 1e-5
+        )  # to handle the floating point subtraction precision
+
+        return action_mask
+
+    def _RESET(self):
+        self.visited = np.zeros((self.n_traj, self.max_nodes + 1), dtype=bool)
+        self.visited[:, 0] = True
+        self.num_steps = 0
+        self.last = np.zeros(self.n_traj, dtype=int)  # idx of the last elem
+        self.load = np.ones(self.n_traj, dtype=float)  # current load
+
+        if self.eval_data:
+            self._load_orders()
+        else:
+            self._generate_orders()
+        self.state = self._update_state()
+        self.info = {}
+        self.done = np.array([False] * self.n_traj)
+        return self.state
+
+    def _load_orders(self):
+        data = VRPDataset[self.eval_partition, self.max_nodes, self.eval_data_idx]
+        self.nodes = np.concatenate((data["depot"][None, ...], data["loc"]))
+        self.demands = data["demand"]
+        self.demands_with_depot = self.demands.copy()
+
+    def _generate_orders(self):
+        self.nodes = np.random.rand(self.max_nodes + 1, 2)
+        self.demands = (
+            np.random.randint(low=1, high=self.demand_limit, size=self.max_nodes)
+            / self.capacity_limit
+        )
+        self.demands_with_depot = self.demands.copy()
+
+    def _go_to(self, destination):
+        dest_node = self.nodes[destination]
+        dist = self.cost(dest_node, self.nodes[self.last])
+        self.last = destination
+        self.load[destination == 0] = 1
+        self.load[destination > 0] -= self.demands[destination[destination > 0] - 1]
+        self.demands_with_depot[destination[destination > 0] - 1] = 0
+        self.visited[np.arange(self.n_traj), destination] = True
+        self.reward = -dist
+
+    def step(self, action):
+        # return last state after done,
+        # for the sake of PPO's abuse of ff on done observation
+        # see https://github.com/opendilab/DI-engine/issues/497
+        # Not needed for CleanRL
+        # if self.done.all():
+        #     return self.state, self.reward, self.done, self.info
+
+        return self._STEP(action)
+
+    def reset(self):
+        return self._RESET()

envs/tsp_data.py

@@ -0,0 +1,43 @@
+import logging
+import pickle
+
+root_dir = "/home/cwan5/OR/attention-learn-to-route/data/tsp/"
+
+
+def load(filename, root_dir=root_dir):
+    return pickle.load(open(root_dir + filename, "rb"))
+
+
+file_catalog = {
+    "test": {
+        20: "tsp20_test_seed1234.pkl",
+        50: "tsp50_test_seed1234.pkl",
+        100: "tsp100_test_seed1234.pkl",
+    },
+    "eval": {
+        20: "tsp20_validation_seed4321.pkl",
+        50: "tsp50_validation_seed4321.pkl",
+        100: "tsp100_validation_seed4321.pkl",
+    },
+}
+
+
+class lazyClass:
+    data = {
+        "test": {},
+        "eval": {},
+    }
+
+    def __getitem__(self, index):
+        partition, nodes, idx = index
+        if not (partition in self.data) or not (nodes in self.data[partition]):
+            logging.warning(
+                f"Data sepecified by ({partition}, {nodes}) was not initialized. Attepmting to load it for the first time."
+            )
+            data = load(file_catalog[partition][nodes])
+            self.data[partition][nodes] = [tuple(instance) for instance in data]
+
+        return self.data[partition][nodes][idx]
+
+
+TSPDataset = lazyClass()

envs/tsp_vector_env.py

@@ -0,0 +1,113 @@
+import gym
+import numpy as np
+from gym import spaces
+
+from .tsp_data import TSPDataset
+
+
+def assign_env_config(self, kwargs):
+    """
+    Set self.key = value, for each key in kwargs
+    """
+    for key, value in kwargs.items():
+        setattr(self, key, value)
+
+
+def dist(loc1, loc2):
+    return ((loc1[:, 0] - loc2[:, 0]) ** 2 + (loc1[:, 1] - loc2[:, 1]) ** 2) ** 0.5
+
+
+class TSPVectorEnv(gym.Env):
+    def __init__(self, *args, **kwargs):
+        self.max_nodes = 50
+        self.n_traj = 50
+        # if eval_data==True, load from 'test' set, the '0'th data
+        self.eval_data = False
+        self.eval_partition = "test"
+        self.eval_data_idx = 0
+        assign_env_config(self, kwargs)
+
+        obs_dict = {"observations": spaces.Box(low=0, high=1, shape=(self.max_nodes, 2))}
+        obs_dict["action_mask"] = spaces.MultiBinary(
+            [self.n_traj, self.max_nodes]
+        )  # 1: OK, 0: cannot go
+        obs_dict["first_node_idx"] = spaces.MultiDiscrete([self.max_nodes] * self.n_traj)
+        obs_dict["last_node_idx"] = spaces.MultiDiscrete([self.max_nodes] * self.n_traj)
+        obs_dict["is_initial_action"] = spaces.Discrete(1)
+
+        self.observation_space = spaces.Dict(obs_dict)
+        self.action_space = spaces.MultiDiscrete([self.max_nodes] * self.n_traj)
+        self.reward_space = None
+
+        self.reset()
+
+    def seed(self, seed):
+        np.random.seed(seed)
+
+    def reset(self):
+        self.visited = np.zeros((self.n_traj, self.max_nodes), dtype=bool)
+        self.num_steps = 0
+        self.last = np.zeros(self.n_traj, dtype=int)  # idx of the first elem
+        self.first = np.zeros(self.n_traj, dtype=int)  # idx of the first elem
+
+        if self.eval_data:
+            self._load_orders()
+        else:
+            self._generate_orders()
+        self.state = self._update_state()
+        self.info = {}
+        self.done = False
+        return self.state
+
+    def _load_orders(self):
+        self.nodes = np.array(TSPDataset[self.eval_partition, self.max_nodes, self.eval_data_idx])
+
+    def _generate_orders(self):
+        self.nodes = np.random.rand(self.max_nodes, 2)
+
+    def step(self, action):
+
+        self._go_to(action)  # Go to node 'action', modify the reward
+        self.num_steps += 1
+        self.state = self._update_state()
+
+        # need to revisit the first node after visited all other nodes
+        self.done = (action == self.first) & self.is_all_visited()
+
+        return self.state, self.reward, self.done, self.info
+
+    # Euclidean cost function
+    def cost(self, loc1, loc2):
+        return dist(loc1, loc2)
+
+    def is_all_visited(self):
+        # assumes no repetition in the first `max_nodes` steps
+        return self.visited[:, :].all(axis=1)
+
+    def _go_to(self, destination):
+        dest_node = self.nodes[destination]
+        if self.num_steps != 0:
+            dist = self.cost(dest_node, self.nodes[self.last])
+        else:
+            dist = np.zeros(self.n_traj)
+            self.first = destination
+
+        self.last = destination
+
+        self.visited[np.arange(self.n_traj), destination] = True
+        self.reward = -dist
+
+    def _update_state(self):
+        obs = {"observations": self.nodes}  # n x 2 array
+        obs["action_mask"] = self._update_mask()
+        obs["first_node_idx"] = self.first
+        obs["last_node_idx"] = self.last
+        obs["is_initial_action"] = self.num_steps == 0
+        return obs
+
+    def _update_mask(self):
+        # Only allow to visit unvisited nodes
+        action_mask = ~self.visited
+        # can only visit first node when all nodes have been visited
+        action_mask[np.arange(self.n_traj), self.first] |= self.is_all_visited()
+        return action_mask

envs/vrp_data.py

@@ -0,0 +1,57 @@
+import logging
+import pickle
+
+import numpy as np
+
+root_dir = "/home/cwan5/OR/attention-learn-to-route/data/vrp/"
+
+
+def load(filename, root_dir=root_dir):
+    return pickle.load(open(root_dir + filename, "rb"))
+
+
+file_catalog = {
+    "test": {
+        20: "vrp20_test_seed1234.pkl",
+        50: "vrp50_test_seed1234.pkl",
+        100: "vrp100_test_seed1234.pkl",
+    },
+    "eval": {
+        20: "vrp20_validation_seed4321.pkl",
+        50: "vrp50_validation_seed4321.pkl",
+        100: "vrp100_validation_seed4321.pkl",
+    },
+}
+
+
+def make_instance(args):
+    depot, loc, demand, capacity, *args = args
+    grid_size = 1
+    if len(args) > 0:
+        depot_types, customer_types, grid_size = args
+    return {
+        "loc": np.array(loc) / grid_size,
+        "demand": np.array(demand) / capacity,
+        "depot": np.array(depot) / grid_size,
+    }
+
+
+class lazyClass:
+    data = {
+        "test": {},
+        "eval": {},
+    }
+
+    def __getitem__(self, index):
+        partition, nodes, idx = index
+        if not (partition in self.data) or not (nodes in self.data[partition]):
+            logging.warning(
+                f"Data sepecified by ({partition}, {nodes}) was not initialized. Attepmting to load it for the first time."
+            )
+            data = load(file_catalog[partition][nodes])
+            self.data[partition][nodes] = [make_instance(instance) for instance in data]
+
+        return self.data[partition][nodes][idx]
+
+
+VRPDataset = lazyClass()

models/attention_model_wrapper.py

@@ -0,0 +1,152 @@
+import torch
+
+from .nets.attention_model.attention_model import *
+
+
+class Problem:
+    def __init__(self, name):
+        self.NAME = name
+
+
+class Backbone(nn.Module):
+    def __init__(
+        self,
+        embedding_dim=128,
+        problem_name="tsp",
+        n_encode_layers=3,
+        tanh_clipping=10.0,
+        n_heads=8,
+        device="cpu",
+    ):
+        super(Backbone, self).__init__()
+        self.device = device
+        self.problem = Problem(problem_name)
+        self.embedding = AutoEmbedding(self.problem.NAME, {"embedding_dim": embedding_dim})
+
+        self.encoder = GraphAttentionEncoder(
+            n_heads=n_heads,
+            embed_dim=embedding_dim,
+            n_layers=n_encode_layers,
+        )
+
+        self.decoder = Decoder(
+            embedding_dim, self.embedding.context_dim, n_heads, self.problem, tanh_clipping
+        )
+
+    def forward(self, obs):
+        state = stateWrapper(obs, device=self.device, problem=self.problem.NAME)
+        input = state.states["observations"]
+        embedding = self.embedding(input)
+        encoded_inputs, _ = self.encoder(embedding)
+
+        # decoding
+        cached_embeddings = self.decoder._precompute(encoded_inputs)
+        logits, glimpse = self.decoder.advance(cached_embeddings, state)
+
+        return logits, glimpse
+
+    def encode(self, obs):
+        state = stateWrapper(obs, device=self.device, problem=self.problem.NAME)
+        input = state.states["observations"]
+        embedding = self.embedding(input)
+        encoded_inputs, _ = self.encoder(embedding)
+        cached_embeddings = self.decoder._precompute(encoded_inputs)
+        return cached_embeddings
+
+    def decode(self, obs, cached_embeddings):
+        state = stateWrapper(obs, device=self.device, problem=self.problem.NAME)
+        logits, glimpse = self.decoder.advance(cached_embeddings, state)
+
+        return logits, glimpse
+
+
+class Actor(nn.Module):
+    def __init__(self):
+        super(Actor, self).__init__()
+
+    def forward(self, x):
+        logits = x[0]  # .squeeze(1) # not needed for pomo
+        return logits
+
+
+class Critic(nn.Module):
+    def __init__(self, *args, **kwargs):
+        super(Critic, self).__init__()
+        hidden_size = kwargs["hidden_size"]
+        self.mlp = nn.Sequential(
+            nn.Linear(hidden_size, hidden_size), nn.ReLU(), nn.Linear(hidden_size, 1)
+        )
+
+    def forward(self, x):
+        out = self.mlp(x[1])  # B x T x h_dim --mlp--> B x T X 1
+        return out
+
+
+class Agent(nn.Module):
+    def __init__(self, embedding_dim=128, device="cpu", name="tsp"):
+        super().__init__()
+        self.backbone = Backbone(embedding_dim=embedding_dim, device=device, problem_name=name)
+        self.critic = Critic(hidden_size=embedding_dim)
+        self.actor = Actor()
+
+    def forward(self, x):  # only actor
+        x = self.backbone(x)
+        logits = self.actor(x)
+        action = logits.max(2)[1]
+        return action, logits
+
+    def get_value(self, x):
+        x = self.backbone(x)
+        return self.critic(x)
+
+    def get_action_and_value(self, x, action=None):
+        x = self.backbone(x)
+        logits = self.actor(x)
+        probs = torch.distributions.Categorical(logits=logits)
+        if action is None:
+            action = probs.sample()
+        return action, probs.log_prob(action), probs.entropy(), self.critic(x)
+
+    def get_value_cached(self, x, state):
+        x = self.backbone.decode(x, state)
+        return self.critic(x)
+
+    def get_action_and_value_cached(self, x, action=None, state=None):
+        if state is None:
+            state = self.backbone.encode(x)
+            x = self.backbone.decode(x, state)
+        else:
+            x = self.backbone.decode(x, state)
+        logits = self.actor(x)
+        probs = torch.distributions.Categorical(logits=logits)
+        if action is None:
+            action = probs.sample()
+        return action, probs.log_prob(action), probs.entropy(), self.critic(x), state
+
+
+class stateWrapper:
+    """
+    from dict of numpy arrays to an object that supplies function and data
+    """
+
+    def __init__(self, states, device, problem="tsp"):
+        self.device = device
+        self.states = {k: torch.tensor(v, device=self.device) for k, v in states.items()}
+        if problem == "tsp":
+            self.is_initial_action = self.states["is_initial_action"].to(torch.bool)
+            self.first_a = self.states["first_node_idx"]
+        elif problem == "cvrp":
+            input = {
+                "loc": self.states["observations"],
+                "depot": self.states["depot"].squeeze(-1),
+                "demand": self.states["demand"],
+            }
+            self.states["observations"] = input
+            self.VEHICLE_CAPACITY = 0
+            self.used_capacity = -self.states["current_load"]
+
+    def get_current_node(self):
+        return self.states["last_node_idx"]
+
+    def get_mask(self):
+        return (1 - self.states["action_mask"]).to(torch.bool)

models/nets/attention_model/attention_model.py

@@ -0,0 +1,101 @@
+from torch import nn
+
+from ...nets.attention_model.decoder import Decoder
+from ...nets.attention_model.embedding import AutoEmbedding
+from ...nets.attention_model.encoder import GraphAttentionEncoder
+
+
+class AttentionModel(nn.Module):
+    r"""
+    The Attention Model from Kool et al.,2019.
+    `Link to paper <https://arxiv.org/abs/1803.08475>`_.
+
+    For an instance :math:`s`,
+
+    .. math::
+        \log(p_\theta(\pi|s)),\pi = \mathrm{AttentionModel}(s)
+
+    The following is executed:
+
+    .. math::
+        \begin{aligned}
+        \pmb{x} &= \mathrm{Embedding}(s)                                      \\
+        \pmb{h} &=  \mathrm{Encoder}(\pmb{x})                           \\
+        \{\log(\pmb{p}_t)\},\pi &=  \mathrm{Decoder}(s, \pmb{h}) \\
+        \log(p_\theta(\pi|s)) &= \sum\nolimits_t\log(\pmb{p}_{t,\pi_t})
+        \end{aligned}
+    where :math:`\pmb{h}_i` is the node embedding for each node :math:`i` in the graph.
+
+    In a nutshell, :math:`\mathrm{Embedding}` is a linear projection.
+    The :math:`\mathrm{Encoder}` is a transformer.
+    The :math:`\mathrm{Decoder}` uses (multi-head) attentions.
+    The policy (sequence of action) :math:`\pi` is decoded autoregressively.
+    The log-likelihood :math:`\log(p_\theta(\pi|s))` of the policy is also returned.
+
+    .. seealso::
+        The definition of :math:`\mathrm{Embedding}`, :math:`\mathrm{Encoder}`, and
+        :math:`\mathrm{Decoder}` can be found in the
+        :mod:`.embedding`, :mod:`.encoder`, :mod:`.decoder` modules.
+
+    Args:
+        embedding_dim : the dimension of the embedded inputs
+        hidden_dim : the dimension of the hidden state of the encoder
+        problem : an object defining the state of the problem
+        n_encode_layers: number of encoder layers
+        tanh_clipping : the clipping scale for the decoder
+    Inputs: inputs
+        * **inputs**: problem instance :math:`s`. [batch, graph_size, input_dim]
+    Outputs: ll, pi
+        * **ll**: :math:`\log(p_\theta(\pi|s))` the log-likelihood of the policy. [batch, T]
+        * **pi**: the policy generated :math:`\pi`. [batch, T]
+    """
+
+    def __init__(
+        self,
+        embedding_dim,
+        hidden_dim,
+        problem,
+        n_encode_layers=2,
+        tanh_clipping=10.0,
+        normalization="batch",
+        n_heads=8,
+    ):
+        super(AttentionModel, self).__init__()
+
+        self.embedding_dim = embedding_dim
+        self.hidden_dim = hidden_dim
+        self.n_encode_layers = n_encode_layers
+        self.decode_type = None
+        self.n_heads = n_heads
+
+        self.problem = problem
+
+        self.embedding = AutoEmbedding(problem.NAME, {"embedding_dim": embedding_dim})
+        step_context_dim = self.embedding.context_dim
+
+        self.encoder = GraphAttentionEncoder(
+            n_heads=n_heads,
+            embed_dim=embedding_dim,
+            n_layers=self.n_encode_layers,
+        )
+        self.decoder = Decoder(embedding_dim, step_context_dim, n_heads, problem, tanh_clipping)
+
+    def set_decode_type(self, decode_type):
+        self.decoder.set_decode_type(decode_type)
+
+    def forward(self, input):
+        embedding = self.embedding(input)
+        encoded_inputs, _ = self.encoder(embedding)
+        _log_p, pi = self.decoder(input, encoded_inputs)
+        ll = self._calc_log_likelihood(_log_p, pi)
+        return ll, pi
+
+    def _calc_log_likelihood(self, _log_p, pi):
+
+        # Get log_p corresponding to selected actions
+        log_p = _log_p.gather(2, pi.unsqueeze(-1)).squeeze(-1)
+
+        assert (log_p > -1000).data.all(), "Logprobs should not be -inf, check sampling procedure!"
+
+        # Calculate log_likelihood
+        return log_p.sum(1)

models/nets/attention_model/context.py

@@ -0,0 +1,232 @@
+"""
+Problem specific global embedding for global context.
+"""
+
+import torch
+from torch import nn
+
+
+def AutoContext(problem_name, config):
+    """
+    Automatically select the corresponding module according to ``problem_name``
+    """
+    mapping = {
+        "tsp": TSPContext,
+        "cvrp": VRPContext,
+        "sdvrp": VRPContext,
+        "pctsp": PCTSPContext,
+        "op": OPContext,
+    }
+    embeddingClass = mapping[problem_name]
+    embedding = embeddingClass(**config)
+    return embedding
+
+
+def _gather_by_index(source, index):
+    """
+    target[i,1,:] = source[i,index[i],:]
+    Inputs:
+        source: [B x H x D]
+        index: [B x 1] or [B]
+    Outpus:
+        target: [B x 1 x D]
+    """
+
+    target = torch.gather(source, 1, index.unsqueeze(-1).expand(-1, -1, source.size(-1)))
+    return target
+
+
+class PrevNodeContext(nn.Module):
+    """
+    Abstract class for Context.
+    Any subclass, by default, will return a concatenation of
+
+    +---------------------+-----------------+
+    | prev_node_embedding | state_embedding |
+    +---------------------+-----------------+
+
+    The ``prev_node_embedding`` is the node embedding of the last visited node.
+    It is obtained by ``_prev_node_embedding`` method.
+    It requires ``state.get_current_node()`` to provide the index of the last visited node.
+
+    The ``state_embedding`` is the global context we want to include, such as the remaining capacity in VRP.
+    It is obtained by ``_state_embedding`` method.
+    It is not implemented. The subclass of this abstract class needs to implement this method.
+
+    Args:
+        problem: an object defining the settings of the environment
+        context_dim: the dimension of the output
+    Inputs: embeddings, state
+        * **embeddings** : [batch x graph size x embed dim]
+        * **state**: An object providing observations in the environment. \
+                    Needs to supply ``state.get_current_node()``
+    Outputs: context_embedding
+        * **context_embedding**: [batch x 1 x context_dim]
+
+    """
+
+    def __init__(self, context_dim):
+        super(PrevNodeContext, self).__init__()
+        self.context_dim = context_dim
+
+    def _prev_node_embedding(self, embeddings, state):
+        current_node = state.get_current_node()
+        prev_node_embedding = _gather_by_index(embeddings, current_node)
+        return prev_node_embedding
+
+    def _state_embedding(self, embeddings, state):
+        raise NotImplementedError("Please implement the embedding for your own problem.")
+
+    def forward(self, embeddings, state):
+        prev_node_embedding = self._prev_node_embedding(embeddings, state)
+        state_embedding = self._state_embedding(embeddings, state)
+        # Embedding of previous node + remaining capacity
+        context_embedding = torch.cat((prev_node_embedding, state_embedding), -1)
+        return context_embedding
+
+
+class TSPContext(PrevNodeContext):
+    """
+    Context node embedding for traveling salesman problem.
+    Return a concatenation of
+
+    +------------------------+---------------------------+
+    | first node's embedding | previous node's embedding |
+    +------------------------+---------------------------+
+
+    .. note::
+        Subclass of :class:`.PrevNodeContext`. The argument, inputs, outputs follow the same specification.
+
+        In addition to supplying  ``state.get_current_node()`` for the index of the previous visited node.
+        The input ``state`` needs to supply ``state.first_a`` for the index of the first visited node.
+
+    .. warning::
+        The official implementation concatenates the context with [first node, prev node].
+        However, if we follow the paper closely, it should instead be [prev node, first node].
+        Please check ``forward_code`` and ``forward_paper`` for the different implementations.
+        We follow the official implementation in this class.
+    """
+
+    def __init__(self, context_dim):
+        super(TSPContext, self).__init__(context_dim)
+        self.W_placeholder = nn.Parameter(torch.Tensor(self.context_dim).uniform_(-1, 1))
+
+    def _state_embedding(self, embeddings, state):
+        first_node = state.first_a
+        state_embedding = _gather_by_index(embeddings, first_node)
+        return state_embedding
+
+    def forward_paper(self, embeddings, state):
+        batch_size = embeddings.size(0)
+        if state.i.item() == 0:
+            context_embedding = self.W_placeholder[None, None, :].expand(
+                batch_size, 1, self.W_placeholder.size(-1)
+            )
+        else:
+            context_embedding = super().forward(embeddings, state)
+        return context_embedding
+
+    def forward_code(self, embeddings, state):
+        batch_size = embeddings.size(0)
+        if state.i.item() == 0:
+            context_embedding = self.W_placeholder[None, None, :].expand(
+                batch_size, 1, self.W_placeholder.size(-1)
+            )
+        else:
+            context_embedding = _gather_by_index(
+                embeddings, torch.cat([state.first_a, state.get_current_node()], -1)
+            ).view(batch_size, 1, -1)
+        return context_embedding
+
+    def forward_vectorized(self, embeddings, state):
+        n_queries = state.states["first_node_idx"].shape[-1]
+        batch_size = embeddings.size(0)
+        out_shape = (batch_size, n_queries, self.context_dim)
+
+        switch = state.is_initial_action  # tensor, 1 if is initial action
+        switch = switch[:, None, None].expand(out_shape)  # mask for each data
+
+        # only used for the first action
+        placeholder_embedding = self.W_placeholder[None, None, :].expand(out_shape)
+        # used after first action
+        indexes = torch.stack([state.first_a, state.get_current_node()], -1).flatten(-2)
+        normal_embedding = _gather_by_index(embeddings, indexes).view(out_shape)
+
+        context_embedding = switch * placeholder_embedding + (~switch) * normal_embedding
+        return context_embedding
+
+    def forward(self, embeddings, state):
+        return self.forward_vectorized(embeddings, state)
+
+
+class VRPContext(PrevNodeContext):
+    """
+    Context node embedding for capacitated vehicle routing problem.
+    Return a concatenation of
+
+    +---------------------------+--------------------+
+    | previous node's embedding | remaining capacity |
+    +---------------------------+--------------------+
+
+    .. note::
+        Subclass of :class:`.PrevNodeContext`. The argument, inputs, outputs follow the same specification.
+
+        In addition to supplying  ``state.get_current_node()`` for the index of the previous visited node.
+        The input ``state`` needs to supply ``state.VEHICLE_CAPACITY`` and ``state.used_capacity``
+        for calculating the remaining capcacity.
+    """
+
+    def __init__(self, context_dim):
+        super(VRPContext, self).__init__(context_dim)
+
+    def _state_embedding(self, embeddings, state):
+        state_embedding = state.VEHICLE_CAPACITY - state.used_capacity[:, :, None]
+        return state_embedding
+
+
+class PCTSPContext(PrevNodeContext):
+    """
+    Context node embedding for prize collecting traveling salesman problem.
+    Return a concatenation of
+
+    +---------------------------+----------------------------+
+    | previous node's embedding | remaining prize to collect |
+    +---------------------------+----------------------------+
+
+    .. note::
+        Subclass of :class:`.PrevNodeContext`. The argument, inputs, outputs follow the same specification.
+
+        In addition to supplying  ``state.get_current_node()`` for the index of the previous visited node.
+        The input ``state`` needs to supply ``state.get_remaining_prize_to_collect()``.
+    """
+
+    def __init__(self, context_dim):
+        super(PCTSPContext, self).__init__(context_dim)
+
+    def _state_embedding(self, embeddings, state):
+        state_embedding = state.get_remaining_prize_to_collect()[:, :, None]
+        return state_embedding
+
+
+class OPContext(PrevNodeContext):
+    """
+    Context node embedding for orienteering problem.
+    Return a concatenation of
+
+    +---------------------------+---------------------------------+
+    | previous node's embedding | remaining tour length to travel |
+    +---------------------------+---------------------------------+
+
+    .. note::
+        Subclass of :class:`.PrevNodeContext`. The argument, inputs, outputs follow the same specification.
+
+        In addition to supplying  ``state.get_current_node()`` for the index of the previous visited node.
+        The input ``state`` needs to supply ``state.get_remaining_length()``.
+    """
+
+    def __init__(self, context_dim):
+        super(OPContext, self).__init__(context_dim)
+
+    def _state_embedding(self, embeddings, state):
+        state_embedding = state.get_remaining_length()[:, :, None]
+        return state_embedding

models/nets/attention_model/decoder.py

@@ -0,0 +1,211 @@
+import torch
+from torch import nn
+
+from ...nets.attention_model.context import AutoContext
+from ...nets.attention_model.dynamic_embedding import AutoDynamicEmbedding
+from ...nets.attention_model.multi_head_attention import (
+    AttentionScore,
+    MultiHeadAttention,
+)
+
+
+class Decoder(nn.Module):
+    r"""
+    The decoder of the Attention Model.
+
+    .. math::
+        \{\log(\pmb{p}_t)\},\pi =  \mathrm{Decoder}(s, \pmb{h})
+
+    First of all, precompute the keys and values for the embedding :math:`\pmb{h}`:
+
+    .. math::
+        \pmb{k}, \pmb{v}, \pmb{k}^\prime = W^K\pmb{h}, W^V\pmb{h}, W^{K^\prime}\pmb{h}
+    and the projection of the graph embedding:
+
+    .. math::
+         W_{gc}\bar{\pmb{h}} \quad \text{ for } \bar{\pmb{h}} = \frac{1}{N}\sum\nolimits_i \pmb{h}_i.
+
+    Then, the decoder iterates the decoding process autoregressively.
+    In each decoding step, we perform multiple attentions to get the logits for each node.
+
+    .. math::
+        \begin{aligned}
+        \pmb{h}_{(c)} &= [\bar{\pmb{h}}, \text{Context}(s,\pmb{h})]                                                 \\
+        q & = W^Q \pmb{h}_{(c)} = W_{gc}\bar{\pmb{h}} + W_{sc}\text{Context}(s,\pmb{h})                               \\
+        q_{gl} &= \mathrm{MultiHeadAttention}(q,\pmb{k},\pmb{v},\mathrm{mask}_t)                                    \\
+        \pmb{p}_t &= \mathrm{Softmax}(\mathrm{AttentionScore}_{\text{clip}}(q_{gl},\pmb{k}^\prime, \mathrm{mask}_t))\\
+        \pi_{t} &= \mathrm{DecodingStartegy}(\pmb{p}_t)                                                             \\
+        \mathrm{mask}_{t+1} &= \mathrm{mask}_t.update(\pi_t).
+        \end{aligned}
+
+
+
+    .. note::
+        If there are dynamic node features specified by :mod:`.dynamic_embedding` ,
+        the keys and values projections are updated in each decoding step by
+
+        .. math::
+            \begin{aligned}
+            \pmb{k}_{\text{dynamic}}, \pmb{v}_{\text{dynamic}}, \pmb{k}^\prime_{\text{dynamic}} &= \mathrm{DynamicEmbedding}(s)\\
+            \pmb{k} &= \pmb{k} + \pmb{k}_{\text{dynamic}}\\
+            \pmb{v} &= \pmb{v} +\pmb{v}_{\text{dynamic}} \\
+            \pmb{k}^\prime &= \pmb{k}^\prime +\pmb{k}^\prime_{\text{dynamic}}.
+            \end{aligned}
+    .. seealso::
+        * The :math:`\text{Context}` is defined in the :mod:`.context` module.
+        * The :math:`\text{AttentionScore}` is defined by the :class:`.AttentionScore` class.
+        * The :math:`\text{MultiHeadAttention}` is defined by the :class:`.MultiHeadAttention` class.
+
+    Args:
+         embedding_dim : the dimension of the embedded inputs
+         step_context_dim : the dimension of the context :math:`\text{Context}(\pmb{x})`
+         n_heads: number of heads in the :math:`\mathrm{MultiHeadAttention}`
+         problem: an object defining the state and the mask updating rule of the problem
+         tanh_clipping : the clipping scale of the pointer (attention layer before output)
+    Inputs: input, embeddings
+        * **input** : dict of inputs, for example ``{'loc': tensor, 'depot': tensor, 'demand': tensor}`` for CVRP.
+        * **embeddings**: [batch, graph_size, embedding_dim]
+    Outputs: log_ps, pi
+        * **log_ps**: [batch, graph_size, T]
+        * **pi**: [batch, T]
+
+    """
+
+    def __init__(self, embedding_dim, step_context_dim, n_heads, problem, tanh_clipping):
+        super(Decoder, self).__init__()
+        # For each node we compute (glimpse key, glimpse value, logit key) so 3 * embedding_dim
+
+        self.project_node_embeddings = nn.Linear(embedding_dim, 3 * embedding_dim, bias=False)
+        self.project_fixed_context = nn.Linear(embedding_dim, embedding_dim, bias=False)
+        self.project_step_context = nn.Linear(step_context_dim, embedding_dim, bias=False)
+
+        self.context = AutoContext(problem.NAME, {"context_dim": step_context_dim})
+        self.dynamic_embedding = AutoDynamicEmbedding(
+            problem.NAME, {"embedding_dim": embedding_dim}
+        )
+        self.glimpse = MultiHeadAttention(embedding_dim=embedding_dim, n_heads=n_heads)
+        self.pointer = AttentionScore(use_tanh=True, C=tanh_clipping)
+
+        self.decode_type = None
+        self.problem = problem
+
+    def forward(self, input, embeddings):
+        outputs = []
+        sequences = []
+
+        state = self.problem.make_state(input)
+
+        # Compute keys, values for the glimpse and keys for the logits once as they can be reused in every step
+        cached_embeddings = self._precompute(embeddings)
+
+        # Perform decoding steps
+        while not (state.all_finished()):
+
+            log_p, mask = self.advance(cached_embeddings, state)
+
+            # Select the indices of the next nodes in the sequences, result (batch_size) long
+            # Squeeze out steps dimension
+            action = self.decode(log_p.exp(), mask)
+            state = state.update(action)
+
+            # Collect output of step
+            outputs.append(log_p)
+            sequences.append(action)
+
+        # Collected lists, return Tensor
+        return torch.stack(outputs, 1), torch.stack(sequences, 1)
+
+    def set_decode_type(self, decode_type):
+        r"""
+        Currently support
+
+        .. code-block:: python
+
+            ["greedy", "sampling"]
+
+        """
+        self.decode_type = decode_type
+
+    def decode(self, probs, mask):
+        r"""
+        Execute the decoding strategy specified by ``self.decode_type``.
+
+        Inputs:
+            * **probs**: [batch_size, graph_size]
+            * **mask** (bool): [batch_size, graph_size]
+        Outputs:
+            * **idxs** (int): index of action chosen. [batch_size]
+        """
+        assert (probs == probs).all(), "Probs should not contain any nans"
+
+        if self.decode_type == "greedy":
+            _, selected = probs.max(1)
+            assert not mask.gather(
+                1, selected.unsqueeze(-1)
+            ).data.any(), "Decode greedy: infeasible action has maximum probability"
+
+        elif self.decode_type == "sampling":
+            selected = probs.multinomial(1).squeeze(1)
+
+            # Check if sampling went OK, can go wrong due to bug on GPU
+            # See https://discuss.pytorch.org/t/bad-behavior-of-multinomial-function/10232
+            while mask.gather(1, selected.unsqueeze(-1)).data.any():
+                print("Sampled bad values, resampling!")
+                selected = probs.multinomial(1).squeeze(1)
+
+        else:
+            assert False, "Unknown decode type"
+        return selected
+
+    def _precompute(self, embeddings):
+
+        # The fixed context projection of the graph embedding is calculated only once for efficiency
+        graph_embed = embeddings.mean(1)
+        # fixed context = (batch_size, 1, embed_dim) to make broadcastable with parallel timesteps
+        graph_context = self.project_fixed_context(graph_embed).unsqueeze(-2)
+        # The projection of the node embeddings for the attention is calculated once up front
+        glimpse_key, glimpse_val, logit_key = self.project_node_embeddings(embeddings).chunk(
+            3, dim=-1
+        )
+
+        cache = (
+            embeddings,
+            graph_context,
+            glimpse_key,
+            glimpse_val,
+            logit_key,
+        )  # single head for the final logit
+        return cache
+
+    def advance(self, cached_embeddings, state):
+
+        node_embeddings, graph_context, glimpse_K, glimpse_V, logit_K = cached_embeddings
+
+        # Compute context node embedding: [graph embedding| prev node| problem-state-context]
+        # [batch, 1, context dim]
+        context = self.context(node_embeddings, state)
+        step_context = self.project_step_context(context)  # [batch, 1, embed_dim]
+        query = graph_context + step_context  # [batch, 1, embed_dim]
+
+        glimpse_key_dynamic, glimpse_val_dynamic, logit_key_dynamic = self.dynamic_embedding(state)
+        glimpse_K = glimpse_K + glimpse_key_dynamic
+        glimpse_V = glimpse_V + glimpse_val_dynamic
+        logit_K = logit_K + logit_key_dynamic
+        # Compute the mask
+        mask = state.get_mask()
+
+        # Compute logits (unnormalized log_p)
+        logits, glimpse = self.calc_logits(query, glimpse_K, glimpse_V, logit_K, mask)
+
+        return logits, glimpse
+
+    def calc_logits(self, query, glimpse_K, glimpse_V, logit_K, mask):
+        # Compute glimpse with multi-head-attention.
+        # Then use glimpse as a query to compute logits for each node
+
+        # [batch, 1, embed dim]
+        glimpse = self.glimpse(query, glimpse_K, glimpse_V, mask)
+
+        logits = self.pointer(glimpse, logit_K, mask)
+
+        return logits, glimpse

models/nets/attention_model/dynamic_embedding.py

@@ -0,0 +1,73 @@
+"""
+Problem specific node embedding for dynamic feature.
+"""
+
+import torch.nn as nn
+
+
+def AutoDynamicEmbedding(problem_name, config):
+    """
+    Automatically select the corresponding module according to ``problem_name``
+    """
+    mapping = {
+        "tsp": NonDyanmicEmbedding,
+        "cvrp": NonDyanmicEmbedding,
+        "sdvrp": SDVRPDynamicEmbedding,
+        "pctsp": NonDyanmicEmbedding,
+        "op": NonDyanmicEmbedding,
+    }
+    embeddingClass = mapping[problem_name]
+    embedding = embeddingClass(**config)
+    return embedding
+
+
+class SDVRPDynamicEmbedding(nn.Module):
+    """
+    Embedding for dynamic node feature for the split delivery vehicle routing problem.
+
+    It is implemented as a linear projection of the demands left in each node.
+
+    Args:
+        embedding_dim: dimension of output
+    Inputs: state
+        * **state** : a class that provide ``state.demands_with_depot`` tensor
+    Outputs: glimpse_key_dynamic, glimpse_val_dynamic, logit_key_dynamic
+        * **glimpse_key_dynamic** : [batch, graph_size, embedding_dim]
+        * **glimpse_val_dynamic** : [batch, graph_size, embedding_dim]
+        * **logit_key_dynamic** : [batch, graph_size, embedding_dim]
+
+    """
+
+    def __init__(self, embedding_dim):
+        super(SDVRPDynamicEmbedding, self).__init__()
+        self.projection = nn.Linear(1, 3 * embedding_dim, bias=False)
+
+    def forward(self, state):
+        glimpse_key_dynamic, glimpse_val_dynamic, logit_key_dynamic = self.projection(
+            state.demands_with_depot[:, 0, :, None].clone()
+        ).chunk(3, dim=-1)
+        return glimpse_key_dynamic, glimpse_val_dynamic, logit_key_dynamic
+
+
+class NonDyanmicEmbedding(nn.Module):
+    """
+    Embedding for problems that do not have any dynamic node feature.
+
+    It is implemented as simply returning zeros.
+
+    Args:
+        embedding_dim: dimension of output
+    Inputs: state
+        * **state** : not used, just for consistency
+    Outputs: glimpse_key_dynamic, glimpse_val_dynamic, logit_key_dynamic
+        * **glimpse_key_dynamic** : [batch, graph_size, embedding_dim]
+        * **glimpse_val_dynamic** : [batch, graph_size, embedding_dim]
+        * **logit_key_dynamic** : [batch, graph_size, embedding_dim]
+
+    """
+
+    def __init__(self, embedding_dim):
+        super(NonDyanmicEmbedding, self).__init__()
+
+    def forward(self, state):
+        return 0, 0, 0

models/nets/attention_model/embedding.py

@@ -0,0 +1,185 @@
+"""
+Problem specific node embedding for static feature.
+"""
+
+import torch
+import torch.nn as nn
+
+
+def AutoEmbedding(problem_name, config):
+    """
+    Automatically select the corresponding module according to ``problem_name``
+    """
+    mapping = {
+        "tsp": TSPEmbedding,
+        "cvrp": VRPEmbedding,
+        "sdvrp": VRPEmbedding,
+        "pctsp": PCTSPEmbedding,
+        "op": OPEmbedding,
+    }
+    embeddingClass = mapping[problem_name]
+    embedding = embeddingClass(**config)
+    return embedding
+
+
+class TSPEmbedding(nn.Module):
+    """
+    Embedding for the traveling salesman problem.
+
+    Args:
+        embedding_dim: dimension of output
+    Inputs: input
+        * **input** : [batch, n_customer, 2]
+    Outputs: out
+        * **out** : [batch, n_customer, embedding_dim]
+    """
+
+    def __init__(self, embedding_dim):
+        super(TSPEmbedding, self).__init__()
+        node_dim = 2  # x, y
+        self.context_dim = 2 * embedding_dim  # Embedding of first and last node
+
+        self.init_embed = nn.Linear(node_dim, embedding_dim)
+
+    def forward(self, input):
+        out = self.init_embed(input)
+        return out
+
+
+class VRPEmbedding(nn.Module):
+    """
+    Embedding for the capacitated vehicle routing problem.
+    The shape of tensors in ``input`` is summarized as following:
+
+    +-----------+-------------------------+
+    | key       | size of tensor          |
+    +===========+=========================+
+    | 'loc'     | [batch, n_customer, 2]  |
+    +-----------+-------------------------+
+    | 'depot'   | [batch, 2]              |
+    +-----------+-------------------------+
+    | 'demand'  | [batch, n_customer, 1]  |
+    +-----------+-------------------------+
+
+    Args:
+        embedding_dim: dimension of output
+    Inputs: input
+        * **input** : dict of ['loc', 'depot', 'demand']
+    Outputs: out
+        * **out** : [batch, n_customer+1, embedding_dim]
+    """
+
+    def __init__(self, embedding_dim):
+        super(VRPEmbedding, self).__init__()
+        node_dim = 3  # x, y, demand
+
+        self.context_dim = embedding_dim + 1  # Embedding of last node + remaining_capacity
+
+        self.init_embed = nn.Linear(node_dim, embedding_dim)
+        self.init_embed_depot = nn.Linear(2, embedding_dim)  # depot embedding
+
+    def forward(self, input):  # dict of 'loc', 'demand', 'depot'
+        # batch, 1, 2 -> batch, 1, embedding_dim
+        depot_embedding = self.init_embed_depot(input["depot"])[:, None, :]
+        # [batch, n_customer, 2, batch, n_customer, 1]  -> batch, n_customer, embedding_dim
+        node_embeddings = self.init_embed(
+            torch.cat((input["loc"], input["demand"][:, :, None]), -1)
+        )
+        # batch, n_customer+1, embedding_dim
+        out = torch.cat((depot_embedding, node_embeddings), 1)
+        return out
+
+
+class PCTSPEmbedding(nn.Module):
+    """
+    Embedding for the prize collecting traveling salesman problem.
+    The shape of tensors in ``input`` is summarized as following:
+
+    +------------------------+-------------------------+
+    | key                    | size of tensor          |
+    +========================+=========================+
+    | 'loc'                  | [batch, n_customer, 2]  |
+    +------------------------+-------------------------+
+    | 'depot'                | [batch, 2]              |
+    +------------------------+-------------------------+
+    | 'deterministic_prize'  | [batch, n_customer, 1]  |
+    +------------------------+-------------------------+
+    | 'penalty'              | [batch, n_customer, 1]  |
+    +------------------------+-------------------------+
+
+    Args:
+        embedding_dim: dimension of output
+    Inputs: input
+        * **input** : dict of ['loc', 'depot', 'deterministic_prize', 'penalty']
+    Outputs: out
+        * **out** : [batch, n_customer+1, embedding_dim]
+    """
+
+    def __init__(self, embedding_dim):
+        super(PCTSPEmbedding, self).__init__()
+        node_dim = 4  # x, y, prize, penalty
+        self.context_dim = embedding_dim + 1  # Embedding of last node + remaining prize to collect
+
+        self.init_embed = nn.Linear(node_dim, embedding_dim)
+        self.init_embed_depot = nn.Linear(2, embedding_dim)  # depot embedding
+
+    def forward(self, input):  # dict of 'loc', 'deterministic_prize', 'penalty', 'depot'
+        # batch, 1, 2 -> batch, 1, embedding_dim
+        depot_embedding = self.init_embed_depot(input["depot"])[:, None, :]
+        # [batch, n_customer, 2, batch, n_customer, 1, batch, n_customer, 1]  -> batch, n_customer, embedding_dim
+        node_embeddings = self.init_embed(
+            torch.cat(
+                (
+                    input["loc"],
+                    input["deterministic_prize"][:, :, None],
+                    input["penalty"][:, :, None],
+                ),
+                -1,
+            )
+        )
+        # batch, n_customer+1, embedding_dim
+        out = torch.cat((depot_embedding, node_embeddings), 1)
+        return out
+
+
+class OPEmbedding(nn.Module):
+    """
+    Embedding for the orienteering problem.
+    The shape of tensors in ``input`` is summarized as following:
+
+    +----------+-------------------------+
+    | key      | size of tensor          |
+    +==========+=========================+
+    | 'loc'    | [batch, n_customer, 2]  |
+    +----------+-------------------------+
+    | 'depot'  | [batch, 2]              |
+    +----------+-------------------------+
+    | 'prize'  | [batch, n_customer, 1]  |
+    +----------+-------------------------+
+
+    Args:
+        embedding_dim: dimension of output
+    Inputs: input
+        * **input** : dict of ['loc', 'depot', 'prize']
+    Outputs: out
+        * **out** : [batch, n_customer+1, embedding_dim]
+    """
+
+    def __init__(self, embedding_dim):
+        super(OPEmbedding, self).__init__()
+        node_dim = 3  # x, y, prize
+        self.context_dim = embedding_dim + 1  # Embedding of last node + remaining prize to collect
+
+        self.init_embed = nn.Linear(node_dim, embedding_dim)
+        self.init_embed_depot = nn.Linear(2, embedding_dim)  # depot embedding
+
+    def forward(self, input):  # dict of 'loc', 'prize', 'depot'
+        # batch, 1, 2 -> batch, 1, embedding_dim
+        depot_embedding = self.init_embed_depot(input["depot"])[:, None, :]
+        # [batch, n_customer, 2, batch, n_customer, 1, batch, n_customer, 1]  -> batch, n_customer, embedding_dim
+        node_embeddings = self.init_embed(
+            torch.cat((input["loc"], input["prize"][:, :, None]), -1)
+        )
+        # batch, n_customer+1, embedding_dim
+        out = torch.cat((depot_embedding, node_embeddings), 1)
+        return out

models/nets/attention_model/encoder.py

@@ -0,0 +1,128 @@
+from torch import nn
+
+from ...nets.attention_model.multi_head_attention import MultiHeadAttentionProj
+
+
+class SkipConnection(nn.Module):
+    def __init__(self, module):
+        super(SkipConnection, self).__init__()
+        self.module = module
+
+    def forward(self, input):
+        return input + self.module(input)
+
+
+class Normalization(nn.Module):
+    def __init__(self, embedding_dim):
+        super(Normalization, self).__init__()
+
+        self.normalizer = nn.BatchNorm1d(embedding_dim, affine=True)
+
+    def forward(self, input):
+        #         out = self.normalizer(input.permute(0,2,1)).permute(0,2,1) # slightly different 3e-6
+        #         return out
+        return self.normalizer(input.view(-1, input.size(-1))).view(input.size())
+
+
+class MultiHeadAttentionLayer(nn.Sequential):
+    r"""
+    A layer with attention mechanism and normalization.
+
+    For an embedding :math:`\pmb{x}`,
+
+    .. math::
+        \pmb{h} = \mathrm{MultiHeadAttentionLayer}(\pmb{x})
+
+    The following is executed:
+
+    .. math::
+        \begin{aligned}
+        \pmb{x}_0&=\pmb{x}+\mathrm{MultiHeadAttentionProj}(\pmb{x})  \\
+        \pmb{x}_1&=\mathrm{BatchNorm}(\pmb{x}_0)                      \\
+        \pmb{x}_2&=\pmb{x}_1+\mathrm{MLP_{\text{2 layers}}}(\pmb{x}_1)\\
+        \pmb{h} &=\mathrm{BatchNorm}(\pmb{x}_2)
+        \end{aligned}
+
+
+
+    .. seealso::
+        The :math:`\mathrm{MultiHeadAttentionProj}` computes the self attention
+        of the embedding  :math:`\pmb{x}`. Check :class:`~.MultiHeadAttentionProj` for details.
+
+    Args:
+        n_heads : number of heads
+        embedding_dim : dimension of the query, keys, values
+        feed_forward_hidden : size of the hidden layer in the MLP
+    Inputs: inputs
+        * **inputs**: embeddin :math:`\pmb{x}`. [batch, graph_size, embedding_dim]
+    Outputs: out
+        * **out**: the output :math:`\pmb{h}` [batch, graph_size, embedding_dim]
+    """
+
+    def __init__(
+        self,
+        n_heads,
+        embedding_dim,
+        feed_forward_hidden=512,
+    ):
+        super(MultiHeadAttentionLayer, self).__init__(
+            SkipConnection(
+                MultiHeadAttentionProj(
+                    embedding_dim=embedding_dim,
+                    n_heads=n_heads,
+                )
+            ),
+            Normalization(embedding_dim),
+            SkipConnection(
+                nn.Sequential(
+                    nn.Linear(embedding_dim, feed_forward_hidden),
+                    nn.ReLU(),
+                    nn.Linear(feed_forward_hidden, embedding_dim),
+                )
+                if feed_forward_hidden > 0
+                else nn.Linear(embedding_dim, embedding_dim)
+            ),
+            Normalization(embedding_dim),
+        )
+
+
+class GraphAttentionEncoder(nn.Module):
+    r"""
+    Graph attention by self attention on graph nodes.
+
+    For an embedding :math:`\pmb{x}`, repeat ``n_layers`` time:
+
+    .. math::
+        \pmb{h} = \mathrm{MultiHeadAttentionLayer}(\pmb{x})
+
+    .. seealso::
+        Check :class:`~.MultiHeadAttentionLayer` for details.
+
+    Args:
+        n_heads : number of heads
+        embedding_dim : dimension of the query, keys, values
+        n_layers : number of :class:`~.MultiHeadAttentionLayer` to iterate.
+        feed_forward_hidden : size of the hidden layer in the MLP
+    Inputs: x
+        * **x**: embeddin :math:`\pmb{x}`. [batch, graph_size, embedding_dim]
+    Outputs: (h, h_mean)
+        * **h**: the output :math:`\pmb{h}` [batch, graph_size, embedding_dim]
+    """
+
+    def __init__(self, n_heads, embed_dim, n_layers, feed_forward_hidden=512):
+        super(GraphAttentionEncoder, self).__init__()
+
+        self.layers = nn.Sequential(
+            *(
+                MultiHeadAttentionLayer(n_heads, embed_dim, feed_forward_hidden)
+                for _ in range(n_layers)
+            )
+        )
+
+    def forward(self, x, mask=None):
+
+        assert mask is None, "TODO mask not yet supported!"
+
+        h = self.layers(x)
+
+        return (h, h.mean(dim=1))

models/nets/attention_model/multi_head_attention.py

@@ -0,0 +1,188 @@
+import math
+
+import torch
+from torch import nn
+
+
+class AttentionScore(nn.Module):
+    r"""
+    A helper class for attention operations.
+    There are no parameters in this module.
+    This module computes the alignment score with mask
+    and return only the attention score.
+
+    The default operation is
+
+    .. math::
+         \pmb{u} = \mathrm{Attention}(q,\pmb{k}, \mathrm{mask})
+
+    where for each key :math:`k_j`, we have
+
+    .. math::
+        u_j =
+        \begin{cases}
+             &\frac{q^Tk_j}{\sqrt{\smash{d_q}}} & \text{ if } j \notin \mathrm{mask}\\
+             &-\infty & \text{ otherwise. }
+        \end{cases}
+
+    If ``use_tanh`` is ``True``, apply clipping on the logits :math:`u_j` before masking:
+
+    .. math::
+        u_j =
+        \begin{cases}
+             &C\mathrm{tanh}\left(\frac{q^Tk_j}{\sqrt{\smash{d_q}}}\right) & \text{ if } j \notin \mathrm{mask}\\
+             &-\infty & \text{ otherwise. }
+        \end{cases}
+
+    Args:
+        use_tanh: if True, use clipping on the logits
+        C: the range of the clipping [-C,C]
+    Inputs: query, keys, mask
+        * **query** : [..., 1, h_dim]
+        * **keys**: [..., graph_size, h_dim]
+        * **mask**: [..., graph_size] ``logits[...,j]==-inf`` if ``mask[...,j]==True``.
+    Outputs: logits
+        * **logits**: [..., 1, graph_size] The attention score for each key.
+    """
+
+    def __init__(self, use_tanh=False, C=10):
+        super(AttentionScore, self).__init__()
+        self.use_tanh = use_tanh
+        self.C = C
+
+    def forward(self, query, key, mask=torch.zeros([], dtype=torch.bool)):
+        u = torch.matmul(query, key.transpose(-2, -1)) / math.sqrt(query.size(-1))
+        if self.use_tanh:
+            logits = torch.tanh(u) * self.C
+        else:
+            logits = u
+
+        logits[mask.expand_as(logits)] = float("-inf")  # masked after clipping
+        return logits
+
+
+class MultiHeadAttention(nn.Module):
+    r"""
+    Compute the multi-head attention.
+
+    .. math::
+        q^\prime = \mathrm{MultiHeadAttention}(q,\pmb{k},\pmb{v},\mathrm{mask})
+
+    The following is computed:
+
+    .. math::
+        \begin{aligned}
+        \pmb{a}^{(j)} &= \mathrm{Softmax}(\mathrm{AttentionScore}(q^{(j)},\pmb{k}^{(j)}, \mathrm{mask}))\\
+        h^{(j)} &= \sum\nolimits_i \pmb{a}^{(j)}_i\pmb{v}_i \\
+        q^\prime &= W^O \left[h^{(1)},...,h^{(J)}\right]
+        \end{aligned}
+
+    Args:
+        embedding_dim: dimension of the query, keys, values
+        n_head: number of heads
+    Inputs: query, keys, value, mask
+        * **query** : [batch, n_querys, embedding_dim]
+        * **keys**: [batch, n_keys, embedding_dim]
+        * **value**: [batch, n_keys, embedding_dim]
+        * **mask**: [batch, 1, n_keys] ``logits[batch,j]==-inf`` if ``mask[batch, 0, j]==True``
+    Outputs: logits, out
+        * **out**: [batch, 1, embedding_dim] The output of the multi-head attention
+    """
+
+    def __init__(self, embedding_dim, n_heads=8):
+        super(MultiHeadAttention, self).__init__()
+        self.n_heads = n_heads
+        self.attentionScore = AttentionScore()
+        self.project_out = nn.Linear(embedding_dim, embedding_dim, bias=False)
+
+    def forward(self, query, key, value, mask):
+        query_heads = self._make_heads(query)
+        key_heads = self._make_heads(key)
+        value_heads = self._make_heads(value)
+
+        # [n_heads, batch, 1, nkeys]
+        compatibility = self.attentionScore(query_heads, key_heads, mask)
+
+        # [n_heads, batch, 1, head_dim]
+        out_heads = torch.matmul(torch.softmax(compatibility, dim=-1), value_heads)
+
+        # from multihead [nhead, batch, 1, head_dim] -> [batch, 1, nhead* head_dim]
+        out = self.project_out(self._unmake_heads(out_heads))
+        return out
+
+    def _make_heads(self, v):
+        batch_size, nkeys, h_dim = v.shape
+        #  [batch_size, ..., n_heads* head_dim] --> [n_heads, batch_size, ..., head_dim]
+        out = v.reshape(batch_size, nkeys, self.n_heads, h_dim // self.n_heads).movedim(-2, 0)
+        return out
+
+    def _unmake_heads(self, v):
+        #  [n_heads, batch_size, ..., head_dim] --> [batch_size, ..., n_heads* head_dim]
+        out = v.movedim(0, -2).flatten(-2)
+        return out
+
+
+class MultiHeadAttentionProj(nn.Module):
+    r"""
+    Compute the multi-head attention with projection.
+    Different from :class:`.MultiHeadAttention` which accepts precomputed query, keys, and values,
+    this module computes linear projections from the inputs to query, keys, and values.
+
+    .. math::
+        q^\prime = \mathrm{MultiHeadAttentionProj}(q_0,\pmb{h},\mathrm{mask})
+
+    The following is computed:
+
+    .. math::
+        \begin{aligned}
+        q, \pmb{k}, \pmb{v} &= W^Qq_0, W^K\pmb{h}, W^V\pmb{h}\\
+        \pmb{a}^{(j)} &= \mathrm{Softmax}(\mathrm{AttentionScore}(q^{(j)},\pmb{k}^{(j)}, \mathrm{mask}))\\
+        h^{(j)} &= \sum\nolimits_i \pmb{a}^{(j)}_i\pmb{v}_i \\
+        q^\prime &= W^O \left[h^{(1)},...,h^{(J)}\right]
+        \end{aligned}
+
+    if :math:`\pmb{h}` is not given. This module will compute the self attention of :math:`q_0`.
+
+    .. warning::
+        The results of the in-projection of query, key, value are
+        slightly different (order of ``1e-6``) with the original implementation.
+        This is due to the numerical accuracy.
+        The two implementations differ by the way of multiplying matrix.
+        Thus, different internal implementation libraries of pytorch are called
+        and the results are slightly different.
+        See the pytorch docs on `numerical accruacy <https://pytorch.org/docs/stable/notes/numerical_accuracy.html>`_ for detail.
+
+    Args:
+        embedding_dim: dimension of the query, keys, values
+        n_head: number of heads
+    Inputs: q, h, mask
+        * **q** : [batch, n_querys, embedding_dim]
+        * **h**: [batch, n_keys, embedding_dim]
+        * **mask**: [batch, n_keys] ``logits[batch,j]==-inf`` if ``mask[batch,j]==True``
+    Outputs: out
+        * **out**: [batch, n_querys, embedding_dim] The output of the multi-head attention
+
+
+    """
+
+    def __init__(self, embedding_dim, n_heads=8):
+        super(MultiHeadAttentionProj, self).__init__()
+
+        self.queryEncoder = nn.Linear(embedding_dim, embedding_dim, bias=False)
+        self.keyEncoder = nn.Linear(embedding_dim, embedding_dim, bias=False)
+        self.valueEncoder = nn.Linear(embedding_dim, embedding_dim, bias=False)
+
+        self.MHA = MultiHeadAttention(embedding_dim, n_heads)
+
+    def forward(self, q, h=None, mask=torch.zeros([], dtype=torch.bool)):
+
+        if h is None:
+            h = q  # compute self-attention
+
+        query = self.queryEncoder(q)
+        key = self.keyEncoder(h)
+        value = self.valueEncoder(h)
+
+        out = self.MHA(query, key, value, mask)
+
+        return out

ppo.py

@@ -0,0 +1,311 @@
+# Retrieved from https://github.com/vwxyzjn/cleanrl/blob/28fd178ca182bd83c75ed0d49d52e235ca6cdc88/cleanrl/ppo.py
+# docs and experiment results can be found at https://docs.cleanrl.dev/rl-algorithms/ppo/#ppopy
+
+import argparse
+import os
+import random
+import time
+from distutils.util import strtobool
+
+import gym
+import numpy as np
+import torch
+import torch.nn as nn
+import torch.optim as optim
+from torch.distributions.categorical import Categorical
+from torch.utils.tensorboard import SummaryWriter
+
+
+def parse_args():
+    # fmt: off
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--exp-name", type=str, default=os.path.basename(__file__).rstrip(".py"),
+        help="the name of this experiment")
+    parser.add_argument("--seed", type=int, default=1,
+        help="seed of the experiment")
+    parser.add_argument("--torch-deterministic", type=lambda x: bool(strtobool(x)), default=True, nargs="?", const=True,
+        help="if toggled, `torch.backends.cudnn.deterministic=False`")
+    parser.add_argument("--cuda", type=lambda x: bool(strtobool(x)), default=True, nargs="?", const=True,
+        help="if toggled, cuda will be enabled by default")
+    parser.add_argument("--track", type=lambda x: bool(strtobool(x)), default=False, nargs="?", const=True,
+        help="if toggled, this experiment will be tracked with Weights and Biases")
+    parser.add_argument("--wandb-project-name", type=str, default="cleanRL",
+        help="the wandb's project name")
+    parser.add_argument("--wandb-entity", type=str, default=None,
+        help="the entity (team) of wandb's project")
+    parser.add_argument("--capture-video", type=lambda x: bool(strtobool(x)), default=False, nargs="?", const=True,
+        help="whether to capture videos of the agent performances (check out `videos` folder)")
+
+    # Algorithm specific arguments
+    parser.add_argument("--env-id", type=str, default="CartPole-v1",
+        help="the id of the environment")
+    parser.add_argument("--total-timesteps", type=int, default=500000,
+        help="total timesteps of the experiments")
+    parser.add_argument("--learning-rate", type=float, default=2.5e-4,
+        help="the learning rate of the optimizer")
+    parser.add_argument("--num-envs", type=int, default=4,
+        help="the number of parallel game environments")
+    parser.add_argument("--num-steps", type=int, default=128,
+        help="the number of steps to run in each environment per policy rollout")
+    parser.add_argument("--anneal-lr", type=lambda x: bool(strtobool(x)), default=True, nargs="?", const=True,
+        help="Toggle learning rate annealing for policy and value networks")
+    parser.add_argument("--gamma", type=float, default=0.99,
+        help="the discount factor gamma")
+    parser.add_argument("--gae-lambda", type=float, default=0.95,
+        help="the lambda for the general advantage estimation")
+    parser.add_argument("--num-minibatches", type=int, default=4,
+        help="the number of mini-batches")
+    parser.add_argument("--update-epochs", type=int, default=4,
+        help="the K epochs to update the policy")
+    parser.add_argument("--norm-adv", type=lambda x: bool(strtobool(x)), default=True, nargs="?", const=True,
+        help="Toggles advantages normalization")
+    parser.add_argument("--clip-coef", type=float, default=0.2,
+        help="the surrogate clipping coefficient")
+    parser.add_argument("--clip-vloss", type=lambda x: bool(strtobool(x)), default=True, nargs="?", const=True,
+        help="Toggles whether or not to use a clipped loss for the value function, as per the paper.")
+    parser.add_argument("--ent-coef", type=float, default=0.01,
+        help="coefficient of the entropy")
+    parser.add_argument("--vf-coef", type=float, default=0.5,
+        help="coefficient of the value function")
+    parser.add_argument("--max-grad-norm", type=float, default=0.5,
+        help="the maximum norm for the gradient clipping")
+    parser.add_argument("--target-kl", type=float, default=None,
+        help="the target KL divergence threshold")
+    args = parser.parse_args()
+    args.batch_size = int(args.num_envs * args.num_steps)
+    args.minibatch_size = int(args.batch_size // args.num_minibatches)
+    # fmt: on
+    return args
+
+
+def make_env(env_id, seed, idx, capture_video, run_name):
+    def thunk():
+        env = gym.make(env_id)
+        if capture_video:
+            if idx == 0:
+                env = gym.wrappers.RecordVideo(env, f"videos/{run_name}")
+        env.action_space.seed(seed)
+        env.observation_space.seed(seed)
+        return env
+
+    return thunk
+
+
+def layer_init(layer, std=np.sqrt(2), bias_const=0.0):
+    torch.nn.init.orthogonal_(layer.weight, std)
+    torch.nn.init.constant_(layer.bias, bias_const)
+    return layer
+
+
+class Agent(nn.Module):
+    def __init__(self, envs):
+        super().__init__()
+        self.critic = nn.Sequential(
+            layer_init(nn.Linear(np.array(envs.single_observation_space.shape).prod(), 64)),
+            nn.Tanh(),
+            layer_init(nn.Linear(64, 64)),
+            nn.Tanh(),
+            layer_init(nn.Linear(64, 1), std=1.0),
+        )
+        self.actor = nn.Sequential(
+            layer_init(nn.Linear(np.array(envs.single_observation_space.shape).prod(), 64)),
+            nn.Tanh(),
+            layer_init(nn.Linear(64, 64)),
+            nn.Tanh(),
+            layer_init(nn.Linear(64, envs.single_action_space.n), std=0.01),
+        )
+
+    def get_value(self, x):
+        return self.critic(x)
+
+    def get_action_and_value(self, x, action=None):
+        logits = self.actor(x)
+        probs = Categorical(logits=logits)
+        if action is None:
+            action = probs.sample()
+        return action, probs.log_prob(action), probs.entropy(), self.critic(x)
+
+
+if __name__ == "__main__":
+    args = parse_args()
+    run_name = f"{args.env_id}__{args.exp_name}__{args.seed}__{int(time.time())}"
+    if args.track:
+        import wandb
+
+        wandb.init(
+            project=args.wandb_project_name,
+            entity=args.wandb_entity,
+            sync_tensorboard=True,
+            config=vars(args),
+            name=run_name,
+            monitor_gym=True,
+            save_code=True,
+        )
+    writer = SummaryWriter(f"runs/{run_name}")
+    writer.add_text(
+        "hyperparameters",
+        "|param|value|\n|-|-|\n%s" % ("\n".join([f"|{key}|{value}|" for key, value in vars(args).items()])),
+    )
+
+    # TRY NOT TO MODIFY: seeding
+    random.seed(args.seed)
+    np.random.seed(args.seed)
+    torch.manual_seed(args.seed)
+    torch.backends.cudnn.deterministic = args.torch_deterministic
+
+    device = torch.device("cuda" if torch.cuda.is_available() and args.cuda else "cpu")
+
+    # env setup
+    envs = gym.vector.SyncVectorEnv(
+        [make_env(args.env_id, args.seed + i, i, args.capture_video, run_name) for i in range(args.num_envs)]
+    )
+    envs = gym.wrappers.RecordEpisodeStatistics(envs)
+    assert isinstance(envs.single_action_space, gym.spaces.Discrete), "only discrete action space is supported"
+
+    agent = Agent(envs).to(device)
+    optimizer = optim.Adam(agent.parameters(), lr=args.learning_rate, eps=1e-5)
+
+    # ALGO Logic: Storage setup
+    obs = torch.zeros((args.num_steps, args.num_envs) + envs.single_observation_space.shape).to(device)
+    actions = torch.zeros((args.num_steps, args.num_envs) + envs.single_action_space.shape).to(device)
+    logprobs = torch.zeros((args.num_steps, args.num_envs)).to(device)
+    rewards = torch.zeros((args.num_steps, args.num_envs)).to(device)
+    terminateds = torch.zeros((args.num_steps, args.num_envs)).to(device)
+    values = torch.zeros((args.num_steps, args.num_envs)).to(device)
+
+    # TRY NOT TO MODIFY: start the game
+    global_step = 0
+    start_time = time.time()
+    next_obs = torch.Tensor(envs.reset()[0]).to(device)
+    next_terminated = torch.zeros(args.num_envs).to(device)
+    num_updates = args.total_timesteps // args.batch_size
+
+    for update in range(1, num_updates + 1):
+        # Annealing the rate if instructed to do so.
+        if args.anneal_lr:
+            frac = 1.0 - (update - 1.0) / num_updates
+            lrnow = frac * args.learning_rate
+            optimizer.param_groups[0]["lr"] = lrnow
+
+        for step in range(0, args.num_steps):
+            global_step += 1 * args.num_envs
+            obs[step] = next_obs
+            terminateds[step] = next_terminated
+
+            # ALGO LOGIC: action logic
+            with torch.no_grad():
+                action, logprob, _, value = agent.get_action_and_value(next_obs)
+                values[step] = value.flatten()
+            actions[step] = action
+            logprobs[step] = logprob
+
+            # TRY NOT TO MODIFY: execute the game and log data.
+            next_obs, reward, terminated, _, info = envs.step(action.cpu().numpy())
+            rewards[step] = torch.tensor(reward).to(device).view(-1)
+            next_obs, next_terminated = torch.Tensor(next_obs).to(device), torch.Tensor(terminated).to(device)
+
+            if "episode" in info:
+                first_idx = info["_episode"].nonzero()[0][0]
+                r = info["episode"]["r"][first_idx]
+                l = info["episode"]["l"][first_idx]
+                print(f"global_step={global_step}, episodic_return={r}")
+                writer.add_scalar("charts/episodic_return", r, global_step)
+                writer.add_scalar("charts/episodic_length", l, global_step)
+
+        # bootstrap value if not terminated
+        with torch.no_grad():
+            next_value = agent.get_value(next_obs).reshape(1, -1)
+            advantages = torch.zeros_like(rewards).to(device)
+            lastgaelam = 0
+            for t in reversed(range(args.num_steps)):
+                if t == args.num_steps - 1:
+                    nextnonterminal = 1.0 - next_terminated
+                    nextvalues = next_value
+                else:
+                    nextnonterminal = 1.0 - terminateds[t + 1]
+                    nextvalues = values[t + 1]
+                delta = rewards[t] + args.gamma * nextvalues * nextnonterminal - values[t]
+                advantages[t] = lastgaelam = delta + args.gamma * args.gae_lambda * nextnonterminal * lastgaelam
+            returns = advantages + values
+
+        # flatten the batch
+        b_obs = obs.reshape((-1,) + envs.single_observation_space.shape)
+        b_logprobs = logprobs.reshape(-1)
+        b_actions = actions.reshape((-1,) + envs.single_action_space.shape)
+        b_advantages = advantages.reshape(-1)
+        b_returns = returns.reshape(-1)
+        b_values = values.reshape(-1)
+
+        # Optimizing the policy and value network
+        b_inds = np.arange(args.batch_size)
+        clipfracs = []
+        for epoch in range(args.update_epochs):
+            np.random.shuffle(b_inds)
+            for start in range(0, args.batch_size, args.minibatch_size):
+                end = start + args.minibatch_size
+                mb_inds = b_inds[start:end]
+
+                _, newlogprob, entropy, newvalue = agent.get_action_and_value(b_obs[mb_inds], b_actions.long()[mb_inds])
+                logratio = newlogprob - b_logprobs[mb_inds]
+                ratio = logratio.exp()
+
+                with torch.no_grad():
+                    # calculate approx_kl http://joschu.net/blog/kl-approx.html
+                    old_approx_kl = (-logratio).mean()
+                    approx_kl = ((ratio - 1) - logratio).mean()
+                    clipfracs += [((ratio - 1.0).abs() > args.clip_coef).float().mean().item()]
+
+                mb_advantages = b_advantages[mb_inds]
+                if args.norm_adv:
+                    mb_advantages = (mb_advantages - mb_advantages.mean()) / (mb_advantages.std() + 1e-8)
+
+                # Policy loss
+                pg_loss1 = -mb_advantages * ratio
+                pg_loss2 = -mb_advantages * torch.clamp(ratio, 1 - args.clip_coef, 1 + args.clip_coef)
+                pg_loss = torch.max(pg_loss1, pg_loss2).mean()
+
+                # Value loss
+                newvalue = newvalue.view(-1)
+                if args.clip_vloss:
+                    v_loss_unclipped = (newvalue - b_returns[mb_inds]) ** 2
+                    v_clipped = b_values[mb_inds] + torch.clamp(
+                        newvalue - b_values[mb_inds],
+                        -args.clip_coef,
+                        args.clip_coef,
+                    )
+                    v_loss_clipped = (v_clipped - b_returns[mb_inds]) ** 2
+                    v_loss_max = torch.max(v_loss_unclipped, v_loss_clipped)
+                    v_loss = 0.5 * v_loss_max.mean()
+                else:
+                    v_loss = 0.5 * ((newvalue - b_returns[mb_inds]) ** 2).mean()
+
+                entropy_loss = entropy.mean()
+                loss = pg_loss - args.ent_coef * entropy_loss + v_loss * args.vf_coef
+
+                optimizer.zero_grad()
+                loss.backward()
+                nn.utils.clip_grad_norm_(agent.parameters(), args.max_grad_norm)
+                optimizer.step()
+
+            if args.target_kl is not None:
+                if approx_kl > args.target_kl:
+                    break
+
+        y_pred, y_true = b_values.cpu().numpy(), b_returns.cpu().numpy()
+        var_y = np.var(y_true)
+        explained_var = np.nan if var_y == 0 else 1 - np.var(y_true - y_pred) / var_y
+
+        # TRY NOT TO MODIFY: record rewards for plotting purposes
+        writer.add_scalar("charts/learning_rate", optimizer.param_groups[0]["lr"], global_step)
+        writer.add_scalar("losses/value_loss", v_loss.item(), global_step)
+        writer.add_scalar("losses/policy_loss", pg_loss.item(), global_step)
+        writer.add_scalar("losses/entropy", entropy_loss.item(), global_step)
+        writer.add_scalar("losses/old_approx_kl", old_approx_kl.item(), global_step)
+        writer.add_scalar("losses/approx_kl", approx_kl.item(), global_step)
+        writer.add_scalar("losses/clipfrac", np.mean(clipfracs), global_step)
+        writer.add_scalar("losses/explained_variance", explained_var, global_step)
+        print("SPS:", int(global_step / (time.time() - start_time)))
+        writer.add_scalar("charts/SPS", int(global_step / (time.time() - start_time)), global_step)
+
+    envs.close()
+    writer.close()

ppo_or.py

@@ -0,0 +1,397 @@
+# docs and experiment results can be found at https://docs.cleanrl.dev/rl-algorithms/ppo/#ppopy
+
+import argparse
+import os
+import random
+import shutil
+import time
+from distutils.util import strtobool
+
+import gym
+import numpy as np
+import torch
+import torch.nn as nn
+import torch.optim as optim
+from torch.utils.tensorboard import SummaryWriter
+
+
+def parse_args():
+    # fmt: off
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--exp-name", type=str, default=os.path.basename(__file__).rstrip(".py"),
+        help="the name of this experiment")
+    parser.add_argument("--seed", type=int, default=1,
+        help="seed of the experiment")
+    parser.add_argument("--torch-deterministic", type=lambda x: bool(strtobool(x)), default=True, nargs="?", const=True,
+        help="if toggled, `torch.backends.cudnn.deterministic=False`")
+    parser.add_argument("--cuda", type=lambda x: bool(strtobool(x)), default=True, nargs="?", const=True,
+        help="if toggled, cuda will be enabled by default")
+    parser.add_argument("--track", type=lambda x: bool(strtobool(x)), default=False, nargs="?", const=True,
+        help="if toggled, this experiment will be tracked with Weights and Biases")
+    parser.add_argument("--wandb-project-name", type=str, default="cleanRL",
+        help="the wandb's project name")
+    parser.add_argument("--wandb-entity", type=str, default=None,
+        help="the entity (team) of wandb's project")
+    parser.add_argument("--capture-video", type=lambda x: bool(strtobool(x)), default=False, nargs="?", const=True,
+        help="whether to capture videos of the agent performances (check out `videos` folder)")
+
+    # Algorithm specific arguments
+    parser.add_argument("--problem", type=str, default="cvrp",
+        help="the OR problem we are trying to solve, it will be passed to the agent")
+    parser.add_argument("--env-id", type=str, default="cvrp-v0",
+        help="the id of the environment")
+    parser.add_argument("--env-entry-point", type=str, default="envs.cvrp_vector_env:CVRPVectorEnv",
+        help="the path to the definition of the environment, for example `envs.cvrp_vector_env:CVRPVectorEnv` if the `CVRPVectorEnv` class is defined in ./envs/cvrp_vector_env.py")
+    parser.add_argument("--total-timesteps", type=int, default=6_000_000_000,
+        help="total timesteps of the experiments")
+    parser.add_argument("--learning-rate", type=float, default=1e-3,
+        help="the learning rate of the optimizer")
+    parser.add_argument("--weight-decay", type=float, default=0,
+        help="the weight decay of the optimizer")
+    parser.add_argument("--num-envs", type=int, default=1024,
+        help="the number of parallel game environments")
+    parser.add_argument("--num-steps", type=int, default=100,
+        help="the number of steps to run in each environment per policy rollout")
+    parser.add_argument("--anneal-lr", type=lambda x: bool(strtobool(x)), default=True, nargs="?", const=True,
+        help="Toggle learning rate annealing for policy and value networks")
+    parser.add_argument("--gamma", type=float, default=0.99,
+        help="the discount factor gamma")
+    parser.add_argument("--gae-lambda", type=float, default=0.95,
+        help="the lambda for the general advantage estimation")
+    parser.add_argument("--num-minibatches", type=int, default=8,
+        help="the number of mini-batches")
+    parser.add_argument("--update-epochs", type=int, default=2,
+        help="the K epochs to update the policy")
+    parser.add_argument("--norm-adv", type=lambda x: bool(strtobool(x)), default=True, nargs="?", const=True,
+        help="Toggles advantages normalization")
+    parser.add_argument("--clip-coef", type=float, default=0.2,
+        help="the surrogate clipping coefficient")
+    parser.add_argument("--clip-vloss", type=lambda x: bool(strtobool(x)), default=True, nargs="?", const=True,
+        help="Toggles whether or not to use a clipped loss for the value function, as per the paper.")
+    parser.add_argument("--ent-coef", type=float, default=0.01,
+        help="coefficient of the entropy")
+    parser.add_argument("--vf-coef", type=float, default=0.5,
+        help="coefficient of the value function")
+    parser.add_argument("--max-grad-norm", type=float, default=0.5,
+        help="the maximum norm for the gradient clipping")
+    parser.add_argument("--target-kl", type=float, default=None,
+        help="the target KL divergence threshold")
+    parser.add_argument("--n-traj", type=int, default=50,
+        help="number of trajectories in a vectorized sub-environment")
+    parser.add_argument("--n-test", type=int, default=1000,
+        help="how many test instance")
+    parser.add_argument("--multi-greedy-inference", type=lambda x: bool(strtobool(x)), default=True, nargs="?", const=True,
+        help="whether to use multiple trajectory greedy inference")
+    args = parser.parse_args()
+    args.batch_size = int(args.num_envs * args.num_steps)
+    args.minibatch_size = int(args.batch_size // args.num_minibatches)
+    # fmt: on
+    return args
+
+
+from wrappers.recordWrapper import RecordEpisodeStatistics
+
+
+def make_env(env_id, seed, cfg={}):
+    def thunk():
+        env = gym.make(env_id, **cfg)
+        env = RecordEpisodeStatistics(env)
+        env.seed(seed)
+        env.action_space.seed(seed)
+        env.observation_space.seed(seed)
+        return env
+
+    return thunk
+
+
+from models.attention_model_wrapper import Agent
+from wrappers.syncVectorEnvPomo import SyncVectorEnv
+
+if __name__ == "__main__":
+    args = parse_args()
+    run_name = f"{args.env_id}__{args.exp_name}__{args.seed}__{int(time.time())}"
+    if args.track:
+        import wandb
+
+        wandb.init(
+            project=args.wandb_project_name,
+            entity=args.wandb_entity,
+            sync_tensorboard=True,
+            config=vars(args),
+            name=run_name,
+            monitor_gym=True,
+            save_code=True,
+        )
+    writer = SummaryWriter(f"runs/{run_name}")
+    writer.add_text(
+        "hyperparameters",
+        "|param|value|\n|-|-|\n%s"
+        % ("\n".join([f"|{key}|{value}|" for key, value in vars(args).items()])),
+    )
+    os.makedirs(os.path.join(f"runs/{run_name}", "ckpt"), exist_ok=True)
+    shutil.copy(__file__, os.path.join(f"runs/{run_name}", "main.py"))
+    # TRY NOT TO MODIFY: seeding
+    random.seed(args.seed)
+    np.random.seed(args.seed)
+    torch.manual_seed(args.seed)
+    torch.backends.cudnn.deterministic = args.torch_deterministic
+
+    device = torch.device("cuda" if torch.cuda.is_available() and args.cuda else "cpu")
+
+    #######################
+    #### Env defintion ####
+    #######################
+
+    gym.envs.register(
+        id=args.env_id,
+        entry_point=args.env_entry_point,
+    )
+
+    # training env setup
+    envs = SyncVectorEnv([make_env(args.env_id, args.seed + i) for i in range(args.num_envs)])
+    # evaluation env setup: 1.) from a fix dataset, or 2.) generated with seed
+   # 1.) use test instance from a fix dataset
+    test_envs = SyncVectorEnv(
+        [
+            make_env(
+                args.env_id,
+                args.seed + i,
+                cfg={"eval_data": True, "eval_partition": "eval", "eval_data_idx": i},
+            )
+            for i in range(args.n_test)
+        ]
+    )
+#     # 2.) use generated evaluation instance instead
+#     import logging
+#     logging.warning('Using generated evaluation instance. For benchmarking, please download the fix dataset.')
+#     test_envs = SyncVectorEnv([make_env(args.env_id, args.seed + args.num_envs + i) for i in range(args.n_test)])
+    
+    assert isinstance(
+        envs.single_action_space, gym.spaces.MultiDiscrete
+    ), "only discrete action space is supported"
+
+    #######################
+    ### Agent defintion ###
+    #######################
+
+    agent = Agent(device=device, name=args.problem).to(device)
+    # agent.backbone.load_state_dict(torch.load('./vrp50.pt'))
+    optimizer = optim.Adam(
+        agent.parameters(), lr=args.learning_rate, eps=1e-5, weight_decay=args.weight_decay
+    )
+
+    #######################
+    # Algorithm defintion #
+    #######################
+
+    # ALGO Logic: Storage setup
+    obs = [None] * args.num_steps
+    actions = torch.zeros((args.num_steps, args.num_envs) + envs.single_action_space.shape).to(
+        device
+    )
+    logprobs = torch.zeros((args.num_steps, args.num_envs, args.n_traj)).to(device)
+    rewards = torch.zeros((args.num_steps, args.num_envs, args.n_traj)).to(device)
+    dones = torch.zeros((args.num_steps, args.num_envs, args.n_traj)).to(device)
+    values = torch.zeros((args.num_steps, args.num_envs, args.n_traj)).to(device)
+
+    # TRY NOT TO MODIFY: start the game
+    global_step = 0
+    start_time = time.time()
+    next_obs = envs.reset()
+    next_done = torch.zeros(args.num_envs, args.n_traj).to(device)
+    num_updates = args.total_timesteps // args.batch_size
+    for update in range(1, num_updates + 1):
+        agent.train()
+        # Annealing the rate if instructed to do so.
+        if args.anneal_lr:
+            frac = 1.0 - (update - 1.0) / num_updates
+            lrnow = frac * args.learning_rate
+            optimizer.param_groups[0]["lr"] = lrnow
+        next_obs = envs.reset()
+        encoder_state = agent.backbone.encode(next_obs)
+        next_done = torch.zeros(args.num_envs, args.n_traj).to(device)
+        r = []
+        for step in range(0, args.num_steps):
+            global_step += 1 * args.num_envs
+            obs[step] = next_obs
+            dones[step] = next_done
+
+            # ALGO LOGIC: action logic
+            with torch.no_grad():
+                action, logprob, _, value, _ = agent.get_action_and_value_cached(
+                    next_obs, state=encoder_state
+                )
+                action = action.view(args.num_envs, args.n_traj)
+                values[step] = value.view(args.num_envs, args.n_traj)
+            actions[step] = action
+            logprobs[step] = logprob.view(args.num_envs, args.n_traj)
+            # TRY NOT TO MODIFY: execute the game and log data.
+            next_obs, reward, done, info = envs.step(action.cpu().numpy())
+            rewards[step] = torch.tensor(reward).to(device)
+            next_obs, next_done = next_obs, torch.Tensor(done).to(device)
+
+            for item in info:
+                if "episode" in item.keys():
+                    r.append(item)
+        print("completed_episodes=", len(r))
+        avg_episodic_return = np.mean([rollout["episode"]["r"].mean() for rollout in r])
+        max_episodic_return = np.mean([rollout["episode"]["r"].max() for rollout in r])
+        avg_episodic_length = np.mean([rollout["episode"]["l"].mean() for rollout in r])
+        print(
+            f"[Train] global_step={global_step}\n \
+            avg_episodic_return={avg_episodic_return}\n \
+            max_episodic_return={max_episodic_return}\n \
+            avg_episodic_length={avg_episodic_length}"
+        )
+        writer.add_scalar("charts/episodic_return_mean", avg_episodic_return, global_step)
+        writer.add_scalar("charts/episodic_return_max", max_episodic_return, global_step)
+        writer.add_scalar("charts/episodic_length", avg_episodic_length, global_step)
+        # bootstrap value if not done
+        with torch.no_grad():
+            next_value = agent.get_value_cached(next_obs, encoder_state).squeeze(-1)  # B x T
+            advantages = torch.zeros_like(rewards).to(device)  # steps x B x T
+            lastgaelam = torch.zeros(args.num_envs, args.n_traj).to(device)  # B x T
+            for t in reversed(range(args.num_steps)):
+                if t == args.num_steps - 1:
+                    nextnonterminal = 1.0 - next_done  # next_done: B
+                    nextvalues = next_value  # B x T
+                else:
+                    nextnonterminal = 1.0 - dones[t + 1]
+                    nextvalues = values[t + 1]  # B x T
+                delta = rewards[t] + args.gamma * nextvalues * nextnonterminal - values[t]
+                advantages[t] = lastgaelam = (
+                    delta + args.gamma * args.gae_lambda * nextnonterminal * lastgaelam
+                )
+            returns = advantages + values
+
+        # flatten the batch
+        b_obs = {
+            k: np.concatenate([obs_[k] for obs_ in obs]) for k in envs.single_observation_space
+        }
+
+        # Edited
+        b_logprobs = logprobs.reshape(-1, args.n_traj)
+        b_actions = actions.reshape((-1,) + envs.single_action_space.shape)
+        b_advantages = advantages.reshape(-1, args.n_traj)
+        b_returns = returns.reshape(-1, args.n_traj)
+        b_values = values.reshape(-1, args.n_traj)
+
+        # Optimizing the policy and value network
+        assert args.num_envs % args.num_minibatches == 0
+        envsperbatch = args.num_envs // args.num_minibatches
+        envinds = np.arange(args.num_envs)
+        flatinds = np.arange(args.batch_size).reshape(args.num_steps, args.num_envs)
+
+        clipfracs = []
+        for epoch in range(args.update_epochs):
+            np.random.shuffle(envinds)
+            for start in range(0, args.num_envs, envsperbatch):
+                end = start + envsperbatch
+                mbenvinds = envinds[start:end]  # mini batch env id
+                mb_inds = flatinds[:, mbenvinds].ravel()  # be really careful about the index
+                r_inds = np.tile(np.arange(envsperbatch), args.num_steps)
+
+                cur_obs = {k: v[mbenvinds] for k, v in obs[0].items()}
+                encoder_state = agent.backbone.encode(cur_obs)
+                _, newlogprob, entropy, newvalue, _ = agent.get_action_and_value_cached(
+                    {k: v[mb_inds] for k, v in b_obs.items()},
+                    b_actions.long()[mb_inds],
+                    (embedding[r_inds, :] for embedding in encoder_state),
+                )
+                # _, newlogprob, entropy, newvalue = agent.get_action_and_value({k:v[mb_inds] for k,v in b_obs.items()}, b_actions.long()[mb_inds])
+                logratio = newlogprob - b_logprobs[mb_inds]
+                ratio = logratio.exp()
+
+                with torch.no_grad():
+                    # calculate approx_kl http://joschu.net/blog/kl-approx.html
+                    old_approx_kl = (-logratio).mean()
+                    approx_kl = ((ratio - 1) - logratio).mean()
+                    clipfracs += [((ratio - 1.0).abs() > args.clip_coef).float().mean().item()]
+
+                mb_advantages = b_advantages[mb_inds]
+                if args.norm_adv:
+                    mb_advantages = (mb_advantages - mb_advantages.mean()) / (
+                        mb_advantages.std() + 1e-8
+                    )
+
+                # Policy loss
+                pg_loss1 = -mb_advantages * ratio
+                pg_loss2 = -mb_advantages * torch.clamp(
+                    ratio, 1 - args.clip_coef, 1 + args.clip_coef
+                )
+                pg_loss = torch.max(pg_loss1, pg_loss2).mean()
+                # Value loss
+                newvalue = newvalue.view(-1, args.n_traj)
+                if args.clip_vloss:
+                    v_loss_unclipped = (newvalue - b_returns[mb_inds]) ** 2
+                    v_clipped = b_values[mb_inds] + torch.clamp(
+                        newvalue - b_values[mb_inds],
+                        -args.clip_coef,
+                        args.clip_coef,
+                    )
+                    v_loss_clipped = (v_clipped - b_returns[mb_inds]) ** 2
+                    v_loss_max = torch.max(v_loss_unclipped, v_loss_clipped)
+                    v_loss = 0.5 * v_loss_max.mean()
+                else:
+                    v_loss = 0.5 * ((newvalue - b_returns[mb_inds]) ** 2).mean()
+
+                entropy_loss = entropy.mean()
+                loss = pg_loss - args.ent_coef * entropy_loss + v_loss * args.vf_coef
+
+                optimizer.zero_grad()
+                loss.backward()
+                nn.utils.clip_grad_norm_(agent.parameters(), args.max_grad_norm)
+                optimizer.step()
+
+            if args.target_kl is not None:
+                if approx_kl > args.target_kl:
+                    break
+
+        y_pred, y_true = b_values.cpu().numpy(), b_returns.cpu().numpy()
+        var_y = np.var(y_true)
+        explained_var = np.nan if var_y == 0 else 1 - np.var(y_true - y_pred) / var_y
+
+        # TRY NOT TO MODIFY: record rewards for plotting purposes
+        writer.add_scalar("charts/learning_rate", optimizer.param_groups[0]["lr"], global_step)
+        writer.add_scalar("losses/value_loss", v_loss.item(), global_step)
+        writer.add_scalar("losses/policy_loss", pg_loss.item(), global_step)
+        writer.add_scalar("losses/entropy", entropy_loss.item(), global_step)
+        writer.add_scalar("losses/old_approx_kl", old_approx_kl.item(), global_step)
+        writer.add_scalar("losses/approx_kl", approx_kl.item(), global_step)
+        writer.add_scalar("losses/clipfrac", np.mean(clipfracs), global_step)
+        writer.add_scalar("losses/explained_variance", explained_var, global_step)
+        print("SPS:", int(global_step / (time.time() - start_time)))
+        writer.add_scalar("charts/SPS", int(global_step / (time.time() - start_time)), global_step)
+        if update % 1000 == 0 or update == num_updates:
+            torch.save(agent.state_dict(), f"runs/{run_name}/ckpt/{update}.pt")
+        if update % 100 == 0 or update == num_updates:
+            agent.eval()
+            test_obs = test_envs.reset()
+            r = []
+            for step in range(0, args.num_steps):
+                # ALGO LOGIC: action logic
+                with torch.no_grad():
+                    action, logits = agent(test_obs)
+                if step == 0:
+                    if args.multi_greedy_inference:
+                        if args.problem == 'tsp':
+                            action = torch.arange(args.n_traj).repeat(args.n_test, 1)
+                        elif args.problem == 'cvrp':
+                            action = torch.arange(1, args.n_traj + 1).repeat(args.n_test, 1)
+                # TRY NOT TO MODIFY: execute the game and log data.
+                test_obs, _, _, test_info = test_envs.step(action.cpu().numpy())
+
+                for item in test_info:
+                    if "episode" in item.keys():
+                        r.append(item)
+
+            avg_episodic_return = np.mean([rollout["episode"]["r"].mean() for rollout in r])
+            max_episodic_return = np.mean([rollout["episode"]["r"].max() for rollout in r])
+            avg_episodic_length = np.mean([rollout["episode"]["l"].mean() for rollout in r])
+            print(f"[test] episodic_return={max_episodic_return}")
+            writer.add_scalar("test/episodic_return_mean", avg_episodic_return, global_step)
+            writer.add_scalar("test/episodic_return_max", max_episodic_return, global_step)
+            writer.add_scalar("test/episodic_length", avg_episodic_length, global_step)
+
+    envs.close()
+    writer.close()

runs/cvrp-v0__ppo_or__1__1678159979/ckpt/12000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4f3be054c6d5517a8bb522713cfb8b8a5b8841a6c2729dedb65f0286f54df856
+size 2865239

runs/tsp-v0__ppo_or__1__1678160003/ckpt/12000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ab6ea48bdcef97438227ad0751836ee7fe8e49ee11d220201c4fbb8a6ddc5bd6
+size 2928924

wrappers/recordWrapper.py

@@ -0,0 +1,62 @@
+import time
+from collections import deque
+
+import gym
+import numpy as np
+
+
+class RecordEpisodeStatistics(gym.Wrapper):
+    def __init__(self, env, deque_size=100):
+        super().__init__(env)
+        self.num_envs = getattr(env, "num_envs", 1)
+        self.n_traj = env.n_traj
+        self.t0 = time.perf_counter()
+        self.episode_count = 0
+        self.episode_returns = None
+        self.episode_lengths = None
+        self.return_queue = deque(maxlen=deque_size)
+        self.length_queue = deque(maxlen=deque_size)
+        self.is_vector_env = getattr(env, "is_vector_env", False)
+
+    def reset(self, **kwargs):
+        observations = super().reset(**kwargs)
+        self.episode_returns = np.zeros((self.num_envs, self.n_traj), dtype=np.float32)
+        self.episode_lengths = np.zeros(self.num_envs, dtype=np.int32)
+        self.finished = [False] * self.num_envs
+        return observations
+
+    def step(self, action):
+        observations, rewards, dones, infos = super().step(action)
+        self.episode_returns += rewards
+        self.episode_lengths += 1
+        if not self.is_vector_env:
+            infos = [infos]
+            dones = [dones]
+        else:
+            infos = list(infos)  # Convert infos to mutable type
+        for i in range(len(dones)):
+            if dones[i].all() and not self.finished[i]:
+                infos[i] = infos[i].copy()
+                episode_return = self.episode_returns[i]
+                episode_length = self.episode_lengths[i]
+                episode_info = {
+                    "r": episode_return.copy(),
+                    "l": episode_length,
+                    "t": round(time.perf_counter() - self.t0, 6),
+                }
+                infos[i]["episode"] = episode_info
+                self.return_queue.append(episode_return)
+                self.length_queue.append(episode_length)
+                self.episode_count += 1
+                self.episode_returns[i] = 0
+                self.episode_lengths[i] = 0
+                self.finished[i] = True
+
+        if self.is_vector_env:
+            infos = tuple(infos)
+        return (
+            observations,
+            rewards,
+            dones if self.is_vector_env else dones[0],
+            infos if self.is_vector_env else infos[0],
+        )

wrappers/syncVectorEnvPomo.py

@@ -0,0 +1,195 @@
+from copy import deepcopy
+from typing import List, Optional, Union
+
+import numpy as np
+from gym.vector.utils import concatenate, create_empty_array, iterate
+from gym.vector.vector_env import VectorEnv
+
+__all__ = ["SyncVectorEnv"]
+
+
+class SyncVectorEnv(VectorEnv):
+    """Vectorized environment that serially runs multiple environments.
+
+    Parameters
+    ----------
+    env_fns : iterable of callable
+        Functions that create the environments.
+
+    observation_space : :class:`gym.spaces.Space`, optional
+        Observation space of a single environment. If ``None``, then the
+        observation space of the first environment is taken.
+
+    action_space : :class:`gym.spaces.Space`, optional
+        Action space of a single environment. If ``None``, then the action space
+        of the first environment is taken.
+
+    copy : bool
+        If ``True``, then the :meth:`reset` and :meth:`step` methods return a
+        copy of the observations.
+
+    Raises
+    ------
+    RuntimeError
+        If the observation space of some sub-environment does not match
+        :obj:`observation_space` (or, by default, the observation space of
+        the first sub-environment).
+
+    Example
+    -------
+
+    .. code-block::
+
+        >>> env = gym.vector.SyncVectorEnv([
+        ...     lambda: gym.make("Pendulum-v0", g=9.81),
+        ...     lambda: gym.make("Pendulum-v0", g=1.62)
+        ... ])
+        >>> env.reset()
+        array([[-0.8286432 ,  0.5597771 ,  0.90249056],
+               [-0.85009176,  0.5266346 ,  0.60007906]], dtype=float32)
+    """
+
+    def __init__(self, env_fns, observation_space=None, action_space=None, copy=True):
+        self.env_fns = env_fns
+        self.envs = [env_fn() for env_fn in env_fns]
+        self.copy = copy
+        self.metadata = self.envs[0].metadata
+        self.n_traj = self.envs[0].n_traj
+
+        if (observation_space is None) or (action_space is None):
+            observation_space = observation_space or self.envs[0].observation_space
+            action_space = action_space or self.envs[0].action_space
+        super().__init__(
+            num_envs=len(env_fns),
+            observation_space=observation_space,
+            action_space=action_space,
+        )
+
+        self._check_spaces()
+        self.observations = create_empty_array(
+            self.single_observation_space, n=self.num_envs, fn=np.zeros
+        )
+        self._rewards = np.zeros((self.num_envs, self.n_traj), dtype=np.float64)
+        self._dones = np.zeros((self.num_envs, self.n_traj), dtype=np.bool_)
+        self._actions = None
+
+    def seed(self, seed=None):
+        super().seed(seed=seed)
+        if seed is None:
+            seed = [None for _ in range(self.num_envs)]
+        if isinstance(seed, int):
+            seed = [seed + i for i in range(self.num_envs)]
+        assert len(seed) == self.num_envs
+
+        for env, single_seed in zip(self.envs, seed):
+            env.seed(single_seed)
+
+    def reset_wait(
+        self,
+        seed: Optional[Union[int, List[int]]] = None,
+        return_info: bool = False,
+        options: Optional[dict] = None,
+    ):
+        if seed is None:
+            seed = [None for _ in range(self.num_envs)]
+        if isinstance(seed, int):
+            seed = [seed + i for i in range(self.num_envs)]
+        assert len(seed) == self.num_envs
+
+        self._dones[:] = False
+        observations = []
+        data_list = []
+        for env, single_seed in zip(self.envs, seed):
+
+            kwargs = {}
+            if single_seed is not None:
+                kwargs["seed"] = single_seed
+            if options is not None:
+                kwargs["options"] = options
+            if return_info == True:
+                kwargs["return_info"] = return_info
+
+            if not return_info:
+                observation = env.reset(**kwargs)
+                observations.append(observation)
+            else:
+                observation, data = env.reset(**kwargs)
+                observations.append(observation)
+                data_list.append(data)
+
+        self.observations = concatenate(
+            self.single_observation_space, observations, self.observations
+        )
+        if not return_info:
+            return deepcopy(self.observations) if self.copy else self.observations
+        else:
+            return (deepcopy(self.observations) if self.copy else self.observations), data_list
+
+    def step_async(self, actions):
+        self._actions = iterate(self.action_space, actions)
+
+    def step_wait(self):
+        observations, infos = [], []
+        for i, (env, action) in enumerate(zip(self.envs, self._actions)):
+            observation, self._rewards[i], self._dones[i], info = env.step(action)
+            # if self._dones[i].all():
+            #    observation = env.reset()
+            observations.append(observation)
+            infos.append(info)
+        self.observations = concatenate(
+            self.single_observation_space, observations, self.observations
+        )
+
+        return (
+            deepcopy(self.observations) if self.copy else self.observations,
+            np.copy(self._rewards),
+            np.copy(self._dones),
+            infos,
+        )
+
+    def call(self, name, *args, **kwargs):
+        results = []
+        for env in self.envs:
+            function = getattr(env, name)
+            if callable(function):
+                results.append(function(*args, **kwargs))
+            else:
+                results.append(function)
+
+        return tuple(results)
+
+    def set_attr(self, name, values):
+        if not isinstance(values, (list, tuple)):
+            values = [values for _ in range(self.num_envs)]
+        if len(values) != self.num_envs:
+            raise ValueError(
+                "Values must be a list or tuple with length equal to the "
+                f"number of environments. Got `{len(values)}` values for "
+                f"{self.num_envs} environments."
+            )
+
+        for env, value in zip(self.envs, values):
+            setattr(env, name, value)
+
+    def close_extras(self, **kwargs):
+        """Close the environments."""
+        [env.close() for env in self.envs]
+
+    def _check_spaces(self):
+        for env in self.envs:
+            if not (env.observation_space == self.single_observation_space):
+                raise RuntimeError(
+                    "Some environments have an observation space different from "
+                    f"`{self.single_observation_space}`. In order to batch observations, "
+                    "the observation spaces from all environments must be equal."
+                )
+
+            if not (env.action_space == self.single_action_space):
+                raise RuntimeError(
+                    "Some environments have an action space different from "
+                    f"`{self.single_action_space}`. In order to batch actions, the "
+                    "action spaces from all environments must be equal."
+                )
+
+        else:
+            return True