Asymmetric self-play

docs/Training-Self-Play.md

@@ -15,7 +15,7 @@ Self-play is triggered by including the self-play hyperparameter hierarchy in th
 
 ![Team ID](images/team_id.png)
 
-See the trainer configuration and agent prefabs for our Tennis environment for an example.
+***Team ID must be 0 or an integer greater than 0. Negative numbers will cause unpredictable behavior.*** See the trainer configuration and agent prefabs for our Tennis environment for an example.
 
 ## Best Practices Training with Self-Play
 

ml-agents/mlagents/trainers/behavior_id_utils.py

@@ -1,10 +1,10 @@
-from typing import Dict, NamedTuple
+from typing import NamedTuple
 
 
 class BehaviorIdentifiers(NamedTuple):
-    name_behavior_id: str
+    behavior_id: str
     brain_name: str
-    behavior_ids: Dict[str, int]
+    team_id: int
 
     @staticmethod
     def from_name_behavior_id(name_behavior_id: str) -> "BehaviorIdentifiers":
@@ -17,20 +17,14 @@ def from_name_behavior_id(name_behavior_id: str) -> "BehaviorIdentifiers":
         :returns: A BehaviorIdentifiers object.
         """
 
-        ids: Dict[str, int] = {}
+        team_id: int = 0
         if "?" in name_behavior_id:
-            name, identifiers = name_behavior_id.rsplit("?", 1)
-            if "&" in identifiers:
-                list_of_identifiers = identifiers.split("&")
-            else:
-                list_of_identifiers = [identifiers]
-
-            for identifier in list_of_identifiers:
-                key, value = identifier.split("=")
-                ids[key] = int(value)
+            name, team_and_id = name_behavior_id.rsplit("?", 1)
+            _, team_id_str = team_and_id.split("=")
+            team_id = int(team_id_str)
         else:
             name = name_behavior_id
 
         return BehaviorIdentifiers(
-            name_behavior_id=name_behavior_id, brain_name=name, behavior_ids=ids
+            behavior_id=name_behavior_id, brain_name=name, team_id=team_id
         )

ml-agents/mlagents/trainers/ghost/controller.py

@@ -0,0 +1,48 @@
+from typing import Deque, Dict
+from collections import deque
+from mlagents.trainers.ghost.trainer import GhostTrainer
+
+
+class GhostController(object):
+    def __init__(self, swap_interval: int, maxlen: int = 10):
+        self._swap_interval = swap_interval
+        self._last_swap: int = 0
+        self._queue: Deque[int] = deque(maxlen=maxlen)
+        self._learning_team: int = -1
+        self._ghost_trainers: Dict[int, GhostTrainer] = {}
+
+    def subscribe_team_id(self, team_id: int, trainer: GhostTrainer) -> None:
+        if team_id not in self._ghost_trainers:
+            self._queue.append(team_id)
+            self._ghost_trainers[team_id] = trainer
+            if self._learning_team < 0:
+                self._learning_team = team_id
+
+    def get_learning_team(self, step: int) -> int:
+        if step >= self._swap_interval + self._last_swap:
+            self._last_swap = step
+            self._learning_team = self._queue.popleft()
+            self._queue.append(self._learning_team)
+        return self._learning_team
+
+    # Adapted from https://github.com/Unity-Technologies/ml-agents/pull/1975 and
+    # https://metinmediamath.wordpress.com/2013/11/27/how-to-calculate-the-elo-rating-including-example/
+    # ELO calculation
+
+    def compute_elo_rating_changes(self, rating: float, result: float) -> float:
+        opponent_rating: float = 0.0
+        for team_id, trainer in self._ghost_trainers.items():
+            if team_id != self._learning_team:
+                opponent_rating = trainer.get_opponent_elo()
+        r1 = pow(10, rating / 400)
+        r2 = pow(10, opponent_rating / 400)
+
+        summed = r1 + r2
+        e1 = r1 / summed
+
+        change = result - e1
+        for team_id, trainer in self._ghost_trainers.items():
+            if team_id != self._learning_team:
+                trainer.change_opponent_elo(change)
+
+        return change