Merge pull request #77 from Axelrod-Python/indexing

WIP: Add automatic allocation of copies of the share library on demand for multiple copies of players.

.travis.yml

@@ -10,7 +10,9 @@ before_install:
   - git clone https://github.com/Axelrod-Python/TourExec.git /tmp/TourExec
   - cd /tmp/TourExec
   - sudo make install
-  - export LD_LIBRARY_PATH=/usr/local/lib
+  - echo "/usr/lib/libstrategies.so" | sudo tee /etc/ld.so.conf.d/strategies-lib.conf
+  - sudo ldconfig
+  - ldconfig -p | grep libstrategies.so
   - cd $TRAVIS_BUILD_DIR
 install:
   - pip install -r requirements.txt

src/axelrod_fortran/player.py

@@ -1,23 +1,37 @@
+from ctypes import byref, c_float, c_int, POINTER
 import random
 import warnings
 
 import axelrod as axl
 from axelrod.interaction_utils import compute_final_score
 from axelrod.action import Action
-from ctypes import cdll, c_int, c_float, byref, POINTER
+
 from .strategies import characteristics
+from .shared_library_manager import MultiprocessManager, load_library
 
 C, D = Action.C, Action.D
 actions = {0: C, 1: D}
 original_actions = {C: 0, D: 1}
 
 
+self_interaction_message = """
+You are playing a match with the same player against itself. However
+axelrod_fortran players share memory. You can initialise another instance of an
+Axelrod_fortran player with player.clone().
+"""
+
+
+# Initialize a module-wide manager for loading copies of the shared library.
+manager = MultiprocessManager()
+manager.start()
+shared_library_manager = manager.SharedLibraryManager("libstrategies.so")
+
+
 class Player(axl.Player):
 
     classifier = {"stochastic": True}
 
-    def __init__(self, original_name,
-                 shared_library_name='libstrategies.so'):
+    def __init__(self, original_name):
         """
         Parameters
         ----------
@@ -28,14 +42,16 @@ def __init__(self, original_name,
             A instance of an axelrod Game
         """
         super().__init__()
-        self.shared_library_name = shared_library_name
-        self.shared_library = cdll.LoadLibrary(shared_library_name)
+        self.index, self.shared_library_filename = \
+            shared_library_manager.get_filename_for_player(original_name)
+        self.shared_library = load_library(self.shared_library_filename)
         self.original_name = original_name
         self.original_function = self.original_name
         is_stochastic = characteristics[self.original_name]['stochastic']
         if is_stochastic is not None:
             self.classifier['stochastic'] = is_stochastic
 
+
     def __enter__(self):
         return self
 
@@ -75,17 +91,8 @@ def original_strategy(
         return self.original_function(*[byref(arg) for arg in args])
 
     def strategy(self, opponent):
-        if type(opponent) is Player \
-          and (opponent.original_name == self.original_name) \
-          and (opponent.shared_library_name == self.shared_library_name):
-
-            message = """
-You are playing a match with two copies of the same player.
-However the axelrod fortran players share memory.
-You can initialise an instance of an Axelrod_fortran player with a
-`shared_library_name`
-variable that points to a copy of the shared library."""
-            warnings.warn(message=message)
+        if self is opponent:
+            warnings.warn(message=self_interaction_message)
 
         if not self.history:
             their_last_move = 0
@@ -106,6 +113,25 @@ def strategy(self, opponent):
             my_last_move)
         return actions[original_action]
 
+    def _release_shared_library(self):
+        # While this looks like we're checking that the shared library file
+        # isn't deleted, the exception is actually thrown if the manager
+        # thread closes before the player class is garbage collected, which
+        # tends to happen at the end of a script.
+        try:
+            shared_library_manager.release(self.original_name, self.index)
+        except FileNotFoundError:
+            pass
+
     def reset(self):
+        # Release the shared library since the object is rebuilt on reset.
+        self._release_shared_library()
         super().reset()
         self.original_function = self.original_name
+
+    def __del__(self):
+        # Release the library before deletion.
+        self._release_shared_library()
+
+    def __repr__(self):
+        return self.original_name

src/axelrod_fortran/shared_library_manager.py

@@ -0,0 +1,120 @@
+from collections import defaultdict
+from ctypes import cdll
+from ctypes.util import find_library
+from multiprocessing.managers import BaseManager
+from pathlib import Path
+import platform
+import shutil
+import subprocess
+import tempfile
+import uuid
+
+
+def load_library(filename):
+    """Loads a shared library."""
+    lib = None
+    if Path(filename).exists():
+        lib = cdll.LoadLibrary(filename)
+    return lib
+
+
+class SharedLibraryManager(object):
+    """LibraryManager creates (and deletes) copies of a shared library, which
+    enables multiple copies of the same strategy to be run without the end user
+    having to maintain many copies of the shared library.
+
+    This works by making a copy of the shared library file and loading it into
+    memory again. Loading the same file again will return a reference to the
+    same memory addresses. To be thread-safe, this class just passes filenames
+    back to the Player class (which actually loads a reference to the library),
+    ensuring that multiple copies of a given player type do not use the same
+    copy of the shared library.
+    """
+
+    def __init__(self, shared_library_name, verbose=False):
+        self.shared_library_name = shared_library_name
+        self.verbose = verbose
+        self.filenames = []
+        self.player_indices = defaultdict(set)
+        self.player_next = defaultdict(set)
+        # Generate a random prefix for tempfile generation
+        self.prefix = str(uuid.uuid4())
+        self.library_path = self.find_shared_library(shared_library_name)
+
+    def find_shared_library(self, shared_library_name):
+        # Hack for Linux since find_library doesn't return the full path.
+        if 'Linux' in platform.system():
+            output = subprocess.check_output(["ldconfig", "-p"])
+            for line in str(output).split(r"\n"):
+                rhs = line.split(" => ")[-1]
+                if shared_library_name in rhs:
+                    return rhs
+            raise ValueError("{} not found".format(shared_library_name))
+        else:
+            return find_library(
+                shared_library_name.replace("lib", "").replace(".so", ""))
+
+    def create_library_copy(self):
+        """Create a new copy of the shared library."""
+        # Copy the library file to a new (temp) location.
+        temp_directory = tempfile.gettempdir()
+        copy_number = len(self.filenames)
+        filename =  "{}-{}-{}".format(
+            self.prefix,
+            str(copy_number),
+            self.shared_library_name)
+        new_filename = str(Path(temp_directory, filename))
+        if self.verbose:
+            print("Loading {}".format(new_filename))
+        shutil.copy2(self.library_path, new_filename)
+        self.filenames.append(new_filename)
+
+    def next_player_index(self, name):
+        """Determine the index of the next free shared library copy to
+        allocate for the player. If none is available then make another copy."""
+        # Is there a free index?
+        if len(self.player_next[name]) > 0:
+            return self.player_next[name].pop()
+        # Do we need to load a new copy?
+        player_count = len(self.player_indices[name])
+        if player_count == len(self.filenames):
+            self.create_library_copy()
+            return player_count
+        # Find the first unused index
+        for i in range(len(self.filenames)):
+            if i not in self.player_indices[name]:
+                return i
+        raise ValueError("We shouldn't be here.")
+
+    def get_filename_for_player(self, name):
+        """For a given player return a filename for a copy of the shared library
+        for use in a Player class, along with an index for later releasing."""
+        index = self.next_player_index(name)
+        self.player_indices[name].add(index)
+        if self.verbose:
+            print("allocating {}".format(index))
+        return index, self.filenames[index]
+
+    def release(self, name, index):
+        """Release the copy of the library so that it can be re-allocated."""
+        self.player_indices[name].remove(index)
+        if self.verbose:
+            print("releasing {}".format(index))
+        self.player_next[name].add(index)
+
+    def __del__(self):
+        """Cleanup temp files on object deletion."""
+        for filename in self.filenames:
+            path = Path(filename)
+            if path.exists():
+                if self.verbose:
+                    print("deleting", str(path))
+                path.unlink()
+
+
+# Setup up thread safe library manager.
+class MultiprocessManager(BaseManager):
+    pass
+
+
+MultiprocessManager.register('SharedLibraryManager', SharedLibraryManager)

tests/test_player.py

@@ -1,12 +1,14 @@
-from axelrod_fortran import Player, characteristics, all_strategies
-from axelrod import (Alternator, Cooperator, Defector,
-                     Match, Game, basic_strategies, seed)
-from axelrod.action import Action
 from ctypes import c_int, c_float, POINTER, CDLL
-
 import itertools
+
 import pytest
 
+from axelrod_fortran import Player, characteristics, all_strategies
+from axelrod import (Alternator, Cooperator, Defector, Match, MoranProcess,
+                     Game, basic_strategies, seed)
+from axelrod.action import Action
+
+
 C, D = Action.C, Action.D
 
 
@@ -22,15 +24,6 @@ def test_init():
         assert player.original_function.restype == c_int
         with pytest.raises(ValueError):
             player = Player('test')
-        assert "libstrategies.so" == player.shared_library_name
-        assert type(player.shared_library) is CDLL
-        assert "libstrategies.so" in str(player.shared_library)
-
-def test_init_with_shared():
-    player = Player("k42r", shared_library_name="libstrategies.so")
-    assert "libstrategies.so" == player.shared_library_name
-    assert type(player.shared_library) is CDLL
-    assert "libstrategies.so" in str(player.shared_library)
 
 
 def test_matches():
@@ -106,6 +99,7 @@ def test_original_strategy():
                 my_score += scores[0]
                 their_score += scores[1]
 
+
 def test_deterministic_strategies():
     """
     Test that the strategies classified as deterministic indeed act
@@ -139,6 +133,7 @@ def test_implemented_strategies():
                 axl_match = Match((axl_player, opponent))
                 assert interactions == axl_match.play(), (player, opponent)
 
+
 def test_champion_v_alternator():
     """
     Specific regression test for a bug.
@@ -155,18 +150,20 @@ def test_champion_v_alternator():
     seed(0)
     assert interactions == match.play()
 
+
 def test_warning_for_self_interaction(recwarn):
     """
     Test that a warning is given for a self interaction.
     """
     player = Player("k42r")
-    opponent = Player("k42r")
+    opponent = player
 
     match = Match((player, opponent))
 
     interactions = match.play()
     assert len(recwarn) == 1
 
+
 def test_no_warning_for_normal_interaction(recwarn):
     """
     Test that a warning is not given for a normal interaction
@@ -180,3 +177,11 @@ def test_no_warning_for_normal_interaction(recwarn):
 
         interactions = match.play()
         assert len(recwarn) == 0
+
+
+def test_multiple_copies(recwarn):
+    players = [Player('ktitfortatc') for _ in range(5)] + [
+        Player('k42r') for _ in range(5)]
+    mp = MoranProcess(players)
+    mp.play()
+    mp.populations_plot()

tests/test_titfortat.py

@@ -24,3 +24,10 @@ def test_versus_defector():
     match = axl.Match(players, 5)
     expected = [(C, D), (D, D), (D, D), (D, D), (D, D)]
     assert match.play() == expected
+
+
+def test_versus_itself():
+    players = (Player('ktitfortatc'), Player('ktitfortatc'))
+    match = axl.Match(players, 5)
+    expected = [(C, C), (C, C), (C, C), (C, C), (C, C)]
+    assert match.play() == expected