Fix nondeterminism in fixture collection order

fixtures.reorder_items is non-deterministic because it reorders based
on iteration over an (unordered) set.  Change the code to use an
OrderedDict instead so that we get deterministic behaviour, fixes #920.

Author: wence-

AUTHORS

@@ -91,6 +91,7 @@ Kale Kundert
 Katarzyna Jachim
 Kevin Cox
 Kodi B. Arfer
+Lawrence Mitchell
 Lee Kamentsky
 Lev Maximov
 Llandy Riveron Del Risco

_pytest/fixtures.py

@@ -19,6 +19,11 @@
 from _pytest.runner import fail
 from _pytest.compat import FuncargnamesCompatAttr
 
+if sys.version_info[:2] == (2, 6):
+    from ordereddict import OrderedDict
+else:
+    from collections import OrderedDict
+
 
 def pytest_sessionstart(session):
     import _pytest.python
@@ -136,10 +141,10 @@ def get_parametrized_fixture_keys(item, scopenum):
     except AttributeError:
         pass
     else:
-        # cs.indictes.items() is random order of argnames but
-        # then again different functions (items) can change order of
-        # arguments so it doesn't matter much probably
-        for argname, param_index in cs.indices.items():
+        # cs.indices.items() is random order of argnames.  Need to
+        # sort this so that different calls to
+        # get_parametrized_fixture_keys will be deterministic.
+        for argname, param_index in sorted(cs.indices.items()):
             if cs._arg2scopenum[argname] != scopenum:
                 continue
             if scopenum == 0:    # session
@@ -161,7 +166,7 @@ def reorder_items(items):
     for scopenum in range(0, scopenum_function):
         argkeys_cache[scopenum] = d = {}
         for item in items:
-            keys = set(get_parametrized_fixture_keys(item, scopenum))
+            keys = OrderedDict.fromkeys(get_parametrized_fixture_keys(item, scopenum))
             if keys:
                 d[item] = keys
     return reorder_items_atscope(items, set(), argkeys_cache, 0)
@@ -196,9 +201,9 @@ def slice_items(items, ignore, scoped_argkeys_cache):
         for i, item in enumerate(it):
             argkeys = scoped_argkeys_cache.get(item)
             if argkeys is not None:
-                argkeys = argkeys.difference(ignore)
-                if argkeys:  # found a slicing key
-                    slicing_argkey = argkeys.pop()
+                newargkeys = OrderedDict.fromkeys(k for k in argkeys if k not in ignore)
+                if newargkeys:  # found a slicing key
+                    slicing_argkey, _ = newargkeys.popitem()
                     items_before = items[:i]
                     items_same = [item]
                     items_other = []

changelog/920.bugfix

@@ -0,0 +1 @@
+Fix non-determinism in order of fixture collection.

testing/python/fixture.py

@@ -2548,7 +2548,6 @@ def test_foo(fix):
             '*test_foo*beta*'])
 
     @pytest.mark.issue920
-    @pytest.mark.xfail(reason="Fixture reordering not deterministic with hash randomisation")
     def test_deterministic_fixture_collection(self, testdir, monkeypatch):
         testdir.makepyfile("""
             import pytest

tox.ini

@@ -34,6 +34,7 @@ deps =
     hypothesis<3.0
     nose
     mock<1.1
+    ordereddict
 
 [testenv:py27-subprocess]
 changedir = .