diff --git a/pandas/src/hash.pyx b/pandas/src/hash.pyx index a393e0df96954..06ed947808e39 100644 --- a/pandas/src/hash.pyx +++ b/pandas/src/hash.pyx @@ -40,7 +40,8 @@ def hash_object_array(ndarray[object] arr, object key, object encoding='utf8'): Py_ssize_t i, l, n ndarray[uint64_t] result bytes data, k - uint8_t *kb, *lens + uint8_t *kb + uint64_t *lens char **vecs, *cdata object val @@ -55,7 +56,7 @@ def hash_object_array(ndarray[object] arr, object key, object encoding='utf8'): # create an array of bytes vecs = malloc(n * sizeof(char *)) - lens = malloc(n * sizeof(uint8_t)) + lens = malloc(n * sizeof(uint64_t)) cdef list datas = [] for i in range(n): diff --git a/pandas/tools/tests/test_hashing.py b/pandas/tools/tests/test_hashing.py index 4e05ae7007c80..6e5f30fb7a52d 100644 --- a/pandas/tools/tests/test_hashing.py +++ b/pandas/tools/tests/test_hashing.py @@ -142,7 +142,36 @@ def test_alternate_encoding(self): obj = Series(list('abc')) self.check_equal(obj, encoding='ascii') - def test_long_strings(self): - - obj = Index(tm.rands_array(nchars=10000, size=100)) - self.check_equal(obj) + def test_same_len_hash_collisions(self): + + for l in range(8): + length = 2**(l + 8) + 1 + s = tm.rands_array(length, 2) + result = hash_array(s, 'utf8') + self.assertFalse(result[0] == result[1]) + + for l in range(8): + length = 2**(l + 8) + s = tm.rands_array(length, 2) + result = hash_array(s, 'utf8') + self.assertFalse(result[0] == result[1]) + + def test_hash_collisions(self): + + # hash collisions are bad + # https://github.com/pandas-dev/pandas/issues/14711#issuecomment-264885726 + L = ['Ingrid-9Z9fKIZmkO7i7Cn51Li34pJm44fgX6DYGBNj3VPlOH50m7HnBlPxfIwFMrcNJNMP6PSgLmwWnInciMWrCSAlLEvt7JkJl4IxiMrVbXSa8ZQoVaq5xoQPjltuJEfwdNlO6jo8qRRHvD8sBEBMQASrRa6TsdaPTPCBo3nwIBpE7YzzmyH0vMBhjQZLx1aCT7faSEx7PgFxQhHdKFWROcysamgy9iVj8DO2Fmwg1NNl93rIAqC3mdqfrCxrzfvIY8aJdzin2cHVzy3QUJxZgHvtUtOLxoqnUHsYbNTeq0xcLXpTZEZCxD4PGubIuCNf32c33M7HFsnjWSEjE2yVdWKhmSVodyF8hFYVmhYnMCztQnJrt3O8ZvVRXd5IKwlLexiSp4h888w7SzAIcKgc3g5XQJf6MlSMftDXm9lIsE1mJNiJEv6uY6pgvC3fUPhatlR5JPpVAHNSbSEE73MBzJrhCAbOLXQumyOXigZuPoME7QgJcBalliQol7YZ9', # noqa + 'Tim-b9MddTxOWW2AT1Py6vtVbZwGAmYCjbp89p8mxsiFoVX4FyDOF3wFiAkyQTUgwg9sVqVYOZo09Dh1AzhFHbgij52ylF0SEwgzjzHH8TGY8Lypart4p4onnDoDvVMBa0kdthVGKl6K0BDVGzyOXPXKpmnMF1H6rJzqHJ0HywfwS4XYpVwlAkoeNsiicHkJUFdUAhG229INzvIAiJuAHeJDUoyO4DCBqtoZ5TDend6TK7Y914yHlfH3g1WZu5LksKv68VQHJriWFYusW5e6ZZ6dKaMjTwEGuRgdT66iU5nqWTHRH8WSzpXoCFwGcTOwyuqPSe0fTe21DVtJn1FKj9F9nEnR9xOvJUO7E0piCIF4Ad9yAIDY4DBimpsTfKXCu1vdHpKYerzbndfuFe5AhfMduLYZJi5iAw8qKSwR5h86ttXV0Mc0QmXz8dsRvDgxjXSmupPxBggdlqUlC828hXiTPD7am0yETBV0F3bEtvPiNJfremszcV8NcqAoARMe'] # noqa + + # these should be different! + result1 = hash_array(np.asarray(L[0:1], dtype=object), 'utf8') + expected1 = np.array([14963968704024874985], dtype=np.uint64) + self.assert_numpy_array_equal(result1, expected1) + + result2 = hash_array(np.asarray(L[1:2], dtype=object), 'utf8') + expected2 = np.array([16428432627716348016], dtype=np.uint64) + self.assert_numpy_array_equal(result2, expected2) + + result = hash_array(np.asarray(L, dtype=object), 'utf8') + self.assert_numpy_array_equal( + result, np.concatenate([expected1, expected2], axis=0))