bernsteinbear.com
this repo has no description
1#!/usr/bin/env python3
2# Copyright (c) Facebook, Inc. and its affiliates. (http://www.facebook.com)
3# WARNING: This is a temporary copy of code from the cpython library to
4# facilitate bringup. Please file a task for anything you change!
5# flake8: noqa
6# fmt: off
7import gc
8import pickle
9import unittest
10from test import seq_tests, support
11
12# For tuple hashes, we normally only run a test to ensure that we get
13# the same results across platforms in a handful of cases. If that's
14# so, there's no real point to running more. Set RUN_ALL_HASH_TESTS to
15# run more anyway. That's usually of real interest only when analyzing,
16# or changing, the hash algorithm. In which case it's usually also
17# most useful to set JUST_SHOW_HASH_RESULTS, to see all the results
18# instead of wrestling with test "failures". See the bottom of the
19# file for extensive notes on what we're testing here and why.
20RUN_ALL_HASH_TESTS = False
21JUST_SHOW_HASH_RESULTS = False # if RUN_ALL_HASH_TESTS, just display
22
23class TupleTest(seq_tests.CommonTest):
24 type2test = tuple
25
26 def test_getitem_error(self):
27 t = ()
28 msg = "tuple indices must be integers or slices"
29 with self.assertRaisesRegex(TypeError, msg):
30 t['a']
31
32 def test_constructors(self):
33 super().test_constructors()
34 # calling built-in types without argument must return empty
35 self.assertEqual(tuple(), ())
36 t0_3 = (0, 1, 2, 3)
37 t0_3_bis = tuple(t0_3)
38 self.assertTrue(t0_3 is t0_3_bis)
39 self.assertEqual(tuple([]), ())
40 self.assertEqual(tuple([0, 1, 2, 3]), (0, 1, 2, 3))
41 self.assertEqual(tuple(''), ())
42 self.assertEqual(tuple('spam'), ('s', 'p', 'a', 'm'))
43 self.assertEqual(tuple(x for x in range(10) if x % 2),
44 (1, 3, 5, 7, 9))
45
46 def test_keyword_args(self):
47 with self.assertRaisesRegex(TypeError, 'keyword argument'):
48 tuple(sequence=())
49
50 def test_truth(self):
51 super().test_truth()
52 self.assertTrue(not ())
53 self.assertTrue((42, ))
54
55 def test_len(self):
56 super().test_len()
57 self.assertEqual(len(()), 0)
58 self.assertEqual(len((0,)), 1)
59 self.assertEqual(len((0, 1, 2)), 3)
60
61 def test_iadd(self):
62 super().test_iadd()
63 u = (0, 1)
64 u2 = u
65 u += (2, 3)
66 self.assertTrue(u is not u2)
67
68 def test_imul(self):
69 super().test_imul()
70 u = (0, 1)
71 u2 = u
72 u *= 3
73 self.assertTrue(u is not u2)
74
75 def test_tupleresizebug(self):
76 # Check that a specific bug in _PyTuple_Resize() is squashed.
77 def f():
78 for i in range(1000):
79 yield i
80 self.assertEqual(list(tuple(f())), list(range(1000)))
81
82 # We expect tuples whose base components have deterministic hashes to
83 # have deterministic hashes too - and, indeed, the same hashes across
84 # platforms with hash codes of the same bit width.
85 @unittest.skip("Pyro hash size differs from CPython")
86 def test_hash_exact(self):
87 def check_one_exact(t, e32, e64):
88 got = hash(t)
89 expected = e32 if support.NHASHBITS == 32 else e64
90 if got != expected:
91 msg = f"FAIL hash({t!r}) == {got} != {expected}"
92 self.fail(msg)
93
94 check_one_exact((), 750394483, 5740354900026072187)
95 check_one_exact((0,), 1214856301, -8753497827991233192)
96 check_one_exact((0, 0), -168982784, -8458139203682520985)
97 check_one_exact((0.5,), 2077348973, -408149959306781352)
98 check_one_exact((0.5, (), (-2, 3, (4, 6))), 714642271,
99 -1845940830829704396)
100
101 # Various tests for hashing of tuples to check that we get few collisions.
102 # Does something only if RUN_ALL_HASH_TESTS is true.
103 #
104 # Earlier versions of the tuple hash algorithm had massive collisions
105 # reported at:
106 # - https://bugs.python.org/issue942952
107 # - https://bugs.python.org/issue34751
108 def test_hash_optional(self):
109 from itertools import product
110
111 if not RUN_ALL_HASH_TESTS:
112 return
113
114 # If specified, `expected` is a 2-tuple of expected
115 # (number_of_collisions, pileup) values, and the test fails if
116 # those aren't the values we get. Also if specified, the test
117 # fails if z > `zlimit`.
118 def tryone_inner(tag, nbins, hashes, expected=None, zlimit=None):
119 from collections import Counter
120
121 nballs = len(hashes)
122 mean, sdev = support.collision_stats(nbins, nballs)
123 c = Counter(hashes)
124 collisions = nballs - len(c)
125 z = (collisions - mean) / sdev
126 pileup = max(c.values()) - 1
127 del c
128 got = (collisions, pileup)
129 failed = False
130 prefix = ""
131 if zlimit is not None and z > zlimit:
132 failed = True
133 prefix = f"FAIL z > {zlimit}; "
134 if expected is not None and got != expected:
135 failed = True
136 prefix += f"FAIL {got} != {expected}; "
137 if failed or JUST_SHOW_HASH_RESULTS:
138 msg = f"{prefix}{tag}; pileup {pileup:,} mean {mean:.1f} "
139 msg += f"coll {collisions:,} z {z:+.1f}"
140 if JUST_SHOW_HASH_RESULTS:
141 import sys
142 print(msg, file=sys.__stdout__)
143 else:
144 self.fail(msg)
145
146 def tryone(tag, xs,
147 native32=None, native64=None, hi32=None, lo32=None,
148 zlimit=None):
149 NHASHBITS = support.NHASHBITS
150 hashes = list(map(hash, xs))
151 tryone_inner(tag + f"; {NHASHBITS}-bit hash codes",
152 1 << NHASHBITS,
153 hashes,
154 native32 if NHASHBITS == 32 else native64,
155 zlimit)
156
157 if NHASHBITS > 32:
158 shift = NHASHBITS - 32
159 tryone_inner(tag + "; 32-bit upper hash codes",
160 1 << 32,
161 [h >> shift for h in hashes],
162 hi32,
163 zlimit)
164
165 mask = (1 << 32) - 1
166 tryone_inner(tag + "; 32-bit lower hash codes",
167 1 << 32,
168 [h & mask for h in hashes],
169 lo32,
170 zlimit)
171
172 # Tuples of smallish positive integers are common - nice if we
173 # get "better than random" for these.
174 tryone("range(100) by 3", list(product(range(100), repeat=3)),
175 (0, 0), (0, 0), (4, 1), (0, 0))
176
177 # A previous hash had systematic problems when mixing integers of
178 # similar magnitude but opposite sign, obscurely related to that
179 # j ^ -2 == -j when j is odd.
180 cands = list(range(-10, -1)) + list(range(9))
181
182 # Note: -1 is omitted because hash(-1) == hash(-2) == -2, and
183 # there's nothing the tuple hash can do to avoid collisions
184 # inherited from collisions in the tuple components' hashes.
185 tryone("-10 .. 8 by 4", list(product(cands, repeat=4)),
186 (0, 0), (0, 0), (0, 0), (0, 0))
187 del cands
188
189 # The hashes here are a weird mix of values where all the
190 # variation is in the lowest bits and across a single high-order
191 # bit - the middle bits are all zeroes. A decent hash has to
192 # both propagate low bits to the left and high bits to the
193 # right. This is also complicated a bit in that there are
194 # collisions among the hashes of the integers in L alone.
195 L = [n << 60 for n in range(100)]
196 tryone("0..99 << 60 by 3", list(product(L, repeat=3)),
197 (0, 0), (0, 0), (0, 0), (324, 1))
198 del L
199
200 # Used to suffer a massive number of collisions.
201 tryone("[-3, 3] by 18", list(product([-3, 3], repeat=18)),
202 (7, 1), (0, 0), (7, 1), (6, 1))
203
204 # And even worse. hash(0.5) has only a single bit set, at the
205 # high end. A decent hash needs to propagate high bits right.
206 tryone("[0, 0.5] by 18", list(product([0, 0.5], repeat=18)),
207 (5, 1), (0, 0), (9, 1), (12, 1))
208
209 # Hashes of ints and floats are the same across platforms.
210 # String hashes vary even on a single platform across runs, due
211 # to hash randomization for strings. So we can't say exactly
212 # what this should do. Instead we insist that the # of
213 # collisions is no more than 4 sdevs above the theoretically
214 # random mean. Even if the tuple hash can't achieve that on its
215 # own, the string hash is trying to be decently pseudo-random
216 # (in all bit positions) on _its_ own. We can at least test
217 # that the tuple hash doesn't systematically ruin that.
218 tryone("4-char tuples",
219 list(product("abcdefghijklmnopqrstuvwxyz", repeat=4)),
220 zlimit=4.0)
221
222 # The "old tuple test". See https://bugs.python.org/issue942952.
223 # Ensures, for example, that the hash:
224 # is non-commutative
225 # spreads closely spaced values
226 # doesn't exhibit cancellation in tuples like (x,(x,y))
227 N = 50
228 base = list(range(N))
229 xp = list(product(base, repeat=2))
230 inps = base + list(product(base, xp)) + \
231 list(product(xp, base)) + xp + list(zip(base))
232 tryone("old tuple test", inps,
233 (2, 1), (0, 0), (52, 49), (7, 1))
234 del base, xp, inps
235
236 # The "new tuple test". See https://bugs.python.org/issue34751.
237 # Even more tortured nesting, and a mix of signed ints of very
238 # small magnitude.
239 n = 5
240 A = [x for x in range(-n, n+1) if x != -1]
241 B = A + [(a,) for a in A]
242 L2 = list(product(A, repeat=2))
243 L3 = L2 + list(product(A, repeat=3))
244 L4 = L3 + list(product(A, repeat=4))
245 # T = list of testcases. These consist of all (possibly nested
246 # at most 2 levels deep) tuples containing at most 4 items from
247 # the set A.
248 T = A
249 T += [(a,) for a in B + L4]
250 T += product(L3, B)
251 T += product(L2, repeat=2)
252 T += product(B, L3)
253 T += product(B, B, L2)
254 T += product(B, L2, B)
255 T += product(L2, B, B)
256 T += product(B, repeat=4)
257 assert len(T) == 345130
258 tryone("new tuple test", T,
259 (9, 1), (0, 0), (21, 5), (6, 1))
260
261 def test_repr(self):
262 l0 = tuple()
263 l2 = (0, 1, 2)
264 a0 = self.type2test(l0)
265 a2 = self.type2test(l2)
266
267 self.assertEqual(str(a0), repr(l0))
268 self.assertEqual(str(a2), repr(l2))
269 self.assertEqual(repr(a0), "()")
270 self.assertEqual(repr(a2), "(0, 1, 2)")
271
272 def _not_tracked(self, t):
273 # Nested tuples can take several collections to untrack
274 gc.collect()
275 gc.collect()
276 self.assertFalse(gc.is_tracked(t), t)
277
278 def _tracked(self, t):
279 self.assertTrue(gc.is_tracked(t), t)
280 gc.collect()
281 gc.collect()
282 self.assertTrue(gc.is_tracked(t), t)
283
284 @unittest.skip("garbage collection is different in pyro")
285 def test_track_literals(self):
286 # Test GC-optimization of tuple literals
287 x, y, z = 1.5, "a", []
288
289 self._not_tracked(())
290 self._not_tracked((1,))
291 self._not_tracked((1, 2))
292 self._not_tracked((1, 2, "a"))
293 self._not_tracked((1, 2, (None, True, False, ()), int))
294 self._not_tracked((object(),))
295 self._not_tracked(((1, x), y, (2, 3)))
296
297 # Tuples with mutable elements are always tracked, even if those
298 # elements are not tracked right now.
299 self._tracked(([],))
300 self._tracked(([1],))
301 self._tracked(({},))
302 self._tracked((set(),))
303 self._tracked((x, y, z))
304
305 def check_track_dynamic(self, tp, always_track):
306 x, y, z = 1.5, "a", []
307
308 check = self._tracked if always_track else self._not_tracked
309 check(tp())
310 check(tp([]))
311 check(tp(set()))
312 check(tp([1, x, y]))
313 check(tp(obj for obj in [1, x, y]))
314 check(tp(set([1, x, y])))
315 check(tp(tuple([obj]) for obj in [1, x, y]))
316 check(tuple(tp([obj]) for obj in [1, x, y]))
317
318 self._tracked(tp([z]))
319 self._tracked(tp([[x, y]]))
320 self._tracked(tp([{x: y}]))
321 self._tracked(tp(obj for obj in [x, y, z]))
322 self._tracked(tp(tuple([obj]) for obj in [x, y, z]))
323 self._tracked(tuple(tp([obj]) for obj in [x, y, z]))
324
325 @unittest.skip("garbage collection is different in pyro")
326 def test_track_dynamic(self):
327 # Test GC-optimization of dynamically constructed tuples.
328 self.check_track_dynamic(tuple, False)
329
330 @unittest.skip("garbage collection is different in pyro")
331 def test_track_subtypes(self):
332 # Tuple subtypes must always be tracked
333 class MyTuple(tuple):
334 pass
335 self.check_track_dynamic(MyTuple, True)
336
337 @unittest.skip("garbage collection is different in pyro")
338 def test_bug7466(self):
339 # Trying to untrack an unfinished tuple could crash Python
340 self._not_tracked(tuple(gc.collect() for i in range(101)))
341
342 def test_repr_large(self):
343 # Check the repr of large list objects
344 def check(n):
345 l = (0,) * n
346 s = repr(l)
347 self.assertEqual(s,
348 '(' + ', '.join(['0'] * n) + ')')
349 check(10) # check our checking code
350 check(1000000)
351
352 def test_iterator_pickle(self):
353 # Userlist iterators don't support pickling yet since
354 # they are based on generators.
355 data = self.type2test([4, 5, 6, 7])
356 for proto in range(pickle.HIGHEST_PROTOCOL + 1):
357 itorg = iter(data)
358 d = pickle.dumps(itorg, proto)
359 it = pickle.loads(d)
360 self.assertEqual(type(itorg), type(it))
361 self.assertEqual(self.type2test(it), self.type2test(data))
362
363 it = pickle.loads(d)
364 next(it)
365 d = pickle.dumps(it, proto)
366 self.assertEqual(self.type2test(it), self.type2test(data)[1:])
367
368 def test_reversed_pickle(self):
369 data = self.type2test([4, 5, 6, 7])
370 for proto in range(pickle.HIGHEST_PROTOCOL + 1):
371 itorg = reversed(data)
372 d = pickle.dumps(itorg, proto)
373 it = pickle.loads(d)
374 self.assertEqual(type(itorg), type(it))
375 self.assertEqual(self.type2test(it), self.type2test(reversed(data)))
376
377 it = pickle.loads(d)
378 next(it)
379 d = pickle.dumps(it, proto)
380 self.assertEqual(self.type2test(it), self.type2test(reversed(data))[1:])
381
382 def test_no_comdat_folding(self):
383 # Issue 8847: In the PGO build, the MSVC linker's COMDAT folding
384 # optimization causes failures in code that relies on distinct
385 # function addresses.
386 class T(tuple): pass
387 with self.assertRaises(TypeError):
388 [3,] + T((1,2))
389
390 def test_lexicographic_ordering(self):
391 # Issue 21100
392 a = self.type2test([1, 2])
393 b = self.type2test([1, 2, 0])
394 c = self.type2test([1, 3])
395 self.assertLess(a, b)
396 self.assertLess(b, c)
397
398# Notes on testing hash codes. The primary thing is that Python doesn't
399# care about "random" hash codes. To the contrary, we like them to be
400# very regular when possible, so that the low-order bits are as evenly
401# distributed as possible. For integers this is easy: hash(i) == i for
402# all not-huge i except i==-1.
403#
404# For tuples of mixed type there's really no hope of that, so we want
405# "randomish" here instead. But getting close to pseudo-random in all
406# bit positions is more expensive than we've been willing to pay for.
407#
408# We can tolerate large deviations from random - what we don't want is
409# catastrophic pileups on a relative handful of hash codes. The dict
410# and set lookup routines remain effective provided that full-width hash
411# codes for not-equal objects are distinct.
412#
413# So we compute various statistics here based on what a "truly random"
414# hash would do, but don't automate "pass or fail" based on those
415# results. Instead those are viewed as inputs to human judgment, and the
416# automated tests merely ensure we get the _same_ results across
417# platforms. In fact, we normally don't bother to run them at all -
418# set RUN_ALL_HASH_TESTS to force it.
419#
420# When global JUST_SHOW_HASH_RESULTS is True, the tuple hash statistics
421# are just displayed to stdout. A typical output line looks like:
422#
423# old tuple test; 32-bit upper hash codes; \
424# pileup 49 mean 7.4 coll 52 z +16.4
425#
426# "old tuple test" is just a string name for the test being run.
427#
428# "32-bit upper hash codes" means this was run under a 64-bit build and
429# we've shifted away the lower 32 bits of the hash codes.
430#
431# "pileup" is 0 if there were no collisions across those hash codes.
432# It's 1 less than the maximum number of times any single hash code was
433# seen. So in this case, there was (at least) one hash code that was
434# seen 50 times: that hash code "piled up" 49 more times than ideal.
435#
436# "mean" is the number of collisions a perfectly random hash function
437# would have yielded, on average.
438#
439# "coll" is the number of collisions actually seen.
440#
441# "z" is "coll - mean" divided by the standard deviation of the number
442# of collisions a perfectly random hash function would suffer. A
443# positive value is "worse than random", and negative value "better than
444# random". Anything of magnitude greater than 3 would be highly suspect
445# for a hash function that claimed to be random. It's essentially
446# impossible that a truly random function would deliver a result 16.4
447# sdevs "worse than random".
448#
449# But we don't care here! That's why the test isn't coded to fail.
450# Knowing something about how the high-order hash code bits behave
451# provides insight, but is irrelevant to how the dict and set lookup
452# code performs. The low-order bits are much more important to that,
453# and on the same test those did "just like random":
454#
455# old tuple test; 32-bit lower hash codes; \
456# pileup 1 mean 7.4 coll 7 z -0.2
457#
458# So there are always tradeoffs to consider. For another:
459#
460# 0..99 << 60 by 3; 32-bit hash codes; \
461# pileup 0 mean 116.4 coll 0 z -10.8
462#
463# That was run under a 32-bit build, and is spectacularly "better than
464# random". On a 64-bit build the wider hash codes are fine too:
465#
466# 0..99 << 60 by 3; 64-bit hash codes; \
467# pileup 0 mean 0.0 coll 0 z -0.0
468#
469# but their lower 32 bits are poor:
470#
471# 0..99 << 60 by 3; 32-bit lower hash codes; \
472# pileup 1 mean 116.4 coll 324 z +19.2
473#
474# In a statistical sense that's waaaaay too many collisions, but (a) 324
475# collisions out of a million hash codes isn't anywhere near being a
476# real problem; and, (b) the worst pileup on a single hash code is a measly
477# 1 extra. It's a relatively poor case for the tuple hash, but still
478# fine for practical use.
479#
480# This isn't, which is what Python 3.7.1 produced for the hashes of
481# itertools.product([0, 0.5], repeat=18). Even with a fat 64-bit
482# hashcode, the highest pileup was over 16,000 - making a dict/set
483# lookup on one of the colliding values thousands of times slower (on
484# average) than we expect.
485#
486# [0, 0.5] by 18; 64-bit hash codes; \
487# pileup 16,383 mean 0.0 coll 262,128 z +6073641856.9
488# [0, 0.5] by 18; 32-bit lower hash codes; \
489# pileup 262,143 mean 8.0 coll 262,143 z +92683.6
490
491if __name__ == "__main__":
492 unittest.main()