jcs.org
"Das U-Boot" Source Tree
1# SPDX-License-Identifier: GPL-2.0+
2# Copyright (c) 2012 The Chromium OS Authors.
3#
4
5from filelock import FileLock
6import os
7import shutil
8import sys
9import tempfile
10import time
11import unittest
12from unittest.mock import patch
13
14from buildman import board
15from buildman import boards
16from buildman import bsettings
17from buildman import builder
18from buildman import cfgutil
19from buildman import control
20from buildman import toolchain
21from patman import commit
22from u_boot_pylib import command
23from u_boot_pylib import terminal
24from u_boot_pylib import test_util
25from u_boot_pylib import tools
26
27use_network = True
28
29settings_data = '''
30# Buildman settings file
31
32[toolchain]
33main: /usr/sbin
34
35[toolchain-alias]
36x86: i386 x86_64
37'''
38
39settings_data_wrapper = '''
40# Buildman settings file
41
42[toolchain]
43main: /usr/sbin
44
45[toolchain-wrapper]
46wrapper = ccache
47'''
48
49settings_data_homedir = '''
50# Buildman settings file
51
52[toolchain]
53main = ~/mypath
54
55[toolchain-prefix]
56x86 = ~/mypath-x86-
57'''
58
59migration = '''===================== WARNING ======================
60This board does not use CONFIG_DM. CONFIG_DM will be
61compulsory starting with the v2020.01 release.
62Failure to update may result in board removal.
63See doc/develop/driver-model/migration.rst for more info.
64====================================================
65'''
66
67errors = [
68 '''main.c: In function 'main_loop':
69main.c:260:6: warning: unused variable 'joe' [-Wunused-variable]
70''',
71 '''main.c: In function 'main_loop2':
72main.c:295:2: error: 'fred' undeclared (first use in this function)
73main.c:295:2: note: each undeclared identifier is reported only once for each function it appears in
74make[1]: *** [main.o] Error 1
75make: *** [common/libcommon.o] Error 2
76Make failed
77''',
78 '''arch/arm/dts/socfpga_arria10_socdk_sdmmc.dtb: Warning \
79(avoid_unnecessary_addr_size): /clocks: unnecessary #address-cells/#size-cells \
80without "ranges" or child "reg" property
81''',
82 '''powerpc-linux-ld: warning: dot moved backwards before `.bss'
83powerpc-linux-ld: warning: dot moved backwards before `.bss'
84powerpc-linux-ld: u-boot: section .text lma 0xfffc0000 overlaps previous sections
85powerpc-linux-ld: u-boot: section .rodata lma 0xfffef3ec overlaps previous sections
86powerpc-linux-ld: u-boot: section .reloc lma 0xffffa400 overlaps previous sections
87powerpc-linux-ld: u-boot: section .data lma 0xffffcd38 overlaps previous sections
88powerpc-linux-ld: u-boot: section .u_boot_cmd lma 0xffffeb40 overlaps previous sections
89powerpc-linux-ld: u-boot: section .bootpg lma 0xfffff198 overlaps previous sections
90''',
91 '''In file included from %(basedir)sarch/sandbox/cpu/cpu.c:9:0:
92%(basedir)sarch/sandbox/include/asm/state.h:44:0: warning: "xxxx" redefined [enabled by default]
93%(basedir)sarch/sandbox/include/asm/state.h:43:0: note: this is the location of the previous definition
94%(basedir)sarch/sandbox/cpu/cpu.c: In function 'do_reset':
95%(basedir)sarch/sandbox/cpu/cpu.c:27:1: error: unknown type name 'blah'
96%(basedir)sarch/sandbox/cpu/cpu.c:28:12: error: expected declaration specifiers or '...' before numeric constant
97make[2]: *** [arch/sandbox/cpu/cpu.o] Error 1
98make[1]: *** [arch/sandbox/cpu] Error 2
99make[1]: *** Waiting for unfinished jobs....
100In file included from %(basedir)scommon/board_f.c:55:0:
101%(basedir)sarch/sandbox/include/asm/state.h:44:0: warning: "xxxx" redefined [enabled by default]
102%(basedir)sarch/sandbox/include/asm/state.h:43:0: note: this is the location of the previous definition
103make: *** [sub-make] Error 2
104'''
105]
106
107
108# hash, subject, return code, list of errors/warnings
109commits = [
110 ['1234', 'upstream/master, migration warning', 0, []],
111 ['5678', 'Second commit, a warning', 0, errors[0:1]],
112 ['9012', 'Third commit, error', 1, errors[0:2]],
113 ['3456', 'Fourth commit, warning', 0, [errors[0], errors[2]]],
114 ['7890', 'Fifth commit, link errors', 1, [errors[0], errors[3]]],
115 ['abcd', 'Sixth commit, fixes all errors', 0, []],
116 ['ef01', 'Seventh commit, fix migration, check directory suppression', 1,
117 [errors[4]]],
118]
119
120BOARDS = [
121 ['Active', 'arm', 'armv7', '', 'Tester', 'ARM Board 1', 'board0', ''],
122 ['Active', 'arm', 'armv7', '', 'Tester', 'ARM Board 2', 'board1', ''],
123 ['Active', 'powerpc', 'powerpc', '', 'Tester', 'PowerPC board 1', 'board2', ''],
124 ['Active', 'powerpc', 'mpc83xx', '', 'Tester', 'PowerPC board 2', 'board3', ''],
125 ['Active', 'sandbox', 'sandbox', '', 'Tester', 'Sandbox board', 'board4', ''],
126]
127
128BASE_DIR = 'base'
129
130OUTCOME_OK, OUTCOME_WARN, OUTCOME_ERR = range(3)
131
132class Options:
133 """Class that holds build options"""
134 pass
135
136class TestBuild(unittest.TestCase):
137 """Test buildman
138
139 TODO: Write tests for the rest of the functionality
140 """
141 def setUp(self):
142 # Set up commits to build
143 self.commits = []
144 sequence = 0
145 for commit_info in commits:
146 comm = commit.Commit(commit_info[0])
147 comm.subject = commit_info[1]
148 comm.return_code = commit_info[2]
149 comm.error_list = commit_info[3]
150 if sequence < 6:
151 comm.error_list += [migration]
152 comm.sequence = sequence
153 sequence += 1
154 self.commits.append(comm)
155
156 # Set up boards to build
157 self.brds = boards.Boards()
158 for brd in BOARDS:
159 self.brds.add_board(board.Board(*brd))
160 self.brds.select_boards([])
161
162 # Add some test settings
163 bsettings.setup(None)
164 bsettings.add_file(settings_data)
165
166 # Set up the toolchains
167 self.toolchains = toolchain.Toolchains()
168 self.toolchains.Add('arm-linux-gcc', test=False)
169 self.toolchains.Add('sparc-linux-gcc', test=False)
170 self.toolchains.Add('powerpc-linux-gcc', test=False)
171 self.toolchains.Add('/path/to/aarch64-linux-gcc', test=False)
172 self.toolchains.Add('gcc', test=False)
173
174 # Avoid sending any output
175 terminal.set_print_test_mode()
176 self._col = terminal.Color()
177
178 self.base_dir = tempfile.mkdtemp()
179 if not os.path.isdir(self.base_dir):
180 os.mkdir(self.base_dir)
181
182 self.cur_time = 0
183 self.valid_pids = []
184 self.finish_time = None
185 self.finish_pid = None
186
187 def tearDown(self):
188 shutil.rmtree(self.base_dir)
189
190 def Make(self, commit, brd, stage, *args, **kwargs):
191 result = command.CommandResult()
192 boardnum = int(brd.target[-1])
193 result.return_code = 0
194 result.stderr = ''
195 result.stdout = ('This is the test output for board %s, commit %s' %
196 (brd.target, commit.hash))
197 if ((boardnum >= 1 and boardnum >= commit.sequence) or
198 boardnum == 4 and commit.sequence == 6):
199 result.return_code = commit.return_code
200 result.stderr = (''.join(commit.error_list)
201 % {'basedir' : self.base_dir + '/.bm-work/00/'})
202 elif commit.sequence < 6:
203 result.stderr = migration
204
205 result.combined = result.stdout + result.stderr
206 return result
207
208 def assertSummary(self, text, arch, plus, brds, outcome=OUTCOME_ERR):
209 col = self._col
210 expected_colour = (col.GREEN if outcome == OUTCOME_OK else
211 col.YELLOW if outcome == OUTCOME_WARN else col.RED)
212 expect = '%10s: ' % arch
213 # TODO(sjg@chromium.org): If plus is '', we shouldn't need this
214 expect += ' ' + col.build(expected_colour, plus)
215 expect += ' '
216 for brd in brds:
217 expect += col.build(expected_colour, ' %s' % brd)
218 self.assertEqual(text, expect)
219
220 def _SetupTest(self, echo_lines=False, threads=1, **kwdisplay_args):
221 """Set up the test by running a build and summary
222
223 Args:
224 echo_lines: True to echo lines to the terminal to aid test
225 development
226 kwdisplay_args: Dict of arguments to pass to
227 Builder.SetDisplayOptions()
228
229 Returns:
230 Iterator containing the output lines, each a PrintLine() object
231 """
232 build = builder.Builder(self.toolchains, self.base_dir, None, threads,
233 2, checkout=False, show_unknown=False)
234 build.do_make = self.Make
235 board_selected = self.brds.get_selected_dict()
236
237 # Build the boards for the pre-defined commits and warnings/errors
238 # associated with each. This calls our Make() to inject the fake output.
239 build.build_boards(self.commits, board_selected, keep_outputs=False,
240 verbose=False)
241 lines = terminal.get_print_test_lines()
242 count = 0
243 for line in lines:
244 if line.text.strip():
245 count += 1
246
247 # We should get two starting messages, an update for every commit built
248 # and a summary message
249 self.assertEqual(count, len(commits) * len(BOARDS) + 3)
250 build.set_display_options(**kwdisplay_args);
251 build.show_summary(self.commits, board_selected)
252 if echo_lines:
253 terminal.echo_print_test_lines()
254 return iter(terminal.get_print_test_lines())
255
256 def _CheckOutput(self, lines, list_error_boards=False,
257 filter_dtb_warnings=False,
258 filter_migration_warnings=False):
259 """Check for expected output from the build summary
260
261 Args:
262 lines: Iterator containing the lines returned from the summary
263 list_error_boards: Adjust the check for output produced with the
264 --list-error-boards flag
265 filter_dtb_warnings: Adjust the check for output produced with the
266 --filter-dtb-warnings flag
267 """
268 def add_line_prefix(prefix, brds, error_str, colour):
269 """Add a prefix to each line of a string
270
271 The training \n in error_str is removed before processing
272
273 Args:
274 prefix: String prefix to add
275 error_str: Error string containing the lines
276 colour: Expected colour for the line. Note that the board list,
277 if present, always appears in magenta
278
279 Returns:
280 New string where each line has the prefix added
281 """
282 lines = error_str.strip().splitlines()
283 new_lines = []
284 for line in lines:
285 if brds:
286 expect = self._col.build(colour, prefix + '(')
287 expect += self._col.build(self._col.MAGENTA, brds,
288 bright=False)
289 expect += self._col.build(colour, ') %s' % line)
290 else:
291 expect = self._col.build(colour, prefix + line)
292 new_lines.append(expect)
293 return '\n'.join(new_lines)
294
295 col = terminal.Color()
296 boards01234 = ('board0 board1 board2 board3 board4'
297 if list_error_boards else '')
298 boards1234 = 'board1 board2 board3 board4' if list_error_boards else ''
299 boards234 = 'board2 board3 board4' if list_error_boards else ''
300 boards34 = 'board3 board4' if list_error_boards else ''
301 boards4 = 'board4' if list_error_boards else ''
302
303 # Upstream commit: migration warnings only
304 self.assertEqual(next(lines).text, '01: %s' % commits[0][1])
305
306 if not filter_migration_warnings:
307 self.assertSummary(next(lines).text, 'arm', 'w+',
308 ['board0', 'board1'], outcome=OUTCOME_WARN)
309 self.assertSummary(next(lines).text, 'powerpc', 'w+',
310 ['board2', 'board3'], outcome=OUTCOME_WARN)
311 self.assertSummary(next(lines).text, 'sandbox', 'w+', ['board4'],
312 outcome=OUTCOME_WARN)
313
314 self.assertEqual(next(lines).text,
315 add_line_prefix('+', boards01234, migration, col.RED))
316
317 # Second commit: all archs should fail with warnings
318 self.assertEqual(next(lines).text, '02: %s' % commits[1][1])
319
320 if filter_migration_warnings:
321 self.assertSummary(next(lines).text, 'arm', 'w+',
322 ['board1'], outcome=OUTCOME_WARN)
323 self.assertSummary(next(lines).text, 'powerpc', 'w+',
324 ['board2', 'board3'], outcome=OUTCOME_WARN)
325 self.assertSummary(next(lines).text, 'sandbox', 'w+', ['board4'],
326 outcome=OUTCOME_WARN)
327
328 # Second commit: The warnings should be listed
329 self.assertEqual(next(lines).text,
330 add_line_prefix('w+', boards1234, errors[0], col.YELLOW))
331
332 # Third commit: Still fails
333 self.assertEqual(next(lines).text, '03: %s' % commits[2][1])
334 if filter_migration_warnings:
335 self.assertSummary(next(lines).text, 'arm', '',
336 ['board1'], outcome=OUTCOME_OK)
337 self.assertSummary(next(lines).text, 'powerpc', '+',
338 ['board2', 'board3'])
339 self.assertSummary(next(lines).text, 'sandbox', '+', ['board4'])
340
341 # Expect a compiler error
342 self.assertEqual(next(lines).text,
343 add_line_prefix('+', boards234, errors[1], col.RED))
344
345 # Fourth commit: Compile errors are fixed, just have warning for board3
346 self.assertEqual(next(lines).text, '04: %s' % commits[3][1])
347 if filter_migration_warnings:
348 expect = '%10s: ' % 'powerpc'
349 expect += ' ' + col.build(col.GREEN, '')
350 expect += ' '
351 expect += col.build(col.GREEN, ' %s' % 'board2')
352 expect += ' ' + col.build(col.YELLOW, 'w+')
353 expect += ' '
354 expect += col.build(col.YELLOW, ' %s' % 'board3')
355 self.assertEqual(next(lines).text, expect)
356 else:
357 self.assertSummary(next(lines).text, 'powerpc', 'w+',
358 ['board2', 'board3'], outcome=OUTCOME_WARN)
359 self.assertSummary(next(lines).text, 'sandbox', 'w+', ['board4'],
360 outcome=OUTCOME_WARN)
361
362 # Compile error fixed
363 self.assertEqual(next(lines).text,
364 add_line_prefix('-', boards234, errors[1], col.GREEN))
365
366 if not filter_dtb_warnings:
367 self.assertEqual(
368 next(lines).text,
369 add_line_prefix('w+', boards34, errors[2], col.YELLOW))
370
371 # Fifth commit
372 self.assertEqual(next(lines).text, '05: %s' % commits[4][1])
373 if filter_migration_warnings:
374 self.assertSummary(next(lines).text, 'powerpc', '', ['board3'],
375 outcome=OUTCOME_OK)
376 self.assertSummary(next(lines).text, 'sandbox', '+', ['board4'])
377
378 # The second line of errors[3] is a duplicate, so buildman will drop it
379 expect = errors[3].rstrip().split('\n')
380 expect = [expect[0]] + expect[2:]
381 expect = '\n'.join(expect)
382 self.assertEqual(next(lines).text,
383 add_line_prefix('+', boards4, expect, col.RED))
384
385 if not filter_dtb_warnings:
386 self.assertEqual(
387 next(lines).text,
388 add_line_prefix('w-', boards34, errors[2], col.CYAN))
389
390 # Sixth commit
391 self.assertEqual(next(lines).text, '06: %s' % commits[5][1])
392 if filter_migration_warnings:
393 self.assertSummary(next(lines).text, 'sandbox', '', ['board4'],
394 outcome=OUTCOME_OK)
395 else:
396 self.assertSummary(next(lines).text, 'sandbox', 'w+', ['board4'],
397 outcome=OUTCOME_WARN)
398
399 # The second line of errors[3] is a duplicate, so buildman will drop it
400 expect = errors[3].rstrip().split('\n')
401 expect = [expect[0]] + expect[2:]
402 expect = '\n'.join(expect)
403 self.assertEqual(next(lines).text,
404 add_line_prefix('-', boards4, expect, col.GREEN))
405 self.assertEqual(next(lines).text,
406 add_line_prefix('w-', boards4, errors[0], col.CYAN))
407
408 # Seventh commit
409 self.assertEqual(next(lines).text, '07: %s' % commits[6][1])
410 if filter_migration_warnings:
411 self.assertSummary(next(lines).text, 'sandbox', '+', ['board4'])
412 else:
413 self.assertSummary(next(lines).text, 'arm', '', ['board0', 'board1'],
414 outcome=OUTCOME_OK)
415 self.assertSummary(next(lines).text, 'powerpc', '',
416 ['board2', 'board3'], outcome=OUTCOME_OK)
417 self.assertSummary(next(lines).text, 'sandbox', '+', ['board4'])
418
419 # Pick out the correct error lines
420 expect_str = errors[4].rstrip().replace('%(basedir)s', '').split('\n')
421 expect = expect_str[3:8] + [expect_str[-1]]
422 expect = '\n'.join(expect)
423 if not filter_migration_warnings:
424 self.assertEqual(
425 next(lines).text,
426 add_line_prefix('-', boards01234, migration, col.GREEN))
427
428 self.assertEqual(next(lines).text,
429 add_line_prefix('+', boards4, expect, col.RED))
430
431 # Now the warnings lines
432 expect = [expect_str[0]] + expect_str[10:12] + [expect_str[9]]
433 expect = '\n'.join(expect)
434 self.assertEqual(next(lines).text,
435 add_line_prefix('w+', boards4, expect, col.YELLOW))
436
437 def testOutput(self):
438 """Test basic builder operation and output
439
440 This does a line-by-line verification of the summary output.
441 """
442 lines = self._SetupTest(show_errors=True)
443 self._CheckOutput(lines, list_error_boards=False,
444 filter_dtb_warnings=False)
445
446 def testErrorBoards(self):
447 """Test output with --list-error-boards
448
449 This does a line-by-line verification of the summary output.
450 """
451 lines = self._SetupTest(show_errors=True, list_error_boards=True)
452 self._CheckOutput(lines, list_error_boards=True)
453
454 def testFilterDtb(self):
455 """Test output with --filter-dtb-warnings
456
457 This does a line-by-line verification of the summary output.
458 """
459 lines = self._SetupTest(show_errors=True, filter_dtb_warnings=True)
460 self._CheckOutput(lines, filter_dtb_warnings=True)
461
462 def testFilterMigration(self):
463 """Test output with --filter-migration-warnings
464
465 This does a line-by-line verification of the summary output.
466 """
467 lines = self._SetupTest(show_errors=True,
468 filter_migration_warnings=True)
469 self._CheckOutput(lines, filter_migration_warnings=True)
470
471 def testSingleThread(self):
472 """Test operation without threading"""
473 lines = self._SetupTest(show_errors=True, threads=0)
474 self._CheckOutput(lines, list_error_boards=False,
475 filter_dtb_warnings=False)
476
477 def _testGit(self):
478 """Test basic builder operation by building a branch"""
479 options = Options()
480 options.git = os.getcwd()
481 options.summary = False
482 options.jobs = None
483 options.dry_run = False
484 #options.git = os.path.join(self.base_dir, 'repo')
485 options.branch = 'test-buildman'
486 options.force_build = False
487 options.list_tool_chains = False
488 options.count = -1
489 options.git_dir = None
490 options.threads = None
491 options.show_unknown = False
492 options.quick = False
493 options.show_errors = False
494 options.keep_outputs = False
495 args = ['tegra20']
496 control.do_buildman(options, args)
497
498 def testBoardSingle(self):
499 """Test single board selection"""
500 self.assertEqual(self.brds.select_boards(['sandbox']),
501 ({'all': ['board4'], 'sandbox': ['board4']}, []))
502
503 def testBoardArch(self):
504 """Test single board selection"""
505 self.assertEqual(self.brds.select_boards(['arm']),
506 ({'all': ['board0', 'board1'],
507 'arm': ['board0', 'board1']}, []))
508
509 def testBoardArchSingle(self):
510 """Test single board selection"""
511 self.assertEqual(self.brds.select_boards(['arm sandbox']),
512 ({'sandbox': ['board4'],
513 'all': ['board0', 'board1', 'board4'],
514 'arm': ['board0', 'board1']}, []))
515
516
517 def testBoardArchSingleMultiWord(self):
518 """Test single board selection"""
519 self.assertEqual(self.brds.select_boards(['arm', 'sandbox']),
520 ({'sandbox': ['board4'],
521 'all': ['board0', 'board1', 'board4'],
522 'arm': ['board0', 'board1']}, []))
523
524 def testBoardSingleAnd(self):
525 """Test single board selection"""
526 self.assertEqual(self.brds.select_boards(['Tester & arm']),
527 ({'Tester&arm': ['board0', 'board1'],
528 'all': ['board0', 'board1']}, []))
529
530 def testBoardTwoAnd(self):
531 """Test single board selection"""
532 self.assertEqual(self.brds.select_boards(['Tester', '&', 'arm',
533 'Tester' '&', 'powerpc',
534 'sandbox']),
535 ({'sandbox': ['board4'],
536 'all': ['board0', 'board1', 'board2', 'board3',
537 'board4'],
538 'Tester&powerpc': ['board2', 'board3'],
539 'Tester&arm': ['board0', 'board1']}, []))
540
541 def testBoardAll(self):
542 """Test single board selection"""
543 self.assertEqual(self.brds.select_boards([]),
544 ({'all': ['board0', 'board1', 'board2', 'board3',
545 'board4']}, []))
546
547 def testBoardRegularExpression(self):
548 """Test single board selection"""
549 self.assertEqual(self.brds.select_boards(['T.*r&^Po']),
550 ({'all': ['board2', 'board3'],
551 'T.*r&^Po': ['board2', 'board3']}, []))
552
553 def testBoardDuplicate(self):
554 """Test single board selection"""
555 self.assertEqual(self.brds.select_boards(['sandbox sandbox',
556 'sandbox']),
557 ({'all': ['board4'], 'sandbox': ['board4']}, []))
558 def CheckDirs(self, build, dirname):
559 self.assertEqual('base%s' % dirname, build.get_output_dir(1))
560 self.assertEqual('base%s/fred' % dirname,
561 build.get_build_dir(1, 'fred'))
562 self.assertEqual('base%s/fred/done' % dirname,
563 build.get_done_file(1, 'fred'))
564 self.assertEqual('base%s/fred/u-boot.sizes' % dirname,
565 build.get_func_sizes_file(1, 'fred', 'u-boot'))
566 self.assertEqual('base%s/fred/u-boot.objdump' % dirname,
567 build.get_objdump_file(1, 'fred', 'u-boot'))
568 self.assertEqual('base%s/fred/err' % dirname,
569 build.get_err_file(1, 'fred'))
570
571 def testOutputDir(self):
572 build = builder.Builder(self.toolchains, BASE_DIR, None, 1, 2,
573 checkout=False, show_unknown=False)
574 build.commits = self.commits
575 build.commit_count = len(self.commits)
576 subject = self.commits[1].subject.translate(builder.trans_valid_chars)
577 dirname ='/%02d_g%s_%s' % (2, commits[1][0], subject[:20])
578 self.CheckDirs(build, dirname)
579
580 def testOutputDirCurrent(self):
581 build = builder.Builder(self.toolchains, BASE_DIR, None, 1, 2,
582 checkout=False, show_unknown=False)
583 build.commits = None
584 build.commit_count = 0
585 self.CheckDirs(build, '/current')
586
587 def testOutputDirNoSubdirs(self):
588 build = builder.Builder(self.toolchains, BASE_DIR, None, 1, 2,
589 checkout=False, show_unknown=False,
590 no_subdirs=True)
591 build.commits = None
592 build.commit_count = 0
593 self.CheckDirs(build, '')
594
595 def testToolchainAliases(self):
596 self.assertTrue(self.toolchains.Select('arm') != None)
597 with self.assertRaises(ValueError):
598 self.toolchains.Select('no-arch')
599 with self.assertRaises(ValueError):
600 self.toolchains.Select('x86')
601
602 self.toolchains = toolchain.Toolchains()
603 self.toolchains.Add('x86_64-linux-gcc', test=False)
604 self.assertTrue(self.toolchains.Select('x86') != None)
605
606 self.toolchains = toolchain.Toolchains()
607 self.toolchains.Add('i386-linux-gcc', test=False)
608 self.assertTrue(self.toolchains.Select('x86') != None)
609
610 def testToolchainDownload(self):
611 """Test that we can download toolchains"""
612 if use_network:
613 with test_util.capture_sys_output() as (stdout, stderr):
614 url = self.toolchains.LocateArchUrl('arm')
615 self.assertRegex(url, 'https://www.kernel.org/pub/tools/'
616 'crosstool/files/bin/x86_64/.*/'
617 'x86_64-gcc-.*-nolibc[-_]arm-.*linux-gnueabi.tar.xz')
618
619 def testGetEnvArgs(self):
620 """Test the GetEnvArgs() function"""
621 tc = self.toolchains.Select('arm')
622 self.assertEqual('arm-linux-',
623 tc.GetEnvArgs(toolchain.VAR_CROSS_COMPILE))
624 self.assertEqual('', tc.GetEnvArgs(toolchain.VAR_PATH))
625 self.assertEqual('arm',
626 tc.GetEnvArgs(toolchain.VAR_ARCH))
627 self.assertEqual('', tc.GetEnvArgs(toolchain.VAR_MAKE_ARGS))
628
629 tc = self.toolchains.Select('sandbox')
630 self.assertEqual('', tc.GetEnvArgs(toolchain.VAR_CROSS_COMPILE))
631
632 self.toolchains.Add('/path/to/x86_64-linux-gcc', test=False)
633 tc = self.toolchains.Select('x86')
634 self.assertEqual('/path/to',
635 tc.GetEnvArgs(toolchain.VAR_PATH))
636 tc.override_toolchain = 'clang'
637 self.assertEqual('HOSTCC=clang CC=clang',
638 tc.GetEnvArgs(toolchain.VAR_MAKE_ARGS))
639
640 # Test config with ccache wrapper
641 bsettings.setup(None)
642 bsettings.add_file(settings_data_wrapper)
643
644 tc = self.toolchains.Select('arm')
645 self.assertEqual('ccache arm-linux-',
646 tc.GetEnvArgs(toolchain.VAR_CROSS_COMPILE))
647
648 tc = self.toolchains.Select('sandbox')
649 self.assertEqual('', tc.GetEnvArgs(toolchain.VAR_CROSS_COMPILE))
650
651 def testMakeEnvironment(self):
652 """Test the MakeEnvironment function"""
653 tc = self.toolchains.Select('arm')
654 env = tc.MakeEnvironment(False)
655 self.assertEqual(env[b'CROSS_COMPILE'], b'arm-linux-')
656
657 tc = self.toolchains.Select('sandbox')
658 env = tc.MakeEnvironment(False)
659 self.assertTrue(b'CROSS_COMPILE' not in env)
660
661 # Test config with ccache wrapper
662 bsettings.setup(None)
663 bsettings.add_file(settings_data_wrapper)
664
665 tc = self.toolchains.Select('arm')
666 env = tc.MakeEnvironment(False)
667 self.assertEqual(env[b'CROSS_COMPILE'], b'ccache arm-linux-')
668
669 tc = self.toolchains.Select('sandbox')
670 env = tc.MakeEnvironment(False)
671 self.assertTrue(b'CROSS_COMPILE' not in env)
672
673 def testPrepareOutputSpace(self):
674 def _Touch(fname):
675 tools.write_file(os.path.join(base_dir, fname), b'')
676
677 base_dir = tempfile.mkdtemp()
678
679 # Add various files that we want removed and left alone
680 to_remove = ['01_g0982734987_title', '102_g92bf_title',
681 '01_g2938abd8_title']
682 to_leave = ['something_else', '01-something.patch', '01_another']
683 for name in to_remove + to_leave:
684 _Touch(name)
685
686 build = builder.Builder(self.toolchains, base_dir, None, 1, 2)
687 build.commits = self.commits
688 build.commit_count = len(commits)
689 result = set(build._get_output_space_removals())
690 expected = set([os.path.join(base_dir, f) for f in to_remove])
691 self.assertEqual(expected, result)
692
693 def test_adjust_cfg_nop(self):
694 """check various adjustments of config that are nops"""
695 # enable an enabled CONFIG
696 self.assertEqual(
697 'CONFIG_FRED=y',
698 cfgutil.adjust_cfg_line('CONFIG_FRED=y', {'FRED':'FRED'})[0])
699
700 # disable a disabled CONFIG
701 self.assertEqual(
702 '# CONFIG_FRED is not set',
703 cfgutil.adjust_cfg_line(
704 '# CONFIG_FRED is not set', {'FRED':'~FRED'})[0])
705
706 # use the adjust_cfg_lines() function
707 self.assertEqual(
708 ['CONFIG_FRED=y'],
709 cfgutil.adjust_cfg_lines(['CONFIG_FRED=y'], {'FRED':'FRED'}))
710 self.assertEqual(
711 ['# CONFIG_FRED is not set'],
712 cfgutil.adjust_cfg_lines(['CONFIG_FRED=y'], {'FRED':'~FRED'}))
713
714 # handling an empty line
715 self.assertEqual('#', cfgutil.adjust_cfg_line('#', {'FRED':'~FRED'})[0])
716
717 def test_adjust_cfg(self):
718 """check various adjustments of config"""
719 # disable a CONFIG
720 self.assertEqual(
721 '# CONFIG_FRED is not set',
722 cfgutil.adjust_cfg_line('CONFIG_FRED=1' , {'FRED':'~FRED'})[0])
723
724 # enable a disabled CONFIG
725 self.assertEqual(
726 'CONFIG_FRED=y',
727 cfgutil.adjust_cfg_line(
728 '# CONFIG_FRED is not set', {'FRED':'FRED'})[0])
729
730 # enable a CONFIG that doesn't exist
731 self.assertEqual(
732 ['CONFIG_FRED=y'],
733 cfgutil.adjust_cfg_lines([], {'FRED':'FRED'}))
734
735 # disable a CONFIG that doesn't exist
736 self.assertEqual(
737 ['# CONFIG_FRED is not set'],
738 cfgutil.adjust_cfg_lines([], {'FRED':'~FRED'}))
739
740 # disable a value CONFIG
741 self.assertEqual(
742 '# CONFIG_FRED is not set',
743 cfgutil.adjust_cfg_line('CONFIG_FRED="fred"' , {'FRED':'~FRED'})[0])
744
745 # setting a value CONFIG
746 self.assertEqual(
747 'CONFIG_FRED="fred"',
748 cfgutil.adjust_cfg_line('# CONFIG_FRED is not set' ,
749 {'FRED':'FRED="fred"'})[0])
750
751 # changing a value CONFIG
752 self.assertEqual(
753 'CONFIG_FRED="fred"',
754 cfgutil.adjust_cfg_line('CONFIG_FRED="ernie"' ,
755 {'FRED':'FRED="fred"'})[0])
756
757 # setting a value for a CONFIG that doesn't exist
758 self.assertEqual(
759 ['CONFIG_FRED="fred"'],
760 cfgutil.adjust_cfg_lines([], {'FRED':'FRED="fred"'}))
761
762 def test_convert_adjust_cfg_list(self):
763 """Check conversion of the list of changes into a dict"""
764 self.assertEqual({}, cfgutil.convert_list_to_dict(None))
765
766 expect = {
767 'FRED':'FRED',
768 'MARY':'~MARY',
769 'JOHN':'JOHN=0x123',
770 'ALICE':'ALICE="alice"',
771 'AMY':'AMY',
772 'ABE':'~ABE',
773 'MARK':'MARK=0x456',
774 'ANNA':'ANNA="anna"',
775 }
776 actual = cfgutil.convert_list_to_dict(
777 ['FRED', '~MARY', 'JOHN=0x123', 'ALICE="alice"',
778 'CONFIG_AMY', '~CONFIG_ABE', 'CONFIG_MARK=0x456',
779 'CONFIG_ANNA="anna"'])
780 self.assertEqual(expect, actual)
781
782 def test_check_cfg_file(self):
783 """Test check_cfg_file detects conflicts as expected"""
784 # Check failure to disable CONFIG
785 result = cfgutil.check_cfg_lines(['CONFIG_FRED=1'], {'FRED':'~FRED'})
786 self.assertEqual([['~FRED', 'CONFIG_FRED=1']], result)
787
788 result = cfgutil.check_cfg_lines(
789 ['CONFIG_FRED=1', 'CONFIG_MARY="mary"'], {'FRED':'~FRED'})
790 self.assertEqual([['~FRED', 'CONFIG_FRED=1']], result)
791
792 result = cfgutil.check_cfg_lines(
793 ['CONFIG_FRED=1', 'CONFIG_MARY="mary"'], {'MARY':'~MARY'})
794 self.assertEqual([['~MARY', 'CONFIG_MARY="mary"']], result)
795
796 # Check failure to enable CONFIG
797 result = cfgutil.check_cfg_lines(
798 ['# CONFIG_FRED is not set'], {'FRED':'FRED'})
799 self.assertEqual([['FRED', '# CONFIG_FRED is not set']], result)
800
801 # Check failure to set CONFIG value
802 result = cfgutil.check_cfg_lines(
803 ['# CONFIG_FRED is not set', 'CONFIG_MARY="not"'],
804 {'MARY':'MARY="mary"', 'FRED':'FRED'})
805 self.assertEqual([
806 ['FRED', '# CONFIG_FRED is not set'],
807 ['MARY="mary"', 'CONFIG_MARY="not"']], result)
808
809 # Check failure to add CONFIG value
810 result = cfgutil.check_cfg_lines([], {'MARY':'MARY="mary"'})
811 self.assertEqual([
812 ['MARY="mary"', 'Missing expected line: CONFIG_MARY="mary"']], result)
813
814 def get_procs(self):
815 running_fname = os.path.join(self.base_dir, control.RUNNING_FNAME)
816 items = tools.read_file(running_fname, binary=False).split()
817 return [int(x) for x in items]
818
819 def get_time(self):
820 return self.cur_time
821
822 def inc_time(self, amount):
823 self.cur_time += amount
824
825 # Handle a process exiting
826 if self.finish_time == self.cur_time:
827 self.valid_pids = [pid for pid in self.valid_pids
828 if pid != self.finish_pid]
829
830 def kill(self, pid, signal):
831 if pid not in self.valid_pids:
832 raise OSError('Invalid PID')
833
834 def test_process_limit(self):
835 """Test wait_for_process_limit() function"""
836 tmpdir = self.base_dir
837
838 with (patch('time.time', side_effect=self.get_time),
839 patch('time.monotonic', side_effect=self.get_time),
840 patch('time.sleep', side_effect=self.inc_time),
841 patch('os.kill', side_effect=self.kill)):
842 # Grab the process. Since there is no other profcess, this should
843 # immediately succeed
844 control.wait_for_process_limit(1, tmpdir=tmpdir, pid=1)
845 lines = terminal.get_print_test_lines()
846 self.assertEqual(0, self.cur_time)
847 self.assertEqual('Waiting for other buildman processes...',
848 lines[0].text)
849 self.assertEqual(self._col.RED, lines[0].colour)
850 self.assertEqual(False, lines[0].newline)
851 self.assertEqual(True, lines[0].bright)
852
853 self.assertEqual('done...', lines[1].text)
854 self.assertEqual(None, lines[1].colour)
855 self.assertEqual(False, lines[1].newline)
856 self.assertEqual(True, lines[1].bright)
857
858 self.assertEqual('starting build', lines[2].text)
859 self.assertEqual([1], control.read_procs(tmpdir))
860 self.assertEqual(None, lines[2].colour)
861 self.assertEqual(False, lines[2].newline)
862 self.assertEqual(True, lines[2].bright)
863
864 # Try again, with a different PID...this should eventually timeout
865 # and start the build anyway
866 self.cur_time = 0
867 self.valid_pids = [1]
868 control.wait_for_process_limit(1, tmpdir=tmpdir, pid=2)
869 lines = terminal.get_print_test_lines()
870 self.assertEqual('Waiting for other buildman processes...',
871 lines[0].text)
872 self.assertEqual('timeout...', lines[1].text)
873 self.assertEqual(None, lines[1].colour)
874 self.assertEqual(False, lines[1].newline)
875 self.assertEqual(True, lines[1].bright)
876 self.assertEqual('starting build', lines[2].text)
877 self.assertEqual([1, 2], control.read_procs(tmpdir))
878 self.assertEqual(control.RUN_WAIT_S, self.cur_time)
879
880 # Check lock-busting
881 self.cur_time = 0
882 self.valid_pids = [1, 2]
883 lock_fname = os.path.join(tmpdir, control.LOCK_FNAME)
884 lock = FileLock(lock_fname)
885 lock.acquire(timeout=1)
886 control.wait_for_process_limit(1, tmpdir=tmpdir, pid=3)
887 lines = terminal.get_print_test_lines()
888 self.assertEqual('Waiting for other buildman processes...',
889 lines[0].text)
890 self.assertEqual('failed to get lock: busting...', lines[1].text)
891 self.assertEqual(None, lines[1].colour)
892 self.assertEqual(False, lines[1].newline)
893 self.assertEqual(True, lines[1].bright)
894 self.assertEqual('timeout...', lines[2].text)
895 self.assertEqual('starting build', lines[3].text)
896 self.assertEqual([1, 2, 3], control.read_procs(tmpdir))
897 self.assertEqual(control.RUN_WAIT_S, self.cur_time)
898 lock.release()
899
900 # Check handling of dead processes. Here we have PID 2 as a running
901 # process, even though the PID file contains 1, 2 and 3. So we can
902 # add one more PID, to make 2 and 4
903 self.cur_time = 0
904 self.valid_pids = [2]
905 control.wait_for_process_limit(2, tmpdir=tmpdir, pid=4)
906 lines = terminal.get_print_test_lines()
907 self.assertEqual('Waiting for other buildman processes...',
908 lines[0].text)
909 self.assertEqual('done...', lines[1].text)
910 self.assertEqual('starting build', lines[2].text)
911 self.assertEqual([2, 4], control.read_procs(tmpdir))
912 self.assertEqual(0, self.cur_time)
913
914 # Try again, with PID 2 quitting at time 50. This allows the new
915 # build to start
916 self.cur_time = 0
917 self.valid_pids = [2, 4]
918 self.finish_pid = 2
919 self.finish_time = 50
920 control.wait_for_process_limit(2, tmpdir=tmpdir, pid=5)
921 lines = terminal.get_print_test_lines()
922 self.assertEqual('Waiting for other buildman processes...',
923 lines[0].text)
924 self.assertEqual('done...', lines[1].text)
925 self.assertEqual('starting build', lines[2].text)
926 self.assertEqual([4, 5], control.read_procs(tmpdir))
927 self.assertEqual(self.finish_time, self.cur_time)
928
929 def call_make_environment(self, tchn, full_path, in_env=None):
930 """Call Toolchain.MakeEnvironment() and process the result
931
932 Args:
933 tchn (Toolchain): Toolchain to use
934 full_path (bool): True to return the full path in CROSS_COMPILE
935 rather than adding it to the PATH variable
936 in_env (dict): Input environment to use, None to use current env
937
938 Returns:
939 tuple:
940 dict: Changes that MakeEnvironment has made to the environment
941 key: Environment variable that was changed
942 value: New value (for PATH this only includes components
943 which were added)
944 str: Full value of the new PATH variable
945 """
946 env = tchn.MakeEnvironment(full_path, env=in_env)
947
948 # Get the original environment
949 orig_env = dict(os.environb if in_env is None else in_env)
950 orig_path = orig_env[b'PATH'].split(b':')
951
952 # Find new variables
953 diff = dict((k, env[k]) for k in env if orig_env.get(k) != env[k])
954
955 # Find new / different path components
956 diff_path = None
957 new_path = None
958 if b'PATH' in diff:
959 new_path = diff[b'PATH'].split(b':')
960 diff_paths = [p for p in new_path if p not in orig_path]
961 diff_path = b':'.join(p for p in new_path if p not in orig_path)
962 if diff_path:
963 diff[b'PATH'] = diff_path
964 else:
965 del diff[b'PATH']
966 return diff, new_path
967
968 def test_toolchain_env(self):
969 """Test PATH and other environment settings for toolchains"""
970 # Use a toolchain which has a path, so that full_path makes a difference
971 tchn = self.toolchains.Select('aarch64')
972
973 # Normal cases
974 diff = self.call_make_environment(tchn, full_path=False)[0]
975 self.assertEqual(
976 {b'CROSS_COMPILE': b'aarch64-linux-', b'LC_ALL': b'C',
977 b'PATH': b'/path/to'}, diff)
978
979 diff = self.call_make_environment(tchn, full_path=True)[0]
980 self.assertEqual(
981 {b'CROSS_COMPILE': b'/path/to/aarch64-linux-', b'LC_ALL': b'C'},
982 diff)
983
984 # When overriding the toolchain, only LC_ALL should be set
985 tchn.override_toolchain = True
986 diff = self.call_make_environment(tchn, full_path=True)[0]
987 self.assertEqual({b'LC_ALL': b'C'}, diff)
988
989 # Test that virtualenv is handled correctly
990 tchn.override_toolchain = False
991 sys.prefix = '/some/venv'
992 env = dict(os.environb)
993 env[b'PATH'] = b'/some/venv/bin:other/things'
994 tchn.path = '/my/path'
995 diff, diff_path = self.call_make_environment(tchn, False, env)
996
997 self.assertIn(b'PATH', diff)
998 self.assertEqual([b'/some/venv/bin', b'/my/path', b'other/things'],
999 diff_path)
1000 self.assertEqual(
1001 {b'CROSS_COMPILE': b'aarch64-linux-', b'LC_ALL': b'C',
1002 b'PATH': b'/my/path'}, diff)
1003
1004 # Handle a toolchain wrapper
1005 tchn.path = ''
1006 bsettings.add_section('toolchain-wrapper')
1007 bsettings.set_item('toolchain-wrapper', 'my-wrapper', 'fred')
1008 diff = self.call_make_environment(tchn, full_path=True)[0]
1009 self.assertEqual(
1010 {b'CROSS_COMPILE': b'fred aarch64-linux-', b'LC_ALL': b'C'}, diff)
1011
1012 def test_skip_dtc(self):
1013 """Test skipping building the dtc tool"""
1014 old_path = os.getenv('PATH')
1015 try:
1016 os.environ['PATH'] = self.base_dir
1017
1018 # Check a missing tool
1019 with self.assertRaises(ValueError) as exc:
1020 builder.Builder(self.toolchains, self.base_dir, None, 0, 2,
1021 dtc_skip=True)
1022 self.assertIn('Cannot find dtc', str(exc.exception))
1023
1024 # Create a fake tool to use
1025 dtc = os.path.join(self.base_dir, 'dtc')
1026 tools.write_file(dtc, b'xx')
1027 os.chmod(dtc, 0o777)
1028
1029 build = builder.Builder(self.toolchains, self.base_dir, None, 0, 2,
1030 dtc_skip=True)
1031 toolchain = self.toolchains.Select('arm')
1032 env = build.make_environment(toolchain)
1033 self.assertIn(b'DTC', env)
1034
1035 # Try the normal case, i.e. not skipping the dtc build
1036 build = builder.Builder(self.toolchains, self.base_dir, None, 0, 2)
1037 toolchain = self.toolchains.Select('arm')
1038 env = build.make_environment(toolchain)
1039 self.assertNotIn(b'DTC', env)
1040 finally:
1041 os.environ['PATH'] = old_path
1042
1043 def testHomedir(self):
1044 """Test using ~ in a toolchain or toolchain-prefix section"""
1045 # Add some test settings
1046 bsettings.setup(None)
1047 bsettings.add_file(settings_data_homedir)
1048
1049 # Set up the toolchains
1050 home = os.path.expanduser('~')
1051 toolchains = toolchain.Toolchains()
1052 toolchains.GetSettings()
1053 self.assertEqual([f'{home}/mypath'], toolchains.paths)
1054
1055 # Check scanning
1056 with test_util.capture_sys_output() as (stdout, _):
1057 toolchains.Scan(verbose=True, raise_on_error=False)
1058 lines = iter(stdout.getvalue().splitlines() + ['##done'])
1059 self.assertEqual('Scanning for tool chains', next(lines))
1060 self.assertEqual(f" - scanning prefix '{home}/mypath-x86-'",
1061 next(lines))
1062 self.assertEqual(
1063 f"Error: No tool chain found for prefix '{home}/mypath-x86-gcc'",
1064 next(lines))
1065 self.assertEqual(f" - scanning path '{home}/mypath'", next(lines))
1066 self.assertEqual(f" - looking in '{home}/mypath/.'", next(lines))
1067 self.assertEqual(f" - looking in '{home}/mypath/bin'", next(lines))
1068 self.assertEqual(f" - looking in '{home}/mypath/usr/bin'",
1069 next(lines))
1070 self.assertEqual('##done', next(lines))
1071
1072 # Check adding a toolchain
1073 with test_util.capture_sys_output() as (stdout, _):
1074 toolchains.Add('~/aarch64-linux-gcc', test=True, verbose=True)
1075 lines = iter(stdout.getvalue().splitlines() + ['##done'])
1076 self.assertEqual('Tool chain test: BAD', next(lines))
1077 self.assertEqual(f'Command: {home}/aarch64-linux-gcc --version',
1078 next(lines))
1079 self.assertEqual('', next(lines))
1080 self.assertEqual('', next(lines))
1081 self.assertEqual('##done', next(lines))
1082
1083
1084if __name__ == "__main__":
1085 unittest.main()