jcs.org
"Das U-Boot" Source Tree
1# SPDX-License-Identifier: GPL-2.0+
2# Copyright (c) 2012 The Chromium OS Authors.
3#
4
5import re
6import glob
7from html.parser import HTMLParser
8import os
9import sys
10import tempfile
11import urllib.request, urllib.error, urllib.parse
12
13from buildman import bsettings
14from u_boot_pylib import command
15from u_boot_pylib import terminal
16from u_boot_pylib import tools
17
18(PRIORITY_FULL_PREFIX, PRIORITY_PREFIX_GCC, PRIORITY_PREFIX_GCC_PATH,
19 PRIORITY_CALC) = list(range(4))
20
21(VAR_CROSS_COMPILE, VAR_PATH, VAR_ARCH, VAR_MAKE_ARGS) = range(4)
22
23# Simple class to collect links from a page
24class MyHTMLParser(HTMLParser):
25 def __init__(self, arch):
26 """Create a new parser
27
28 After the parser runs, self.links will be set to a list of the links
29 to .xz archives found in the page, and self.arch_link will be set to
30 the one for the given architecture (or None if not found).
31
32 Args:
33 arch: Architecture to search for
34 """
35 HTMLParser.__init__(self)
36 self.arch_link = None
37 self.links = []
38 self.re_arch = re.compile('[-_]%s-' % arch)
39
40 def handle_starttag(self, tag, attrs):
41 if tag == 'a':
42 for tag, value in attrs:
43 if tag == 'href':
44 if value and value.endswith('.xz'):
45 self.links.append(value)
46 if self.re_arch.search(value):
47 self.arch_link = value
48
49
50class Toolchain:
51 """A single toolchain
52
53 Public members:
54 gcc: Full path to C compiler
55 path: Directory path containing C compiler
56 cross: Cross compile string, e.g. 'arm-linux-'
57 arch: Architecture of toolchain as determined from the first
58 component of the filename. E.g. arm-linux-gcc becomes arm
59 priority: Toolchain priority (0=highest, 20=lowest)
60 override_toolchain: Toolchain to use for sandbox, overriding the normal
61 one
62 """
63 def __init__(self, fname, test, verbose=False, priority=PRIORITY_CALC,
64 arch=None, override_toolchain=None):
65 """Create a new toolchain object.
66
67 Args:
68 fname: Filename of the gcc component, possibly with ~ or $HOME in it
69 test: True to run the toolchain to test it
70 verbose: True to print out the information
71 priority: Priority to use for this toolchain, or PRIORITY_CALC to
72 calculate it
73 """
74 fname = os.path.expanduser(fname)
75 self.gcc = fname
76 self.path = os.path.dirname(fname)
77 self.override_toolchain = override_toolchain
78
79 # Find the CROSS_COMPILE prefix to use for U-Boot. For example,
80 # 'arm-linux-gnueabihf-gcc' turns into 'arm-linux-gnueabihf-'.
81 basename = os.path.basename(fname)
82 pos = basename.rfind('-')
83 self.cross = basename[:pos + 1] if pos != -1 else ''
84
85 # The architecture is the first part of the name
86 pos = self.cross.find('-')
87 if arch:
88 self.arch = arch
89 else:
90 self.arch = self.cross[:pos] if pos != -1 else 'sandbox'
91 if self.arch == 'sandbox' and override_toolchain:
92 self.gcc = override_toolchain
93
94 env = self.MakeEnvironment(False)
95
96 # As a basic sanity check, run the C compiler with --version
97 cmd = [fname, '--version']
98 if priority == PRIORITY_CALC:
99 self.priority = self.GetPriority(fname)
100 else:
101 self.priority = priority
102 if test:
103 result = command.run_pipe([cmd], capture=True, env=env,
104 raise_on_error=False)
105 self.ok = result.return_code == 0
106 if verbose:
107 print('Tool chain test: ', end=' ')
108 if self.ok:
109 print("OK, arch='%s', priority %d" % (self.arch,
110 self.priority))
111 else:
112 print('BAD')
113 print(f"Command: {' '.join(cmd)}")
114 print(result.stdout)
115 print(result.stderr)
116 else:
117 self.ok = True
118
119 def GetPriority(self, fname):
120 """Return the priority of the toolchain.
121
122 Toolchains are ranked according to their suitability by their
123 filename prefix.
124
125 Args:
126 fname: Filename of toolchain
127 Returns:
128 Priority of toolchain, PRIORITY_CALC=highest, 20=lowest.
129 """
130 priority_list = ['-elf', '-unknown-linux-gnu', '-linux',
131 '-none-linux-gnueabi', '-none-linux-gnueabihf', '-uclinux',
132 '-none-eabi', '-gentoo-linux-gnu', '-linux-gnueabi',
133 '-linux-gnueabihf', '-le-linux', '-uclinux']
134 for prio in range(len(priority_list)):
135 if priority_list[prio] in fname:
136 return PRIORITY_CALC + prio
137 return PRIORITY_CALC + prio
138
139 def GetWrapper(self, show_warning=True):
140 """Get toolchain wrapper from the setting file.
141 """
142 value = ''
143 for name, value in bsettings.get_items('toolchain-wrapper'):
144 if not value:
145 print("Warning: Wrapper not found")
146 if value:
147 value = value + ' '
148
149 return value
150
151 def GetEnvArgs(self, which):
152 """Get an environment variable/args value based on the the toolchain
153
154 Args:
155 which: VAR_... value to get
156
157 Returns:
158 Value of that environment variable or arguments
159 """
160 if which == VAR_CROSS_COMPILE:
161 wrapper = self.GetWrapper()
162 base = '' if self.arch == 'sandbox' else self.path
163 if (base == '' and self.cross == ''):
164 return ''
165 return wrapper + os.path.join(base, self.cross)
166 elif which == VAR_PATH:
167 return self.path
168 elif which == VAR_ARCH:
169 return self.arch
170 elif which == VAR_MAKE_ARGS:
171 args = self.MakeArgs()
172 if args:
173 return ' '.join(args)
174 return ''
175 else:
176 raise ValueError('Unknown arg to GetEnvArgs (%d)' % which)
177
178 def MakeEnvironment(self, full_path, env=None):
179 """Returns an environment for using the toolchain.
180
181 This takes the current environment and adds CROSS_COMPILE so that
182 the tool chain will operate correctly. This also disables localized
183 output and possibly Unicode encoded output of all build tools by
184 adding LC_ALL=C. For the case where full_path is False, it prepends
185 the toolchain to PATH
186
187 Note that os.environb is used to obtain the environment, since in some
188 cases the environment many contain non-ASCII characters and we see
189 errors like:
190
191 UnicodeEncodeError: 'utf-8' codec can't encode characters in position
192 569-570: surrogates not allowed
193
194 When running inside a Python venv, care is taken not to put the
195 toolchain path before the venv path, so that builds initiated by
196 buildman will still respect the venv.
197
198 Args:
199 full_path: Return the full path in CROSS_COMPILE and don't set
200 PATH
201 env (dict of bytes): Original environment, used for testing
202 Returns:
203 Dict containing the (bytes) environment to use. This is based on the
204 current environment, with changes as needed to CROSS_COMPILE, PATH
205 and LC_ALL.
206 """
207 env = dict(env or os.environb)
208
209 wrapper = self.GetWrapper()
210
211 if self.override_toolchain:
212 # We'll use MakeArgs() to provide this
213 pass
214 elif full_path and self.cross:
215 env[b'CROSS_COMPILE'] = tools.to_bytes(
216 wrapper + os.path.join(self.path, self.cross))
217 elif self.cross:
218 env[b'CROSS_COMPILE'] = tools.to_bytes(wrapper + self.cross)
219
220 # Detect a Python virtualenv and avoid defeating it
221 if sys.prefix != sys.base_prefix:
222 paths = env[b'PATH'].split(b':')
223 new_paths = []
224 to_insert = tools.to_bytes(self.path)
225 insert_after = tools.to_bytes(sys.prefix)
226 for path in paths:
227 new_paths.append(path)
228 if to_insert and path.startswith(insert_after):
229 new_paths.append(to_insert)
230 to_insert = None
231 if to_insert:
232 new_paths.append(to_insert)
233 env[b'PATH'] = b':'.join(new_paths)
234 else:
235 env[b'PATH'] = tools.to_bytes(self.path) + b':' + env[b'PATH']
236
237 env[b'LC_ALL'] = b'C'
238
239 return env
240
241 def MakeArgs(self):
242 """Create the 'make' arguments for a toolchain
243
244 This is only used when the toolchain is being overridden. Since the
245 U-Boot Makefile sets CC and HOSTCC explicitly we cannot rely on the
246 environment (and MakeEnvironment()) to override these values. This
247 function returns the arguments to accomplish this.
248
249 Returns:
250 List of arguments to pass to 'make'
251 """
252 if self.override_toolchain:
253 return ['HOSTCC=%s' % self.override_toolchain,
254 'CC=%s' % self.override_toolchain]
255 return []
256
257
258class Toolchains:
259 """Manage a list of toolchains for building U-Boot
260
261 We select one toolchain for each architecture type
262
263 Public members:
264 toolchains: Dict of Toolchain objects, keyed by architecture name
265 prefixes: Dict of prefixes to check, keyed by architecture. This can
266 be a full path and toolchain prefix, for example
267 {'x86', 'opt/i386-linux/bin/i386-linux-'}, or the name of
268 something on the search path, for example
269 {'arm', 'arm-linux-gnueabihf-'}. Wildcards are not supported.
270 paths: List of paths to check for toolchains (may contain wildcards)
271 """
272
273 def __init__(self, override_toolchain=None):
274 self.toolchains = {}
275 self.prefixes = {}
276 self.paths = []
277 self.override_toolchain = override_toolchain
278 self._make_flags = dict(bsettings.get_items('make-flags'))
279
280 def GetPathList(self, show_warning=True):
281 """Get a list of available toolchain paths
282
283 Args:
284 show_warning: True to show a warning if there are no tool chains.
285
286 Returns:
287 List of strings, each a path to a toolchain mentioned in the
288 [toolchain] section of the settings file.
289 """
290 toolchains = bsettings.get_items('toolchain')
291 if show_warning and not toolchains:
292 print(("Warning: No tool chains. Please run 'buildman "
293 "--fetch-arch all' to download all available toolchains, or "
294 "add a [toolchain] section to your buildman config file "
295 "%s. See buildman.rst for details" %
296 bsettings.config_fname))
297
298 paths = []
299 for name, value in toolchains:
300 fname = os.path.expanduser(value)
301 if '*' in value:
302 paths += glob.glob(fname)
303 else:
304 paths.append(fname)
305 return paths
306
307 def GetSettings(self, show_warning=True):
308 """Get toolchain settings from the settings file.
309
310 Args:
311 show_warning: True to show a warning if there are no tool chains.
312 """
313 self.prefixes = bsettings.get_items('toolchain-prefix')
314 self.paths += self.GetPathList(show_warning)
315
316 def Add(self, fname, test=True, verbose=False, priority=PRIORITY_CALC,
317 arch=None):
318 """Add a toolchain to our list
319
320 We select the given toolchain as our preferred one for its
321 architecture if it is a higher priority than the others.
322
323 Args:
324 fname: Filename of toolchain's gcc driver
325 test: True to run the toolchain to test it
326 priority: Priority to use for this toolchain
327 arch: Toolchain architecture, or None if not known
328 """
329 toolchain = Toolchain(fname, test, verbose, priority, arch,
330 self.override_toolchain)
331 add_it = toolchain.ok
332 if add_it:
333 if toolchain.arch in self.toolchains:
334 add_it = (toolchain.priority <
335 self.toolchains[toolchain.arch].priority)
336 if add_it:
337 self.toolchains[toolchain.arch] = toolchain
338 elif verbose:
339 print(("Toolchain '%s' at priority %d will be ignored because "
340 "another toolchain for arch '%s' has priority %d" %
341 (toolchain.gcc, toolchain.priority, toolchain.arch,
342 self.toolchains[toolchain.arch].priority)))
343
344 def ScanPath(self, path, verbose):
345 """Scan a path for a valid toolchain
346
347 Args:
348 path: Path to scan
349 verbose: True to print out progress information
350 Returns:
351 Filename of C compiler if found, else None
352 """
353 fnames = []
354 for subdir in ['.', 'bin', 'usr/bin']:
355 dirname = os.path.join(path, subdir)
356 if verbose: print(" - looking in '%s'" % dirname)
357 for fname in glob.glob(dirname + '/*gcc'):
358 if verbose: print(" - found '%s'" % fname)
359 fnames.append(fname)
360 return fnames
361
362 def ScanPathEnv(self, fname):
363 """Scan the PATH environment variable for a given filename.
364
365 Args:
366 fname: Filename to scan for
367 Returns:
368 List of matching pathanames, or [] if none
369 """
370 pathname_list = []
371 for path in os.environ["PATH"].split(os.pathsep):
372 path = path.strip('"')
373 pathname = os.path.join(path, fname)
374 if os.path.exists(pathname):
375 pathname_list.append(pathname)
376 return pathname_list
377
378 def Scan(self, verbose, raise_on_error=True):
379 """Scan for available toolchains and select the best for each arch.
380
381 We look for all the toolchains we can file, figure out the
382 architecture for each, and whether it works. Then we select the
383 highest priority toolchain for each arch.
384
385 Args:
386 verbose: True to print out progress information
387 """
388 if verbose: print('Scanning for tool chains')
389 for name, value in self.prefixes:
390 fname = os.path.expanduser(value)
391 if verbose: print(" - scanning prefix '%s'" % fname)
392 if os.path.exists(fname):
393 self.Add(fname, True, verbose, PRIORITY_FULL_PREFIX, name)
394 continue
395 fname += 'gcc'
396 if os.path.exists(fname):
397 self.Add(fname, True, verbose, PRIORITY_PREFIX_GCC, name)
398 continue
399 fname_list = self.ScanPathEnv(fname)
400 for f in fname_list:
401 self.Add(f, True, verbose, PRIORITY_PREFIX_GCC_PATH, name)
402 if not fname_list:
403 msg = f"No tool chain found for prefix '{fname}'"
404 if raise_on_error:
405 raise ValueError(msg)
406 else:
407 print(f'Error: {msg}')
408 for path in self.paths:
409 if verbose: print(" - scanning path '%s'" % path)
410 fnames = self.ScanPath(path, verbose)
411 for fname in fnames:
412 self.Add(fname, True, verbose)
413
414 def List(self):
415 """List out the selected toolchains for each architecture"""
416 col = terminal.Color()
417 print(col.build(col.BLUE, 'List of available toolchains (%d):' %
418 len(self.toolchains)))
419 if len(self.toolchains):
420 for key, value in sorted(self.toolchains.items()):
421 print('%-10s: %s' % (key, value.gcc))
422 else:
423 print('None')
424
425 def Select(self, arch):
426 """Returns the toolchain for a given architecture
427
428 Args:
429 args: Name of architecture (e.g. 'arm', 'ppc_8xx')
430
431 returns:
432 toolchain object, or None if none found
433 """
434 for tag, value in bsettings.get_items('toolchain-alias'):
435 if arch == tag:
436 for alias in value.split():
437 if alias in self.toolchains:
438 return self.toolchains[alias]
439
440 if not arch in self.toolchains:
441 raise ValueError("No tool chain found for arch '%s'" % arch)
442 return self.toolchains[arch]
443
444 def ResolveReferences(self, var_dict, args):
445 """Resolve variable references in a string
446
447 This converts ${blah} within the string to the value of blah.
448 This function works recursively.
449
450 Resolved string
451
452 Args:
453 var_dict: Dictionary containing variables and their values
454 args: String containing make arguments
455 Returns:
456 >>> bsettings.setup(None)
457 >>> tcs = Toolchains()
458 >>> tcs.Add('fred', False)
459 >>> var_dict = {'oblique' : 'OBLIQUE', 'first' : 'fi${second}rst', \
460 'second' : '2nd'}
461 >>> tcs.ResolveReferences(var_dict, 'this=${oblique}_set')
462 'this=OBLIQUE_set'
463 >>> tcs.ResolveReferences(var_dict, 'this=${oblique}_set${first}nd')
464 'this=OBLIQUE_setfi2ndrstnd'
465 """
466 re_var = re.compile(r'(\$\{[-_a-z0-9A-Z]{1,}\})')
467
468 while True:
469 m = re_var.search(args)
470 if not m:
471 break
472 lookup = m.group(0)[2:-1]
473 value = var_dict.get(lookup, '')
474 args = args[:m.start(0)] + value + args[m.end(0):]
475 return args
476
477 def GetMakeArguments(self, brd):
478 """Returns 'make' arguments for a given board
479
480 The flags are in a section called 'make-flags'. Flags are named
481 after the target they represent, for example snapper9260=TESTING=1
482 will pass TESTING=1 to make when building the snapper9260 board.
483
484 References to other boards can be added in the string also. For
485 example:
486
487 [make-flags]
488 at91-boards=ENABLE_AT91_TEST=1
489 snapper9260=${at91-boards} BUILD_TAG=442
490 snapper9g45=${at91-boards} BUILD_TAG=443
491
492 This will return 'ENABLE_AT91_TEST=1 BUILD_TAG=442' for snapper9260
493 and 'ENABLE_AT91_TEST=1 BUILD_TAG=443' for snapper9g45.
494
495 A special 'target' variable is set to the board target.
496
497 Args:
498 brd: Board object for the board to check.
499 Returns:
500 'make' flags for that board, or '' if none
501 """
502 self._make_flags['target'] = brd.target
503 arg_str = self.ResolveReferences(self._make_flags,
504 self._make_flags.get(brd.target, ''))
505 args = re.findall(r"(?:\".*?\"|\S)+", arg_str)
506 i = 0
507 while i < len(args):
508 args[i] = args[i].replace('"', '')
509 if not args[i]:
510 del args[i]
511 else:
512 i += 1
513 return args
514
515 def LocateArchUrl(self, fetch_arch):
516 """Find a toolchain available online
517
518 Look in standard places for available toolchains. At present the
519 only standard place is at kernel.org.
520
521 Args:
522 arch: Architecture to look for, or 'list' for all
523 Returns:
524 If fetch_arch is 'list', a tuple:
525 Machine architecture (e.g. x86_64)
526 List of toolchains
527 else
528 URL containing this toolchain, if avaialble, else None
529 """
530 arch = command.output_one_line('uname', '-m')
531 if arch == 'aarch64':
532 arch = 'arm64'
533 base = 'https://www.kernel.org/pub/tools/crosstool/files/bin'
534 versions = ['13.2.0', '12.2.0']
535 links = []
536 for version in versions:
537 url = '%s/%s/%s/' % (base, arch, version)
538 print('Checking: %s' % url)
539 response = urllib.request.urlopen(url)
540 html = tools.to_string(response.read())
541 parser = MyHTMLParser(fetch_arch)
542 parser.feed(html)
543 if fetch_arch == 'list':
544 links += parser.links
545 elif parser.arch_link:
546 return url + parser.arch_link
547 if fetch_arch == 'list':
548 return arch, links
549 return None
550
551 def Unpack(self, fname, dest):
552 """Unpack a tar file
553
554 Args:
555 fname: Filename to unpack
556 dest: Destination directory
557 Returns:
558 Directory name of the first entry in the archive, without the
559 trailing /
560 """
561 stdout = command.output('tar', 'xvfJ', fname, '-C', dest)
562 dirs = stdout.splitlines()[1].split('/')[:2]
563 return '/'.join(dirs)
564
565 def TestSettingsHasPath(self, path):
566 """Check if buildman will find this toolchain
567
568 Returns:
569 True if the path is in settings, False if not
570 """
571 paths = self.GetPathList(False)
572 return path in paths
573
574 def ListArchs(self):
575 """List architectures with available toolchains to download"""
576 host_arch, archives = self.LocateArchUrl('list')
577 re_arch = re.compile('[-a-z0-9.]*[-_]([^-]*)-.*')
578 arch_set = set()
579 for archive in archives:
580 # Remove the host architecture from the start
581 arch = re_arch.match(archive[len(host_arch):])
582 if arch:
583 if arch.group(1) != '2.0' and arch.group(1) != '64':
584 arch_set.add(arch.group(1))
585 return sorted(arch_set)
586
587 def FetchAndInstall(self, arch):
588 """Fetch and install a new toolchain
589
590 arch:
591 Architecture to fetch, or 'list' to list
592 """
593 # Fist get the URL for this architecture
594 col = terminal.Color()
595 print(col.build(col.BLUE, "Downloading toolchain for arch '%s'" % arch))
596 url = self.LocateArchUrl(arch)
597 if not url:
598 print(("Cannot find toolchain for arch '%s' - use 'list' to list" %
599 arch))
600 return 2
601 home = os.environ['HOME']
602 dest = os.path.join(home, '.buildman-toolchains')
603 if not os.path.exists(dest):
604 os.mkdir(dest)
605
606 # Download the tar file for this toolchain and unpack it
607 tarfile, tmpdir = tools.download(url, '.buildman')
608 if not tarfile:
609 return 1
610 print(col.build(col.GREEN, 'Unpacking to: %s' % dest), end=' ')
611 sys.stdout.flush()
612 path = self.Unpack(tarfile, dest)
613 os.remove(tarfile)
614 os.rmdir(tmpdir)
615 print()
616
617 # Check that the toolchain works
618 print(col.build(col.GREEN, 'Testing'))
619 dirpath = os.path.join(dest, path)
620 compiler_fname_list = self.ScanPath(dirpath, True)
621 if not compiler_fname_list:
622 print('Could not locate C compiler - fetch failed.')
623 return 1
624 if len(compiler_fname_list) != 1:
625 print(col.build(col.RED, 'Warning, ambiguous toolchains: %s' %
626 ', '.join(compiler_fname_list)))
627 toolchain = Toolchain(compiler_fname_list[0], True, True)
628
629 # Make sure that it will be found by buildman
630 if not self.TestSettingsHasPath(dirpath):
631 print(("Adding 'download' to config file '%s'" %
632 bsettings.config_fname))
633 bsettings.set_item('toolchain', 'download', '%s/*/*' % dest)
634 return 0