Something went wrong. Try again.
Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676#!/usr/bin/env python3# SPDX-License-Identifier: GPL-2.0-only# Copyright (C) 2024 ARM Ltd.## Utility providing smaps-like output detailing transparent hugepage usage.# For more info, run:# ./thpmaps --help## Requires numpy:# pip3 install numpy
import argparseimport collectionsimport mathimport osimport reimport resourceimport shutilimport sysimport textwrapimport timeimport numpy as np
with open('/sys/kernel/mm/transparent_hugepage/hpage_pmd_size') as f: PAGE_SIZE = resource.getpagesize() PAGE_SHIFT = int(math.log2(PAGE_SIZE)) PMD_SIZE = int(f.read()) PMD_ORDER = int(math.log2(PMD_SIZE / PAGE_SIZE))
def align_forward(v, a): return (v + (a - 1)) & ~(a - 1)
def align_offset(v, a): return v & (a - 1)
def kbnr(kb): # Convert KB to number of pages. return (kb << 10) >> PAGE_SHIFT
def nrkb(nr): # Convert number of pages to KB. return (nr << PAGE_SHIFT) >> 10
def odkb(order): # Convert page order to KB. return (PAGE_SIZE << order) >> 10
def cont_ranges_all(search, index): # Given a list of arrays, find the ranges for which values are monotonically # incrementing in all arrays. all arrays in search and index must be the # same size. sz = len(search[0]) r = np.full(sz, 2) d = np.diff(search[0]) == 1 for dd in [np.diff(arr) == 1 for arr in search[1:]]: d &= dd r[1:] -= d r[:-1] -= d return [np.repeat(arr, r).reshape(-1, 2) for arr in index]
class ArgException(Exception): pass
class FileIOException(Exception): pass
class BinArrayFile: # Base class used to read /proc/<pid>/pagemap and /proc/kpageflags into a # numpy array. Use inherrited class in a with clause to ensure file is # closed when it goes out of scope. def __init__(self, filename, element_size): self.element_size = element_size self.filename = filename self.fd = os.open(self.filename, os.O_RDONLY)
def cleanup(self): os.close(self.fd)
def __enter__(self): return self
def __exit__(self, exc_type, exc_val, exc_tb): self.cleanup()
def _readin(self, offset, buffer): length = os.preadv(self.fd, (buffer,), offset) if len(buffer) != length: raise FileIOException('error: {} failed to read {} bytes at {:x}' .format(self.filename, len(buffer), offset))
def _toarray(self, buf): assert(self.element_size == 8) return np.frombuffer(buf, dtype=np.uint64)
def getv(self, vec): vec *= self.element_size offsets = vec[:, 0] lengths = (np.diff(vec) + self.element_size).reshape(len(vec)) buf = bytearray(int(np.sum(lengths))) view = memoryview(buf) pos = 0 for offset, length in zip(offsets, lengths): offset = int(offset) length = int(length) self._readin(offset, view[pos:pos+length]) pos += length return self._toarray(buf)
def get(self, index, nr=1): offset = index * self.element_size length = nr * self.element_size buf = bytearray(length) self._readin(offset, buf) return self._toarray(buf)
PM_PAGE_PRESENT = 1 << 63PM_PFN_MASK = (1 << 55) - 1
class PageMap(BinArrayFile): # Read ranges of a given pid's pagemap into a numpy array. def __init__(self, pid='self'): super().__init__(f'/proc/{pid}/pagemap', 8)
KPF_ANON = 1 << 12KPF_COMPOUND_HEAD = 1 << 15KPF_COMPOUND_TAIL = 1 << 16KPF_THP = 1 << 22
class KPageFlags(BinArrayFile): # Read ranges of /proc/kpageflags into a numpy array. def __init__(self): super().__init__(f'/proc/kpageflags', 8)
vma_all_stats = set([ "Size", "Rss", "Pss", "Pss_Dirty", "Shared_Clean", "Shared_Dirty", "Private_Clean", "Private_Dirty", "Referenced", "Anonymous", "KSM", "LazyFree", "AnonHugePages", "ShmemPmdMapped", "FilePmdMapped", "Shared_Hugetlb", "Private_Hugetlb", "Swap", "SwapPss", "Locked",])
vma_min_stats = set([ "Rss", "Anonymous", "AnonHugePages", "ShmemPmdMapped", "FilePmdMapped",])
VMA = collections.namedtuple('VMA', [ 'name', 'start', 'end', 'read', 'write', 'execute', 'private', 'pgoff', 'major', 'minor', 'inode', 'stats',])
class VMAList: # A container for VMAs, parsed from /proc/<pid>/smaps. Iterate over the # instance to receive VMAs. def __init__(self, pid='self', stats=[]): self.vmas = [] with open(f'/proc/{pid}/smaps', 'r') as file: for line in file: elements = line.split() if '-' in elements[0]: start, end = map(lambda x: int(x, 16), elements[0].split('-')) major, minor = map(lambda x: int(x, 16), elements[3].split(':')) self.vmas.append(VMA( name=elements[5] if len(elements) == 6 else '', start=start, end=end, read=elements[1][0] == 'r', write=elements[1][1] == 'w', execute=elements[1][2] == 'x', private=elements[1][3] == 'p', pgoff=int(elements[2], 16), major=major, minor=minor, inode=int(elements[4], 16), stats={}, )) else: param = elements[0][:-1] if param in stats: value = int(elements[1]) self.vmas[-1].stats[param] = {'type': None, 'value': value}
def __iter__(self): yield from self.vmas
def thp_parse(vma, kpageflags, ranges, indexes, vfns, pfns, anons, heads): # Given 4 same-sized arrays representing a range within a page table backed # by THPs (vfns: virtual frame numbers, pfns: physical frame numbers, anons: # True if page is anonymous, heads: True if page is head of a THP), return a # dictionary of statistics describing the mapped THPs. stats = { 'file': { 'partial': 0, 'aligned': [0] * (PMD_ORDER + 1), 'unaligned': [0] * (PMD_ORDER + 1), }, 'anon': { 'partial': 0, 'aligned': [0] * (PMD_ORDER + 1), 'unaligned': [0] * (PMD_ORDER + 1), }, }
for rindex, rpfn in zip(ranges[0], ranges[2]): index_next = int(rindex[0]) index_end = int(rindex[1]) + 1 pfn_end = int(rpfn[1]) + 1
folios = indexes[index_next:index_end][heads[index_next:index_end]]
# Account pages for any partially mapped THP at the front. In that case, # the first page of the range is a tail. nr = (int(folios[0]) if len(folios) else index_end) - index_next stats['anon' if anons[index_next] else 'file']['partial'] += nr
# Account pages for any partially mapped THP at the back. In that case, # the next page after the range is a tail. if len(folios): flags = int(kpageflags.get(pfn_end)[0]) if flags & KPF_COMPOUND_TAIL: nr = index_end - int(folios[-1]) folios = folios[:-1] index_end -= nr stats['anon' if anons[index_end - 1] else 'file']['partial'] += nr
# Account fully mapped THPs in the middle of the range. if len(folios): folio_nrs = np.append(np.diff(folios), np.uint64(index_end - folios[-1])) folio_orders = np.log2(folio_nrs).astype(np.uint64) for index, order in zip(folios, folio_orders): index = int(index) order = int(order) nr = 1 << order vfn = int(vfns[index]) align = 'aligned' if align_forward(vfn, nr) == vfn else 'unaligned' anon = 'anon' if anons[index] else 'file' stats[anon][align][order] += nr
# Account PMD-mapped THPs spearately, so filter out of the stats. There is a # race between acquiring the smaps stats and reading pagemap, where memory # could be deallocated. So clamp to zero incase it would have gone negative. anon_pmd_mapped = vma.stats['AnonHugePages']['value'] file_pmd_mapped = vma.stats['ShmemPmdMapped']['value'] + \ vma.stats['FilePmdMapped']['value'] stats['anon']['aligned'][PMD_ORDER] = max(0, stats['anon']['aligned'][PMD_ORDER] - kbnr(anon_pmd_mapped)) stats['file']['aligned'][PMD_ORDER] = max(0, stats['file']['aligned'][PMD_ORDER] - kbnr(file_pmd_mapped))
rstats = { f"anon-thp-pmd-aligned-{odkb(PMD_ORDER)}kB": {'type': 'anon', 'value': anon_pmd_mapped}, f"file-thp-pmd-aligned-{odkb(PMD_ORDER)}kB": {'type': 'file', 'value': file_pmd_mapped}, }
def flatten_sub(type, subtype, stats): param = f"{type}-thp-pte-{subtype}-{{}}kB" for od, nr in enumerate(stats[2:], 2): rstats[param.format(odkb(od))] = {'type': type, 'value': nrkb(nr)}
def flatten_type(type, stats): flatten_sub(type, 'aligned', stats['aligned']) flatten_sub(type, 'unaligned', stats['unaligned']) rstats[f"{type}-thp-pte-partial"] = {'type': type, 'value': nrkb(stats['partial'])}
flatten_type('anon', stats['anon']) flatten_type('file', stats['file'])
return rstats
def cont_parse(vma, order, ranges, anons, heads): # Given 4 same-sized arrays representing a range within a page table backed # by THPs (vfns: virtual frame numbers, pfns: physical frame numbers, anons: # True if page is anonymous, heads: True if page is head of a THP), return a # dictionary of statistics describing the contiguous blocks. nr_cont = 1 << order nr_anon = 0 nr_file = 0
for rindex, rvfn, rpfn in zip(*ranges): index_next = int(rindex[0]) index_end = int(rindex[1]) + 1 vfn_start = int(rvfn[0]) pfn_start = int(rpfn[0])
if align_offset(pfn_start, nr_cont) != align_offset(vfn_start, nr_cont): continue
off = align_forward(vfn_start, nr_cont) - vfn_start index_next += off
while index_next + nr_cont <= index_end: folio_boundary = heads[index_next+1:index_next+nr_cont].any() if not folio_boundary: if anons[index_next]: nr_anon += nr_cont else: nr_file += nr_cont index_next += nr_cont
# Account blocks that are PMD-mapped spearately, so filter out of the stats. # There is a race between acquiring the smaps stats and reading pagemap, # where memory could be deallocated. So clamp to zero incase it would have # gone negative. anon_pmd_mapped = vma.stats['AnonHugePages']['value'] file_pmd_mapped = vma.stats['ShmemPmdMapped']['value'] + \ vma.stats['FilePmdMapped']['value'] nr_anon = max(0, nr_anon - kbnr(anon_pmd_mapped)) nr_file = max(0, nr_file - kbnr(file_pmd_mapped))
rstats = { f"anon-cont-pmd-aligned-{nrkb(nr_cont)}kB": {'type': 'anon', 'value': anon_pmd_mapped}, f"file-cont-pmd-aligned-{nrkb(nr_cont)}kB": {'type': 'file', 'value': file_pmd_mapped}, }
rstats[f"anon-cont-pte-aligned-{nrkb(nr_cont)}kB"] = {'type': 'anon', 'value': nrkb(nr_anon)} rstats[f"file-cont-pte-aligned-{nrkb(nr_cont)}kB"] = {'type': 'file', 'value': nrkb(nr_file)}
return rstats
def vma_print(vma, pid): # Prints a VMA instance in a format similar to smaps. The main difference is # that the pid is included as the first value. print("{:010d}: {:016x}-{:016x} {}{}{}{} {:08x} {:02x}:{:02x} {:08x} {}" .format( pid, vma.start, vma.end, 'r' if vma.read else '-', 'w' if vma.write else '-', 'x' if vma.execute else '-', 'p' if vma.private else 's', vma.pgoff, vma.major, vma.minor, vma.inode, vma.name ))
def stats_print(stats, tot_anon, tot_file, inc_empty): # Print a statistics dictionary. label_field = 32 for label, stat in stats.items(): type = stat['type'] value = stat['value'] if value or inc_empty: pad = max(0, label_field - len(label) - 1) if type == 'anon' and tot_anon > 0: percent = f' ({value / tot_anon:3.0%})' elif type == 'file' and tot_file > 0: percent = f' ({value / tot_file:3.0%})' else: percent = '' print(f"{label}:{' ' * pad}{value:8} kB{percent}")
def vma_parse(vma, pagemap, kpageflags, contorders): # Generate thp and cont statistics for a single VMA. start = vma.start >> PAGE_SHIFT end = vma.end >> PAGE_SHIFT
pmes = pagemap.get(start, end - start) present = pmes & PM_PAGE_PRESENT != 0 pfns = pmes & PM_PFN_MASK pfns = pfns[present] vfns = np.arange(start, end, dtype=np.uint64) vfns = vfns[present]
pfn_vec = cont_ranges_all([pfns], [pfns])[0] flags = kpageflags.getv(pfn_vec) anons = flags & KPF_ANON != 0 heads = flags & KPF_COMPOUND_HEAD != 0 thps = flags & KPF_THP != 0
vfns = vfns[thps] pfns = pfns[thps] anons = anons[thps] heads = heads[thps]
indexes = np.arange(len(vfns), dtype=np.uint64) ranges = cont_ranges_all([vfns, pfns], [indexes, vfns, pfns])
thpstats = thp_parse(vma, kpageflags, ranges, indexes, vfns, pfns, anons, heads) contstats = [cont_parse(vma, order, ranges, anons, heads) for order in contorders]
tot_anon = vma.stats['Anonymous']['value'] tot_file = vma.stats['Rss']['value'] - tot_anon
return { **thpstats, **{k: v for s in contstats for k, v in s.items()} }, tot_anon, tot_file
def do_main(args): pids = set() rollup = {} rollup_anon = 0 rollup_file = 0
if args.cgroup: strict = False for walk_info in os.walk(args.cgroup): cgroup = walk_info[0] with open(f'{cgroup}/cgroup.procs') as pidfile: for line in pidfile.readlines(): pids.add(int(line.strip())) elif args.pid: strict = True pids = pids.union(args.pid) else: strict = False for pid in os.listdir('/proc'): if pid.isdigit(): pids.add(int(pid))
if not args.rollup: print(" PID START END PROT OFFSET DEV INODE OBJECT")
for pid in pids: try: with PageMap(pid) as pagemap: with KPageFlags() as kpageflags: for vma in VMAList(pid, vma_all_stats if args.inc_smaps else vma_min_stats): if (vma.read or vma.write or vma.execute) and vma.stats['Rss']['value'] > 0: stats, vma_anon, vma_file = vma_parse(vma, pagemap, kpageflags, args.cont) else: stats = {} vma_anon = 0 vma_file = 0 if args.inc_smaps: stats = {**vma.stats, **stats} if args.rollup: for k, v in stats.items(): if k in rollup: assert(rollup[k]['type'] == v['type']) rollup[k]['value'] += v['value'] else: rollup[k] = v rollup_anon += vma_anon rollup_file += vma_file else: vma_print(vma, pid) stats_print(stats, vma_anon, vma_file, args.inc_empty) except (FileNotFoundError, ProcessLookupError, FileIOException): if strict: raise
if args.rollup: stats_print(rollup, rollup_anon, rollup_file, args.inc_empty)
def main(): docs_width = shutil.get_terminal_size().columns docs_width -= 2 docs_width = min(80, docs_width)
def format(string): text = re.sub(r'\s+', ' ', string) text = re.sub(r'\s*\\n\s*', '\n', text) paras = text.split('\n') paras = [textwrap.fill(p, width=docs_width) for p in paras] return '\n'.join(paras)
def formatter(prog): return argparse.RawDescriptionHelpFormatter(prog, width=docs_width)
def size2order(human): units = { "K": 2**10, "M": 2**20, "G": 2**30, "k": 2**10, "m": 2**20, "g": 2**30, } unit = 1 if human[-1] in units: unit = units[human[-1]] human = human[:-1] try: size = int(human) except ValueError: raise ArgException('error: --cont value must be integer size with optional KMG unit') size *= unit order = int(math.log2(size / PAGE_SIZE)) if order < 1: raise ArgException('error: --cont value must be size of at least 2 pages') if (1 << order) * PAGE_SIZE != size: raise ArgException('error: --cont value must be size of power-of-2 pages') if order > PMD_ORDER: raise ArgException('error: --cont value must be less than or equal to PMD order') return order
parser = argparse.ArgumentParser(formatter_class=formatter, description=format("""Prints information about how transparent huge pages are mapped, either system-wide, or for a specified process or cgroup.\\n \\n When run with --pid, the user explicitly specifies the set of pids to scan. e.g. "--pid 10 [--pid 134 ...]". When run with --cgroup, the user passes either a v1 or v2 cgroup and all pids that belong to the cgroup subtree are scanned. When run with neither --pid nor --cgroup, the full set of pids on the system is gathered from /proc and scanned as if the user had provided "--pid 1 --pid 2 ...".\\n \\n A default set of statistics is always generated for THP mappings. However, it is also possible to generate additional statistics for "contiguous block mappings" where the block size is user-defined.\\n \\n Statistics are maintained independently for anonymous and file-backed (pagecache) memory and are shown both in kB and as a percentage of either total anonymous or total file-backed memory as appropriate.\\n \\n THP Statistics\\n --------------\\n \\n Statistics are always generated for fully- and contiguously-mapped THPs whose mapping address is aligned to their size, for each <size> supported by the system. Separate counters describe THPs mapped by PTE vs those mapped by PMD. (Although note a THP can only be mapped by PMD if it is PMD-sized):\\n \\n - anon-thp-pte-aligned-<size>kB\\n - file-thp-pte-aligned-<size>kB\\n - anon-thp-pmd-aligned-<size>kB\\n - file-thp-pmd-aligned-<size>kB\\n \\n Similarly, statistics are always generated for fully- and contiguously-mapped THPs whose mapping address is *not* aligned to their size, for each <size> supported by the system. Due to the unaligned mapping, it is impossible to map by PMD, so there are only PTE counters for this case:\\n \\n - anon-thp-pte-unaligned-<size>kB\\n - file-thp-pte-unaligned-<size>kB\\n \\n Statistics are also always generated for mapped pages that belong to a THP but where the is THP is *not* fully- and contiguously- mapped. These "partial" mappings are all counted in the same counter regardless of the size of the THP that is partially mapped:\\n \\n - anon-thp-pte-partial\\n - file-thp-pte-partial\\n \\n Contiguous Block Statistics\\n ---------------------------\\n \\n An optional, additional set of statistics is generated for every contiguous block size specified with `--cont <size>`. These statistics show how much memory is mapped in contiguous blocks of <size> and also aligned to <size>. A given contiguous block must all belong to the same THP, but there is no requirement for it to be the *whole* THP. Separate counters describe contiguous blocks mapped by PTE vs those mapped by PMD:\\n \\n - anon-cont-pte-aligned-<size>kB\\n - file-cont-pte-aligned-<size>kB\\n - anon-cont-pmd-aligned-<size>kB\\n - file-cont-pmd-aligned-<size>kB\\n \\n As an example, if monitoring 64K contiguous blocks (--cont 64K), there are a number of sources that could provide such blocks: a fully- and contiguously-mapped 64K THP that is aligned to a 64K boundary would provide 1 block. A fully- and contiguously-mapped 128K THP that is aligned to at least a 64K boundary would provide 2 blocks. Or a 128K THP that maps its first 100K, but contiguously and starting at a 64K boundary would provide 1 block. A fully- and contiguously-mapped 2M THP would provide 32 blocks. There are many other possible permutations.\\n"""), epilog=format("""Requires root privilege to access pagemap and kpageflags."""))
group = parser.add_mutually_exclusive_group(required=False) group.add_argument('--pid', metavar='pid', required=False, type=int, default=[], action='append', help="""Process id of the target process. Maybe issued multiple times to scan multiple processes. --pid and --cgroup are mutually exclusive. If neither are provided, all processes are scanned to provide system-wide information.""")
group.add_argument('--cgroup', metavar='path', required=False, help="""Path to the target cgroup in sysfs. Iterates over every pid in the cgroup and its children. --pid and --cgroup are mutually exclusive. If neither are provided, all processes are scanned to provide system-wide information.""")
parser.add_argument('--rollup', required=False, default=False, action='store_true', help="""Sum the per-vma statistics to provide a summary over the whole system, process or cgroup.""")
parser.add_argument('--cont', metavar='size[KMG]', required=False, default=[], action='append', help="""Adds stats for memory that is mapped in contiguous blocks of <size> and also aligned to <size>. May be issued multiple times to track multiple sized blocks. Useful to infer e.g. arm64 contpte and hpa mappings. Size must be a power-of-2 number of pages.""")
parser.add_argument('--inc-smaps', required=False, default=False, action='store_true', help="""Include all numerical, additive /proc/<pid>/smaps stats in the output.""")
parser.add_argument('--inc-empty', required=False, default=False, action='store_true', help="""Show all statistics including those whose value is 0.""")
parser.add_argument('--periodic', metavar='sleep_ms', required=False, type=int, help="""Run in a loop, polling every sleep_ms milliseconds.""")
args = parser.parse_args()
try: args.cont = [size2order(cont) for cont in args.cont] except ArgException as e: parser.print_usage() raise
if args.periodic: while True: do_main(args) print() time.sleep(args.periodic / 1000) else: do_main(args)
if __name__ == "__main__": try: main() except Exception as e: prog = os.path.basename(sys.argv[0]) print(f'{prog}: {e}') exit(1)