Remove C-API header generation from build · bernsteinbear.com/skybison@928fe61

+57 -187

CMakeLists.txt

··· 93 93 -fno-optimize-sibling-calls 94 94 -fno-rtti) 95 95 96 - set(BINARY_DIR_HEADERS "${CMAKE_BINARY_DIR}/Include") 97 - file(MAKE_DIRECTORY "${BINARY_DIR_HEADERS}") 98 96 set( 99 - C_API_HEADERS_INPUT 100 - ${CMAKE_SOURCE_DIR}/capi/Python.h 101 - ${CMAKE_SOURCE_DIR}/capi/cpython-data.h 102 - ${CMAKE_SOURCE_DIR}/capi/cpython-func.h 103 - ${CMAKE_SOURCE_DIR}/capi/cpython-types.h 104 - ${CMAKE_SOURCE_DIR}/capi/pyconfig-linux.h 105 - ${CMAKE_SOURCE_DIR}/capi/pyconfig-osx.h 106 - ${CMAKE_SOURCE_DIR}/capi/pyconfig.h) 107 - set( 108 - C_API_HEADERS_OUTPUT 109 - ${CMAKE_BINARY_DIR}/Include/Python.h 110 - ${CMAKE_BINARY_DIR}/Include/pyconfig-linux.h 111 - ${CMAKE_BINARY_DIR}/Include/pyconfig-osx.h 112 - ${CMAKE_BINARY_DIR}/Include/pyconfig.h) 113 - add_custom_command( 114 - OUTPUT ${C_API_HEADERS_OUTPUT} 115 - COMMAND 116 - cmake -E copy_if_different ${C_API_HEADERS_INPUT} ${BINARY_DIR_HEADERS} 117 - WORKING_DIRECTORY ${CMAKE_SOURCE_DIR} 118 - DEPENDS ${C_API_HEADERS_INPUT} 119 - COMMENT "Generating Pyro Headers") 120 - 121 - add_library(capi-headers INTERFACE) 122 - target_sources( 123 - capi-headers 124 - INTERFACE 125 - ${C_API_HEADERS_OUTPUT} 97 + C_API_HEADERS 126 98 ${CMAKE_SOURCE_DIR}/capi/Python.h 127 99 ${CMAKE_SOURCE_DIR}/capi/abstract.h 128 100 ${CMAKE_SOURCE_DIR}/capi/bltinmodule.h ··· 200 172 ${CMAKE_SOURCE_DIR}/capi/unicodeobject.h 201 173 ${CMAKE_SOURCE_DIR}/capi/warnings.h 202 174 ${CMAKE_SOURCE_DIR}/capi/weakrefobject.h) 175 + add_library(capi-headers INTERFACE) 176 + target_sources( 177 + capi-headers 178 + INTERFACE 179 + ${C_API_HEADERS}) 203 180 target_include_directories( 204 181 capi-headers 205 182 INTERFACE 206 183 capi) 184 + 185 + set(BUILD_INCLUDE_DIR "${CMAKE_BINARY_DIR}/Include") 186 + string( 187 + REGEX 188 + REPLACE "(${CMAKE_SOURCE_DIR}\/capi\/)" "${BUILD_INCLUDE_DIR}/" 189 + C_API_HEADERS_OUTPUT 190 + "${C_API_HEADERS}") 191 + add_custom_command( 192 + OUTPUT ${C_API_HEADERS_OUTPUT} 193 + COMMAND 194 + cmake -E make_directory ${BUILD_INCLUDE_DIR} 195 + COMMAND 196 + cmake -E copy_if_different ${C_API_HEADERS} ${BUILD_INCLUDE_DIR} 197 + DEPENDS ${C_API_HEADERS} 198 + COMMENT "Copying C-API headers") 199 + add_custom_target(copy-capi-headers ALL DEPENDS ${C_API_HEADERS_OUTPUT}) 207 200 208 201 set( 209 202 FROZEN_MODULE_INPUT ··· 454 447 file(WRITE "${CMAKE_BINARY_DIR}/Modules/Setup.local" "Pretend Setup exists") 455 448 456 449 set(CPYTHON_DIR ${CMAKE_SOURCE_DIR}/third-party/cpython) 457 - set( 458 - CPYTHON_HEADERS_INPUT 459 - ${CPYTHON_DIR}/Include/Python-ast.h 460 - ${CPYTHON_DIR}/Include/abstract.h 461 - ${CPYTHON_DIR}/Include/accu.h 462 - ${CPYTHON_DIR}/Include/asdl.h 463 - ${CPYTHON_DIR}/Include/ast.h 464 - ${CPYTHON_DIR}/Include/bitset.h 465 - ${CPYTHON_DIR}/Include/bltinmodule.h 466 - ${CPYTHON_DIR}/Include/boolobject.h 467 - ${CPYTHON_DIR}/Include/bytearrayobject.h 468 - ${CPYTHON_DIR}/Include/bytes_methods.h 469 - ${CPYTHON_DIR}/Include/bytesobject.h 470 - ${CPYTHON_DIR}/Include/cellobject.h 471 - ${CPYTHON_DIR}/Include/ceval.h 472 - ${CPYTHON_DIR}/Include/classobject.h 473 - ${CPYTHON_DIR}/Include/code.h 474 - ${CPYTHON_DIR}/Include/codecs.h 475 - ${CPYTHON_DIR}/Include/compile.h 476 - ${CPYTHON_DIR}/Include/complexobject.h 477 - ${CPYTHON_DIR}/Include/datetime.h 478 - ${CPYTHON_DIR}/Include/descrobject.h 479 - ${CPYTHON_DIR}/Include/dictobject.h 480 - ${CPYTHON_DIR}/Include/dtoa.h 481 - ${CPYTHON_DIR}/Include/dynamic_annotations.h 482 - ${CPYTHON_DIR}/Include/enumobject.h 483 - ${CPYTHON_DIR}/Include/errcode.h 484 - ${CPYTHON_DIR}/Include/eval.h 485 - ${CPYTHON_DIR}/Include/fileobject.h 486 - ${CPYTHON_DIR}/Include/fileutils.h 487 - ${CPYTHON_DIR}/Include/floatobject.h 488 - ${CPYTHON_DIR}/Include/frameobject.h 489 - ${CPYTHON_DIR}/Include/funcobject.h 490 - ${CPYTHON_DIR}/Include/genobject.h 491 - ${CPYTHON_DIR}/Include/graminit.h 492 - ${CPYTHON_DIR}/Include/grammar.h 493 - ${CPYTHON_DIR}/Include/import.h 494 - ${CPYTHON_DIR}/Include/intrcheck.h 495 - ${CPYTHON_DIR}/Include/iterobject.h 496 - ${CPYTHON_DIR}/Include/listobject.h 497 - ${CPYTHON_DIR}/Include/longintrepr.h 498 - ${CPYTHON_DIR}/Include/longobject.h 499 - ${CPYTHON_DIR}/Include/marshal.h 500 - ${CPYTHON_DIR}/Include/memoryobject.h 501 - ${CPYTHON_DIR}/Include/metagrammar.h 502 - ${CPYTHON_DIR}/Include/methodobject.h 503 - ${CPYTHON_DIR}/Include/modsupport.h 504 - ${CPYTHON_DIR}/Include/moduleobject.h 505 - ${CPYTHON_DIR}/Include/namespaceobject.h 506 - ${CPYTHON_DIR}/Include/node.h 507 - ${CPYTHON_DIR}/Include/object.h 508 - ${CPYTHON_DIR}/Include/objimpl.h 509 - ${CPYTHON_DIR}/Include/odictobject.h 510 - ${CPYTHON_DIR}/Include/opcode.h 511 - ${CPYTHON_DIR}/Include/osdefs.h 512 - ${CPYTHON_DIR}/Include/osmodule.h 513 - ${CPYTHON_DIR}/Include/parsetok.h 514 - ${CPYTHON_DIR}/Include/patchlevel.h 515 - ${CPYTHON_DIR}/Include/pgen.h 516 - ${CPYTHON_DIR}/Include/pgenheaders.h 517 - ${CPYTHON_DIR}/Include/py_curses.h 518 - ${CPYTHON_DIR}/Include/pyarena.h 519 - ${CPYTHON_DIR}/Include/pyatomic.h 520 - ${CPYTHON_DIR}/Include/pycapsule.h 521 - ${CPYTHON_DIR}/Include/pyctype.h 522 - ${CPYTHON_DIR}/Include/pydebug.h 523 - ${CPYTHON_DIR}/Include/pydtrace.h 524 - ${CPYTHON_DIR}/Include/pyerrors.h 525 - ${CPYTHON_DIR}/Include/pyexpat.h 526 - ${CPYTHON_DIR}/Include/pyfpe.h 527 - ${CPYTHON_DIR}/Include/pygetopt.h 528 - ${CPYTHON_DIR}/Include/pyhash.h 529 - ${CPYTHON_DIR}/Include/pylifecycle.h 530 - ${CPYTHON_DIR}/Include/pymacro.h 531 - ${CPYTHON_DIR}/Include/pymath.h 532 - ${CPYTHON_DIR}/Include/pymem.h 533 - ${CPYTHON_DIR}/Include/pyport.h 534 - ${CPYTHON_DIR}/Include/pystate.h 535 - ${CPYTHON_DIR}/Include/pystrcmp.h 536 - ${CPYTHON_DIR}/Include/pystrhex.h 537 - ${CPYTHON_DIR}/Include/pystrtod.h 538 - ${CPYTHON_DIR}/Include/pythonrun.h 539 - ${CPYTHON_DIR}/Include/pythread.h 540 - ${CPYTHON_DIR}/Include/pytime.h 541 - ${CPYTHON_DIR}/Include/rangeobject.h 542 - ${CPYTHON_DIR}/Include/setobject.h 543 - ${CPYTHON_DIR}/Include/sliceobject.h 544 - ${CPYTHON_DIR}/Include/structmember.h 545 - ${CPYTHON_DIR}/Include/structseq.h 546 - ${CPYTHON_DIR}/Include/symtable.h 547 - ${CPYTHON_DIR}/Include/sysmodule.h 548 - ${CPYTHON_DIR}/Include/token.h 549 - ${CPYTHON_DIR}/Include/traceback.h 550 - ${CPYTHON_DIR}/Include/tupleobject.h 551 - ${CPYTHON_DIR}/Include/typeslots.h 552 - ${CPYTHON_DIR}/Include/ucnhash.h 553 - ${CPYTHON_DIR}/Include/unicodeobject.h 554 - ${CPYTHON_DIR}/Include/warnings.h 555 - ${CPYTHON_DIR}/Include/weakrefobject.h) 556 - string( 557 - REGEX 558 - REPLACE "(${CPYTHON_DIR}/Include\/)" "${BINARY_DIR_HEADERS}/" 559 - CPYTHON_HEADERS_OUTPUT "${CPYTHON_HEADERS_INPUT}") 560 - add_custom_command( 561 - OUTPUT 562 - ${CPYTHON_HEADERS_OUTPUT} 563 - COMMAND 564 - python3 util/generate_cpython_sources.py 565 - -sources ${CPYTHON_HEADERS_INPUT} 566 - -modified ${C_API_HEADERS_INPUT} 567 - -output_dir ${CMAKE_BINARY_DIR} 568 - WORKING_DIRECTORY 569 - ${CMAKE_SOURCE_DIR} 570 - DEPENDS 571 - ${CPYTHON_HEADERS_INPUT} 572 - ${C_API_HEADERS_INPUT} 573 - util/generate_cpython_sources.py 574 - COMMENT "Generating CPython Headers") 575 - 576 - set( 577 - CPYTHON_SOURCES 450 + add_library( 451 + extension-modules 452 + STATIC 453 + ${CPYTHON_DIR}/Modules/_blake2/blake2b_impl.c 454 + ${CPYTHON_DIR}/Modules/_blake2/blake2module.c 455 + ${CPYTHON_DIR}/Modules/_blake2/blake2s_impl.c 456 + ${CPYTHON_DIR}/Modules/_blake2/clinic/blake2b_impl.c.h 457 + ${CPYTHON_DIR}/Modules/_blake2/clinic/blake2s_impl.c.h 578 458 ${CPYTHON_DIR}/Modules/_bz2module.c 579 459 ${CPYTHON_DIR}/Modules/_hashopenssl.c 580 460 ${CPYTHON_DIR}/Modules/_lzmamodule.c ··· 590 470 ${CPYTHON_DIR}/Modules/addrinfo.h 591 471 ${CPYTHON_DIR}/Modules/atexitmodule.c 592 472 ${CPYTHON_DIR}/Modules/binascii.c 593 - ${CPYTHON_DIR}/Modules/_blake2/blake2module.c 594 - ${CPYTHON_DIR}/Modules/_blake2/blake2b_impl.c 595 - ${CPYTHON_DIR}/Modules/_blake2/blake2s_impl.c 596 - ${CPYTHON_DIR}/Modules/_blake2/clinic/blake2b_impl.c.h 597 - ${CPYTHON_DIR}/Modules/_blake2/clinic/blake2s_impl.c.h 598 473 ${CPYTHON_DIR}/Modules/clinic/_bz2module.c.h 599 474 ${CPYTHON_DIR}/Modules/clinic/_hashopenssl.c.h 600 475 ${CPYTHON_DIR}/Modules/clinic/_lzmamodule.c.h ··· 676 551 ${CPYTHON_DIR}/Python/pystrtod.c 677 552 ${CPYTHON_DIR}/Python/pytime.c 678 553 ${CPYTHON_DIR}/Python/symtable.c 679 - ${CPYTHON_DIR}/Python/wordcode_helpers.h) 680 - 681 - # This target is built in util/run_clang_tidy.py:create_compile_commands() to 682 - # ensure all generated files are present when clang-tidy runs. 683 - add_custom_target( 684 - cpython-sources 685 - DEPENDS 686 - ${C_API_HEADERS_OUTPUT} 687 - ${CPYTHON_HEADERS_OUTPUT} 688 - ${CPYTHON_SOURCES}) 689 - 690 - add_library( 691 - extension-modules 692 - STATIC 693 - ${CPYTHON_SOURCES}) 694 - add_dependencies(extension-modules cpython-sources) 554 + ${CPYTHON_DIR}/Python/wordcode_helpers.h 555 + capi/cpython-internal/Python-ast.h 556 + capi/cpython-internal/accu.h 557 + capi/cpython-internal/asdl.h 558 + capi/cpython-internal/ast.h 559 + capi/cpython-internal/bitset.h 560 + capi/cpython-internal/errcode.h 561 + capi/cpython-internal/graminit.h 562 + capi/cpython-internal/grammar.h 563 + capi/cpython-internal/metagrammar.h 564 + capi/cpython-internal/node.h 565 + capi/cpython-internal/osdefs.h 566 + capi/cpython-internal/pgen.h 567 + capi/cpython-internal/pgenheaders.h 568 + capi/cpython-internal/pystrhex.h 569 + capi/cpython-internal/symtable.h 570 + capi/cpython-internal/token.h 571 + capi/cpython-internal/ucnhash.h) 695 572 target_include_directories( 696 573 extension-modules 697 574 PUBLIC 698 - ${BINARY_DIR_HEADERS} 575 + capi/cpython-internal 699 576 ${CPYTHON_DIR}/Modules 700 577 ${CPYTHON_DIR}/Modules/_sha3 701 578 ${CPYTHON_DIR}/Modules/zlib ··· 705 582 target_compile_definitions(extension-modules PRIVATE _GNU_SOURCE) 706 583 target_compile_options( 707 584 extension-modules 708 - PUBLIC 709 - ${OS_DEFINE} 710 585 PRIVATE 711 586 "-Werror" 712 587 "-DPy_BUILD_CORE" ··· 801 676 ext/Python/traceback.cpp) 802 677 target_compile_options( 803 678 extension 804 - PUBLIC 679 + PRIVATE 805 680 ${OS_DEFINE} 806 - PRIVATE 807 681 ${PYRO_COMPILE_OPTIONS}) 808 682 target_include_directories( 809 683 extension 810 684 PRIVATE 811 - $<TARGET_PROPERTY:gtest,INTERFACE_INCLUDE_DIRECTORIES> 685 + $<TARGET_PROPERTY:capi-headers,INTERFACE_INCLUDE_DIRECTORIES> 812 686 $<TARGET_PROPERTY:gmock,INTERFACE_INCLUDE_DIRECTORIES> 813 - runtime 814 - ${BINARY_DIR_HEADERS}) 815 - add_dependencies(extension cpython-sources) 687 + $<TARGET_PROPERTY:gtest,INTERFACE_INCLUDE_DIRECTORIES> 688 + runtime) 816 689 817 690 set(BINARY_DIR_LIB "${CMAKE_BINARY_DIR}/Lib") 818 691 set(CPYTHON_LIBRARY_DIR "${CMAKE_SOURCE_DIR}/third-party/cpython/Lib") ··· 1517 1390 target_include_directories( 1518 1391 python 1519 1392 PRIVATE 1520 - capi-headers 1521 1393 runtime) 1522 1394 target_link_libraries( 1523 1395 python 1524 1396 PRIVATE 1397 + capi-headers 1525 1398 runtime 1526 1399 modules) 1527 1400 set_target_properties( ··· 1619 1492 runtime-tests 1620 1493 PRIVATE 1621 1494 $<TARGET_PROPERTY:benchmark,INTERFACE_INCLUDE_DIRECTORIES> 1622 - $<TARGET_PROPERTY:gtest,INTERFACE_INCLUDE_DIRECTORIES> 1623 1495 $<TARGET_PROPERTY:gmock,INTERFACE_INCLUDE_DIRECTORIES> 1496 + $<TARGET_PROPERTY:gtest,INTERFACE_INCLUDE_DIRECTORIES> 1624 1497 $<TARGET_PROPERTY:runtime,INTERFACE_INCLUDE_DIRECTORIES> 1625 1498 ${FROZEN_MODULE_OUTPUT_DIR}) 1626 1499 add_dependencies( ··· 1699 1572 extension-tests 1700 1573 PRIVATE 1701 1574 ext/test 1702 - $<TARGET_PROPERTY:gtest,INTERFACE_INCLUDE_DIRECTORIES> 1575 + $<TARGET_PROPERTY:capi-headers,INTERFACE_INCLUDE_DIRECTORIES> 1703 1576 $<TARGET_PROPERTY:gmock,INTERFACE_INCLUDE_DIRECTORIES> 1704 - ${BINARY_DIR_HEADERS}) 1705 - add_dependencies( 1706 - extension-tests 1707 - frozen-sources) 1577 + $<TARGET_PROPERTY:gtest,INTERFACE_INCLUDE_DIRECTORIES>) 1708 1578 1709 1579 add_executable( 1710 1580 python-tests

+1

capi/cpython-internal/Python-ast.h

··· 1 + ../../third-party/cpython/Include/Python-ast.h

+1

capi/cpython-internal/accu.h

··· 1 + ../../third-party/cpython/Include/accu.h

+1

capi/cpython-internal/asdl.h

··· 1 + ../../third-party/cpython/Include/asdl.h

+1

capi/cpython-internal/ast.h

··· 1 + ../../third-party/cpython/Include/ast.h

+1

capi/cpython-internal/bitset.h

··· 1 + ../../third-party/cpython/Include/bitset.h

+1

capi/cpython-internal/errcode.h

··· 1 + ../../third-party/cpython/Include/errcode.h

+1

capi/cpython-internal/graminit.h

··· 1 + ../../third-party/cpython/Include/graminit.h

+1

capi/cpython-internal/grammar.h

··· 1 + ../../third-party/cpython/Include/grammar.h

+1

capi/cpython-internal/metagrammar.h

··· 1 + ../../third-party/cpython/Include/metagrammar.h

+1

capi/cpython-internal/node.h

··· 1 + ../../third-party/cpython/Include/node.h

+1

capi/cpython-internal/osdefs.h

··· 1 + ../../third-party/cpython/Include/osdefs.h

+1

capi/cpython-internal/parsetok.h

··· 1 + ../../third-party/cpython/Include/parsetok.h

+1

capi/cpython-internal/pgen.h

··· 1 + ../../third-party/cpython/Include/pgen.h

+1

capi/cpython-internal/pgenheaders.h

··· 1 + ../../third-party/cpython/Include/pgenheaders.h

+1

capi/cpython-internal/pystrhex.h

··· 1 + ../../third-party/cpython/Include/pystrhex.h

+1

capi/cpython-internal/symtable.h

··· 1 + ../../third-party/cpython/Include/symtable.h

+1

capi/cpython-internal/token.h

··· 1 + ../../third-party/cpython/Include/token.h

+1

capi/cpython-internal/ucnhash.h

··· 1 + ../../third-party/cpython/Include/ucnhash.h

-271

util/generate_cpython_sources.py

··· 1 - #!/usr/bin/env python3 2 - import argparse 3 - import collections 4 - import os 5 - import re 6 - import sys 7 - 8 - 9 - # Tuple that specifies a regex and the position where a symbol will be found 10 - SymbolRegex = collections.namedtuple("SymbolRegex", ["regex", "pos"]) 11 - 12 - 13 - # For sources where only the symbol is required 14 - HEADER_SYMBOL_REGEX = { 15 - "typedef": [ 16 - SymbolRegex(regex=re.compile("^typedef struct.*;", re.MULTILINE), pos=3), 17 - SymbolRegex(regex=re.compile("^typedef(?! struct).*;", re.MULTILINE), pos=2), 18 - SymbolRegex( 19 - regex=re.compile( 20 - "^typedef PyObject\* \(\*_PyCFunctionFast.*\n.*;", re.MULTILINE 21 - ), 22 - pos=2, 23 - ), 24 - SymbolRegex(regex=re.compile("^} .*;", re.MULTILINE), pos=-1), 25 - ], 26 - "struct": [SymbolRegex(regex=re.compile("^struct.*{", re.MULTILINE), pos=1)], 27 - "macro": [ 28 - SymbolRegex(regex=re.compile("^#define.*[^\\\\]\n", re.MULTILINE), pos=1), 29 - SymbolRegex(regex=re.compile("^#define.*\\\\", re.MULTILINE), pos=1), 30 - ], 31 - "pytypeobject_macro": [ 32 - SymbolRegex(regex=re.compile("^#define.*_Type ", re.MULTILINE), pos=1) 33 - ], 34 - "pyexc_macro": [ 35 - SymbolRegex(regex=re.compile("^#define PyExc_\w+ ", re.MULTILINE), pos=1) 36 - ], 37 - "enum": [SymbolRegex(regex=re.compile("^enum.*{", re.MULTILINE), pos=1)], 38 - } 39 - 40 - SOURCE_SYMBOL_REGEX = { 41 - "pytypeobject": [ 42 - SymbolRegex( 43 - regex=re.compile('^extern "C".*PyTypeObject.*_Type', re.MULTILINE), pos=3 44 - ) 45 - ], 46 - "pyfunction": [ 47 - SymbolRegex(regex=re.compile("^PY_EXPORT(?:[^{]|\n)*{", re.MULTILINE), pos=2) 48 - ], 49 - } 50 - 51 - # For sources where the entire definition match is required 52 - HEADER_DEFINITIONS_REGEX = { 53 - "typedef": [ 54 - SymbolRegex(regex=re.compile("^\s*typedef struct.*;", re.MULTILINE), pos=3), 55 - SymbolRegex( 56 - regex=re.compile("^\s*typedef(?! struct).*;.*\n", re.MULTILINE), pos=2 57 - ), 58 - SymbolRegex( 59 - regex=re.compile( 60 - "^typedef PyObject \*$\*_PyCFunctionFast.*\n.*;", re.MULTILINE 61 - ), 62 - pos=2, 63 - ), 64 - SymbolRegex( 65 - regex=re.compile("^typedef.*{(.|\n)*?}.*;.*\n", re.MULTILINE), pos=-1 66 - ), 67 - ], 68 - "struct": [ 69 - SymbolRegex(regex=re.compile("^struct(.|\n)*?};.*\n", re.MULTILINE), pos=1) 70 - ], 71 - "macro": [ 72 - SymbolRegex( 73 - regex=re.compile("^#define[^\\\\]*?(?=[\n\\/])", re.MULTILINE), pos=1 74 - ), 75 - SymbolRegex( 76 - regex=re.compile("^#define.*\\\\(\n.*\\\$*\n.*\n", re.MULTILINE), pos=1 77 - ), 78 - ], 79 - "pytypeobject_macro": [ 80 - SymbolRegex( 81 - regex=re.compile("^PyAPI_DATA$PyTypeObject.*;.*\n", re.MULTILINE), pos=2 82 - ) 83 - ], 84 - "pyexc_macro": [ 85 - SymbolRegex( 86 - regex=re.compile("^PyAPI_DATA\(PyObject.*(PyExc_\w+).*\n", re.MULTILINE), 87 - pos=2, 88 - ) 89 - ], 90 - "enum": [SymbolRegex(regex=re.compile("^enum(.|\n)*?};", re.MULTILINE), pos=1)], 91 - } 92 - 93 - 94 - SPECIAL_CHARS_REGEX = re.compile("[\*|,|;|\(|$|]") 95 - 96 - 97 - def remove_extra_chars(match): 98 - # Modify typedefs with the following signature: 99 - # type (*name)(variables) -> type (*name variables) 100 - modified_match = re.sub("\)\(", " ", match) 101 - # Modify return types longer than one 102 - # unsigned long long -> unsigned_long_long 103 - modified_match = re.sub("static ", "static_", modified_match) 104 - modified_match = re.sub("long long", "long_long", modified_match) 105 - modified_match = re.sub("unsigned ", "unsigned_", modified_match) 106 - modified_match = re.sub("const ", "const_", modified_match) 107 - # Remove extra characters to standardize symbol location 108 - # type (*name variables...) -> type name variables 109 - modified_match = re.sub(SPECIAL_CHARS_REGEX, " ", modified_match) 110 - return modified_match 111 - 112 - 113 - # Given a source file, find the matched patterns 114 - def find_symbols_in_file(lines, regex_dict): 115 - symbols_dict = {x: [] for x in regex_dict.keys()} 116 - for symbol_type, srs in regex_dict.items(): 117 - for sr in srs: 118 - matches = re.findall(sr.regex, lines) 119 - for match in matches: 120 - # Split and locate symbol based on its position 121 - modified_match = remove_extra_chars(match).split() 122 - if len(modified_match) > sr.pos: 123 - symbols_dict[symbol_type].append(modified_match[sr.pos]) 124 - return symbols_dict 125 - 126 - 127 - # Given a list of files, find all the defined symbols 128 - def create_header_symbols_dict(modified_source_paths): 129 - symbols_dict = {} 130 - for path in modified_source_paths: 131 - if not path.endswith(".h"): 132 - continue 133 - with open(path, "r") as f: 134 - lines = f.read() 135 - result_dict = find_symbols_in_file(lines, HEADER_SYMBOL_REGEX) 136 - for k, v in result_dict.items(): 137 - symbols_dict.setdefault(k, []).extend(v) 138 - return symbols_dict 139 - 140 - 141 - # The set of heuristics to determine if a substitution should be performed 142 - def replace_definition(match, pos, symbols): 143 - original_match = match.group(0) 144 - # Split and locate symbol based on its position 145 - symbol = remove_extra_chars(original_match).split()[pos] 146 - # Remove the forward declare of PyTypeObject 147 - if symbol == "PyTypeObject": 148 - modified_match = original_match.replace("typedef struct", "struct") 149 - modified_match = modified_match.replace("} PyTypeObject", "}") 150 - return modified_match 151 - # Check if symbol is redefined or not 152 - if symbol in symbols: 153 - return "" 154 - return original_match 155 - 156 - 157 - # Given a source file, replace the matched patterns 158 - def modify_file(lines, symbols_dict, regex_dict): 159 - # Iterate dictionary of symbol types (i.e. macro, typedef, etc.) 160 - for symbol_type, symbols in symbols_dict.items(): 161 - for sr in regex_dict[symbol_type]: 162 - # Iterate the symbols that will be replaced 163 - lines = re.sub( 164 - sr.regex, lambda m: replace_definition(m, sr.pos, symbols), lines 165 - ) 166 - 167 - return lines 168 - 169 - 170 - # Given a list of sources files, modify the patterns that were annotated 171 - def create_output_file_dict(source_paths, header_symbols_dict): 172 - output_file_dict = {} 173 - # Iterate all source files 174 - for path in source_paths: 175 - output_file_dict[path] = None 176 - 177 - # Modules should not be patched. This quickly skips over them 178 - # to just copy the file directly. 179 - if "Modules" in path: 180 - continue 181 - 182 - if path.endswith(".h"): 183 - with open(path, "r") as f: 184 - lines = f.read() 185 - output_file_dict[path] = modify_file( 186 - lines, header_symbols_dict, HEADER_DEFINITIONS_REGEX 187 - ) 188 - 189 - return output_file_dict 190 - 191 - 192 - def write_if_different(dest_path, new_contents): 193 - """If `dest_path` does not exist or its content are different from 194 - `new_contents` write `new_contents to it.""" 195 - try: 196 - current_contents = open(dest_path, "rb").read() 197 - if current_contents == new_contents: 198 - return 199 - except FileNotFoundError: 200 - pass 201 - with open(dest_path, "wb") as dest_fp: 202 - dest_fp.write(new_contents) 203 - 204 - 205 - # Given a dictionary of files, write the files to the output directory 206 - def output_files_to_directory(output_directory, output_file_dict): 207 - for file_path, lines in output_file_dict.items(): 208 - # Create directory 209 - output_file_path = file_path.split("cpython")[-1] 210 - 211 - # Special handle Python.h 212 - if "Python.h" in file_path: 213 - if "ext" not in file_path: 214 - continue 215 - output_file_path = file_path.split("ext")[-1] 216 - 217 - output_gen_path = f"{output_directory}/{output_file_path}" 218 - if not os.path.exists(os.path.dirname(output_gen_path)): 219 - os.makedirs(os.path.dirname(output_gen_path)) 220 - 221 - # Copy those files that were not modified 222 - if lines is None: 223 - with open(file_path, "rb") as source_fp: 224 - contents = source_fp.read() 225 - write_if_different(output_gen_path, contents) 226 - continue 227 - 228 - # Delete _PyObject_INIT, we are already replacing the macro that uses it 229 - if "objimpl.h" in file_path: 230 - contents = b"".join([line.encode("utf-8") for line in lines]) 231 - contents = re.sub( 232 - b"static.*\n_PyObject_INIT.*\n{(.|\n)*?\n}", b"", contents 233 - ) 234 - write_if_different(output_gen_path, contents) 235 - continue 236 - 237 - # Write files 238 - contents = b"".join([line.encode("utf-8") for line in lines]) 239 - write_if_different(output_gen_path, contents) 240 - 241 - 242 - def main(args): 243 - parser = argparse.ArgumentParser( 244 - description="This strips out the unneeded definitions in the CPython Extension Subsystem" 245 - ) 246 - parser.add_argument( 247 - "-sources", 248 - nargs="*", 249 - required=True, 250 - help="A list of sources from a clean CPython source", 251 - ) 252 - parser.add_argument( 253 - "-modified", 254 - nargs="*", 255 - required=True, 256 - help="A list of sources that modify the clean CPython sources", 257 - ) 258 - parser.add_argument( 259 - "-output_dir", 260 - required=True, 261 - help="The directory in which to output the processed sources", 262 - ) 263 - args = parser.parse_args() 264 - 265 - header_symbols_dict = create_header_symbols_dict(args.modified) 266 - output_file_dict = create_output_file_dict(args.sources, header_symbols_dict) 267 - output_files_to_directory(args.output_dir, output_file_dict) 268 - 269 - 270 - if __name__ == "__main__": 271 - main(sys.argv)

-414

util/generate_cpython_sources_test.py

··· 1 - #!/usr/bin/env python3 2 - import unittest 3 - 4 - import generate_cpython_sources as gcs 5 - 6 - 7 - class TestSymbolRegex(unittest.TestCase): 8 - def test_typdef_regex_PyCFunctionFast_returns_symbol(self): 9 - lines = """ 10 - typedef PyObject* (*_PyCFunctionFast)(PyObject* self, PyObject** args, 11 - Py_ssize_t nargs, PyObject* kwnames); 12 - """ 13 - symbols_dict = gcs.find_symbols_in_file(lines, gcs.HEADER_SYMBOL_REGEX) 14 - res = symbols_dict["typedef"] 15 - self.assertListEqual(res, ["_PyCFunctionFast"]) 16 - 17 - def test_typedef_regex_returns_multiple_symbols(self): 18 - lines = """ 19 - typedef type1 Foo; // Comment 20 - 21 - typedef void (*Bar)(void *); 22 - 23 - typedef struct newtype { 24 - int foo_bar; 25 - } Foo_Bar; 26 - 27 - typedef struct Hello World; 28 - 29 - typedef foo_bar Baz; 30 - 31 - typedef PyObject *(*Func)(PyObject *, PyObject *); 32 - """ 33 - symbols_dict = gcs.find_symbols_in_file(lines, gcs.HEADER_SYMBOL_REGEX) 34 - res = symbols_dict["typedef"] 35 - self.assertListEqual(res, ["World", "Foo", "Bar", "Baz", "Func", "Foo_Bar"]) 36 - 37 - def test_struct_regex_returns_multiple_symbols(self): 38 - lines = """ 39 - struct Foo { 40 - Baz baz1; /* Comment */ 41 - Baz baz2; 42 - }; 43 - 44 - typedef int FooBar; 45 - 46 - #define FooBaz 1, 47 - 48 - struct Bar { 49 - Baz baz; 50 - }; 51 - """ 52 - symbols_dict = gcs.find_symbols_in_file(lines, gcs.HEADER_SYMBOL_REGEX) 53 - res = symbols_dict["struct"] 54 - self.assertListEqual(res, ["Foo", "Bar"]) 55 - 56 - def test_enum_regex_returns_multiple_symbols(self): 57 - lines = """ 58 - enum Foo { 59 - baz1 = 0, /* Comment */ 60 - baz2 = 1, 61 - }; 62 - 63 - typedef int FooBar; 64 - 65 - #define FooBaz 1, 66 - 67 - enum Bar { 68 - baz3 = 0; 69 - }; 70 - """ 71 - symbols_dict = gcs.find_symbols_in_file(lines, gcs.HEADER_SYMBOL_REGEX) 72 - res = symbols_dict["enum"] 73 - self.assertListEqual(res, ["Foo", "Bar"]) 74 - 75 - def test_macro_regex_returns_multiple_symbols(self): 76 - lines = """ 77 - #define Foo 0, // Comment 78 - 79 - typedef int FooBar; 80 - 81 - #define Bar(o) Foo; 82 - 83 - #define FooBaz(o) \\ 84 - { Baz(type) }, 85 - """ 86 - symbols_dict = gcs.find_symbols_in_file(lines, gcs.HEADER_SYMBOL_REGEX) 87 - res = symbols_dict["macro"] 88 - self.assertListEqual(res, ["Foo", "Bar", "FooBaz"]) 89 - 90 - def test_pytypeobject_regex_returns_multiple_symbols(self): 91 - lines = """ 92 - extern "C" PyTypeObject* Foo_Type_Ptr() { 93 - Thread* thread = Thread::currentThread(); 94 - } 95 - 96 - extern "C" PyObject* Foo_Function(void) { 97 - // Some implementation 98 - } 99 - 100 - extern "C" PyTypeObject *Bar_Type_Ptr() { 101 - Thread* thread = Thread::currentThread(); 102 - } 103 - """ 104 - symbols_dict = gcs.find_symbols_in_file(lines, gcs.SOURCE_SYMBOL_REGEX) 105 - res = symbols_dict["pytypeobject"] 106 - self.assertListEqual(res, ["Foo_Type", "Bar_Type"]) 107 - 108 - def test_pytypeobject_macro_regex_returns_multiple_symbols(self): 109 - lines = """ 110 - #define Foo_Type (*Foo_Type_Ptr()) 111 - 112 - #define Foo \\ 113 - { Baz(type) }, 114 - 115 - typedef int FooBar; 116 - 117 - #define Bar_Type (*Bar_Type_Ptr()) 118 - 119 - #define FooBaz(o) Foo, 120 - """ 121 - symbols_dict = gcs.find_symbols_in_file(lines, gcs.HEADER_SYMBOL_REGEX) 122 - res = symbols_dict["pytypeobject_macro"] 123 - self.assertEqual(res, ["Foo_Type", "Bar_Type"]) 124 - 125 - def test_pyexc_macro_regex_returns_multiple_symbols(self): 126 - lines = """ 127 - #define PyExc_FooError ((PyObject *)&(*PyExc_FooError_Ptr())) 128 - 129 - #define Foo_Type (*Foo_Type_Ptr()) 130 - #define PyExc_BarError ((PyObject *)&(*PyExc_BarError_Ptr())) 131 - 132 - typedef int FooBar; 133 - 134 - #define FooBaz(o) Foo, 135 - """ 136 - symbols_dict = gcs.find_symbols_in_file(lines, gcs.HEADER_SYMBOL_REGEX) 137 - res = symbols_dict["pyexc_macro"] 138 - self.assertEqual(res, ["PyExc_FooError", "PyExc_BarError"]) 139 - 140 - def test_pyfunction_regex_returns_multiple_symbols(self): 141 - lines = """ 142 - PY_EXPORT type* foo_function() { 143 - // Implmementation 144 - } 145 - 146 - void Foo_Type_Init(void) { 147 - // Implementation 148 - } 149 - 150 - PY_EXPORT PyTypeObject *bar_function() { 151 - // Implmementation 152 - } 153 - 154 - PY_EXPORT PyObject *baz_function_with_many_args(PyObject *, PyObject *, 155 - PyObject *, PyObject *) { 156 - // Implementation 157 - } 158 - """ 159 - symbols_dict = gcs.find_symbols_in_file(lines, gcs.SOURCE_SYMBOL_REGEX) 160 - res = symbols_dict["pyfunction"] 161 - self.assertEqual( 162 - res, ["foo_function", "bar_function", "baz_function_with_many_args"] 163 - ) 164 - 165 - 166 - class TestDefinitionRegex(unittest.TestCase): 167 - def test_PyCFunctionFast_definition_is_replaced(self): 168 - original_lines = """ 169 - typedef PyObject *(*_PyCFunctionFast)(PyObject *self, PyObject **args, 170 - Py_ssize_t nargs, PyObject *kwnames); 171 - """ 172 - expected_lines = """ 173 - 174 - """ 175 - symbols_to_replace = {"typedef": ["_PyCFunctionFast"]} 176 - res = gcs.modify_file( 177 - original_lines, symbols_to_replace, gcs.HEADER_DEFINITIONS_REGEX 178 - ) 179 - self.assertEqual(res, expected_lines) 180 - 181 - def test_pytypeobject_typedef_is_modified_to_struct_definition(self): 182 - original_lines = """ 183 - typedef struct _typeobject { 184 - int foo; 185 - } PyTypeObject; 186 - """ 187 - expected_lines = """ 188 - struct _typeobject { 189 - int foo; 190 - }; 191 - """ 192 - symbols_to_replace = {"typedef": ["PyTypeObject"]} 193 - res = gcs.modify_file( 194 - original_lines, symbols_to_replace, gcs.HEADER_DEFINITIONS_REGEX 195 - ) 196 - self.assertEqual(res, expected_lines) 197 - 198 - def test_multiple_typedef_definitions_are_replaced(self): 199 - original_lines = """ 200 - typedef type1 Foo; // Comment 201 - 202 - typedef void (*Bar)(void *); 203 - 204 - typedef struct newtype { 205 - int foo_bar; 206 - } Foo_Bar; 207 - 208 - typedef foo_bar Baz; 209 - 210 - typedef struct Foo FooAlias; 211 - 212 - typedef struct Bar BarAlias; 213 - 214 - typedef PyObject *(*Foo)(PyObject *, PyObject *); 215 - """ 216 - expected_lines = """ 217 - typedef struct newtype { 218 - int foo_bar; 219 - } Foo_Bar; 220 - 221 - typedef struct Foo FooAlias; 222 - """ 223 - symbols_to_replace = {"typedef": ["Foo", "Bar", "Baz", "BarAlias"]} 224 - res = gcs.modify_file( 225 - original_lines, symbols_to_replace, gcs.HEADER_DEFINITIONS_REGEX 226 - ) 227 - self.assertEqual(res, expected_lines) 228 - 229 - def test_multiple_multiline_typedef_definitions_are_replaced(self): 230 - original_lines = """ 231 - typedef int Baz; 232 - 233 - typedef struct { 234 - Foo *foo1; 235 - Foo *foo2; /* Comment */ 236 - } Foo; 237 - 238 - typedef struct { 239 - int bar; 240 - } Bar; 241 - 242 - struct FooBarBaz { 243 - Foobar* foobar; 244 - }; 245 - """ 246 - expected_lines = """ 247 - typedef int Baz; 248 - 249 - 250 - 251 - struct FooBarBaz { 252 - Foobar* foobar; 253 - }; 254 - """ 255 - symbols_to_replace = {"typedef": ["Foo", "Bar"]} 256 - res = gcs.modify_file( 257 - original_lines, symbols_to_replace, gcs.HEADER_DEFINITIONS_REGEX 258 - ) 259 - self.assertEqual(res, expected_lines) 260 - 261 - def test_multiple_struct_definitions_are_replaced(self): 262 - original_lines = """ 263 - struct Foo { 264 - Baz baz1; /* Comment */ 265 - Baz baz2; 266 - }; 267 - 268 - typedef int FooBar; 269 - 270 - #define FooBaz 1, 271 - 272 - struct Bar { 273 - Baz baz; 274 - }; 275 - """ 276 - expected_lines = """ 277 - 278 - typedef int FooBar; 279 - 280 - #define FooBaz 1, 281 - 282 - """ 283 - symbols_to_replace = {"struct": ["Foo", "Bar"]} 284 - res = gcs.modify_file( 285 - original_lines, symbols_to_replace, gcs.HEADER_DEFINITIONS_REGEX 286 - ) 287 - self.assertEqual(res, expected_lines) 288 - 289 - def test_multiple_macro_definitions_are_replaced(self): 290 - original_lines = """ 291 - #define Foo 0, // Comment 292 - 293 - typedef int FooBar; 294 - 295 - #define Bar(o) Foo; /* Multi-line 296 - comment */ 297 - 298 - #define FooBaz(o) \\ 299 - { Baz(type) }, 300 - """ 301 - expected_lines = """ 302 - // Comment 303 - 304 - typedef int FooBar; 305 - 306 - /* Multi-line 307 - comment */ 308 - 309 - #define FooBaz(o) \\ 310 - { Baz(type) }, 311 - """ 312 - symbols_to_replace = {"macro": ["Foo", "Bar"]} 313 - res = gcs.modify_file( 314 - original_lines, symbols_to_replace, gcs.HEADER_DEFINITIONS_REGEX 315 - ) 316 - self.assertEqual(res, expected_lines) 317 - 318 - def test_multiline_macro_definitions_are_replaced(self): 319 - original_lines = """ 320 - #define Foo \\ 321 - { Baz(1) }, 322 - 323 - typedef int FooBar; 324 - 325 - #define FooBaz(o) Foo, 326 - 327 - #define Bar(op) \\ 328 - do { \\ 329 - int a = 1; \\ 330 - if (a == 1) \\ 331 - Foo \\ 332 - else \\ 333 - Baz(a) \\ 334 - } while (0) 335 - """ 336 - expected_lines = """ 337 - 338 - typedef int FooBar; 339 - 340 - #define FooBaz(o) Foo, 341 - 342 - """ 343 - symbols_to_replace = {"macro": ["Foo", "Bar"]} 344 - res = gcs.modify_file( 345 - original_lines, symbols_to_replace, gcs.HEADER_DEFINITIONS_REGEX 346 - ) 347 - self.assertEqual(res, expected_lines) 348 - 349 - def test_pytypeobject_macro_definitions_are_replaced(self): 350 - original_lines = """ 351 - PyAPI_DATA(PyTypeObject) Foo_Type; 352 - 353 - #define Foo \\ 354 - { Baz(1) }, 355 - 356 - typedef int FooBar; 357 - 358 - PyAPI_DATA(PyTypeObject) Bar_Type; 359 - 360 - #define FooBaz(o) Foo, 361 - """ 362 - expected_lines = """ 363 - 364 - #define Foo \\ 365 - { Baz(1) }, 366 - 367 - typedef int FooBar; 368 - 369 - 370 - #define FooBaz(o) Foo, 371 - """ 372 - symbols_to_replace = {"pytypeobject_macro": ["Foo_Type", "Bar_Type"]} 373 - res = gcs.modify_file( 374 - original_lines, symbols_to_replace, gcs.HEADER_DEFINITIONS_REGEX 375 - ) 376 - self.assertEqual(res, expected_lines) 377 - 378 - def test_pyexc_macro_definitions_are_replaced(self): 379 - original_lines = """ 380 - # foo 381 - 382 - PyAPI_DATA(PyObject *) PyExc_FooError; 383 - 384 - #define Foo \\ 385 - { Baz(1) }, 386 - 387 - typedef int FooBar; 388 - 389 - PyAPI_DATA(PyTypeObject) Bar_Type; 390 - 391 - #define FooBaz(o) Foo, 392 - """ 393 - expected_lines = """ 394 - # foo 395 - 396 - 397 - #define Foo \\ 398 - { Baz(1) }, 399 - 400 - typedef int FooBar; 401 - 402 - PyAPI_DATA(PyTypeObject) Bar_Type; 403 - 404 - #define FooBaz(o) Foo, 405 - """ 406 - symbols_to_replace = {"pyexc_macro": ["PyExc_FooError"]} 407 - res = gcs.modify_file( 408 - original_lines, symbols_to_replace, gcs.HEADER_DEFINITIONS_REGEX 409 - ) 410 - self.assertEqual(res, expected_lines) 411 - 412 - 413 - if __name__ == "__main__": 414 - unittest.main()