Merge pull request #38881 from obsidiansystems/sierra-hack

cc-wrapper: More intelligent sierra hack

pkgs/build-support/bintools-wrapper/macos-sierra-reexport-hack.bash

···
 
 set -eu -o pipefail
 
+# For cmd | while read; do ...; done
+shopt -s lastpipe
+
 path_backup="$PATH"
 if [ -n "@coreutils_bin@" ]; then
   PATH="@coreutils_bin@/bin"
 fi
 
-declare -r recurThreshold=300
+declare -ri recurThreshold=200
+declare -i overflowCount=0
+
+declare -ar origArgs=("$@")
+
+# Throw away what we won't need
+declare -a parentArgs=()
 
-declare overflowCount=0
-for ((n=0; n < $#; ++n)); do
-    case "${!n}" in
-        -l*) let overflowCount+=1 ;;
-        -reexport-l*) let overflowCount+=1 ;;
-        *) ;;
+while (( $# )); do
+    case "$1" in
+        -l)
+            echo "cctools LD does not support '-l foo'" >&2
+            exit 1
+            ;;
+        -lazy_library | -reexport_library | -upward_library | -weak_library)
+            overflowCount+=1
+            shift 2
+            ;;
+        -l* | *.so.* | *.dylib | -lazy-l* | -reexport-l* | -upward-l* | -weak-l*)
+            overflowCount+=1
+            shift 1
+            ;;
+        *.a | *.o)
+            shift 1
+            ;;
+        -L | -F)
+            # Evidentally ld doesn't like using the child's RPATH, so it still
+            # needs these.
+            parentArgs+=("$1" "$2")
+            shift 2
+            ;;
+        -L?* | -F?*)
+            parentArgs+=("$1")
+            shift 1
+            ;;
+        -o)
+            outputName="$2"
+            parentArgs+=("$1" "$2")
+            shift 2
+            ;;
+        -install_name | -dylib_install_name | -dynamic-linker | -plugin)
+            parentArgs+=("$1" "$2")
+            shift 2
+            ;;
+        -rpath)
+            # Only an rpath to the child is needed, which we will add
+            shift 2
+            ;;
+        *)
+            if [[ -f "$1" ]]; then
+                # Propabably a non-standard object file like Haskell's
+                # `.dyn_o`. Skip it like other inputs
+                :
+            else
+                parentArgs+=("$1")
+            fi
+            shift 1
+            ;;
     esac
 done
 
-declare -a allArgs=()
+
 
 if (( "$overflowCount" <= "$recurThreshold" )); then
-    allArgs=("$@")
-else
-    declare -a childrenLookup=() childrenLink=()
+    if [ -n "${NIX_DEBUG:-}" ]; then
+        echo "ld-wrapper: Only ${overflowCount} inputs counted while ${recurThreshold} is the ceiling, linking normally. " >&2
+    fi
+    PATH="$path_backup"
+    exec @prog@ "${origArgs[@]}"
+fi
 
-    while (( $# )); do
-        case "$1" in
-            -L/*)
-                childrenLookup+=("$1")
-                allArgs+=("$1")
-                ;;
-            -L)
-                echo "cctools LD does not support '-L foo' or '-l foo'" >&2
-                exit 1
-                ;;
-            -l)
-                echo "cctools LD does not support '-L foo' or '-l foo'" >&2
-                exit 1
-                ;;
-            -lazy_library | -lazy_framework | -lto_library)
-                # We aren't linking any "azy_library", "to_library", etc.
-                allArgs+=("$1")
-                ;;
-            -lazy-l | -weak-l)    allArgs+=("$1") ;;
-                # We can't so easily prevent header issues from these.
-            -lSystem)             allArgs+=("$1") ;;
-                # Special case as indirection seems like a bad idea for something
-                # so fundamental. Can be removed for simplicity.
-            -l?* | -reexport-l?*) childrenLink+=("$1") ;;
-            *)                    allArgs+=("$1") ;;
-        esac
+
+
+if [ -n "${NIX_DEBUG:-}" ]; then
+    echo "ld-wrapper: ${overflowCount} inputs counted when ${recurThreshold} is the ceiling, inspecting further. " >&2
+fi
 
-        shift
-    done
+# Collect the normalized linker input
+declare -a norm=()
 
-    declare n=0
-    while (( $n < "${#childrenLink[@]}" )); do
-        if [[ "${childrenLink[n]}" = -l* ]]; then
-            childrenLink[n]="-reexport${childrenLink[n]}"
-        fi
-        let ++n
+# Arguments are null-separated
+@prog@ --dump-normalized-lib-args "${origArgs[@]}" |
+    while IFS= read -r -d '' input; do
+        norm+=("$input")
     done
-    unset n
 
-    declare -r outputNameLibless=$(basename $( \
-        if [[ -z "${outputName:+isUndefined}" ]]; then
-            echo unnamed
-        elif [[ "${outputName:0:3}" = lib ]]; then
-            echo "${outputName:3}"
-        else
-            echo "${outputName}"
-        fi))
-    declare -ra children=("$outputNameLibless-reexport-delegate-0" \
-                          "$outputNameLibless-reexport-delegate-1")
+declare -i leafCount=0
+declare lastLeaf=''
+declare -a childrenInputs=() trailingInputs=()
+while (( "${#norm[@]}" )); do
+    case "${norm[0]}" in
+        -lazy_library | -upward_library)
+            # TODO(@Ericson2314): Don't do that, but intersperse children
+            # between such args.
+            echo "ld-wrapper: Warning: Potentially changing link order" >&2
+            trailingInputs+=("${norm[0]}" "${norm[1]}")
+            norm=("${norm[@]:2}")
+            ;;
+        -reexport_library | -weak_library)
+            childrenInputs+=("${norm[0]}" "${norm[1]}")
+            if [[ "${norm[1]}" != "$lastLeaf" ]]; then
+                leafCount+=1
+                lastLeaf="${norm[1]}"
+            fi
+            norm=("${norm[@]:2}")
+            ;;
+        *.so | *.dylib)
+            childrenInputs+=(-reexport_library "${norm[0]}")
+            if [[ "${norm[0]}" != "$lastLeaf" ]]; then
+                leafCount+=1
+                lastLeaf="${norm[0]}"
+            fi
+            norm=("${norm[@]:1}")
+            ;;
+        *.o | *.a)
+            # Don't delegate object files or static libs
+            parentArgs+=("${norm[0]}")
+            norm=("${norm[@]:1}")
+            ;;
+        *)
+            if [[ -f "${norm[0]}" ]]; then
+                # Propabably a non-standard object file. We'll let it by.
+                parentArgs+=("${norm[0]}")
+                norm=("${norm[@]:1}")
+            else
+                echo "ld-wrapper: Internal Error: Invalid normalized argument" >&2
+                exit -1
+            fi
+            ;;
+    esac
+done
 
-    mkdir -p "$out/lib"
 
-    PATH="$PATH:@out@/bin"
 
-    symbolBloatObject=$outputNameLibless-symbol-hack.o
-    if [[ ! -e $symbolBloatObject ]]; then
-        # `-Q` means use GNU Assembler rather than Clang, avoiding an awkward
-        # dependency cycle.
-        printf '.private_extern _______child_hack_foo\nchild_hack_foo:\n' \
-            | @targetPrefix@as -Q -- -o $symbolBloatObject
+if (( "$leafCount" <= "$recurThreshold" )); then
+    if [ -n "${NIX_DEBUG:-}" ]; then
+        echo "ld-wrapper: Only ${leafCount} *dynamic* inputs counted while ${recurThreshold} is the ceiling, linking normally. " >&2
+    fi
+    PATH="$path_backup"
+    exec @prog@ "${origArgs[@]}"
+fi
+
+
+
+if [ -n "${NIX_DEBUG:-}" ]; then
+    echo "ld-wrapper: ${leafCount} *dynamic* inputs counted when ${recurThreshold} is the ceiling, delegating to children. " >&2
+fi
+
+declare -r outputNameLibless=$( \
+    if [[ -z "${outputName:+isUndefined}" ]]; then
+        echo unnamed
+        return 0;
+    fi
+    baseName=$(basename ${outputName})
+    if [[ "$baseName" = lib* ]]; then
+        baseName="${baseName:3}"
     fi
+    echo "$baseName")
 
-    # first half of libs
-    @targetPrefix@ld -macosx_version_min $MACOSX_DEPLOYMENT_TARGET -arch x86_64 -dylib \
-      -o "$out/lib/lib${children[0]}.dylib" \
-      -install_name "$out/lib/lib${children[0]}.dylib" \
-      "${childrenLookup[@]}" "$symbolBloatObject" \
-      "${childrenLink[@]:0:$((${#childrenLink[@]} / 2 ))}"
+declare -ra children=(
+    "$outputNameLibless-reexport-delegate-0"
+    "$outputNameLibless-reexport-delegate-1"
+)
+
+mkdir -p "$out/lib"
+
+symbolBloatObject=$outputNameLibless-symbol-hack.o
+if [[ ! -f $symbolBloatObject ]]; then
+    # `-Q` means use GNU Assembler rather than Clang, avoiding an awkward
+    # dependency cycle.
+    printf '.private_extern _______child_hack_foo\nchild_hack_foo:\n' |
+        PATH="$PATH:@out@/bin" @targetPrefix@as -Q -- -o $symbolBloatObject
+fi
+
+# Split inputs between children
+declare -a child0Inputs=() child1Inputs=("${childrenInputs[@]}")
+let "countFirstChild = $leafCount / 2" || true
+lastLeaf=''
+while (( "$countFirstChild" )); do
+    case "${child1Inputs[0]}" in
+        -reexport_library | -weak_library)
+            child0Inputs+=("${child1Inputs[0]}" "${child1Inputs[1]}")
+            if [[ "${child1Inputs[1]}" != "$lastLeaf" ]]; then
+                let countFirstChild-=1 || true
+                lastLeaf="${child1Inputs[1]}"
+            fi
+            child1Inputs=("${child1Inputs[@]:2}")
+            ;;
+        *.so | *.dylib)
+            child0Inputs+=(-reexport_library "${child1Inputs[0]}")
+            if [[ "${child1Inputs[0]}" != "$lastLeaf" ]]; then
+                let countFirstChild-=1 || true
+                lastLeaf="${child1Inputs[1]}"
+            fi
+            child1Inputs=("${child1Inputs[@]:2}")
+            ;;
+        *)
+            echo "ld-wrapper: Internal Error: Invalid delegated input" >&2
+            exit -1
+            ;;
+    esac
+done
+
+
+# First half of libs
+@out@/bin/@targetPrefix@ld \
+  -macosx_version_min $MACOSX_DEPLOYMENT_TARGET -arch x86_64 -dylib \
+  -o "$out/lib/lib${children[0]}.dylib" \
+  -install_name "$out/lib/lib${children[0]}.dylib" \
+  "$symbolBloatObject" "${child0Inputs[@]}" "${trailingInputs[@]}"
 
-    # second half of libs
-    @targetPrefix@ld -macosx_version_min $MACOSX_DEPLOYMENT_TARGET -arch x86_64 -dylib \
-      -o "$out/lib/lib${children[1]}.dylib" \
-      -install_name "$out/lib/lib${children[1]}.dylib" \
-      "${childrenLookup[@]}" "$symbolBloatObject" \
-      "${childrenLink[@]:$((${#childrenLink[@]} / 2 ))}"
+# Second half of libs
+@out@/bin/@targetPrefix@ld \
+  -macosx_version_min $MACOSX_DEPLOYMENT_TARGET -arch x86_64 -dylib \
+  -o "$out/lib/lib${children[1]}.dylib" \
+  -install_name "$out/lib/lib${children[1]}.dylib" \
+  "$symbolBloatObject" "${child1Inputs[@]}" "${trailingInputs[@]}"
 
-    allArgs+=("-L$out/lib" "-l${children[0]}" "-l${children[1]}")
+parentArgs+=("-L$out/lib" -rpath "$out/lib")
+if [[ $outputName != *reexport-delegate* ]]; then
+	parentArgs+=("-l${children[0]}" "-l${children[1]}")
+else
+    parentArgs+=("-reexport-l${children[0]}" "-reexport-l${children[1]}")
+fi
+
+parentArgs+=("${trailingInputs[@]}")
+
+if [ -n "${NIX_DEBUG:-}" ]; then
+    echo "flags using delegated children to @prog@:" >&2
+    printf "  %q\n" "${parentArgs[@]}" >&2
 fi
 
 PATH="$path_backup"
-exec @prog@ "${allArgs[@]}"
+exec @prog@ "${parentArgs[@]}"

pkgs/os-specific/darwin/cctools/port.nix

···
 , llvm, libcxx, libcxxabi, clang, libuuid
 , libobjc ? null, maloader ? null, xctoolchain ? null
 , hostPlatform, targetPlatform
+, enableDumpNormalizedLibArgs ? false
 }:
 
 let
···
     "${targetPlatform.config}-";
 in
 
-assert targetPlatform.isDarwin;
-
 # Non-Darwin alternatives
 assert (!hostPlatform.isDarwin) -> (maloader != null && xctoolchain != null);
 
···
     name = "${targetPrefix}cctools-port-${version}";
     version = "895";
 
-    src = fetchFromGitHub {
+    src = fetchFromGitHub (if enableDumpNormalizedLibArgs then {
+      owner  = "tpoechtrager";
+      repo   = "cctools-port";
+      # master with https://github.com/tpoechtrager/cctools-port/pull/34
+      rev    = "8395d4b2c3350356e2fb02f5e04f4f463c7388df";
+      sha256 = "10vbf1cfzx02q8chc77s84fp2kydjpx2y682mr6mrbb7sq5rwh8f";
+    } else {
       owner  = "tpoechtrager";
       repo   = "cctools-port";
       rev    = "2e569d765440b8cd6414a695637617521aa2375b"; # From branch 895-ld64-274.2
       sha256 = "0l45mvyags56jfi24rawms8j2ihbc45mq7v13pkrrwppghqrdn52";
-    };
+    });
 
     outputs = [ "out" "dev" ];
 
···
     };
 
     meta = {
+      broken = !targetPlatform.isDarwin; # Only supports darwin targets
       homepage = http://www.opensource.apple.com/source/cctools/;
       description = "MacOS Compiler Tools (cross-platform port)";
       license = stdenv.lib.licenses.apsl20;

pkgs/top-level/all-packages.nix

···
 
   clang-sierraHack = clang.override {
     name = "clang-wrapper-with-reexport-hack";
-    bintools = clang.bintools.override {
+    bintools = darwin.binutils.override {
       useMacosReexportHack = true;
+      bintools = darwin.binutils.bintools.override {
+        cctools = darwin.cctools.override {
+          enableDumpNormalizedLibArgs = true;
+        };
+      };
     };
   };