Pass -fuse-ld=/path/to/ld if ${LD} != "ld"

This is needed so that setting LD/XLD is not ignored when linking with $CC
instead of directly using $LD. Currently only clang accepts an absolute
path for -fuse-ld= (Clang 12+ will add a new --ld-path flag), so we now
warn when building with GCC and $LD != "ld" since that might result in the
wrong linker being used.

We have been setting XLD=/path/to/cheri/ld.lld in CheriBSD for a long time and
used a similar version of this patch to avoid linking with /usr/bin/ld.
This change is also required when building FreeBSD on an Ubuntu with Clang:
In that case we set XCC=/usr/lib/llvm-10/bin/clang and since
/usr/lib/llvm-10/bin/ does not contain a "ld" binary the build fails with
`clang: error: unable to execute command: Executable "ld" doesn't exist!`
unless we pass -fuse-ld=/usr/lib/llvm-10/bin/ld.lld.

This change passes -fuse-ld instead of copying ${XLD} to WOLRDTMP/bin/ld
since then we would have to ensure that this file does not exist while
building the bootstrap tools. The cross-linker might not be compatible with
the host linker (e.g. when building on macos: host-linker= Mach-O /usr/bin/ld,
cross-linker=LLVM ld.lld).

Reviewed By:	brooks, emaste
Differential Revision: https://reviews.freebsd.org/D26055

share/mk/bsd.sys.mk

@@ -284,6 +284,19 @@ CFLAGS+=	ERROR-tried-to-rebuild-during-make-install
 .endif
 .endif
 
+# Please keep this if in sync with kern.mk
+.if ${LD} != "ld" && (${CC:[1]:H} != ${LD:[1]:H} || ${LD:[1]:T} != "ld")
+# Add -fuse-ld=${LD} if $LD is in a different directory or not called "ld".
+# Note: Clang 12+ will prefer --ld-path= over -fuse-ld=.
+.if ${COMPILER_TYPE} == "clang"
+LDFLAGS+=	-fuse-ld=${LD:[1]}
+.else
+# GCC does not support an absolute path for -fuse-ld so we just print this
+# warning instead and let the user add the required symlinks.
+.warning LD (${LD}) is not the default linker for ${CC} but -fuse-ld= is not supported
+.endif
+.endif
+
 # Tell bmake not to mistake standard targets for things to be searched for
 # or expect to ever be up-to-date.
 PHONY_NOTMAIN = analyze afterdepend afterinstall all beforedepend beforeinstall \

sys/conf/kern.mk

@@ -270,6 +270,22 @@ CFLAGS+=        -std=iso9899:1999
 CFLAGS+=        -std=${CSTD}
 .endif # CSTD
 
+# Please keep this if in sync with bsd.sys.mk
+.if ${LD} != "ld" && (${CC:[1]:H} != ${LD:[1]:H} || ${LD:[1]:T} != "ld")
+# Add -fuse-ld=${LD} if $LD is in a different directory or not called "ld".
+# Note: Clang 12+ will prefer --ld-path= over -fuse-ld=.
+.if ${COMPILER_TYPE} == "clang"
+# Note: unlike bsd.sys.mk we can't use LDFLAGS here since that is used for the
+# flags required when linking the kernel. We don't need those flags when
+# building the vdsos. However, we do need -fuse-ld, so use ${CCLDFLAGS} instead.
+CCLDFLAGS+=	-fuse-ld=${LD:[1]}
+.else
+# GCC does not support an absolute path for -fuse-ld so we just print this
+# warning instead and let the user add the required symlinks.
+.warning LD (${LD}) is not the default linker for ${CC} but -fuse-ld= is not supported
+.endif
+.endif
+
 # Set target-specific linker emulation name.
 LD_EMULATION_aarch64=aarch64elf
 LD_EMULATION_amd64=elf_x86_64_fbsd

sys/conf/kern.post.mk

@@ -228,7 +228,7 @@ kernel-clean:
 # in the a.out ld.  For now, this works.
 hack.pico: Makefile
 	:> hack.c
-	${CC} -shared ${CFLAGS} -nostdlib hack.c -o hack.pico
+	${CC} ${CCLDFLAGS} -shared ${CFLAGS} -nostdlib hack.c -o hack.pico
 	rm -f hack.c
 
 offset.inc: $S/kern/genoffset.sh genoffset.o

sys/modules/cloudabi32/Makefile

@@ -33,7 +33,7 @@ BINARY_ARCHITECTURE=i386
 .endif
 
 cloudabi32_vdso.o: ${VDSO_SRCS}
-	${CC} -x assembler-with-cpp -m32 -shared -nostdinc -nostdlib \
+	${CC} ${CCLDFLAGS} -x assembler-with-cpp -m32 -shared -nostdinc -nostdlib \
 	    -Wl,-T${SYSDIR}/compat/cloudabi/cloudabi_vdso.lds \
 	    ${VDSO_SRCS} -o ${.TARGET}
 

sys/modules/cloudabi64/Makefile

@@ -25,7 +25,7 @@ BINARY_ARCHITECTURE=i386
 .endif
 
 cloudabi64_vdso.o: ${VDSO_SRCS}
-	${CC} -x assembler-with-cpp -shared -nostdinc -nostdlib \
+	${CC} ${CCLDFLAGS} -x assembler-with-cpp -shared -nostdinc -nostdlib \
 	    -Wl,-T${SYSDIR}/compat/cloudabi/cloudabi_vdso.lds \
 	    ${VDSO_SRCS} -o ${.TARGET}
 

sys/modules/linux/Makefile

@@ -54,7 +54,7 @@ linux${SFX}_assym.h: linux${SFX}_genassym.o
 	sh ${SYSDIR}/kern/genassym.sh linux${SFX}_genassym.o > ${.TARGET}
 
 linux${SFX}_locore.o: linux${SFX}_assym.h assym.inc
-	${CC} -x assembler-with-cpp -DLOCORE -m32 -shared -s			\
+	${CC} ${CCLDFLAGS} -x assembler-with-cpp -DLOCORE -m32 -shared -s			\
 	-pipe -I. -I${SYSDIR} ${WERROR} -Wall -fno-common -nostdinc -nostdlib	\
 	-fno-omit-frame-pointer -fPIC						\
 	-Wl,-T${SRCTOP}/sys/${MACHINE_CPUARCH}/linux${SFX}/${VDSO}.lds.s	\

sys/modules/linux64/Makefile

@@ -30,7 +30,7 @@ linux_assym.h: linux_genassym.o
 	sh ${SYSDIR}/kern/genassym.sh linux_genassym.o > ${.TARGET}
 
 linux_locore.o: linux_locore.asm linux_assym.h
-	${CC} -x assembler-with-cpp -DLOCORE -shared -mcmodel=small	\
+	${CC} ${CCLDFLAGS} -x assembler-with-cpp -DLOCORE -shared -mcmodel=small	\
 	-pipe -I. -I${SYSDIR} ${WERROR} -Wall -fno-common -fPIC -nostdinc	\
 	-Wl,-T${SRCTOP}/sys/${MACHINE}/linux/${VDSO}.lds.s	\
 	-Wl,-soname=${VDSO}.so.1,-warn-common -nostdlib			\