META_MODE: Add framework to force rebuilding for major ABI changes.

Normally META_MODE ignores host files for "meta mode" decisions on whether a
file should be rebuilt or not.  This is because a simple installworld can
update timestamps and cause the next build to rebuild all host tools, when the
previous ones may not have any changes in the source tree.  These tools are
normally still ABI compatible.  They are only rebuilt if NO_META_IGNORE_HOST is
set from the workaround/hack in r301467.

One of the major problems with this is when a host tool has objects spread
across many revisions that have mixed-ABI.  For example, if struct stat were to
change on the host, some objects for a tool may have different ideas of that
struct's definition.  If just 1 source file were modified and rebuilt and
linked into the tool, then that toll will have mixed-ABI objects and crash.
This exact thing happened with the ino64 commit in r301467 followed by a
trivial update to libbfd in r318750.  The resulting binary would crash in
buildworld.

Sponsored by:	Dell EMC Isilon

Makefile

@@ -127,7 +127,7 @@ TGTS=	all all-man buildenv buildenvvars buildkernel buildworld \
 	installworld kernel-toolchain libraries lint maninstall \
 	obj objlink rerelease showconfig tags toolchain update \
 	_worldtmp _legacy _bootstrap-tools _cleanobj _obj \
-	_build-tools _compiler-metadata _cross-tools _includes _libraries \
+	_build-tools _build-metadata _cross-tools _includes _libraries \
 	build32 distribute32 install32 buildsoft distributesoft installsoft \
 	builddtb xdev xdev-build xdev-install \
 	xdev-links native-xtools stageworld stagekernel stage-packages \

Makefile.inc1

@@ -643,6 +643,49 @@ LIBCOMPAT= SOFT
 .include "Makefile.libcompat"
 .endif
 
+# META_MODE normally ignores host file changes since every build updates
+# timestamps (see NO_META_IGNORE_HOST in sys.mk).  There are known times
+# when the ABI breaks though that we want to force rebuilding WORLDTMP
+# to get updated host tools.
+.if ${MK_META_MODE} == "yes" && defined(NO_CLEAN) && \
+    !defined(NO_META_IGNORE_HOST) && !defined(NO_META_IGNORE_HOST_HEADERS)
+
+.if !defined(OBJDIR_HOST_OSRELDATE)
+.if exists(${OBJTREE}${.CURDIR}/host-osreldate.h)
+OBJDIR_HOST_OSRELDATE!=	\
+    awk '/^\#define[[:space:]]*__FreeBSD_version/ { print $$3 }' \
+    ${OBJTREE}${.CURDIR}/host-osreldate.h
+.else
+OBJDIR_HOST_OSRELDATE=	0
+.endif
+.export OBJDIR_HOST_OSRELDATE
+.endif
+
+# Note that this logic is the opposite of normal BOOTSTRAP handling.  We want
+# to compare the WORLDTMP's OSRELDATE to the host's OSRELDATE.  If the WORLDTMP
+# is older than the ABI-breakage OSRELDATE of the HOST then we rebuild.
+.for _ver in ${META_MODE_BAD_ABI_VERS}
+.if ${OSRELDATE} >= ${_ver} && ${OBJDIR_HOST_OSRELDATE} < ${_ver}
+_meta_mode_need_rebuild=	${_ver}
+.endif
+.endfor
+.if defined(_meta_mode_need_rebuild)
+.info META_MODE: Rebuilding host tools due to ABI breakage in __FreeBSD_version ${_meta_mode_need_rebuild}.
+NO_META_IGNORE_HOST_HEADERS=	1
+.export NO_META_IGNORE_HOST_HEADERS
+.endif
+.endif
+# This is only used for META_MODE+filemon to track what the oldest
+# __FreeBSD_version is in WORLDTMP.  This purposely does NOT have
+# a make dependency on /usr/include/osreldate.h as the file should
+# only be copied when it is missing or meta mode determines it has changed.
+# Since host files are normally ignored without NO_META_IGNORE_HOST
+# the file will never be updated unless that flag is specified.  This
+# allows tracking the oldest osreldate to force rebuilds via
+# META_MODE_BADABI_REVS above.
+host-osreldate.h: # DO NOT ADD /usr/include/osreldate.h here
+	@cp -f /usr/include/osreldate.h ${.TARGET}
+
 WMAKE=		${WMAKEENV} ${MAKE} ${WORLD_FLAGS} -f Makefile.inc1 DESTDIR=${WORLDTMP}
 
 IMAKEENV=	${CROSSENV}
@@ -820,12 +863,13 @@ _cross-tools:
 	@rm -f ${.OBJDIR}/compiler-metadata.mk
 	${_+_}cd ${.CURDIR}; ${XMAKE} cross-tools
 	${_+_}cd ${.CURDIR}; ${XMAKE} kernel-tools
-_compiler-metadata:
+_build-metadata:
 	@echo
 	@echo "--------------------------------------------------------------"
-	@echo ">>> stage 3.1: recording compiler metadata"
+	@echo ">>> stage 3.1: recording build metadata"
 	@echo "--------------------------------------------------------------"
 	${_+_}cd ${.CURDIR}; ${WMAKE} compiler-metadata.mk
+	${_+_}cd ${.CURDIR}; ${WMAKE} host-osreldate.h
 _includes:
 	@echo
 	@echo "--------------------------------------------------------------"
@@ -864,7 +908,7 @@ WMAKE_TGTS+=	_cleanobj
 WMAKE_TGTS+=	_obj
 .endif
 WMAKE_TGTS+=	_build-tools _cross-tools
-WMAKE_TGTS+=	_compiler-metadata
+WMAKE_TGTS+=	_build-metadata
 WMAKE_TGTS+=	_includes
 .endif
 .if !defined(NO_LIBS)

share/mk/sys.mk

@@ -78,6 +78,7 @@ META_MODE?= normal
 # this should not be a real problem for incremental builds.
 # XXX: This relies on the existing host tools retaining ABI compatibility
 # through upgrades since they won't be rebuilt on header/library changes.
+# This is mitigated by Makefile.inc1 for known-ABI-breaking revisions.
 # Note that these are prefix matching, so /lib matches /libexec.
 .MAKE.META.IGNORE_PATHS+= \
 	${__MAKE_SHELL} \