From 945078deae448e0a13c34b3393d836087719fb16 Mon Sep 17 00:00:00 2001 From: "Simon J. Gerraty" Date: Sat, 13 May 2023 10:03:50 -0700 Subject: [PATCH] Import bmake-20230510 Relevant/interesting changes: o parse.c: don't print null filename in stack traces o for.c: skip syntactically wrong .for loops o var.c: allow for :gmtime=${mtime} add :mtime[=timestamp] where timestamp is used if stat(2) fails, if :mtime=error stat(2) failure causes error. o make.1: fix documentation of .PREFIX to match reality and POSIX o unit-tests: improved var-scope-local --- ChangeLog | 24 ++ FILES | 4 +- VERSION | 2 +- bmake.1 | 33 ++- bmake.cat1 | 17 +- compat.c | 6 +- for.c | 60 ++-- make.1 | 33 ++- mk/ChangeLog | 35 +++ mk/FILES | 2 + mk/dirdeps-targets.mk | 11 +- mk/dirdeps.mk | 16 +- mk/install-mk | 4 +- mk/jobs.mk | 17 +- mk/meta.autodep.mk | 6 +- mk/meta.sys.mk | 105 ++----- mk/mk-files.txt | 55 +++- mk/newlog.sh | 412 ++++++++++++++++++++++++++++ mk/sys.dependfile.mk | 11 +- mk/sys.dirdeps.mk | 183 ++++++++++++ mk/sys.mk | 5 +- parse.c | 27 +- unit-tests/Makefile | 6 +- unit-tests/cond-func.mk | 12 +- unit-tests/cond-late.mk | 15 +- unit-tests/dep-var.mk | 7 +- unit-tests/directive-for-errors.exp | 33 +-- unit-tests/directive-for-errors.mk | 57 ++-- unit-tests/directive-for-escape.exp | 80 +++--- unit-tests/directive-for-escape.mk | 62 ++++- unit-tests/directive-for.exp | 63 +++-- unit-tests/directive-for.mk | 149 +++++++--- unit-tests/forloop.exp | 20 -- unit-tests/forloop.mk | 53 ---- unit-tests/parse.mk | 32 ++- unit-tests/var-eval-short.exp | 8 +- unit-tests/var-eval-short.mk | 12 +- unit-tests/var-scope-local.exp | 54 +++- unit-tests/var-scope-local.mk | 101 ++++--- unit-tests/varmod-gmtime.exp | 20 +- unit-tests/varmod-gmtime.mk | 33 +-- unit-tests/varmod-localtime.exp | 20 +- unit-tests/varmod-localtime.mk | 33 +-- unit-tests/varmod-mtime.exp | 1 + unit-tests/varmod-mtime.mk | 30 ++ unit-tests/varmod-path.mk | 14 +- var.c | 119 ++++++-- 47 files changed, 1545 insertions(+), 557 deletions(-) create mode 100755 mk/newlog.sh create mode 100644 mk/sys.dirdeps.mk delete mode 100644 unit-tests/forloop.exp delete mode 100644 unit-tests/forloop.mk create mode 100644 unit-tests/varmod-mtime.exp create mode 100644 unit-tests/varmod-mtime.mk diff --git a/ChangeLog b/ChangeLog index 7e9d10654840..fbaf247d3848 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,27 @@ +2023-05-10 Simon J Gerraty + + * VERSION (_MAKE_VERSION): 20230510 + Merge with NetBSD make, pick up + o parse.c: don't print null filename in stack traces + o var.c: :mtime operate on each word in variable value + +2023-05-09 Simon J Gerraty + + * VERSION (_MAKE_VERSION): 20230509 + Merge with NetBSD make, pick up + o for.c: skip syntactically wrong .for loops + o var.c: allow for :gmtime=${mtime} + add :mtime[=timestamp] where timestamp is used if stat(2) + fails, if :mtime=error stat(2) failure causes error. + +2023-05-05 Simon J Gerraty + + * VERSION (_MAKE_VERSION): 20230504 + Merge with NetBSD make, pick up + o compat.c: fix compile on NetBSD 7.2 + o make.1: fix documentation of .PREFIX to match reality and POSIX + o unit-tests: improved var-scope-local + 2023-04-14 Simon J Gerraty * VERSION (_MAKE_VERSION): 20230414 diff --git a/FILES b/FILES index 642b1bae7a6b..5ec98a096339 100644 --- a/FILES +++ b/FILES @@ -392,8 +392,6 @@ unit-tests/export-variants.exp unit-tests/export-variants.mk unit-tests/export.exp unit-tests/export.mk -unit-tests/forloop.exp -unit-tests/forloop.mk unit-tests/forsubst.exp unit-tests/forsubst.mk unit-tests/gnode-submake.exp @@ -713,6 +711,8 @@ unit-tests/varmod-match-escape.exp unit-tests/varmod-match-escape.mk unit-tests/varmod-match.exp unit-tests/varmod-match.mk +unit-tests/varmod-mtime.exp +unit-tests/varmod-mtime.mk unit-tests/varmod-no-match.exp unit-tests/varmod-no-match.mk unit-tests/varmod-order-numeric.exp diff --git a/VERSION b/VERSION index 2631e0deeb1e..b49eb8cc7f26 100644 --- a/VERSION +++ b/VERSION @@ -1,2 +1,2 @@ # keep this compatible with sh and make -_MAKE_VERSION=20230414 +_MAKE_VERSION=20230510 diff --git a/bmake.1 b/bmake.1 index 1d5ee8f1f3aa..bfa0f2d74d0a 100644 --- a/bmake.1 +++ b/bmake.1 @@ -1,4 +1,4 @@ -.\" $NetBSD: make.1,v 1.361 2023/03/23 03:29:28 sjg Exp $ +.\" $NetBSD: make.1,v 1.366 2023/05/10 18:22:33 sjg Exp $ .\" .\" Copyright (c) 1990, 1993 .\" The Regents of the University of California. All rights reserved. @@ -29,7 +29,7 @@ .\" .\" from: @(#)make.1 8.4 (Berkeley) 3/19/94 .\" -.Dd March 22, 2023 +.Dd May 10, 2023 .Dt BMAKE 1 .Os .Sh NAME @@ -796,12 +796,10 @@ The list of sources for this target that were deemed out-of-date; also known as .Sq Va \&? . .It Va .PREFIX -The file prefix of the target, containing only the file portion, no suffix -or preceding directory components; also known as +The name of the target with suffix (if declared in +.Ic .SUFFIXES ) +removed; also known as .Sq Va * . -The suffix must be one of the known suffixes declared with -.Ic .SUFFIXES , -or it is not recognized. .It Va .TARGET The name of the target; also known as .Sq Va @ . @@ -1513,6 +1511,25 @@ producing the formatted timestamp. If a .Ar timestamp value is not provided or is 0, the current time is used. +.It Cm \&:mtime Ns Oo Cm = Ns Ar timestamp Oc +Call +.Xr stat 2 +with each word as pathname; +use +.Ql st_mtime +as the new value. +If +.Xr stat 2 +fails; use +.Ar timestamp +or current time. +If +.Ar timestamp +is set to +.Ql error , +then +.Xr stat 2 +failure will cause an error. .It Cm \&:tA Attempts to convert the value to an absolute path using .Xr realpath 3 . @@ -2735,5 +2752,3 @@ using that token pool to abort the build and exit with error code 6. Sometimes the attempt to suppress a cascade of unnecessary errors, can result in a seemingly unexplained .Ql *** Error code 6 - - diff --git a/bmake.cat1 b/bmake.cat1 index 9715bd908f18..6f69dfb9f3ad 100644 --- a/bmake.cat1 +++ b/bmake.cat1 @@ -515,11 +515,8 @@ BMAKE(1) FreeBSD General Commands Manual BMAKE(1) .OODATE The list of sources for this target that were deemed out- of-date; also known as `?'. - .PREFIX The file prefix of the target, containing only the file - portion, no suffix or preceding directory components; - also known as `*'. The suffix must be one of the known - suffixes declared with .SUFFIXES, or it is not recog- - nized. + .PREFIX The name of the target with suffix (if declared in + .SUFFIXES) removed; also known as `*'. .TARGET The name of the target; also known as `@'. For compati- bility with other makes this is an alias for .ARCHIVE in @@ -986,6 +983,12 @@ BMAKE(1) FreeBSD General Commands Manual BMAKE(1) localtime(3), producing the formatted timestamp. If a timestamp value is not provided or is 0, the current time is used. + :mtime[=timestamp] + Call stat(2) with each word as pathname; use `st_mtime' as the new + value. If stat(2) fails; use timestamp or current time. If + timestamp is set to `error', then stat(2) failure will cause an er- + ror. + :tA Attempts to convert the value to an absolute path using realpath(3). If that fails, the value is unchanged. @@ -1750,6 +1753,4 @@ BMAKE(1) FreeBSD General Commands Manual BMAKE(1) attempt to suppress a cascade of unnecessary errors, can result in a seemingly unexplained `*** Error code 6' - - -FreeBSD 13.0 March 22, 2023 FreeBSD 13.0 +FreeBSD 13.0 May 10, 2023 FreeBSD 13.0 diff --git a/compat.c b/compat.c index 521c3a364028..221eb64959e6 100644 --- a/compat.c +++ b/compat.c @@ -1,4 +1,4 @@ -/* $NetBSD: compat.c,v 1.246 2023/03/18 22:20:11 sjg Exp $ */ +/* $NetBSD: compat.c,v 1.247 2023/05/04 22:31:17 sjg Exp $ */ /* * Copyright (c) 1988, 1989, 1990 The Regents of the University of California. @@ -94,7 +94,7 @@ #include "pathnames.h" /* "@(#)compat.c 8.2 (Berkeley) 3/19/94" */ -MAKE_RCSID("$NetBSD: compat.c,v 1.246 2023/03/18 22:20:11 sjg Exp $"); +MAKE_RCSID("$NetBSD: compat.c,v 1.247 2023/05/04 22:31:17 sjg Exp $"); static GNode *curTarg = NULL; static pid_t compatChild; @@ -224,7 +224,7 @@ bool Compat_RunCommand(const char *cmdp, GNode *gn, StringListNode *ln) { char *cmdStart; /* Start of expanded command */ - char *bp; + char *volatile bp; bool silent; /* Don't print command */ bool doIt; /* Execute even if -n */ volatile bool errCheck; /* Check errors */ diff --git a/for.c b/for.c index 089d32efed74..7c090bab5e42 100644 --- a/for.c +++ b/for.c @@ -1,4 +1,4 @@ -/* $NetBSD: for.c,v 1.171 2023/02/14 21:38:31 rillig Exp $ */ +/* $NetBSD: for.c,v 1.174 2023/05/09 19:43:12 rillig Exp $ */ /* * Copyright (c) 1992, The Regents of the University of California. @@ -58,7 +58,7 @@ #include "make.h" /* "@(#)for.c 8.1 (Berkeley) 6/6/93" */ -MAKE_RCSID("$NetBSD: for.c,v 1.171 2023/02/14 21:38:31 rillig Exp $"); +MAKE_RCSID("$NetBSD: for.c,v 1.174 2023/05/09 19:43:12 rillig Exp $"); typedef struct ForLoop { @@ -72,6 +72,22 @@ typedef struct ForLoop { static ForLoop *accumFor; /* Loop being accumulated */ +/* See LK_FOR_BODY. */ +static void +skip_whitespace_or_line_continuation(const char **pp) +{ + const char *p = *pp; + for (;;) { + if (ch_isspace(*p)) + p++; + else if (p[0] == '\\' && p[1] == '\n') + p += 2; + else + break; + } + *pp = p; +} + static ForLoop * ForLoop_New(void) { @@ -123,6 +139,13 @@ ForLoop_Details(ForLoop *f) } static bool +IsValidInVarname(char c) +{ + return c != '$' && c != ':' && c != '\\' && + c != '(' && c != '{' && c != ')' && c != '}'; +} + +static void ForLoop_ParseVarnames(ForLoop *f, const char **pp) { const char *p = *pp; @@ -133,15 +156,20 @@ ForLoop_ParseVarnames(ForLoop *f, const char **pp) cpp_skip_whitespace(&p); if (*p == '\0') { Parse_Error(PARSE_FATAL, "missing `in' in for"); - return false; + f->vars.len = 0; + return; } - /* - * XXX: This allows arbitrary variable names; - * see directive-for.mk. - */ - for (len = 1; p[len] != '\0' && !ch_isspace(p[len]); len++) - continue; + for (len = 0; p[len] != '\0' && !ch_isspace(p[len]); len++) { + if (!IsValidInVarname(p[len])) { + Parse_Error(PARSE_FATAL, + "invalid character '%c' " + "in .for loop variable name", + p[len]); + f->vars.len = 0; + return; + } + } if (len == 2 && p[0] == 'i' && p[1] == 'n') { p += 2; @@ -154,11 +182,10 @@ ForLoop_ParseVarnames(ForLoop *f, const char **pp) if (f->vars.len == 0) { Parse_Error(PARSE_FATAL, "no iteration variables in for"); - return false; + return; } *pp = p; - return true; } static bool @@ -221,17 +248,14 @@ For_Eval(const char *line) ForLoop *f; p = line + 1; /* skip the '.' */ - cpp_skip_whitespace(&p); + skip_whitespace_or_line_continuation(&p); if (IsFor(p)) { p += 3; f = ForLoop_New(); - if (!ForLoop_ParseVarnames(f, &p)) { - ForLoop_Free(f); - return -1; - } - if (!ForLoop_ParseItems(f, p)) + ForLoop_ParseVarnames(f, &p); + if (f->vars.len > 0 && !ForLoop_ParseItems(f, p)) f->items.len = 0; /* don't iterate */ accumFor = f; @@ -254,7 +278,7 @@ For_Accum(const char *line, int *forLevel) if (*p == '.') { p++; - cpp_skip_whitespace(&p); + skip_whitespace_or_line_continuation(&p); if (IsEndfor(p)) { DEBUG1(FOR, "For: end for %d\n", *forLevel); diff --git a/make.1 b/make.1 index c9cb38313030..eff8fed491fa 100644 --- a/make.1 +++ b/make.1 @@ -1,4 +1,4 @@ -.\" $NetBSD: make.1,v 1.361 2023/03/23 03:29:28 sjg Exp $ +.\" $NetBSD: make.1,v 1.366 2023/05/10 18:22:33 sjg Exp $ .\" .\" Copyright (c) 1990, 1993 .\" The Regents of the University of California. All rights reserved. @@ -29,7 +29,7 @@ .\" .\" from: @(#)make.1 8.4 (Berkeley) 3/19/94 .\" -.Dd March 22, 2023 +.Dd May 10, 2023 .Dt MAKE 1 .Os .Sh NAME @@ -796,12 +796,10 @@ The list of sources for this target that were deemed out-of-date; also known as .Sq Va \&? . .It Va .PREFIX -The file prefix of the target, containing only the file portion, no suffix -or preceding directory components; also known as +The name of the target with suffix (if declared in +.Ic .SUFFIXES ) +removed; also known as .Sq Va * . -The suffix must be one of the known suffixes declared with -.Ic .SUFFIXES , -or it is not recognized. .It Va .TARGET The name of the target; also known as .Sq Va @ . @@ -1513,6 +1511,25 @@ producing the formatted timestamp. If a .Ar timestamp value is not provided or is 0, the current time is used. +.It Cm \&:mtime Ns Oo Cm = Ns Ar timestamp Oc +Call +.Xr stat 2 +with each word as pathname; +use +.Ql st_mtime +as the new value. +If +.Xr stat 2 +fails; use +.Ar timestamp +or current time. +If +.Ar timestamp +is set to +.Ql error , +then +.Xr stat 2 +failure will cause an error. .It Cm \&:tA Attempts to convert the value to an absolute path using .Xr realpath 3 . @@ -2730,5 +2747,3 @@ using that token pool to abort the build and exit with error code 6. Sometimes the attempt to suppress a cascade of unnecessary errors, can result in a seemingly unexplained .Ql *** Error code 6 - - diff --git a/mk/ChangeLog b/mk/ChangeLog index 2e4b6516c5cc..15c3861ea701 100644 --- a/mk/ChangeLog +++ b/mk/ChangeLog @@ -1,3 +1,38 @@ +2023-05-10 Simon J Gerraty + + * meta.autodep.mk: if LOCAL_DEPENDS_GUARD is "no" + suppress processing of .depend + +2023-05-09 Simon J Gerraty + + * dirdeps.mk: do not add _CURDIR to DIRDEPS for SRCTOP + + * meta.sys.mk sys.dirdeps.mk: + originally DIRDEPS_BUILD and META_MODE were the same thing, + but META_MODE is useful by itself. + Move things from meta.sys.mk which actually pertain to + DIRDEPS_BUILD to sys.dirdeps.mk + +2023-05-04 Simon J Gerraty + + * install-mk (MK_VERSION): 20230504 May the Forth be with you + + * dirdeps.mk: as with meta.sys.mk we treat "host" as special. + DEP_TARGET_SPEC is just ${DEP_MACHINE} + + * meta.sys.mk: ensure DEP_* for TARGET_SPEC_VARS are set at + level > 0 since these are often refered to in Makefile.depend* + +2023-04-26 Simon J Gerraty + + * jobs.mk: report ${.TARGET} ${JOB_ARGS} ${JOB_LOG} and + anything in ${JOB_LOG_START} + + * jobs.mk: look for newlog.sh in ${.SYSPATH:U${.PARSEDIR}} + or a scripts subdir before searching $PATH. + + * FILES: include newlog.sh for jobs.mk + 2023-04-20 Simon J Gerraty * install-mk (MK_VERSION): 20230420 diff --git a/mk/FILES b/mk/FILES index 45450595f390..aa84d59dd8cf 100644 --- a/mk/FILES +++ b/mk/FILES @@ -30,6 +30,7 @@ man.mk manifest.mk mk-files.txt mkopt.sh +newlog.sh nls.mk obj.mk options.mk @@ -47,6 +48,7 @@ sys.mk sys.clean-env.mk sys.debug.mk sys.dependfile.mk +sys.dirdeps.mk sys.vars.mk sys/AIX.mk sys/Darwin.mk diff --git a/mk/dirdeps-targets.mk b/mk/dirdeps-targets.mk index 6201efe1e402..821ae50e3ffa 100644 --- a/mk/dirdeps-targets.mk +++ b/mk/dirdeps-targets.mk @@ -1,5 +1,5 @@ # RCSid: -# $Id: dirdeps-targets.mk,v 1.24 2020/12/11 18:15:43 sjg Exp $ +# $Id: dirdeps-targets.mk,v 1.25 2023/05/11 05:07:28 sjg Exp $ # # @(#) Copyright (c) 2019-2020 Simon J. Gerraty # @@ -113,16 +113,17 @@ tqtdeps := ${DIRDEPS_TARGETS_MACHINE_LIST:@m@${tdeps:M*.$m,*}@:S,/${.MAKE.DEPEND .endif # now work out what we want in DIRDEPS +DIRDEPS = ${ptdeps} .if empty(REQUESTED_MACHINE) # we want them all just as found -DIRDEPS = ${ptdeps} ${mqtdeps} ${tqtdeps} +DIRDEPS += ${mqtdeps} ${tqtdeps} .else # we only want those that match REQUESTED_MACHINE/REQUESTED_TARGET_SPEC # or REQUESTED_TARGET_SPEC (TARGET_SPEC) -DIRDEPS = \ - ${ptdeps:@d@$d.${REQUESTED_TARGET_SPEC:U${TARGET_SPEC:U${REQUESTED_MACHINE}}}@} \ +DIRDEPS += \ ${mqtdeps:M*.${REQUESTED_MACHINE}} \ - ${tqtdeps:M*.${REQUESTED_TARGET_SPEC:U${TARGET_SPEC}}} + ${tqtdeps:M*.${REQUESTED_TARGET_SPEC:U${TARGET_SPEC}}} \ + .endif # clean up DIRDEPS := ${DIRDEPS:O:u} diff --git a/mk/dirdeps.mk b/mk/dirdeps.mk index 6fedd00310e9..aafa5ab47557 100644 --- a/mk/dirdeps.mk +++ b/mk/dirdeps.mk @@ -1,4 +1,4 @@ -# $Id: dirdeps.mk,v 1.157 2023/04/22 21:07:51 sjg Exp $ +# $Id: dirdeps.mk,v 1.160 2023/05/10 20:44:58 sjg Exp $ # SPDX-License-Identifier: BSD-2-Clause # @@ -273,6 +273,10 @@ _machine_dependfiles := ${.MAKE.DEPENDFILE_PREFERENCE:T:M*${MACHINE}*} .endif .endif +# turn a list into a set of :N modifiers +# NskipFoo = ${Foo:${M_ListToSkip}} +M_ListToSkip ?= O:u:S,^,N,:ts: + # this is how we identify non-machine specific dependfiles N_notmachine := ${.MAKE.DEPENDFILE_PREFERENCE:E:N*${MACHINE}*:${M_ListToSkip}} @@ -333,6 +337,14 @@ DEP_${TARGET_SPEC_VARS:[$i]} := ${_tspec:[$i]} DEP_MACHINE := ${_DEP_TARGET_SPEC} .endif +# host is special +.if ${DEP_MACHINE:Mhost*} != "" +DEP_TARGET_SPEC = ${DEP_MACHINE} +.for v in ${TARGET_SPEC_VARS:O:u:NMACHINE} +.undef DEP_$v +.endfor +.endif + # reset each time through _build_all_dirs = _build_xtra_dirs = @@ -653,7 +665,7 @@ _machines := ${_machines:${M_dep_qual_fixes:ts:}:O:u} # reset each time through _build_dirs = -.if ${DEP_RELDIR} == ${_DEP_RELDIR} +.if ${DEP_RELDIR} == ${_DEP_RELDIR} && ${_CURDIR} != ${SRCTOP} # pickup other machines for this dir if necessary _build_dirs += ${_machines:@m@${_CURDIR}.$m@} .endif diff --git a/mk/install-mk b/mk/install-mk index 89c222cd0ee3..e198d52a8702 100644 --- a/mk/install-mk +++ b/mk/install-mk @@ -59,7 +59,7 @@ # Simon J. Gerraty # RCSid: -# $Id: install-mk,v 1.231 2023/04/20 17:45:03 sjg Exp $ +# $Id: install-mk,v 1.234 2023/05/13 15:52:24 sjg Exp $ # # @(#) Copyright (c) 1994-2023 Simon J. Gerraty # @@ -74,7 +74,7 @@ # sjg@crufty.net # -MK_VERSION=20230420 +MK_VERSION=20230510 OWNER= GROUP= MODE=444 diff --git a/mk/jobs.mk b/mk/jobs.mk index f465ea06310b..62fe9eeae030 100644 --- a/mk/jobs.mk +++ b/mk/jobs.mk @@ -1,4 +1,4 @@ -# $Id: jobs.mk,v 1.7 2023/04/18 23:32:28 sjg Exp $ +# $Id: jobs.mk,v 1.9 2023/04/27 18:10:27 sjg Exp $ # # @(#) Copyright (c) 2012-2023, Simon J. Gerraty # @@ -38,21 +38,30 @@ now_utc ?= ${%s:L:gmtime} start_utc := ${now_utc} .endif -.info ${.newline}${TIME_STAMP} Start ${.TARGETS} - .if make(*-jobs) +.info ${.newline}${TIME_STAMP} Start ${.TARGETS} JOB_LOGDIR ?= ${SRCTOP:H} JOB_LOG = ${JOB_LOGDIR}/${.TARGET:S,-jobs,,:S,/,_,g}.log JOB_LOG_GENS ?= 4 # we like to rotate logs .if empty(NEWLOG_SH) +.for d in ${.SYSPATH:U${.PARSEDIR}:@x@$x $x/scripts@} +.if exists($d/newlog.sh) +NEWLOG_SH := $d/newlog.sh +.if ${MAKE_VERSION} > 20220924 +.break +.endif +.endif +.endfor +.if empty(NEWLOG_SH) .ifdef M_whence NEWLOG_SH := ${newlog.sh:L:${M_whence}} .else NEWLOG_SH := ${(type newlog.sh) 2> /dev/null:L:sh:M/*} .endif .endif +.endif .if !empty(NEWLOG_SH) && exists(${NEWLOG_SH}) NEWLOG := sh ${NEWLOG_SH} JOB_NEWLOG_ARGS ?= -S -n ${JOB_LOG_GENS} @@ -72,7 +81,7 @@ JOB_ARGS+= -j${JOB_MAX} # build orchestration works as expected (DIRDEPS_BUILD) ${.TARGETS:M*-jobs}: @${NEWLOG} ${JOB_NEWLOG_ARGS} ${JOB_LOG} - @echo Logging to ${JOB_LOG} + @echo "${TIME_STAMP} Start ${.TARGET:S,-jobs,,} ${JOB_ARGS} ${JOB_LOG_START} log=${JOB_LOG}" | tee ${JOB_LOG} @cd ${.CURDIR} && env MAKELEVEL=0 \ ${.MAKE} ${JOB_ARGS} _TARGETS=${.TARGET:S,-jobs,,} ${.TARGET:S,-jobs,,} >> ${JOB_LOG} 2>&1 diff --git a/mk/meta.autodep.mk b/mk/meta.autodep.mk index cd08ac3b3520..5f012cdec003 100644 --- a/mk/meta.autodep.mk +++ b/mk/meta.autodep.mk @@ -1,4 +1,4 @@ -# $Id: meta.autodep.mk,v 1.56 2022/09/09 17:44:29 sjg Exp $ +# $Id: meta.autodep.mk,v 1.57 2023/05/13 15:52:24 sjg Exp $ # # @(#) Copyright (c) 2010, Simon J. Gerraty @@ -139,6 +139,10 @@ FORCE_DPADD += ${_nonlibs:@x@${DPADD:M*/$x}@} .END: gendirdeps .endif +.if ${LOCAL_DEPENDS_GUARD:U} == "no" +.depend: +.endif + # if we don't have OBJS, then .depend isn't useful .if !target(.depend) && (!empty(OBJS) || ${.ALLTARGETS:M*.o} != "") # some makefiles and/or targets contain diff --git a/mk/meta.sys.mk b/mk/meta.sys.mk index 1fc58a226cb1..ba213dd49da4 100644 --- a/mk/meta.sys.mk +++ b/mk/meta.sys.mk @@ -1,7 +1,7 @@ -# $Id: meta.sys.mk,v 1.46 2023/04/18 18:43:00 sjg Exp $ +# $Id: meta.sys.mk,v 1.51 2023/05/11 20:05:32 sjg Exp $ # -# @(#) Copyright (c) 2010-2021, Simon J. Gerraty +# @(#) Copyright (c) 2010-2023, Simon J. Gerraty # # This file is provided in the hope that it will # be of use. There is absolutely NO WARRANTY. @@ -17,56 +17,21 @@ # include this if you want to enable meta mode # for maximum benefit, requires filemon(4) driver. -.if ${MAKE_VERSION:U0} > 20100901 -.if !target(.ERROR) +# absolute path to what we are reading. +_PARSEDIR ?= ${.PARSEDIR:tA} .-include -# If TARGET_SPEC_VARS is other than just MACHINE -# it should be set by now. -# TARGET_SPEC must not contain any '.'s. -TARGET_SPEC_VARS ?= MACHINE - -.if !target(_meta_tspec_env_done_) -_meta_tspec_env_done_: .NOTMAIN -# Allow for local.meta.sys.env.mk to have done this - -.if ${TARGET_SPEC:Uno:M*,*} != "" -# deal with TARGET_SPEC from env -_tspec := ${TARGET_SPEC:S/,/ /g} -.for i in ${TARGET_SPEC_VARS:${M_RANGE:Urange}} -${TARGET_SPEC_VARS:[$i]} := ${_tspec:[$i]} -.endfor -# We need to stop that TARGET_SPEC affecting any submakes -TARGET_SPEC= -# so export but do not track -.export-env TARGET_SPEC -.export ${TARGET_SPEC_VARS} -.for v in ${TARGET_SPEC_VARS:O:u} -.if empty($v) -.undef $v -.endif -.endfor -.endif -.endif - -# Now make sure we know what TARGET_SPEC is -# as we may need it to find Makefile.depend* -.if ${MACHINE:Mhost*} != "" -# host is special -TARGET_SPEC = ${MACHINE} -.else -TARGET_SPEC = ${TARGET_SPEC_VARS:@v@${$v:U}@:ts,} -.endif - -# absolute path to what we are reading. -_PARSEDIR = ${.PARSEDIR:tA} - .if !defined(SYS_MK_DIR) SYS_MK_DIR := ${_PARSEDIR} .endif -META_MODE += meta verbose +.if !target(.ERROR) + +META_MODE += meta +.if empty(.MAKEFLAGS:M-s) +META_MODE += verbose +.endif .if ${MAKE_VERSION:U0} > 20130323 && empty(.MAKE.PATH_FILEMON) # we do not support filemon META_MODE += nofilemon @@ -102,19 +67,7 @@ META_MODE += silent=yes .endif .endif -# we use the pseudo machine "host" for the build host. -# this should be taken care of before we get here -.if ${OBJTOP:Ua} == ${HOST_OBJTOP:Ub} -MACHINE = host -.endif - -.if !defined(MACHINE0) -# it can be handy to know which MACHINE kicked off the build -# for example, if using Makefild.depend for multiple machines, -# allowing only MACHINE0 to update can keep things simple. -MACHINE0 := ${MACHINE} -.export MACHINE0 -.endif +.if ${MK_DIRDEPS_BUILD:Uno} == "yes" .if !defined(META2DEPS) .if defined(PYTHON) && exists(${PYTHON}) @@ -134,6 +87,11 @@ MAKE_PRINT_VAR_ON_ERROR += \ MAKEFILE \ .MAKE.MODE +MK_META_ERROR_TARGET = yes +.endif + +.if ${MK_META_ERROR_TARGET:Uno} == "yes" + .if !defined(SB) && defined(SRCTOP) SB = ${SRCTOP:H} .endif @@ -150,21 +108,12 @@ _metaError: .NOMETA .NOTMAIN echo "ERROR: log ${meta_error_log}" >&2; }; : .endif +.endif # Are we, after all, in meta mode? .if ${.MAKE.MODE:Uno:Mmeta*} != "" MKDEP_MK ?= meta.autodep.mk -.if ${.MAKE.MAKEFILES:M*sys.dependfile.mk} == "" -# this does all the smarts of setting .MAKE.DEPENDFILE -.-include -# check if we got anything sane -.if ${.MAKE.DEPENDFILE} == ".depend" -.undef .MAKE.DEPENDFILE -.endif -.MAKE.DEPENDFILE ?= Makefile.depend -.endif - # we can afford to use cookies to prevent some targets # re-running needlessly META_COOKIE_TOUCH?= touch ${COOKIE.${.TARGET}:U${.OBJDIR}/${.TARGET:T}} @@ -192,27 +141,13 @@ UPDATE_DEPENDFILE= NO .endif .endif -.if ${.MAKE.LEVEL} == 0 -.if ${MK_DIRDEPS_BUILD:Uyes} == "yes" -# make sure dirdeps target exists and do it first -all: dirdeps .WAIT -dirdeps: -.NOPATH: dirdeps +.else # in meta mode? -.if defined(ALL_MACHINES) -# the first .MAIN: is what counts -# by default dirdeps is all we want at level0 -.MAIN: dirdeps -.endif -.endif - -.endif -.else META_COOKIE_TOUCH= # some targets need to be .PHONY in non-meta mode META_NOPHONY= .PHONY META_NOECHO= echo -.endif -.endif + +.endif # in meta mode? .-include diff --git a/mk/mk-files.txt b/mk/mk-files.txt index 337df19613e0..0afcea189470 100644 --- a/mk/mk-files.txt +++ b/mk/mk-files.txt @@ -441,6 +441,8 @@ Leverages ``bmake`` to compute optimal link order for libraries. This works nicely and makes refactoring a breeze - so long as you have no (or few) cicular dependencies between libraries. +Consider this experimental. + man.mk ------ @@ -509,17 +511,58 @@ then ``jobs.mk`` will run:: this ensures you get a build log and JOB_MAX is assumed to be set optimally for the host. -Meta mode +META_MODE ========= The 20110505 and later versions of ``mk-files`` include a number of makefiles contributed by Juniper Networks, Inc. These allow the latest version of bmake_ to run in `meta mode`_ -see `dirdeps.mk`_ +see `dirdeps.mk`_ and DIRDEPS_BUILD_ below. .. _`dirdeps.mk`: /help/sjg/dirdeps.htm .. _`meta mode`: bmake-meta-mode.htm +DIRDEPS_BUILD +============= + +When the `meta mode`_ was originally done, there was no distinction +between META_MODE_ and ``DIRDEPS_BUILD``, but as these were integrated +into FreeBSD it became clear that META_MODE_ could be useful to many +developers independently of ``DIRDEPS_BUILD``. + +Thus today we distinguish between the two. +We have the following makefiles which are relevant to +``DIRDEPS_BUILD`` or META_MODE_:: + + share/mk/auto.obj.mk + share/mk/dirdeps-cache-update.mk + share/mk/dirdeps-only.mk + share/mk/dirdeps-options.mk + share/mk/dirdeps-targets.mk + share/mk/dirdeps.mk + share/mk/gendirdeps.mk + share/mk/host-target.mk + share/mk/install-new.mk + share/mk/meta.autodep.mk + share/mk/meta.stage.mk + share/mk/meta.sys.mk + share/mk/meta2deps.py + share/mk/meta2deps.sh + share/mk/sys.dependfile.mk + share/mk/sys.dirdeps.mk + +and the following are typically used for customization. +See `freebsd-meta-mode`_ and `netbsd-meta-mode`_:: + + share/mk/local.dirdeps-build.mk + share/mk/local.dirdeps-missing.mk + share/mk/local.dirdeps.mk + share/mk/local.meta.sys.mk + share/mk/local.sys.dirdeps.env.mk + share/mk/local.sys.dirdeps.mk + share/mk/local.sys.mk + + Install ======= @@ -538,9 +581,11 @@ where you unpacked the tar file, you can:: .. _bmake: bmake.htm .. _NetBSD: http://www.netbsd.org/ -.. _mkdeps.sh: http://www.crufty.net/ftp/pub/sjg/mkdeps.sh -.. _mk.tar.gz: http://www.crufty.net/ftp/pub/sjg/mk.tar.gz +.. _mkdeps.sh: https://www.crufty.net/ftp/pub/sjg/mkdeps.sh +.. _mk.tar.gz: https://www.crufty.net/ftp/pub/sjg/mk.tar.gz +.. _`freebsd-meta-mode`: https://www.crufty.net/sjg/docs/freebsd-meta-mode.htm +.. _`netbsd-meta-mode`: https://www.crufty.net/sjg/docs/netbsd-meta-mode.htm :Author: sjg@crufty.net -:Revision: $Id: mk-files.txt,v 1.22 2023/04/16 23:43:33 sjg Exp $ +:Revision: $Id: mk-files.txt,v 1.23 2023/05/11 22:55:08 sjg Exp $ :Copyright: Crufty.NET diff --git a/mk/newlog.sh b/mk/newlog.sh new file mode 100755 index 000000000000..526d1700d98f --- /dev/null +++ b/mk/newlog.sh @@ -0,0 +1,412 @@ +#!/bin/sh + +# NAME: +# newlog - rotate log files +# +# SYNOPSIS: +# newlog.sh [options] "log"[:"num"] ... +# +# DESCRIPTION: +# This script saves multiple generations of each "log". +# The "logs" are kept compressed except for the current and +# previous ones. +# +# Options: +# +# -C "compress" +# Compact old logs (other than .0) with "compress" +# (default is 'gzip' or 'compress' if no 'gzip'). +# +# -E "ext" +# If "compress" produces a file extention other than +# '.Z' or '.gz' we need to know. +# +# -G "gens" +# "gens" is a comma separated list of "log":"num" pairs +# that allows certain logs to handled differently. +# +# -N Don't actually do anything, just show us. +# +# -R Rotate rather than save logs by default. +# This is the default anyway. +# +# -S Save rather than rotate logs by default. +# Each log is saved to a unique name that remains +# unchanged. This results in far less churn. +# +# -f "fmt" +# Format ('%Y%m%d.%H%M%S') for suffix added to "log" to +# uniquely name it when using the '-S' option. +# If a "log" is saved more than once per second we add +# an extra suffix of our process-id. +# +# -d The "log" to be rotated/saved is a directory. +# We leave the mode of old directories alone. +# +# -e Normally logs are only cycled if non-empty, this +# option forces empty logs to be cycled as well. +# +# -g "group" +# Set the group of "log" to "group". +# +# -m "mode" +# Set the mode of "log". +# +# -M "mode" +# Set the mode of old logs (default 444). +# +# -n "num" +# Keep "num" generations of "log". +# +# -o "owner" +# Set the owner of "log". +# +# Regardless of whether '-R' or '-S' is provided, we attempt to +# choose the correct behavior based on observation of "log.0" if +# it exists; if it is a symbolic link, we save, otherwise +# we rotate. +# +# BUGS: +# 'Newlog.sh' tries to avoid being fooled by symbolic links, but +# multiply indirect symlinks are only handled on machines where +# test(1) supports a check for symlinks. +# +# AUTHOR: +# Simon J. Gerraty +# + +# RCSid: +# $Id: newlog.sh,v 1.26 2021/04/30 16:29:02 sjg Exp $ +# +# @(#) Copyright (c) 1993-2016 Simon J. Gerraty +# +# This file is provided in the hope that it will +# be of use. There is absolutely NO WARRANTY. +# Permission to copy, redistribute or otherwise +# use this file is hereby granted provided that +# the above copyright notice and this notice are +# left intact. +# +# Please send copies of changes and bug-fixes to: +# sjg@crufty.net +# + +Mydir=`dirname $0` +case $Mydir in +/*) ;; +*) Mydir=`cd $Mydir; pwd`;; +esac + +# places to find chown (and setopts.sh) +PATH=$PATH:/usr/etc:/sbin:/usr/sbin:/usr/local/share/bin:/share/bin:$Mydir + +# linux doesn't necessarily have compress, +# and gzip appears in various locations... +Which() { + case "$1" in + -*) t=$1; shift;; + *) t=-x;; + esac + case "$1" in + /*) test $t $1 && echo $1;; + *) + for d in `IFS=:; echo ${2:-$PATH}` + do + test $t $d/$1 && { echo $d/$1; break; } + done + ;; + esac +} + +# shell's typically have test(1) as built-in +# and not all support all options. +test_opt() { + _o=$1 + _a=$2 + _t=${3:-/} + + case `test -$_o $_t 2>&1` in + *:*) eval test_$_o=$_a;; + *) eval test_$_o=-$_o;; + esac +} + +# convert find/ls mode to octal +fmode() { + eval `echo $1 | + sed 's,\(.\)\(...\)\(...\)\(...\),ft=\1 um=\2 gm=\3 om=\4,'` + sm= + case "$um" in + *s*) sm=r + um=`echo $um | sed 's,s,x,'` + ;; + *) sm=-;; + esac + case "$gm" in + *[Ss]*) + sm=${sm}w + gm=`echo $gm | sed 's,s,x,;s,S,-,'` + ;; + *) sm=${sm}-;; + esac + case "$om" in + *t) + sm=${sm}x + om=`echo $om | sed 's,t,x,'` + ;; + *) sm=${sm}-;; + esac + echo $sm $um $gm $om | + sed 's,rwx,7,g;s,rw-,6,g;s,r-x,5,g;s,r--,4,g;s,-wx,3,g;s,-w-,2,g;s,--x,1,g;s,---,0,g;s, ,,g' +} + +get_mode() { + case "$OS,$STAT" in + FreeBSD,*) + $STAT -f %Op $1 | sed 's,.*\(....\),\1,' + return + ;; + esac + # fallback to find + fmode `find $1 -ls -prune | awk '{ print $3 }'` +} + +get_mtime_suffix() { + case "$OS,$STAT" in + FreeBSD,*) + $STAT -t "${2:-$opt_f}" -f %Sm $1 + return + ;; + esac + # this will have to do + date "+${2:-$opt_f}" +} + +case /$0 in +*/newlog*) rotate_func=rotate_log;; +*/save*) rotate_func=save_log;; +*) rotate_func=rotate_log;; +esac + +opt_n=7 +opt_m= +opt_M=444 +opt_f=%Y%m%d.%H%M%S +opt_str=dNn:o:g:G:C:M:m:eE:f:RS + +. setopts.sh + +test $# -gt 0 || exit 0 # nothing to do. + +OS=${OS:-`uname`} +STAT=${STAT:-`Which stat`} + +# sorry, setops semantics for booleans changed. +case "${opt_d:-0}" in +0) rm_f=-f + opt_d=-f + for x in $opt_C gzip compress + do + opt_C=`Which $x "/bin:/usr/bin:$PATH"` + test -x $opt_C && break + done + empty() { test ! -s $1; } + ;; +*) rm_f=-rf + opt_d=-d + opt_M= + opt_C=: + empty() { + if [ -d $1 ]; then + n=`'ls' -a1 $1/. | wc -l` + [ $n -gt 2 ] && return 1 + fi + return 0 + } + ;; +esac +case "${opt_N:-0}" in +0) ECHO=;; +*) ECHO=echo;; +esac +case "${opt_e:-0}" in +0) force=;; +*) force=yes;; +esac +case "${opt_R:-0}" in +0) ;; +*) rotate_func=rotate_log;; +esac +case "${opt_S:-0}" in +0) ;; +*) rotate_func=save_log opt_S=;; +esac + +# see whether test handles -h or -L +test_opt L -h +test_opt h "" +case "$test_L,$test_h" in +-h,) test_L= ;; # we don't support either! +esac + +case "$test_L" in +"") # No, so this is about all we can do... + logs=`'ls' -ld $* | awk '{ print $NF }'` + ;; +*) # it does + logs="$*" + ;; +esac + +read_link() { + case "$test_L" in + "") 'ls' -ld $1 | awk '{ print $NF }'; return;; + esac + if test $test_L $1; then + 'ls' -ld $1 | sed 's,.*> ,,' + else + echo $1 + fi +} + +# create the new log +new_log() { + log=$1 + mode=$2 + if test "x$opt_M" != x; then + $ECHO chmod $opt_M $log.0 2> /dev/null + fi + # someone may have managed to write to it already + # so don't truncate it. + case "$opt_d" in + -d) $ECHO mkdir -p $log;; + *) $ECHO touch $log;; + esac + # the order here matters + test "x$opt_o" = x || $ECHO chown $opt_o $log + test "x$opt_g" = x || $ECHO chgrp $opt_g $log + test "x$mode" = x || $ECHO chmod $mode $log +} + +rotate_log() { + log=$1 + n=${2:-$opt_n} + + # make sure excess generations are trimmed + $ECHO rm $rm_f `echo $log.$n | sed 's/\([0-9]\)$/[\1-9]*/'` + + mode=${opt_m:-`get_mode $log`} + while test $n -gt 0 + do + p=`expr $n - 1` + if test -s $log.$p; then + $ECHO rm $rm_f $log.$p.* + $ECHO $opt_C $log.$p + if test "x$opt_M" != x; then + $ECHO chmod $opt_M $log.$p.* 2> /dev/null + fi + fi + for ext in $opt_E .gz .Z "" + do + test $opt_d $log.$p$ext || continue + $ECHO mv $log.$p$ext $log.$n$ext + done + n=$p + done + # leave $log.0 uncompressed incase some one still has it open. + $ECHO mv $log $log.0 + new_log $log $mode +} + +# unlike rotate_log we do not rotate files, +# but give each log a unique (but stable name). +# This avoids churn for folk who rsync things. +# We make log.0 a symlink to the most recent log +# so it can be found and compressed next time around. +save_log() { + log=$1 + n=${2:-$opt_n} + fmt=$3 + + last=`read_link $log.0` + case "$last" in + $log.0) # should never happen + test -s $last && $ECHO mv $last $log.$$;; + $log.*) + $ECHO $opt_C $last + ;; + *.*) $ECHO $opt_C `dirname $log`/$last + ;; + esac + $ECHO rm -f $log.0 + # remove excess logs - we rely on mtime! + $ECHO rm $rm_f `'ls' -1td $log.* 2> /dev/null | sed "1,${n}d"` + + mode=${opt_m:-`get_mode $log`} + # this is our default suffix + opt_S=${opt_S:-`get_mtime_suffix $log $fmt`} + case "$fmt" in + ""|$opt_f) suffix=$opt_S;; + *) suffix=`get_mtime_suffix $log $fmt`;; + esac + + # find a unique name to save current log as + for nlog in $log.$suffix $log.$suffix.$$ + do + for f in $nlog* + do + break + done + test $opt_d $f || break + done + # leave $log.0 uncompressed incase some one still has it open. + $ECHO mv $log $nlog + test "x$opt_M" = x || $ECHO chmod $opt_M $nlog 2> /dev/null + $ECHO ln -s `basename $nlog` $log.0 + new_log $log $mode +} + +for f in $logs +do + n=$opt_n + save= + case "$f" in + *:[1-9]*) + set -- `IFS=:; echo $f`; f=$1; n=$2;; + *:n=*|*:save=*) + eval `echo "f=$f" | tr ':' ' '`;; + esac + # try and pick the right function to use + rfunc=$rotate_func # default + if test $opt_d $f.0; then + case `read_link $f.0` in + $f.0) rfunc=rotate_log;; + *) rfunc=save_log;; + esac + fi + case "$test_L" in + -?) + while test $test_L $f # it is [still] a symlink + do + f=`read_link $f` + done + ;; + esac + case ",${opt_G}," in + *,${f}:n=*|,${f}:save=*) + eval `echo ",${opt_G}," | sed "s!.*,${f}:\([^,]*\),.*!\1!;s,:, ,g"` + ;; + *,${f}:*) + # opt_G is a , separated list of log:n pairs + n=`echo ,$opt_G, | sed -e "s,.*${f}:\([0-9][0-9]*\).*,\1,"` + ;; + esac + + if empty $f; then + test "$force" || continue + fi + + test "$save" && rfunc=save_log + + $rfunc $f $n $save +done diff --git a/mk/sys.dependfile.mk b/mk/sys.dependfile.mk index 7c1fd94d3eb8..49232646d569 100644 --- a/mk/sys.dependfile.mk +++ b/mk/sys.dependfile.mk @@ -1,6 +1,6 @@ -# $Id: sys.dependfile.mk,v 1.9 2020/08/19 17:51:53 sjg Exp $ +# $Id: sys.dependfile.mk,v 1.10 2023/05/10 19:23:26 sjg Exp $ # -# @(#) Copyright (c) 2012, Simon J. Gerraty +# @(#) Copyright (c) 2012-2023, Simon J. Gerraty # # This file is provided in the hope that it will # be of use. There is absolutely NO WARRANTY. @@ -13,7 +13,10 @@ # sjg@crufty.net # -# This only makes sense in meta mode. +.if !target(__${.PARSEFILE}__) +__${.PARSEFILE}__: .NOTMAIN + +# This only makes sense for DIRDEPS_BUILD. # This allows a mixture of auto generated as well as manually edited # dependency files, which can be differentiated by their names. # As per dirdeps.mk we only require: @@ -57,3 +60,5 @@ MACHINE := ${_m} .endif .endif .MAKE.DEPENDFILE ?= ${.MAKE.DEPENDFILE_DEFAULT} + +.endif diff --git a/mk/sys.dirdeps.mk b/mk/sys.dirdeps.mk new file mode 100644 index 000000000000..845eda181b90 --- /dev/null +++ b/mk/sys.dirdeps.mk @@ -0,0 +1,183 @@ +# $Id: sys.dirdeps.mk,v 1.9 2023/05/11 20:05:42 sjg Exp $ +# +# @(#) Copyright (c) 2012-2023, Simon J. Gerraty +# +# This file is provided in the hope that it will +# be of use. There is absolutely NO WARRANTY. +# Permission to copy, redistribute or otherwise +# use this file is hereby granted provided that +# the above copyright notice and this notice are +# left intact. +# +# Please send copies of changes and bug-fixes to: +# sjg@crufty.net +# + +# Originally DIRDEPS_BUILD and META_MODE were the same thing. +# So, much of this was done in *meta.sys.mk and local*mk +# but properly belongs here. + +# Include from [local.]sys.mk - if doing DIRDEPS_BUILD +# we should not be here otherwise +MK_DIRDEPS_BUILD ?= yes +# these are all implied +MK_AUTO_OBJ ?= yes +MK_META_MODE ?= yes +MK_STAGING ?= yes + +_PARSEDIR ?= ${.PARSEDIR:tA} + +.-include + +.if ${.MAKE.LEVEL} == 0 +# make sure dirdeps target exists and do it first +dirdeps: +# first .MAIN is what counts +.MAIN: dirdeps +.NOPATH: dirdeps +all: dirdeps .WAIT +.endif + +.if empty(SRCTOP) +# fallback assumes share/mk! +SRCTOP := ${SB_SRC:U${.PARSEDIR:tA:H:H}} +.export SRCTOP +.endif + +# fake SB if not using mk wrapper +.if !defined(SB) +SB := ${SRCTOP:H} +.export SB +.endif + +.if empty(OBJROOT) +OBJROOT := ${SB_OBJROOT:U${MAKEOBJDIRPREFIX:U${SB}/obj}/} +.export OBJROOT +.endif + +.if empty(STAGE_ROOT) +STAGE_ROOT ?= ${OBJROOT}stage +.export STAGE_ROOT +.endif + +# We should be included before meta.sys.mk +# If TARGET_SPEC_VARS is other than just MACHINE +# it should be set by now. +# TARGET_SPEC must not contain any '.'s. +TARGET_SPEC_VARS ?= MACHINE + +.if !target(_tspec_env_done_) +_tspec_env_done_: .NOTMAIN + +.if ${TARGET_SPEC:Uno:M*,*} != "" +# deal with TARGET_SPEC from env +_tspec := ${TARGET_SPEC:S/,/ /g} +.for i in ${TARGET_SPEC_VARS:${M_RANGE:Urange}} +${TARGET_SPEC_VARS:[$i]} := ${_tspec:[$i]} +.endfor +# We need to stop that TARGET_SPEC affecting any submakes +TARGET_SPEC= +# so export but do not track +.export-env TARGET_SPEC +.export ${TARGET_SPEC_VARS} +.for v in ${TARGET_SPEC_VARS:O:u} +.if empty($v) +.undef $v +.endif +.endfor +.endif +.endif + +# Now make sure we know what TARGET_SPEC is +# as we may need it to find Makefile.depend* +.if ${MACHINE:Mhost*} != "" +# host is special +TARGET_SPEC = ${MACHINE} +.else +TARGET_SPEC = ${TARGET_SPEC_VARS:@v@${$v:U}@:ts,} +.endif + +.if ${TARGET_SPEC_VARS:[#]} > 1 +TARGET_OBJ_SPEC ?= ${TARGET_SPEC_VARS:@v@${$v:U}@:ts.} +.else +TARGET_OBJ_SPEC ?= ${MACHINE} +.endif + +MAKE_PRINT_VAR_ON_ERROR += ${TARGET_SPEC_VARS} + +.if !defined(MACHINE0) +# it can be handy to know which MACHINE kicked off the build +# for example, if using Makefild.depend for multiple machines, +# allowing only MACHINE0 to update can keep things simple. +MACHINE0 := ${MACHINE} +.export MACHINE0 +.endif + +.if ${MACHINE} == "host" +OBJTOP = ${HOST_OBJTOP} +.elif ${MACHINE} == "host32" +OBJTOP = ${HOST_OBJTOP32} +.endif + +MACHINE_OBJ.host = ${HOST_TARGET} +MACHINE_OBJ.host32 = ${HOST_TARGET32} +MACHINE_OBJ.${MACHINE} ?= ${TARGET_OBJ_SPEC} +MACHINE_OBJDIR = ${MACHINE_OBJ.${MACHINE}} +OBJTOP = ${OBJROOT}/${MACHINE_OBJDIR} + +# we do not use MAKEOBJDIRPREFIX +.undef MAKEOBJDIRPREFIX +# we use this +MAKEOBJDIR ?= ${.CURDIR:S,${SRCTOP},${OBJTOP},} + +STAGE_MACHINE ?= ${MACHINE_OBJDIR} +STAGE_OBJTOP ?= ${STAGE_ROOT}/${STAGE_MACHINE} +STAGE_COMMON_OBJTOP ?= ${STAGE_ROOT}/common +STAGE_HOST_OBJTOP ?= ${STAGE_ROOT}/${HOST_TARGET} +STAGE_HOST_OBJTOP32 ?= ${STAGE_ROOT}/${HOST_TARGET32} + +STAGE_INCLUDEDIR ?= ${STAGE_OBJTOP}${INCLUDEDIR:U/usr/include} +STAGE_LIBDIR ?= ${STAGE_OBJTOP}${LIBDIR:U/lib} + +TIME_STAMP_FMT ?= @ %s [%Y-%m-%d %T] ${:U} +DATE_TIME_STAMP ?= `date '+${TIME_STAMP_FMT}'` +TIME_STAMP ?= ${TIME_STAMP_FMT:localtime} + +.if ${MK_TIME_STAMPS:Uyes} == "yes" +TRACER = ${TIME_STAMP} +ECHO_DIR = echo ${TIME_STAMP} +ECHO_TRACE = echo ${TIME_STAMP} +.endif + +.if ${.CURDIR} == ${SRCTOP} +RELDIR= . +RELTOP= . +.elif ${.CURDIR:M${SRCTOP}/*} +RELDIR:= ${.CURDIR:S,${SRCTOP}/,,} +.else +RELDIR:= ${.OBJDIR:S,${OBJTOP}/,,} +.endif +RELTOP?= ${RELDIR:C,[^/]+,..,g} +RELOBJTOP?= ${RELTOP} +RELSRCTOP?= ${RELTOP} + +# this does all the smarts of setting .MAKE.DEPENDFILE +.-include + +.-include + +# check if we got anything sane +.if ${.MAKE.DEPENDFILE} == ".depend" +.undef .MAKE.DEPENDFILE +.endif +# just in case +.MAKE.DEPENDFILE ?= Makefile.depend + +.if ${.MAKE.LEVEL} > 0 +# Makefile.depend* also get read at level 1+ +# and often refer to DEP_MACHINE etc, +# so ensure DEP_* (for TARGET_SPEC_VARS anyway) are set +.for V in ${TARGET_SPEC_VARS} +DEP_$V = ${$V} +.endfor +.endif diff --git a/mk/sys.mk b/mk/sys.mk index 966409870cc5..b08f6010746d 100644 --- a/mk/sys.mk +++ b/mk/sys.mk @@ -1,4 +1,4 @@ -# $Id: sys.mk,v 1.54 2022/09/09 17:44:29 sjg Exp $ +# $Id: sys.mk,v 1.55 2023/05/10 19:23:26 sjg Exp $ # # @(#) Copyright (c) 2003-2009, Simon J. Gerraty # @@ -85,6 +85,9 @@ OPTIONS_DEFAULT_DEPENDENT += \ .-include # :Uno incase options.mk not installed +.if ${MK_DIRDEPS_BUILD:Uno} == "yes" +.-include +.endif .if ${MK_META_MODE:Uno} == "yes" .-include .MAKE.MODE ?= meta verbose {META_MODE} diff --git a/parse.c b/parse.c index 9d8b840c4418..f0644ea67a14 100644 --- a/parse.c +++ b/parse.c @@ -1,4 +1,4 @@ -/* $NetBSD: parse.c,v 1.696 2023/02/15 06:52:58 rillig Exp $ */ +/* $NetBSD: parse.c,v 1.698 2023/05/10 16:10:02 rillig Exp $ */ /* * Copyright (c) 1988, 1989, 1990, 1993 @@ -91,7 +91,7 @@ * Parse_Error Report a parse error, a warning or an informational * message. * - * Parse_MainName Returns a list of the single main target to create. + * Parse_MainName Populate the list of targets to create. */ #include @@ -121,7 +121,7 @@ #include "pathnames.h" /* "@(#)parse.c 8.3 (Berkeley) 3/19/94" */ -MAKE_RCSID("$NetBSD: parse.c,v 1.696 2023/02/15 06:52:58 rillig Exp $"); +MAKE_RCSID("$NetBSD: parse.c,v 1.698 2023/05/10 16:10:02 rillig Exp $"); /* * A file being read. @@ -257,10 +257,7 @@ SearchPath *defSysIncPath; /* default for sysIncPath */ /* * The parseKeywords table is searched using binary search when deciding - * if a target or source is special. The 'spec' field is the ParseSpecial - * type of the keyword (SP_NOT if the keyword isn't special as a target) while - * the 'op' field is the operator to apply to the list of targets if the - * keyword is used as a source ("0" if the keyword isn't special as a source) + * if a target or source is special. */ static const struct { const char name[17]; @@ -325,7 +322,7 @@ GetInclude(size_t i) return Vector_Get(&includes, i); } -/* The file that is currently being read. */ +/* The makefile that is currently being read. */ static IncludedFile * CurFile(void) { @@ -403,8 +400,11 @@ PrintStackTrace(bool includingInnermost) const char *fname = entry->name.str; char dirbuf[MAXPATHLEN + 1]; - if (fname[0] != '/' && strcmp(fname, "(stdin)") != 0) - fname = realpath(fname, dirbuf); + if (fname[0] != '/' && strcmp(fname, "(stdin)") != 0) { + const char *realPath = realpath(fname, dirbuf); + if (realPath != NULL) + fname = realPath; + } if (entry->forLoop != NULL) { char *details = ForLoop_Details(entry->forLoop); @@ -658,7 +658,7 @@ TryApplyDependencyOperator(GNode *gn, GNodeType op) if (op == OP_DOUBLEDEP && (gn->type & OP_OPMASK) == OP_DOUBLEDEP) { /* - * If the node was of the left-hand side of a '::' operator, + * If the node was on the left-hand side of a '::' operator, * we need to create a new instance of it for the children * and commands on this dependency line since each of these * dependency groups has its own attributes and commands, @@ -3004,10 +3004,7 @@ Parse_End(void) } -/* - * Return a list containing the single main target to create. - * If no such target exists, we Punt with an obnoxious error message. - */ +/* Populate the list with the single main target to create, or error out. */ void Parse_MainName(GNodeList *mainList) { diff --git a/unit-tests/Makefile b/unit-tests/Makefile index bcfe853642a5..a3f069133739 100644 --- a/unit-tests/Makefile +++ b/unit-tests/Makefile @@ -1,6 +1,6 @@ -# $Id: Makefile,v 1.193 2023/02/25 20:03:25 sjg Exp $ +# $Id: Makefile,v 1.195 2023/05/10 18:26:24 sjg Exp $ # -# $NetBSD: Makefile,v 1.333 2023/02/25 19:30:32 sjg Exp $ +# $NetBSD: Makefile,v 1.335 2023/05/10 13:03:06 rillig Exp $ # # Unit tests for make(1) # @@ -211,7 +211,6 @@ TESTS+= export TESTS+= export-all TESTS+= export-env TESTS+= export-variants -TESTS+= forloop TESTS+= forsubst TESTS+= gnode-submake TESTS+= hanoi-include @@ -375,6 +374,7 @@ TESTS+= varmod-loop-delete TESTS+= varmod-loop-varname TESTS+= varmod-match TESTS+= varmod-match-escape +TESTS+= varmod-mtime TESTS+= varmod-no-match TESTS+= varmod-order TESTS+= varmod-order-numeric diff --git a/unit-tests/cond-func.mk b/unit-tests/cond-func.mk index 959367f5c6ab..e09b363edb64 100644 --- a/unit-tests/cond-func.mk +++ b/unit-tests/cond-func.mk @@ -1,12 +1,12 @@ -# $NetBSD: cond-func.mk,v 1.11 2022/01/07 19:30:17 rillig Exp $ +# $NetBSD: cond-func.mk,v 1.12 2023/05/10 15:53:32 rillig Exp $ # # Tests for those parts of the functions in .if conditions that are common # among several functions. # -# The below test uses the function defined(...) since it has no side-effects, -# the other functions (except empty(...)) would work equally well. The -# function empty is special because it uses a different parsing algorithm for -# its argument. +# The below test uses the 'defined' function since it has no side-effects. +# The other functions would work equally well, except for 'empty', which +# parses its argument differently from the other functions. +# DEF= defined ${:UA B}= variable name with spaces @@ -74,7 +74,7 @@ ${VARNAME_UNBALANCED_BRACES}= variable name with unbalanced braces # There may be spaces around the operators and parentheses, and even # inside the parentheses. The spaces inside the parentheses are not -# allowed for the empty() function (see cond-func-empty.mk), therefore +# allowed for the 'empty' function (see cond-func-empty.mk), therefore # they are typically omitted for the other functions as well. .if ! defined ( DEF ) . error diff --git a/unit-tests/cond-late.mk b/unit-tests/cond-late.mk index 4df3df2cf1d4..1cfaaa2ee4e9 100644 --- a/unit-tests/cond-late.mk +++ b/unit-tests/cond-late.mk @@ -1,11 +1,12 @@ -# $NetBSD: cond-late.mk,v 1.3 2020/11/15 14:07:53 rillig Exp $ +# $NetBSD: cond-late.mk,v 1.4 2023/05/10 15:53:32 rillig Exp $ # # Using the :? modifier, variable expressions can contain conditional # expressions that are evaluated late, at expansion time. # -# Any variables appearing in these -# conditions are expanded before parsing the condition. This is -# different from many other places. +# Any expressions appearing in these conditions are expanded before parsing +# the condition. This is different from conditions in .if directives, where +# expressions are evaluated individually and only as far as necessary, see +# cond-short.mk. # # Because of this, variables that are used in these lazy conditions # should not contain double-quotes, or the parser will probably fail. @@ -22,10 +23,14 @@ COND.false= "yes" != "yes" # If the order of evaluation were to change to first parse the condition # and then expand the variables, the output would change from the # current "yes no" to "yes yes", since both variables are non-empty. +# expect: yes +# expect: no cond-literal: @echo ${ ${COND.true} :?yes:no} @echo ${ ${COND.false} :?yes:no} -VAR+= ${${UNDEF} != "no":?:} +VAR= ${${UNDEF} != "no":?:} +# expect-reset +# expect: make: Bad conditional expression ' != "no"' in ' != "no"?:' .if empty(VAR:Mpattern) .endif diff --git a/unit-tests/dep-var.mk b/unit-tests/dep-var.mk index f4a724f0ce7c..16f7498fd5cc 100755 --- a/unit-tests/dep-var.mk +++ b/unit-tests/dep-var.mk @@ -1,4 +1,4 @@ -# $NetBSD: dep-var.mk,v 1.7 2023/02/13 21:01:46 rillig Exp $ +# $NetBSD: dep-var.mk,v 1.8 2023/05/10 15:53:32 rillig Exp $ # # Tests for variable references in dependency declarations. # @@ -91,5 +91,6 @@ undef1 def2 a-def2-b 1-2-$$INDIRECT_2-2-1 ${:U\$)}: .MAKEFLAGS: -d0 -# XXX: Why is the exit status still 0, even though Parse_Error is called -# with PARSE_FATAL in SuffExpandChildren? +# XXX: The exit status is still 0, even though Parse_Error is called with +# PARSE_FATAL in SuffExpandChildren. The exit status is only affected by +# parse errors when they occur in the parsing phase, see Parse_File. diff --git a/unit-tests/directive-for-errors.exp b/unit-tests/directive-for-errors.exp index da5eee473ec2..36ed569c7932 100644 --- a/unit-tests/directive-for-errors.exp +++ b/unit-tests/directive-for-errors.exp @@ -1,22 +1,17 @@ -make: "directive-for-errors.mk" line 7: Unknown directive "fori" -make: "directive-for-errors.mk" line 8: warning: -make: "directive-for-errors.mk" line 9: for-less endfor -make: "directive-for-errors.mk" line 19: Unknown directive "for" -make: "directive-for-errors.mk" line 20: warning: -make: "directive-for-errors.mk" line 21: for-less endfor -make: "directive-for-errors.mk" line 37: Dollar $ 1 1 and backslash 2 2 2. -make: "directive-for-errors.mk" line 37: Dollar $ 3 3 and backslash 4 4 4. -make: "directive-for-errors.mk" line 43: no iteration variables in for -make: "directive-for-errors.mk" line 47: warning: Should not be reached. -make: "directive-for-errors.mk" line 48: for-less endfor -make: "directive-for-errors.mk" line 53: Wrong number of words (5) in .for substitution list with 3 variables -make: "directive-for-errors.mk" line 64: missing `in' in for -make: "directive-for-errors.mk" line 66: warning: Should not be reached. -make: "directive-for-errors.mk" line 67: for-less endfor -make: "directive-for-errors.mk" line 73: Unknown modifier "Z" -make: "directive-for-errors.mk" line 74: warning: Should not be reached. -make: "directive-for-errors.mk" line 74: warning: Should not be reached. -make: "directive-for-errors.mk" line 74: warning: Should not be reached. +make: "directive-for-errors.mk" line 11: Unknown directive "fori" +make: "directive-for-errors.mk" line 12: warning: <> +make: "directive-for-errors.mk" line 13: for-less endfor +make: "directive-for-errors.mk" line 27: Unknown directive "for" +make: "directive-for-errors.mk" line 28: warning: <> +make: "directive-for-errors.mk" line 29: for-less endfor +make: "directive-for-errors.mk" line 46: invalid character '$' in .for loop variable name +make: "directive-for-errors.mk" line 54: no iteration variables in for +make: "directive-for-errors.mk" line 66: Wrong number of words (5) in .for substitution list with 3 variables +make: "directive-for-errors.mk" line 80: missing `in' in for +make: "directive-for-errors.mk" line 91: Unknown modifier "Z" +make: "directive-for-errors.mk" line 92: warning: Should not be reached. +make: "directive-for-errors.mk" line 92: warning: Should not be reached. +make: "directive-for-errors.mk" line 92: warning: Should not be reached. make: Fatal errors encountered -- cannot continue make: stopped in unit-tests exit status 1 diff --git a/unit-tests/directive-for-errors.mk b/unit-tests/directive-for-errors.mk index 602ecbf32e4e..24df5e131839 100644 --- a/unit-tests/directive-for-errors.mk +++ b/unit-tests/directive-for-errors.mk @@ -1,45 +1,56 @@ -# $NetBSD: directive-for-errors.mk,v 1.3 2021/04/04 10:13:09 rillig Exp $ +# $NetBSD: directive-for-errors.mk,v 1.5 2023/05/09 19:43:12 rillig Exp $ # # Tests for error handling in .for loops. +# expect-all + + # A .for directive must be followed by whitespace, everything else results # in a parse error. +# expect+1: Unknown directive "fori" .fori in 1 2 3 -. warning ${i} +. warning <${i}> .endfor +# expect-2: <> +# expect-2: for-less endfor + # A slash is not whitespace, therefore this is not parsed as a .for loop. # # XXX: The error message is misleading though. As of 2020-12-31, it says -# "Unknown directive "for"", but that directive is actually known. This is +# 'Unknown directive "for"', but that directive is actually known. This is # because ForEval does not detect the .for loop as such, so parsing # continues in ParseLine > ParseDependencyLine > ParseDependency > # ParseDependencyTargets > ParseErrorNoDependency, and there the directive # name is parsed a bit differently. +# expect+1: Unknown directive "for" .for/i in 1 2 3 -. warning ${i} +. warning <${i}> .endfor +# expect-2: warning: <> +# expect-2: for-less endfor -# As of 2020-12-31, the variable name can be an arbitrary word, it just needs -# to be separated by whitespace. Even '$' and '\' are valid variable names, -# which is not useful in practice. + +# Before for.c 1.173 from 2023-05-08, the variable name could be an arbitrary +# word, it only needed to be separated by whitespace. Even '$' and '\' were +# valid variable names, which was not useful in practice. # -# The '$$' is not replaced with the values '1' or '3' from the .for loop, -# instead it is kept as-is, and when the .info directive expands its argument, -# each '$$' gets replaced with a single '$'. The "long variable expression" -# ${$} gets replaced though, even though this would be a parse error everywhere -# outside a .for loop. -# -# The '\' on the other hand is treated as a normal variable name. +# The '$$' was not replaced with the values '1' or '3' from the .for loop, +# instead it was kept as-is, and when the .info directive expanded its +# argument, each '$$' got replaced with a single '$'. The "long variable +# expression" ${$} got replaced though, even though this would be a parse +# error everywhere outside a .for loop. ${:U\$}= dollar # see whether the "variable" '$' is local ${:U\\}= backslash # see whether the "variable" '\' is local +# expect+1: invalid character '$' in .for loop variable name .for $ \ in 1 2 3 4 . info Dollar $$ ${$} $($) and backslash $\ ${\} $(\). .endfor # If there are no variables, there is no point in expanding the .for loop -# since this would end up in an endless loop, each time consuming 0 of the -# 3 values. +# since this would end up in an endless loop, consuming 0 of the 3 values in +# each iteration. +# expect+1: no iteration variables in for .for in 1 2 3 # XXX: This should not be reached. It should be skipped, as already done # when the number of values is not a multiple of the number of variables, @@ -47,29 +58,39 @@ ${:U\\}= backslash # see whether the "variable" '\' is local . warning Should not be reached. .endfor + # There are 3 variables and 5 values. These 5 values cannot be split evenly # among the variables, therefore the loop is not expanded at all, it is -# rather skipped. +# skipped instead. +# expect+1: Wrong number of words (5) in .for substitution list with 3 variables .for a b c in 1 2 3 4 5 . warning Should not be reached. .endfor + # The list of values after the 'in' may be empty, no matter if this emptiness # comes from an empty expansion or even from a syntactically empty line. .for i in . info Would be reached if there were items to loop over. .endfor + # A missing 'in' should parse the .for loop but skip the body. -.for i : k +# expect+1: missing `in' in for +.for i over k # XXX: As of 2020-12-31, this line is reached once. . warning Should not be reached. .endfor + # A malformed modifier should be detected and skip the body of the loop. # # XXX: As of 2020-12-31, Var_Subst doesn't report any errors, therefore # the loop body is expanded as if no error had happened. +# expect+1: Unknown modifier "Z" .for i in 1 2 ${:U3:Z} 4 . warning Should not be reached. .endfor +# expect-2: Should not be reached. +# expect-3: Should not be reached. +# expect-4: Should not be reached. diff --git a/unit-tests/directive-for-escape.exp b/unit-tests/directive-for-escape.exp index d679e0756e33..0326b62377f3 100644 --- a/unit-tests/directive-for-escape.exp +++ b/unit-tests/directive-for-escape.exp @@ -2,28 +2,28 @@ For: end for 1 For: loop body: . info ${:U!"#$%&'()*+,-./0-9\:;<=>?@A-Z[\\]_^a-z{|\}~} make: Unclosed variable expression, expecting '}' for modifier "U!"" of variable "" with value "!"" -make: "directive-for-escape.mk" line 19: !" +make: "directive-for-escape.mk" line 21: !" For: end for 1 For: loop body: . info ${:U!"\\\\#$%&'()*+,-./0-9\:;<=>?@A-Z[\\]_^a-z{|\}~} make: Unclosed variable expression, expecting '}' for modifier "U!"\\\\" of variable "" with value "!"\\" -make: "directive-for-escape.mk" line 29: !"\\ +make: "directive-for-escape.mk" line 32: !"\\ For: end for 1 For: loop body: . info ${:U\$} -make: "directive-for-escape.mk" line 43: $ +make: "directive-for-escape.mk" line 47: $ For: loop body: . info ${:U${V}} -make: "directive-for-escape.mk" line 43: value +make: "directive-for-escape.mk" line 47: value For: loop body: . info ${:U${V:=-with-modifier}} -make: "directive-for-escape.mk" line 43: value-with-modifier +make: "directive-for-escape.mk" line 47: value-with-modifier For: loop body: . info ${:U$(V)} -make: "directive-for-escape.mk" line 43: value +make: "directive-for-escape.mk" line 47: value For: loop body: . info ${:U$(V:=-with-modifier)} -make: "directive-for-escape.mk" line 43: value-with-modifier +make: "directive-for-escape.mk" line 47: value-with-modifier For: end for 1 For: loop body: # ${:U\${UNDEF\:U\\$\\$} @@ -34,29 +34,25 @@ For: loop body: For: end for 1 For: loop body: . info ${:U\${UNDEF\:U\\$\\$} -make: "directive-for-escape.mk" line 92: ${UNDEF:U\backslash$ +make: "directive-for-escape.mk" line 101: ${UNDEF:U\backslash$ For: loop body: . info ${:U{{\}\}} -make: "directive-for-escape.mk" line 92: {{}} +make: "directive-for-escape.mk" line 101: {{}} For: loop body: . info ${:Uend\}} -make: "directive-for-escape.mk" line 92: end} +make: "directive-for-escape.mk" line 101: end} For: end for 1 For: loop body: . info ${:Ubegin<${UNDEF:Ufallback:N{{{}}}}>end} -make: "directive-for-escape.mk" line 113: beginend +make: "directive-for-escape.mk" line 122: beginend For: end for 1 For: loop body: . info ${:U\$} -make: "directive-for-escape.mk" line 121: $ +make: "directive-for-escape.mk" line 131: $ +make: "directive-for-escape.mk" line 140: invalid character ':' in .for loop variable name For: end for 1 -For: loop body: -. info ${NUMBERS} ${:Ureplaced} -make: "directive-for-escape.mk" line 129: one two three replaced +make: "directive-for-escape.mk" line 150: invalid character '}' in .for loop variable name For: end for 1 -For: loop body: -. info ${:Ureplaced} -make: "directive-for-escape.mk" line 139: replaced For: end for 1 For: loop body: . info . $$i: ${:Uinner} @@ -69,46 +65,42 @@ For: loop body: . info . $${i2}: ${i2} . info . $${i,}: ${i,} . info . adjacent: ${:Uinner}${:Uinner}${:Uinner:M*}${:Uinner} -make: "directive-for-escape.mk" line 147: . $i: inner -make: "directive-for-escape.mk" line 148: . ${i}: inner -make: "directive-for-escape.mk" line 149: . ${i:M*}: inner -make: "directive-for-escape.mk" line 150: . $(i): inner -make: "directive-for-escape.mk" line 151: . $(i:M*): inner -make: "directive-for-escape.mk" line 152: . ${i${:U}}: outer -make: "directive-for-escape.mk" line 153: . ${i\}}: inner} -make: "directive-for-escape.mk" line 154: . ${i2}: two -make: "directive-for-escape.mk" line 155: . ${i,}: comma -make: "directive-for-escape.mk" line 156: . adjacent: innerinnerinnerinner +make: "directive-for-escape.mk" line 159: . $i: inner +make: "directive-for-escape.mk" line 160: . ${i}: inner +make: "directive-for-escape.mk" line 161: . ${i:M*}: inner +make: "directive-for-escape.mk" line 162: . $(i): inner +make: "directive-for-escape.mk" line 163: . $(i:M*): inner +make: "directive-for-escape.mk" line 164: . ${i${:U}}: outer +make: "directive-for-escape.mk" line 165: . ${i\}}: inner} +make: "directive-for-escape.mk" line 166: . ${i2}: two +make: "directive-for-escape.mk" line 167: . ${i,}: comma +make: "directive-for-escape.mk" line 168: . adjacent: innerinnerinnerinner +make: "directive-for-escape.mk" line 187: invalid character '$' in .for loop variable name For: end for 1 -For: loop body: -. info eight $$$$$$$$ and no cents. -. info eight ${:Udollar}${:Udollar}${:Udollar}${:Udollar} and no cents. -make: "directive-for-escape.mk" line 164: eight $$$$ and no cents. -make: "directive-for-escape.mk" line 165: eight dollardollardollardollar and no cents. -make: "directive-for-escape.mk" line 174: eight and no cents. +make: "directive-for-escape.mk" line 199: eight and no cents. For: end for 1 -make: "directive-for-escape.mk" line 181: newline in .for value -make: "directive-for-escape.mk" line 181: newline in .for value +make: "directive-for-escape.mk" line 212: newline in .for value +make: "directive-for-escape.mk" line 212: newline in .for value For: loop body: . info short: ${:U" "} . info long: ${:U" "} -make: "directive-for-escape.mk" line 182: short: " " -make: "directive-for-escape.mk" line 183: long: " " +make: "directive-for-escape.mk" line 213: short: " " +make: "directive-for-escape.mk" line 214: long: " " For: end for 1 For: loop body: For: end for 1 -Parse_PushInput: .for loop in directive-for-escape.mk, line 196 -make: "directive-for-escape.mk" line 196: newline in .for value - in .for loop from directive-for-escape.mk:196 with i = " +Parse_PushInput: .for loop in directive-for-escape.mk, line 230 +make: "directive-for-escape.mk" line 230: newline in .for value + in .for loop from directive-for-escape.mk:230 with i = " " For: loop body: : ${:U" "} SetFilenameVars: ${.PARSEDIR} = ${.PARSEFILE} = `directive-for-escape.mk' -Parsing line 197: : ${:U" "} +Parsing line 231: : ${:U" "} ParseDependency(: " ") -ParseEOF: returning to file directive-for-escape.mk, line 199 +ParseEOF: returning to file directive-for-escape.mk, line 233 SetFilenameVars: ${.PARSEDIR} = ${.PARSEFILE} = `directive-for-escape.mk' -Parsing line 199: .MAKEFLAGS: -d0 +Parsing line 233: .MAKEFLAGS: -d0 ParseDependency(.MAKEFLAGS: -d0) For: end for 1 For: loop body: diff --git a/unit-tests/directive-for-escape.mk b/unit-tests/directive-for-escape.mk index fe704c453046..7fbd09131d2c 100644 --- a/unit-tests/directive-for-escape.mk +++ b/unit-tests/directive-for-escape.mk @@ -1,9 +1,10 @@ -# $NetBSD: directive-for-escape.mk,v 1.16 2022/06/12 16:09:21 rillig Exp $ +# $NetBSD: directive-for-escape.mk,v 1.18 2023/05/09 19:43:12 rillig Exp $ # # Test escaping of special characters in the iteration values of a .for loop. # These values get expanded later using the :U variable modifier, and this -# escaping and unescaping must pass all characters and strings effectively -# unmodified. +# escaping and unescaping must pass all characters and strings unmodified. + +# expect-all .MAKEFLAGS: -df @@ -12,12 +13,14 @@ # This could be considered a bug. ASCII= !"\#$$%&'()*+,-./0-9:;<=>?@A-Z[\]_^a-z{|}~ + # XXX: As of 2020-12-31, the '#' is not preserved in the expanded body of # the loop. Not only would it need the escaping for the variable modifier # ':U' but also the escaping for the line-end comment. .for chars in ${ASCII} . info ${chars} .endfor +# expect-2: !" # As of 2020-12-31, using 2 backslashes before be '#' would treat the '#' # as comment character. Using 3 backslashes doesn't help either since @@ -28,6 +31,7 @@ ASCII.2020-12-31= !"\\\#$$%&'()*+,-./0-9:;<=>?@A-Z[\]_^a-z{|}~ .for chars in ${ASCII.2020-12-31} . info ${chars} .endfor +# expect-2: !"\\ # Cover the code in ExprLen. # @@ -42,6 +46,11 @@ VALUES= $$ $${V} $${V:=-with-modifier} $$(V) $$(V:=-with-modifier) .for i in ${VALUES} . info $i .endfor +# expect-2: $ +# expect-3: value +# expect-4: value-with-modifier +# expect-5: value +# expect-6: value-with-modifier # Try to cover the code for nested '{}' in ExprLen, without success. @@ -112,6 +121,7 @@ VALUES= begin<$${UNDEF:Ufallback:N{{{}}}}>end .for i in ${VALUES} . info $i .endfor +# expect-2: beginend # A single trailing dollar doesn't happen in practice. # The dollar sign is correctly passed through to the body of the .for loop. @@ -120,21 +130,23 @@ VALUES= begin<$${UNDEF:Ufallback:N{{{}}}}>end .for i in ${:U\$} . info ${i} .endfor +# expect-2: $ -# As of 2020-12-31, the name of the iteration variable can even contain -# colons, which then affects variable expressions having this exact modifier. -# This is clearly an unintended side effect of the implementation. +# Before for.c 1.173 from 2023-05-08, the name of the iteration variable +# could contain colons, which affected variable expressions having this exact +# modifier. This possibility was neither intended nor documented. NUMBERS= one two three +# expect+1: invalid character ':' in .for loop variable name .for NUMBERS:M*e in replaced . info ${NUMBERS} ${NUMBERS:M*e} .endfor -# As of 2020-12-31, the name of the iteration variable can contain braces, -# which gets even more surprising than colons, since it allows to replace -# sequences of variable expressions. There is no practical use case for -# this, though. +# Before for.c 1.173 from 2023-05-08, the name of the iteration variable +# could contain braces, which allowed to replace sequences of variable +# expressions. This possibility was neither intended nor documented. BASENAME= one EXT= .c +# expect+1: invalid character '}' in .for loop variable name .for BASENAME}${EXT in replaced . info ${BASENAME}${EXT} .endfor @@ -155,11 +167,23 @@ i,= comma . info . $${i,}: ${i,} . info . adjacent: $i${i}${i:M*}$i .endfor +# expect-11: . $i: inner +# expect-11: . ${i}: inner +# expect-11: . ${i:M*}: inner +# expect-11: . $(i): inner +# expect-11: . $(i:M*): inner +# expect-11: . ${i${:U}}: outer +# expect-11: . ${i\}}: inner} +# expect-11: . ${i2}: two +# expect-11: . ${i,}: comma +# expect-11: . adjacent: innerinnerinnerinner -# The variable name can be a single '$' since there is no check on valid -# variable names. ForLoop_SubstVarShort skips "stupid" variable names though, -# but ForLoop_SubstVarLong naively parses the body of the loop, substituting -# each '${$}' with an actual 'dollar'. +# Before for.c 1.173 from 2023-05-08, the variable name could be a single '$' +# since there was no check on valid variable names. ForLoop_SubstVarShort +# skipped "stupid" variable names though, but ForLoop_SubstVarLong naively +# parsed the body of the loop, substituting each '${$}' with an actual +# '${:Udollar}'. +# expect+1: invalid character '$' in .for loop variable name .for $ in dollar . info eight $$$$$$$$ and no cents. . info eight ${$}${$}${$}${$} and no cents. @@ -171,6 +195,7 @@ i,= comma # evaluates to an empty string. closing-brace= } # guard against an ${closing-brace}= # alternative interpretation +# expect+1: eight and no cents. .info eight ${$}${$}${$}${$} and no cents. # What happens if the values from the .for loop contain a literal newline? @@ -178,10 +203,18 @@ ${closing-brace}= # alternative interpretation # body of the .for loop, where it was then interpreted as a literal newline, # leading to syntax errors such as "Unclosed variable expression" in the upper # line and "Invalid line type" in the lower line. +# +# The error message occurs in the line of the .for loop since that's the place +# where the body of the .for loop is constructed, and at this point the +# newline character gets replaced with a plain space. +# expect+2: newline in .for value +# expect+1: newline in .for value .for i in "${.newline}" . info short: $i . info long: ${i} .endfor +# expect-3: short: " " +# expect-3: long: " " # No error since the newline character is not actually used. .for i in "${.newline}" @@ -193,6 +226,7 @@ ${closing-brace}= # alternative interpretation # loop is assembled, and at that point, ForLoop.nextItem had already been # advanced. .MAKEFLAGS: -dp +# expect+1: newline in .for value .for i in "${.newline}" : $i .endfor diff --git a/unit-tests/directive-for.exp b/unit-tests/directive-for.exp index dda487917e68..97878ee49f44 100755 --- a/unit-tests/directive-for.exp +++ b/unit-tests/directive-for.exp @@ -1,42 +1,41 @@ -make: "directive-for.mk" line 108: outer -make: "directive-for.mk" line 133: a:\ a:\file.txt -make: "directive-for.mk" line 133: d:\\ -make: "directive-for.mk" line 133: d:\\file.txt -make: "directive-for.mk" line 140: ( ( ( -make: "directive-for.mk" line 140: [ [ [ -make: "directive-for.mk" line 140: { { { -make: "directive-for.mk" line 140: ) ) ) -make: "directive-for.mk" line 140: ] ] ] -make: "directive-for.mk" line 140: } } } -make: "directive-for.mk" line 140: (()) (()) (()) -make: "directive-for.mk" line 140: [[]] [[]] [[]] -make: "directive-for.mk" line 140: {{}} {{}} {{}} -make: "directive-for.mk" line 140: )( )( )( -make: "directive-for.mk" line 140: ][ ][ ][ -make: "directive-for.mk" line 140: }{ }{ }{ -make: "directive-for.mk" line 148: outer value value -make: "directive-for.mk" line 148: outer "quoted" \"quoted\" -make: "directive-for.mk" line 154: Unknown modifier "Z" -make: "directive-for.mk" line 155: XXX: Not reached word1 -make: "directive-for.mk" line 155: XXX: Not reached word3 -make: "directive-for.mk" line 160: no iteration variables in for -make: "directive-for.mk" line 162: Missing argument for ".error" -make: "directive-for.mk" line 163: for-less endfor -make: "directive-for.mk" line 187: 1 open conditional -make: "directive-for.mk" line 203: for-less endfor -make: "directive-for.mk" line 204: if-less endif -make: "directive-for.mk" line 212: if-less endif +make: "directive-for.mk" line 119: outer +make: "directive-for.mk" line 137: a:\ a:\file.txt +make: "directive-for.mk" line 137: d:\\ +make: "directive-for.mk" line 137: d:\\file.txt +make: "directive-for.mk" line 148: ( ( ( +make: "directive-for.mk" line 148: [ [ [ +make: "directive-for.mk" line 148: { { { +make: "directive-for.mk" line 148: ) ) ) +make: "directive-for.mk" line 148: ] ] ] +make: "directive-for.mk" line 148: } } } +make: "directive-for.mk" line 148: (()) (()) (()) +make: "directive-for.mk" line 148: [[]] [[]] [[]] +make: "directive-for.mk" line 148: {{}} {{}} {{}} +make: "directive-for.mk" line 148: )( )( )( +make: "directive-for.mk" line 148: ][ ][ ][ +make: "directive-for.mk" line 148: }{ }{ }{ +make: "directive-for.mk" line 168: invalid character ':' in .for loop variable name +make: "directive-for.mk" line 175: invalid character '$' in .for loop variable name +make: "directive-for.mk" line 187: invalid character '$' in .for loop variable name +make: "directive-for.mk" line 198: Unknown modifier "Z" +make: "directive-for.mk" line 199: XXX: Not reached word1 +make: "directive-for.mk" line 199: XXX: Not reached word3 +make: "directive-for.mk" line 206: no iteration variables in for +make: "directive-for.mk" line 232: 1 open conditional +make: "directive-for.mk" line 248: for-less endfor +make: "directive-for.mk" line 249: if-less endif +make: "directive-for.mk" line 257: if-less endif +For: new loop 2 +For: end for 2 For: end for 1 For: loop body: .\ for inner in i .\ endfor -make: "directive-for.mk" line 229: Unexpected end of file in .for loop +For: end for 1 For: loop body: -.\ - endfor -make: "directive-for.mk" line 227: for-less endfor +make: "directive-for.mk" line 305: newline-item=(a) make: Fatal errors encountered -- cannot continue make: stopped in unit-tests exit status 1 diff --git a/unit-tests/directive-for.mk b/unit-tests/directive-for.mk index 95171c68031f..224a466a7709 100755 --- a/unit-tests/directive-for.mk +++ b/unit-tests/directive-for.mk @@ -1,4 +1,4 @@ -# $NetBSD: directive-for.mk,v 1.15 2022/10/01 09:23:04 rillig Exp $ +# $NetBSD: directive-for.mk,v 1.20 2023/05/10 13:03:06 rillig Exp $ # # Tests for the .for directive. # @@ -8,11 +8,15 @@ # .for _FILE_ in values # .for .FILE. in values # .for _f_ in values - -# Using the .for loop, lists of values can be produced. -# In simple cases, the :@var@${var}@ variable modifier can be used to -# achieve the same effects. # +# See also: +# varmod-loop.mk The ':@var@...@' modifier + +# expect-all + +# A typical use case for a .for loop is to populate a variable with a list of +# values depending on other variables. In simple cases, the same effect can +# be achieved using the ':@var@${var}@' modifier. .undef NUMBERS .for num in 1 2 3 NUMBERS+= ${num} @@ -21,8 +25,9 @@ NUMBERS+= ${num} . error .endif + # The .for loop also works for multiple iteration variables. -# This is something that the variable modifier :@ cannot do. +# This is something that the modifier :@ cannot do. .for name value in VARNAME value NAME2 value2 ${name}= ${value} .endfor @@ -30,12 +35,12 @@ ${name}= ${value} . error .endif + # The .for loop splits the items at whitespace, taking quotes into account, -# just like the :M or :S variable modifiers. -# -# Until 2012-06-03, it had split the items exactly at whitespace, without -# taking the quotes into account. This had resulted in 10 words. +# just like the :M or :S modifiers. # +# Until 2012-06-03, the .for loop had split the items exactly at whitespace, +# without taking the quotes into account. This had resulted in 10 words. .undef WORDS .for var in one t\ w\ o "three three" 'four four' `five six` WORDS+= counted @@ -44,16 +49,19 @@ WORDS+= counted . error .endif + # In the body of the .for loop, the iteration variables can be accessed # like normal variables, even though they are not really variables. # -# Instead, the expression ${var} is transformed into ${:U1}, ${:U2} and so -# on, before the loop body is evaluated. +# Instead, before interpreting the body of the .for loop, the body is +# generated by replacing each expression ${var} with ${:U1}, ${:U2} and so +# on. # -# A notable effect of this implementation technique is that the .for +# A noticeable effect of this implementation technique is that the .for # iteration variables and the normal global variables live in separate -# namespaces and do not influence each other. -# +# namespaces and do not influence each other. The "scope" of the .for loop +# variables is restricted to the current makefile, it does not reach over to +# any included makefiles. var= value before var2= value before .for var var2 in 1 2 3 4 @@ -66,9 +74,8 @@ var2= value before .endif # Everything from the paragraph above also applies if the loop body is -# empty, even if there is no actual iteration since the loop items are -# also empty. -# +# empty. In this particular example, the items to be iterated are empty as +# well. var= value before var2= value before .for var var2 in ${:U} @@ -82,11 +89,13 @@ var2= value before # Until 2008-12-21, the values of the iteration variables were simply # inserted as plain text and then parsed as usual, which made it possible -# to achieve all kinds of strange effects. +# to achieve all kinds of strange effects, such as generating '.if' +# directives or inserting '$' characters in random places, thereby changing +# how following '$' are interpreted. # -# Before that date, the .for loop expanded to: +# Before that date, the .for loop below expanded to: # EXPANSION+= value -# Since that date, the .for loop expands to: +# Since that date, the .for loop below expands to: # EXPANSION${:U+}= value # EXPANSION= before @@ -102,13 +111,16 @@ EXPANSION${plus}= value .endif # When the outer .for loop is expanded, it sees the expression ${i} and -# expands it. The inner loop then has nothing more to expand. +# expands it. The inner loop then only sees the expression ${:Uouter} and +# has nothing more to expand. .for i in outer . for i in inner +# expect+1: outer . info ${i} . endfor .endfor + # From https://gnats.netbsd.org/29985. # # Until 2008-12-21, the .for loop was expanded by replacing the variable @@ -121,17 +133,13 @@ EXPANSION${plus}= value # like "a:\ a:\file.txt" that ended in a single backslash. Since then, the # variable values have been replaced with expressions of the form ${:U...}, # which are not interpreted as code anymore. -# -# As of 2020-09-22, a comment in for.c says that it may be possible to -# produce an "unwanted substitution", but there is no demonstration code yet. -# -# The above changes prevent a backslash at the end of a word from being -# interpreted as part of the code. Because of this, the trailingBackslash -# hack in Var_Subst is no longer needed and as of 2020-09-22, has been -# removed. .for path in a:\ a:\file.txt d:\\ d:\\file.txt . info ${path} .endfor +# expect-2: a:\ a:\file.txt +# expect-3: d:\\ +# expect-4: d:\\file.txt + # Ensure that braces and parentheses are properly escaped by the .for loop. # Each line must print the same word 3 times. @@ -139,28 +147,65 @@ EXPANSION${plus}= value .for v in ( [ { ) ] } (()) [[]] {{}} )( ][ }{ . info $v ${v} $(v) .endfor +# expect-02: ( ( ( +# expect-03: [ [ [ +# expect-04: { { { +# expect-05: ) ) ) +# expect-06: ] ] ] +# expect-07: } } } +# expect-08: (()) (()) (()) +# expect-09: [[]] [[]] [[]] +# expect-10: {{}} {{}} {{}} +# expect-11: )( )( )( +# expect-12: ][ ][ ][ +# expect-13: }{ }{ }{ -# As of 2020-10-25, the variable names may contain arbitrary characters, -# except for whitespace. This allows for creative side effects. Hopefully -# nobody is misusing this "feature". +# Before 2023-05-09, the variable names could contain arbitrary characters, +# except for whitespace, allowing for creative side effects, as usual for +# arbitrary code injection. var= outer +# expect+1: invalid character ':' in .for loop variable name .for var:Q in value "quoted" -. info ${var} ${var:Q} ${var:Q:Q} +. info <${var}> <${var:Q}> <${var:Q:Q}> +.endfor + +# Before 2023-05-09, when variable names could contain '$', the short +# expression '$$' was preserved, the long expressions were substituted. +# expect+1: invalid character '$' in .for loop variable name +.for $ in value +. info <$$> <${$}> <$($)> +.endfor + + +# https://gnats.netbsd.org/53146 mentions the idea of using a dynamic +# variable name in .for loops, based on some other variable. The .for loops +# are already tricky enough to understand in detail, even without this +# possibility, therefore the variable names are restricted to using harmless +# characters only. +INDIRECT= direct +# expect+1: invalid character '$' in .for loop variable name +.for $(INDIRECT) in value +# If the variable name could be chosen dynamically, the iteration variable +# might have been 'direct', thereby expanding the expression '${direct}'. +. info <$(INDIRECT)> <$(direct)> <$($(INDIRECT))> .endfor # XXX: A parse error or evaluation error in the items of the .for loop -# should skip the whole loop. As of 2020-12-27, the loop is expanded twice. +# should skip the whole loop. As of 2023-05-09, the loop is expanded as +# usual. +# expect+1: Unknown modifier "Z" .for var in word1 ${:Uword2:Z} word3 . info XXX: Not reached ${var} .endfor +# expect-2: XXX: Not reached word1 +# expect-3: XXX: Not reached word3 # An empty list of variables to the left of the 'in' is a parse error. .for in value # expect+0: no iteration variables in for -# XXX: The loop body is evaluated once, even with the parse error above. -. error # expect+0: Missing argument for ".error" -.endfor # expect+0: for-less endfor +. error +.endfor # An empty list of iteration values to the right of the 'in' is accepted. # Unlike in the shell, it is not a parse error. @@ -214,12 +259,19 @@ var= outer .endif # no 'if-less endif' -# When make parses a .for loop, it assumes that there is no line break between -# the '.' and the 'for' or 'endfor', as there is no practical reason to break -# the line at this point. When make scans the outer .for loop, it does not -# recognize the inner directives as such. When make scans the inner .for -# loop, it recognizes the '.\n for' but does not recognize the '.\n endfor', -# as LK_FOR_BODY preserves the backslash-newline sequences. +# Before for.c 1.172 from 2023-05-08, when make parsed a .for loop, it +# assumed that there was no line continuation between the '.' and the 'for' +# or 'endfor', as there is no practical reason to break the line at this +# point. +# +# When make scanned the outer .for loop, it did not recognize the inner .for +# loop as such and instead treated it as an unknown directive. The body of +# the outer .for loop thus ended above the '.endfor'. +# +# When make scanned the inner .for loop, it did not recognize the inner +# .endfor as such, which led to a parse error 'Unexpected end of file in .for +# loop' from the '.endfor' line, followed by a second parse error 'for-less +# .endfor' from the '.\\n endfor' line. .MAKEFLAGS: -df .for outer in o .\ @@ -244,3 +296,12 @@ var= outer . error . endif .endfor + + +# Since at least 1993, iteration stops at the first newline. +# Back then, the .newline variable didn't exist, therefore it was unlikely +# that a newline ever occurred. +.for var in a${.newline}b${.newline}c +. info newline-item=(${var}) +.endfor +# expect-2: newline-item=(a) diff --git a/unit-tests/forloop.exp b/unit-tests/forloop.exp deleted file mode 100644 index 422711b41247..000000000000 --- a/unit-tests/forloop.exp +++ /dev/null @@ -1,20 +0,0 @@ -make: "forloop.mk" line 14: x=one -make: "forloop.mk" line 14: x="two and three" -make: "forloop.mk" line 14: x=four -make: "forloop.mk" line 14: x="five" -make: "forloop.mk" line 20: x=-I/this -make: "forloop.mk" line 20: x=-I"This or that" -make: "forloop.mk" line 20: x=-Ithat -make: "forloop.mk" line 20: x="-DTHIS=\"this and that\"" -make: "forloop.mk" line 27: cfl=-I/this -I"This or that" -Ithat "-DTHIS=\"this and that\"" -make: "forloop.mk" line 41: newline-item=(a) -make: "forloop.mk" line 47: a=one b="two and three" -make: "forloop.mk" line 47: a=four b="five" -make: "forloop.mk" line 47: a=ONE b="TWO AND THREE" -make: "forloop.mk" line 47: a=FOUR b="FIVE" -We expect an error next: -make: "forloop.mk" line 46: Wrong number of words (9) in .for substitution list with 2 variables -make: Fatal errors encountered -- cannot continue -make: stopped in unit-tests -OK -exit status 0 diff --git a/unit-tests/forloop.mk b/unit-tests/forloop.mk deleted file mode 100644 index cef05cbe4c61..000000000000 --- a/unit-tests/forloop.mk +++ /dev/null @@ -1,53 +0,0 @@ -# $NetBSD: forloop.mk,v 1.7 2020/11/03 17:37:57 rillig Exp $ - -all: for-loop - -LIST= one "two and three" four "five" - -.if make(for-fail) -for-fail: - -XTRA_LIST= xtra -.else - -. for x in ${LIST} -. info x=$x -. endfor - -CFL= -I/this -I"This or that" -Ithat "-DTHIS=\"this and that\"" -cfl= -. for x in ${CFL} -. info x=$x -. if empty(cfl) -cfl= $x -. else -cfl+= $x -. endif -. endfor -. info cfl=${cfl} - -. if ${cfl} != ${CFL} -. error ${.newline}${cfl} != ${.newline}${CFL} -. endif - -. for a b in ${EMPTY} -. info a=$a b=$b -. endfor - -# Since at least 1993, iteration stops at the first newline. -# Back then, the .newline variable didn't exist, therefore it was unlikely -# that a newline ever occurred. -. for var in a${.newline}b${.newline}c -. info newline-item=(${var}) -. endfor - -.endif # for-fail - -.for a b in ${LIST} ${LIST:tu} ${XTRA_LIST} -. info a=$a b=$b -.endfor - -for-loop: - @echo We expect an error next: - @(cd ${.CURDIR} && ${.MAKE} -f ${MAKEFILE} for-fail) && \ - { echo "Oops that should have failed!"; exit 1; } || echo OK diff --git a/unit-tests/parse.mk b/unit-tests/parse.mk index 986c303083bf..551dc98aaf24 100644 --- a/unit-tests/parse.mk +++ b/unit-tests/parse.mk @@ -1,4 +1,4 @@ -# $NetBSD: parse.mk,v 1.3 2022/07/24 20:25:23 rillig Exp $ +# $NetBSD: parse.mk,v 1.4 2023/04/28 13:09:48 rillig Exp $ # # Test those parts of the parsing that do not belong in any of the other # categories. @@ -22,3 +22,33 @@ # # https://bugs.freebsd.org/265119 one-target ${:U } + + +# Since parse.c 1.656 from 2022-01-27 and before parse.c 1.662 from +# 2022-02-05, there was an out-of-bounds read in Parse_IsVar when looking for +# a variable assignment in a dependency line with trailing whitespace. Lines +# without trailing whitespace were not affected. Global variable assignments +# were guaranteed to have no trailing whitespace and were thus not affected. +# +# Try to reproduce some variants that may lead to a crash, depending on the +# memory allocator. To get a crash, the terminating '\0' of the line must be +# the last byte of a memory page. The expression '${:U}' forces this trailing +# whitespace. + +# On FreeBSD x86_64, a crash could in some cases be forced using the following +# line, which has length 47, and if the memory for the expanded line starts at +# 0xXXXX_XXd0, the terminating '\0' may end up at 0xXXXX_Xfff: +Try_to_crash_FreeBSD.xxxxxxxxxxxxxxxxxx: 12345 ${:U} + +# The following line has length 4095 after being expanded, so line[4095] == +# '\0'. If the line is +# allocated on a page boundary and the following page is not mapped, this line +# leads to a segmentation fault. +${:U:range=511:@_@1234567@:ts.}: 12345 ${:U} + +# The following line has length 8191, so line[8191] == '\0'. If the line is +# allocated on a page boundary and the following page is not mapped, this line +# leads to a segmentation fault. +${:U:range=1023:@_@1234567@:ts.}: 12345 ${:U} + +12345: diff --git a/unit-tests/var-eval-short.exp b/unit-tests/var-eval-short.exp index 34380ec61c41..ea6e0d7485f9 100644 --- a/unit-tests/var-eval-short.exp +++ b/unit-tests/var-eval-short.exp @@ -1,9 +1,5 @@ make: "var-eval-short.mk" line 44: In the :@ modifier of "", the variable name "${FAIL}" must not contain a dollar make: "var-eval-short.mk" line 44: Malformed conditional (0 && ${:Uword:@${FAIL}@expr@}) -make: "var-eval-short.mk" line 84: Invalid time value at "${FAIL}}" -make: "var-eval-short.mk" line 84: Malformed conditional (0 && ${:Uword:gmtime=${FAIL}}) -make: "var-eval-short.mk" line 98: Invalid time value at "${FAIL}}" -make: "var-eval-short.mk" line 98: Malformed conditional (0 && ${:Uword:localtime=${FAIL}}) CondParser_Eval: 0 && ${0:?${FAIL}then:${FAIL}else} Var_Parse: ${0:?${FAIL}then:${FAIL}else} (parse-only) Parsing modifier ${0:?...} @@ -12,7 +8,7 @@ Modifier part: "${FAIL}then" Var_Parse: ${FAIL}else} (parse-only) Modifier part: "${FAIL}else" Result of ${0:?${FAIL}then:${FAIL}else} is "" (parse-only, defined) -Parsing line 163: DEFINED= defined +Parsing line 165: DEFINED= defined Global: DEFINED = defined CondParser_Eval: 0 && ${DEFINED:L:?${FAIL}then:${FAIL}else} Var_Parse: ${DEFINED:L:?${FAIL}then:${FAIL}else} (parse-only) @@ -24,7 +20,7 @@ Modifier part: "${FAIL}then" Var_Parse: ${FAIL}else} (parse-only) Modifier part: "${FAIL}else" Result of ${DEFINED:?${FAIL}then:${FAIL}else} is "defined" (parse-only, regular) -Parsing line 166: .MAKEFLAGS: -d0 +Parsing line 168: .MAKEFLAGS: -d0 ParseDependency(.MAKEFLAGS: -d0) Global: .MAKEFLAGS = -r -k -d cpv -d Global: .MAKEFLAGS = -r -k -d cpv -d 0 diff --git a/unit-tests/var-eval-short.mk b/unit-tests/var-eval-short.mk index a099b6871d1e..7faf1cc5d154 100644 --- a/unit-tests/var-eval-short.mk +++ b/unit-tests/var-eval-short.mk @@ -1,4 +1,4 @@ -# $NetBSD: var-eval-short.mk,v 1.8 2021/12/27 18:54:19 rillig Exp $ +# $NetBSD: var-eval-short.mk,v 1.9 2023/05/09 16:27:00 rillig Exp $ # # Tests for each variable modifier to ensure that they only do the minimum # necessary computations. If the result of the expression is irrelevant, @@ -79,8 +79,9 @@ DEFINED= # defined .if 0 && ${:Uword:E} .endif -# As of 2021-03-14, the error 'Invalid time value: ${FAIL}}' is ok since -# ':gmtime' does not expand its argument. +# Before var.c 1.1050 from 2023-05-09, the ':gmtime' modifier produced the +# error message 'Invalid time value: ${FAIL}}' since it did not expand its +# argument. .if 0 && ${:Uword:gmtime=${FAIL}} .endif @@ -93,8 +94,9 @@ DEFINED= # defined .if 0 && ${value:L} .endif -# As of 2021-03-14, the error 'Invalid time value: ${FAIL}}' is ok since -# ':localtime' does not expand its argument. +# Before var.c 1.1050 from 2023-05-09, the ':localtime' modifier produced the +# error message 'Invalid time value: ${FAIL}}' since it did not expand its +# argument. .if 0 && ${:Uword:localtime=${FAIL}} .endif diff --git a/unit-tests/var-scope-local.exp b/unit-tests/var-scope-local.exp index 403bf83884f7..a9a947a620c7 100644 --- a/unit-tests/var-scope-local.exp +++ b/unit-tests/var-scope-local.exp @@ -1,5 +1,5 @@ -Global: .ALLTARGETS = one -Global: .ALLTARGETS = one two +Global: .ALLTARGETS = all target-rule.ext dir/subdir/target-rule.ext target-rule.ir-gen-from dir/subdir/target-rule-dir.ir-gen-from inference-rule.ir-to dir/subdir/inference-rule.ir-to inference-rule.ir-from dir/subdir/inference-rule.ir-from inference-rule-chain.ir-to dir/subdir/inference-rule-chain.ir-to inference-rule-chain.ir-gen-from dir/subdir/inference-rule-chain.ir-gen-from one +Global: .ALLTARGETS = all target-rule.ext dir/subdir/target-rule.ext target-rule.ir-gen-from dir/subdir/target-rule-dir.ir-gen-from inference-rule.ir-to dir/subdir/inference-rule.ir-to inference-rule.ir-from dir/subdir/inference-rule.ir-from inference-rule-chain.ir-to dir/subdir/inference-rule-chain.ir-to inference-rule-chain.ir-gen-from dir/subdir/inference-rule-chain.ir-gen-from one two Var_Parse: ${.MAKE.TARGET_LOCAL_VARIABLES} (eval) Var_SetExpand: variable name "" expands to empty string, with value "three" - ignored Var_SetExpand: variable name "" expands to empty string, with value "three" - ignored @@ -7,6 +7,56 @@ Global: one two = # (empty) Global: one two = three Global: .MAKEFLAGS = -r -k -d v -d Global: .MAKEFLAGS = -r -k -d v -d 0 +target-rule.ext: @ = +target-rule.ext: % = +target-rule.ext: ? = <> +target-rule.ext: < = +target-rule.ext: * = +dir/subdir/target-rule.ext: @ = +dir/subdir/target-rule.ext: % = +dir/subdir/target-rule.ext: ? = <> +dir/subdir/target-rule.ext: < = +dir/subdir/target-rule.ext: * = +target-rule.ir-gen-from: @ = +target-rule.ir-gen-from: % = +target-rule.ir-gen-from: ? = <> +target-rule.ir-gen-from: < = +target-rule.ir-gen-from: * = +dir/subdir/target-rule-dir.ir-gen-from: @ = +dir/subdir/target-rule-dir.ir-gen-from: % = +dir/subdir/target-rule-dir.ir-gen-from: ? = <> +dir/subdir/target-rule-dir.ir-gen-from: < = +dir/subdir/target-rule-dir.ir-gen-from: * = +inference-rule.ir-to: @ = +inference-rule.ir-to: % = +inference-rule.ir-to: ? = +inference-rule.ir-to: < = +inference-rule.ir-to: * = +dir/subdir/inference-rule.ir-to: @ = +dir/subdir/inference-rule.ir-to: % = +dir/subdir/inference-rule.ir-to: ? = +dir/subdir/inference-rule.ir-to: < = +dir/subdir/inference-rule.ir-to: * = +inference-rule-chain.ir-from: @ = +inference-rule-chain.ir-from: % = +inference-rule-chain.ir-from: ? = +inference-rule-chain.ir-from: < = +inference-rule-chain.ir-from: * = +inference-rule-chain.ir-to: @ = +inference-rule-chain.ir-to: % = +inference-rule-chain.ir-to: ? = +inference-rule-chain.ir-to: < = +inference-rule-chain.ir-to: * = +dir/subdir/inference-rule-chain.ir-from: @ = +dir/subdir/inference-rule-chain.ir-from: % = +dir/subdir/inference-rule-chain.ir-from: ? = +dir/subdir/inference-rule-chain.ir-from: < = +dir/subdir/inference-rule-chain.ir-from: * = +dir/subdir/inference-rule-chain.ir-to: @ = +dir/subdir/inference-rule-chain.ir-to: % = +dir/subdir/inference-rule-chain.ir-to: ? = +dir/subdir/inference-rule-chain.ir-to: < = +dir/subdir/inference-rule-chain.ir-to: * = : Making var-scope-local.c out of nothing. : Making var-scope-local.o from var-scope-local.c. : Making basename "var-scope-local.o" in "." from "var-scope-local.c" in ".". diff --git a/unit-tests/var-scope-local.mk b/unit-tests/var-scope-local.mk index ed1362444504..2cfeda044c40 100644 --- a/unit-tests/var-scope-local.mk +++ b/unit-tests/var-scope-local.mk @@ -1,4 +1,4 @@ -# $NetBSD: var-scope-local.mk,v 1.5 2022/02/09 21:09:24 rillig Exp $ +# $NetBSD: var-scope-local.mk,v 1.7 2023/04/29 10:16:24 rillig Exp $ # # Tests for target-local variables, such as ${.TARGET} or $@. These variables # are relatively short-lived as they are created just before making the @@ -12,6 +12,64 @@ .MAIN: all +# Target-local variables in a target rule +# +# In target rules, '$*' only strips the extension off the pathname if the +# extension is listed in '.SUFFIXES'. +# +# expect: target-rule.ext: * = +all: target-rule.ext dir/subdir/target-rule.ext +target-rule.ext dir/subdir/target-rule.ext: .PHONY + @echo '$@: @ = <${@:Uundefined}>' + @echo '$@: % = <${%:Uundefined}>' + @echo '$@: ? = <${?:Uundefined}>' + @echo '$@: < = <${<:Uundefined}>' + @echo '$@: * = <${*:Uundefined}>' + +.SUFFIXES: .ir-gen-from .ir-from .ir-to + +# In target rules, '$*' strips the extension off the pathname of the target +# if the extension is listed in '.SUFFIXES'. +# +# expect: target-rule.ir-gen-from: * = +all: target-rule.ir-gen-from dir/subdir/target-rule-dir.ir-gen-from +target-rule.ir-gen-from dir/subdir/target-rule-dir.ir-gen-from: + @echo '$@: @ = <${@:Uundefined}>' + @echo '$@: % = <${%:Uundefined}>' + @echo '$@: ? = <${?:Uundefined}>' + @echo '$@: < = <${<:Uundefined}>' + @echo '$@: * = <${*:Uundefined}>' + +.ir-from.ir-to: + @echo '$@: @ = <${@:Uundefined}>' + @echo '$@: % = <${%:Uundefined}>' + @echo '$@: ? = <${?:Uundefined}>' + @echo '$@: < = <${<:Uundefined}>' + @echo '$@: * = <${*:Uundefined}>' +.ir-gen-from.ir-from: + @echo '$@: @ = <${@:Uundefined}>' + @echo '$@: % = <${%:Uundefined}>' + @echo '$@: ? = <${?:Uundefined}>' + @echo '$@: < = <${<:Uundefined}>' + @echo '$@: * = <${*:Uundefined}>' + +# Target-local variables in an inference rule +all: inference-rule.ir-to dir/subdir/inference-rule.ir-to +inference-rule.ir-from: .PHONY +dir/subdir/inference-rule.ir-from: .PHONY + +# Target-local variables in a chain of inference rules +all: inference-rule-chain.ir-to dir/subdir/inference-rule-chain.ir-to +inference-rule-chain.ir-gen-from: .PHONY +dir/subdir/inference-rule-chain.ir-gen-from: .PHONY + +# The run-time 'check' directives from above happen after the parse-time +# 'check' directives from below. +# +# expect-reset + +# Deferred evaluation during parsing +# # The target-local variables can be used in expressions, just like other # variables. When these expressions are evaluated outside of a target, these # expressions are not yet expanded, instead their text is preserved, to allow @@ -20,8 +78,8 @@ # # Conditions from .if directives are evaluated in the scope of the command # line, which means that variables from the command line, from the global -# scope and from the environment are resolved, in this order (but see the -# command line option '-e'). In that phase, expressions involving +# scope and from the environment are resolved, in this precedence order (but +# see the command line option '-e'). In that phase, expressions involving # target-local variables need to be preserved, including the exact names of # the variables. # @@ -77,13 +135,17 @@ .endif +# Custom local variables +# # Additional target-local variables may be defined in dependency lines. .MAKEFLAGS: -dv # In the following line, the ':=' may either be interpreted as an assignment # operator or as the dependency operator ':', followed by an empty variable # name and the assignment operator '='. It is the latter since in an -# assignment, the left-hand side must be at most a single word. The empty -# variable name is expanded twice, once for 'one' and once for 'two'. +# assignment, the left-hand side must be a single word or empty. +# +# The empty variable name is expanded twice, once for 'one' and once for +# 'two'. # expect: Var_SetExpand: variable name "" expands to empty string, with value "three" - ignored # expect: Var_SetExpand: variable name "" expands to empty string, with value "three" - ignored one two:=three @@ -205,32 +267,3 @@ a_use: .USE VAR=use all: var-scope-local-use.o var-scope-local-use.o: a_use - - -# Since parse.c 1.656 from 2022-01-27 and before parse.c 1.662 from -# 2022-02-05, there was an out-of-bounds read in Parse_IsVar when looking for -# a variable assignment in a dependency line with trailing whitespace. Lines -# without trailing whitespace were not affected. Global variable assignments -# were guaranteed to have no trailing whitespace and were thus not affected. -# -# Try to reproduce some variants that may lead to a crash, depending on the -# memory allocator. To get a crash, the terminating '\0' of the line must be -# the last byte of a memory page. The expression '${:U}' forces this trailing -# whitespace. - -# On FreeBSD x86_64, a crash could in some cases be forced using the following -# line, which has length 47, so the terminating '\0' may end up at an address -# of the form 0xXXXX_XXXX_XXXX_Xfff: -Try_to_crash_FreeBSD.xxxxxxxxxxxxxxxxxx: 12345 ${:U} - -# The following line has length 4095, so line[4095] == '\0'. If the line is -# allocated on a page boundary and the following page is not mapped, this line -# leads to a segmentation fault. -${:U:range=511:@_@1234567@:ts.}: 12345 ${:U} - -# The following line has length 8191, so line[8191] == '\0'. If the line is -# allocated on a page boundary and the following page is not mapped, this line -# leads to a segmentation fault. -${:U:range=1023:@_@1234567@:ts.}: 12345 ${:U} - -12345: diff --git a/unit-tests/varmod-gmtime.exp b/unit-tests/varmod-gmtime.exp index fdc9a2170e2f..916e471c462b 100644 --- a/unit-tests/varmod-gmtime.exp +++ b/unit-tests/varmod-gmtime.exp @@ -1,13 +1,13 @@ -make: "varmod-gmtime.mk" line 57: Invalid time value at "${:U1593536400}} != "mtime=11593536400}"" -make: "varmod-gmtime.mk" line 57: Malformed conditional (${%Y:L:gmtime=${:U1593536400}} != "mtime=11593536400}") -make: "varmod-gmtime.mk" line 67: Invalid time value at "-1} != """ -make: "varmod-gmtime.mk" line 67: Malformed conditional (${:L:gmtime=-1} != "") -make: "varmod-gmtime.mk" line 76: Invalid time value at " 1} != """ -make: "varmod-gmtime.mk" line 76: Malformed conditional (${:L:gmtime= 1} != "") -make: "varmod-gmtime.mk" line 119: Invalid time value at "10000000000000000000000000000000} != """ -make: "varmod-gmtime.mk" line 119: Malformed conditional (${:L:gmtime=10000000000000000000000000000000} != "") -make: "varmod-gmtime.mk" line 130: Invalid time value at "error} != """ -make: "varmod-gmtime.mk" line 130: Malformed conditional (${:L:gmtime=error} != "") +make: "varmod-gmtime.mk" line 59: Invalid time value "-1" +make: "varmod-gmtime.mk" line 59: Malformed conditional (${:L:gmtime=-1} != "") +make: "varmod-gmtime.mk" line 68: Invalid time value " 1" +make: "varmod-gmtime.mk" line 68: Malformed conditional (${:L:gmtime= 1} != "") +make: "varmod-gmtime.mk" line 114: Invalid time value "10000000000000000000000000000000" +make: "varmod-gmtime.mk" line 114: Malformed conditional (${:L:gmtime=10000000000000000000000000000000} != "") +make: "varmod-gmtime.mk" line 125: Invalid time value "error" +make: "varmod-gmtime.mk" line 125: Malformed conditional (${:L:gmtime=error} != "") +make: "varmod-gmtime.mk" line 134: Invalid time value "100000S,1970,bad," +make: "varmod-gmtime.mk" line 134: Malformed conditional (${%Y:L:gmtime=100000S,1970,bad,} != "bad") make: Fatal errors encountered -- cannot continue make: stopped in unit-tests exit status 1 diff --git a/unit-tests/varmod-gmtime.mk b/unit-tests/varmod-gmtime.mk index cb3d4e7eb241..639c95b24f6b 100644 --- a/unit-tests/varmod-gmtime.mk +++ b/unit-tests/varmod-gmtime.mk @@ -1,7 +1,10 @@ -# $NetBSD: varmod-gmtime.mk,v 1.10 2021/01/19 05:26:34 rillig Exp $ +# $NetBSD: varmod-gmtime.mk,v 1.14 2023/05/10 15:53:32 rillig Exp $ # # Tests for the :gmtime variable modifier, which formats a timestamp # using strftime(3) in UTC. +# +# See also: +# varmod-localtime.mk .if ${TZ:Uundefined} != "undefined" # see unit-tests/Makefile . error @@ -41,20 +44,9 @@ .endif -# As of 2020-08-16, it is not possible to pass the seconds via a -# variable expression. This is because parsing of the :gmtime -# modifier stops at the '$' and returns to ApplyModifiers. -# -# There, a colon would be skipped but not a dollar. -# Parsing therefore continues at the '$' of the ${:U159...}, looking -# for an ordinary variable modifier. -# -# At this point, the ${:U} is expanded and interpreted as a variable -# modifier, which results in the error message "Unknown modifier '1'". -# -# If ApplyModifier_Gmtime were to pass its argument through -# ParseModifierPart, this would work. -.if ${%Y:L:gmtime=${:U1593536400}} != "mtime=11593536400}" +# Before var.c 1.1050 from 2023-05-09, it was not possible to pass the +# seconds via a variable expression. +.if ${%Y:L:gmtime=${:U1593536400}} != "2020" . error .endif @@ -75,6 +67,8 @@ # because it would make sense but just as a side-effect from using strtoul. .if ${:L:gmtime= 1} != "" . error +.else +. error .endif @@ -115,7 +109,8 @@ # ULONG_MAX, which got converted to -1. This resulted in a time stamp of # the second before 1970. # -# Since var.c 1.631, the overflow is detected and produces a parse error. +# Since var.c 1.631 from 2020-10-31, the overflow is detected and produces a +# parse error. .if ${:L:gmtime=10000000000000000000000000000000} != "" . error .else @@ -133,5 +128,11 @@ . error .endif +# Before var.c 1.1050 from 2023-05-09, the timestamp could be directly +# followed by the next modifier, without a ':' separator. This was the same +# bug as for the ':L' and ':P' modifiers. +.if ${%Y:L:gmtime=100000S,1970,bad,} != "bad" +. error +.endif all: diff --git a/unit-tests/varmod-localtime.exp b/unit-tests/varmod-localtime.exp index 494f160b766e..c734c234f083 100644 --- a/unit-tests/varmod-localtime.exp +++ b/unit-tests/varmod-localtime.exp @@ -1,13 +1,13 @@ -make: "varmod-localtime.mk" line 57: Invalid time value at "${:U1593536400}} != "mtime=11593536400}"" -make: "varmod-localtime.mk" line 57: Malformed conditional (${%Y:L:localtime=${:U1593536400}} != "mtime=11593536400}") -make: "varmod-localtime.mk" line 67: Invalid time value at "-1} != """ -make: "varmod-localtime.mk" line 67: Malformed conditional (${:L:localtime=-1} != "") -make: "varmod-localtime.mk" line 76: Invalid time value at " 1} != """ -make: "varmod-localtime.mk" line 76: Malformed conditional (${:L:localtime= 1} != "") -make: "varmod-localtime.mk" line 119: Invalid time value at "10000000000000000000000000000000} != """ -make: "varmod-localtime.mk" line 119: Malformed conditional (${:L:localtime=10000000000000000000000000000000} != "") -make: "varmod-localtime.mk" line 130: Invalid time value at "error} != """ -make: "varmod-localtime.mk" line 130: Malformed conditional (${:L:localtime=error} != "") +make: "varmod-localtime.mk" line 59: Invalid time value "-1" +make: "varmod-localtime.mk" line 59: Malformed conditional (${:L:localtime=-1} != "") +make: "varmod-localtime.mk" line 68: Invalid time value " 1" +make: "varmod-localtime.mk" line 68: Malformed conditional (${:L:localtime= 1} != "") +make: "varmod-localtime.mk" line 114: Invalid time value "10000000000000000000000000000000" +make: "varmod-localtime.mk" line 114: Malformed conditional (${:L:localtime=10000000000000000000000000000000} != "") +make: "varmod-localtime.mk" line 125: Invalid time value "error" +make: "varmod-localtime.mk" line 125: Malformed conditional (${:L:localtime=error} != "") +make: "varmod-localtime.mk" line 134: Invalid time value "100000S,1970,bad," +make: "varmod-localtime.mk" line 134: Malformed conditional (${%Y:L:localtime=100000S,1970,bad,} != "bad") make: Fatal errors encountered -- cannot continue make: stopped in unit-tests exit status 1 diff --git a/unit-tests/varmod-localtime.mk b/unit-tests/varmod-localtime.mk index ffa09a0bc5fc..7d66745aae8e 100644 --- a/unit-tests/varmod-localtime.mk +++ b/unit-tests/varmod-localtime.mk @@ -1,7 +1,10 @@ -# $NetBSD: varmod-localtime.mk,v 1.8 2021/01/19 05:26:34 rillig Exp $ +# $NetBSD: varmod-localtime.mk,v 1.12 2023/05/10 15:53:32 rillig Exp $ # # Tests for the :localtime variable modifier, which formats a timestamp # using strftime(3) in local time. +# +# See also: +# varmod-gmtime.mk .if ${TZ:Uno:NEurope/Berlin:NUTC-1} != "" # see unit-tests/Makefile . error @@ -41,20 +44,9 @@ .endif -# As of 2020-08-16, it is not possible to pass the seconds via a -# variable expression. This is because parsing of the :localtime -# modifier stops at the '$' and returns to ApplyModifiers. -# -# There, a colon would be skipped but not a dollar. -# Parsing therefore continues at the '$' of the ${:U159...}, looking -# for an ordinary variable modifier. -# -# At this point, the ${:U} is expanded and interpreted as a variable -# modifier, which results in the error message "Unknown modifier '1'". -# -# If ApplyModifier_Localtime were to pass its argument through -# ParseModifierPart, this would work. -.if ${%Y:L:localtime=${:U1593536400}} != "mtime=11593536400}" +# Before var.c 1.1050 from 2023-05-09, it was not possible to pass the +# seconds via a variable expression. +.if ${%Y:L:localtime=${:U1593536400}} != "2020" . error .endif @@ -75,6 +67,8 @@ # because it would make sense but just as a side-effect from using strtoul. .if ${:L:localtime= 1} != "" . error +.else +. error .endif @@ -115,7 +109,8 @@ # ULONG_MAX, which got converted to -1. This resulted in a time stamp of # the second before 1970. # -# Since var.c 1.631, the overflow is detected and produces a parse error. +# Since var.c 1.631 from 2020-10-31, the overflow is detected and produces a +# parse error. .if ${:L:localtime=10000000000000000000000000000000} != "" . error .else @@ -133,5 +128,11 @@ . error .endif +# Before var.c 1.1050 from 2023-05-09, the timestamp could be directly +# followed by the next modifier, without a ':' separator. This was the same +# bug as for the ':L' and ':P' modifiers. +.if ${%Y:L:localtime=100000S,1970,bad,} != "bad" +. error +.endif all: diff --git a/unit-tests/varmod-mtime.exp b/unit-tests/varmod-mtime.exp new file mode 100644 index 000000000000..39a9383953dd --- /dev/null +++ b/unit-tests/varmod-mtime.exp @@ -0,0 +1 @@ +exit status 0 diff --git a/unit-tests/varmod-mtime.mk b/unit-tests/varmod-mtime.mk new file mode 100644 index 000000000000..dd57ef3f0f31 --- /dev/null +++ b/unit-tests/varmod-mtime.mk @@ -0,0 +1,30 @@ +# $NetBSD: varmod-mtime.mk,v 1.1 2023/05/09 20:14:27 sjg Exp $ +# +# Tests for the :mtime variable modifier, which provides mtime +# of variable value assumed to be a pathname. + +all: + +# mtime of this makefile +mtime:= ${MAKEFILE:mtime} + +# if pathname does not exist and timestamp is provided +# that is the result +.if ${no/such:L:mtime=0} != "0" +. error +.endif + +.if ${no/such:L:mtime=42} != "42" +. error +.endif + +# if no timestamp is provided and stat(2) fails use current time +.if ${no/such:L:mtime} < ${mtime} +. error no/such:L:mtime ${no/such:L:mtime} < ${mtime} +.endif + +COOKIE = ${TMPDIR}/varmod-mtime.cookie +x!= touch ${COOKIE} +.if ${COOKIE:mtime=0} < ${mtime} +. error COOKIE:mtime=0 ${COOKIE:mtime=0} < ${mtime} +.endif diff --git a/unit-tests/varmod-path.mk b/unit-tests/varmod-path.mk index ebbf755ddbec..25d4e3899b99 100644 --- a/unit-tests/varmod-path.mk +++ b/unit-tests/varmod-path.mk @@ -1,4 +1,4 @@ -# $NetBSD: varmod-path.mk,v 1.3 2020/08/23 08:10:49 rillig Exp $ +# $NetBSD: varmod-path.mk,v 1.4 2023/05/10 15:53:32 rillig Exp $ # # Tests for the :P variable modifier, which looks up the path for a given # target. @@ -7,11 +7,12 @@ # as of 2020-08-23 it is nevertheless resolved to a path. This is probably # unintended. # -# The real target is located in a subdirectory, and its full path is returned. -# If it had been in the current directory, the difference between its path and -# its name would not be visible. +# In this test, the real target is located in a subdirectory, and its full +# path is returned. If it had been in the current directory, the difference +# between its path and its name would not be visible. # -# The enoent target does not exist, therefore the target name is returned. +# The enoent target does not exist, therefore the plain name of the target +# is returned. .MAIN: all @@ -20,7 +21,8 @@ _!= mkdir varmod-path.subdir _!= > varmod-path.subdir/varmod-path.phony _!= > varmod-path.subdir/varmod-path.real -# To have an effect, this .PATH declaration must be after the directory is created. +# To have an effect, this .PATH declaration must be processed after the +# directory has been created. .PATH: varmod-path.subdir varmod-path.phony: .PHONY diff --git a/var.c b/var.c index 965080bbcddd..074a96639bdb 100644 --- a/var.c +++ b/var.c @@ -1,4 +1,4 @@ -/* $NetBSD: var.c,v 1.1049 2023/03/28 14:39:31 rillig Exp $ */ +/* $NetBSD: var.c,v 1.1054 2023/05/10 18:22:33 sjg Exp $ */ /* * Copyright (c) 1988, 1989, 1990, 1993 @@ -147,7 +147,7 @@ #include "metachar.h" /* "@(#)var.c 8.3 (Berkeley) 3/19/94" */ -MAKE_RCSID("$NetBSD: var.c,v 1.1049 2023/03/28 14:39:31 rillig Exp $"); +MAKE_RCSID("$NetBSD: var.c,v 1.1054 2023/05/10 18:22:33 sjg Exp $"); /* * Variables are defined using one of the VAR=value assignments. Their @@ -2196,6 +2196,8 @@ ParseModifierPartBalanced(const char **pp, LazyBuf *part) static bool ParseModifierPartSubst( const char **pp, + /* If true, parse up to but excluding the next ':' or ch->endc. */ + bool whole, char delim, VarEvalMode emode, ModChain *ch, @@ -2213,11 +2215,14 @@ ParseModifierPartSubst( ) { const char *p; + char end1, end2; p = *pp; LazyBuf_Init(part, p); - while (*p != '\0' && *p != delim) { + end1 = whole ? ':' : delim; + end2 = whole ? ch->endc : delim; + while (*p != '\0' && *p != end1 && *p != end2) { if (IsEscapedModifierPart(p, delim, subst)) { LazyBuf_Add(part, p[1]); p += 2; @@ -2239,15 +2244,15 @@ ParseModifierPartSubst( ParseModifierPartExpr(&p, part, ch, emode); } - if (*p != delim) { - *pp = p; + *pp = p; + if (*p != end1 && *p != end2) { Error("Unfinished modifier for \"%s\" ('%c' missing)", - ch->expr->name, delim); + ch->expr->name, end2); LazyBuf_Done(part); return false; } - - *pp = p + 1; + if (!whole) + (*pp)++; { Substring sub = LazyBuf_Get(part); @@ -2280,7 +2285,8 @@ ParseModifierPart( LazyBuf *part ) { - return ParseModifierPartSubst(pp, delim, emode, ch, part, NULL, NULL); + return ParseModifierPartSubst(pp, false, delim, emode, ch, part, + NULL, NULL); } MAKE_INLINE bool @@ -2576,11 +2582,23 @@ ApplyModifier_Time(const char **pp, ModChain *ch) if (args[0] == '=') { const char *p = args + 1; - if (!TryParseTime(&p, &t)) { - Parse_Error(PARSE_FATAL, - "Invalid time value at \"%s\"", p); + LazyBuf buf; + if (!ParseModifierPartSubst(&p, true, '\0', ch->expr->emode, + ch, &buf, NULL, NULL)) return AMR_CLEANUP; - } + if (ModChain_ShouldEval(ch)) { + Substring arg = LazyBuf_Get(&buf); + const char *arg_p = arg.start; + if (!TryParseTime(&arg_p, &t) || arg_p != arg.end) { + Parse_Error(PARSE_FATAL, + "Invalid time value \"%.*s\"", + (int)Substring_Length(arg), arg.start); + LazyBuf_Done(&buf); + return AMR_CLEANUP; + } + } else + t = 0; + LazyBuf_Done(&buf); *pp = p; } else { t = 0; @@ -2823,6 +2841,71 @@ ApplyModifier_Match(const char **pp, ModChain *ch) return AMR_OK; } +struct ModifyWord_MtimeArgs { + bool error; + bool fallback; + ApplyModifierResult rc; + time_t t; +}; + +static void +ModifyWord_Mtime(Substring word, SepBuf *buf, void *data) +{ + char tbuf[BUFSIZ]; + struct stat st; + struct ModifyWord_MtimeArgs *args = data; + + if (Substring_IsEmpty(word)) + return; + assert(word.end[0] == '\0'); /* assume null-terminated word */ + if (stat(word.start, &st) < 0) { + if (args->error) { + Parse_Error(PARSE_FATAL, + "Cannot determine mtime for '%s': %s", + word.start, strerror(errno)); + args->rc = AMR_CLEANUP; + return; + } + if (args->fallback) + st.st_mtime = args->t; + else + time(&st.st_mtime); + } + snprintf(tbuf, sizeof(tbuf), "%u", (unsigned)st.st_mtime); + SepBuf_AddStr(buf, tbuf); +} + +/* :mtime */ +static ApplyModifierResult +ApplyModifier_Mtime(const char **pp, ModChain *ch) +{ + const char *p, *mod = *pp; + struct ModifyWord_MtimeArgs args; + + if (!ModMatchEq(mod, "mtime", ch)) + return AMR_UNKNOWN; + *pp += 5; + p = *pp; + args.error = args.fallback = false; + args.rc = AMR_OK; + if (p[0] == '=') { + p++; + args.fallback = true; + if (!TryParseTime(&p, &args.t)) { + if (strncmp(p, "error", 5) == 0) { + args.error = true; + p += 5; + } else + return AMR_BAD; + } + *pp = p; + } + if (!ModChain_ShouldEval(ch)) + return AMR_OK; + ModifyWords(ch, ModifyWord_Mtime, &args, ch->oneBigWord); + return args.rc; +} + static void ParsePatternFlags(const char **pp, PatternFlags *pflags, bool *oneBigWord) { @@ -2870,13 +2953,13 @@ ApplyModifier_Subst(const char **pp, ModChain *ch) (*pp)++; } - if (!ParseModifierPartSubst(pp, delim, ch->expr->emode, ch, &lhsBuf, - &args.pflags, NULL)) + if (!ParseModifierPartSubst(pp, + false, delim, ch->expr->emode, ch, &lhsBuf, &args.pflags, NULL)) return AMR_CLEANUP; args.lhs = LazyBuf_Get(&lhsBuf); - if (!ParseModifierPartSubst(pp, delim, ch->expr->emode, ch, &rhsBuf, - NULL, &args)) { + if (!ParseModifierPartSubst(pp, + false, delim, ch->expr->emode, ch, &rhsBuf, NULL, &args)) { LazyBuf_Done(&lhsBuf); return AMR_CLEANUP; } @@ -3805,6 +3888,8 @@ ApplyModifier(const char **pp, ModChain *ch) case 'M': case 'N': return ApplyModifier_Match(pp, ch); + case 'm': + return ApplyModifier_Mtime(pp, ch); case 'O': return ApplyModifier_Order(pp, ch); case 'P':