freebsd-src/awkgram.y
Warner Losh 2e406c584f ota: Import One True Awk from 20231102 (254b979f32df)
This is a rollup of a lot of changes. In summary, lots of bug fixes,
Unicode support and CSV support to match the 2nd Edition of the Awk
Book.

In detail, from AWK's FIXES and FIXES.1e:

From FIXES:
Oct 30, 2023:
	multiple fixes and a minor code cleanup.
	disabled utf-8 for non-multibyte locales, such as C or POSIX.
	fixed a bad char * cast that causes incorrect results on big-endian
	systems. also fixed an out-of-bounds read for empty CCL.
	fixed a buffer overflow in substr with utf-8 strings.
	many thanks to Todd C Miller.

Sep 24, 2023:
	fnematch and getrune have been overhauled to solve issues around
	unicode FS and RS. also fixed gsub null match issue with unicode.
	big thanks to Arnold Robbins.

Sep 12, 2023:
	Fixed a length error in u8_byte2char that set RSTART to
	incorrect (cannot happen) value for EOL match(str, /$/).

-----------------------------------------------------------------

[This entry is a summary, not a precise list of changes.]

	Added --csv option to enable processing of comma-separated
	values inputs.  When --csv is enabled, fields are separated
	by commas, fields may be quoted with " double quotes, fields
	may contain embedded newlines.

	If no explicit separator argument is provided, split() uses
	the setting of --csv to determine how fields are split.

	Strings may now contain UTF-8 code points (not necessarily
	characters).  Functions that operate on characters, like
	length, substr, index, match, etc., use UTF-8, so the length
	of a string of 3 emojis is 3, not 12 as it would be if bytes
	were counted.

	Regular expressions are processes as UTF-8.

	Unicode literals can be written as \u followed by one
	to eight hexadecimal digits.  These may appear in strings and
	regular expressions.

From FIXES.1e:

Sep 06, 2023:
	Fix edge case where FS is changed on commandline. Thanks to
	Gordon Shephard and Miguel Pineiro Jr.

	Fix regular expression clobbering in the lexer, where lexer does
	not make a copy of regexp literals. also makedfa memory leaks have
	been plugged. Thanks to Miguel Pineiro Jr.

Dec 15, 2022:
	Force hex escapes in strings to be no more than two characters,
	as they already are in regular expressions. This brings internal
	consistency, as well as consistency with gawk. Thanks to
	Arnold Robbins.

Sep 12, 2022:
	adjbuf minlen error (cannot be 0) in cat, resulting in NULL pbuf.
	discovered by todd miller. also use-after-free issue with
	tempfree in cat, thanks to Miguel Pineiro Jr and valgrind.

Aug 30, 2022:
	Various leaks and use-after-free issues plugged/fixed.
	Thanks to Miguel Pineiro Jr. <mpj@pineiro.cc>.

May 23, 2022:
	Memory leak when assigning a string to some of the built-in
	variables. allocated string erroneously marked DONTFREE.
	Thanks to Miguel Pineiro Jr. <mpj@pineiro.cc>.

Mar 14, 2022:
	Historic bug: command-line "name=value" assignment had been
	truncating its entry in ARGV. (circa 1989) Thanks to
	Miguel Pineiro Jr. <mpj@pineiro.cc>.

Mar 3, 2022:
	Fixed file management memory leak that appears to have been
	there since the files array was first initialized with stdin,
	stdout, and stderr (circa 1992). Thanks to Miguel Pineiro Jr.
	<mpj@pineiro.cc>.

December 8, 2021:
	The error handling in closefile and closeall was mangled. Long
	standing warnings had been made fatal and some fatal errors went
	undetected. Thanks to Miguel Pineiro Jr. <mpj@pineiro.cc>.

Nov 03, 2021:
        getline accesses uninitialized data after getrec()
	returns 0 on EOF and leaves the contents of buf unchanged.
	Thanks to Volodymyr Gubarkov, and Todd C Miller.

Oct 12, 2021:
	The fix for #83 changed the code to insert 2 chars, but the
	call to adjbuf just above it only allows for 1 char. This can
	cause a heap buffer overflow.

Sponsored by:		Netflix
2023-11-02 10:20:09 -06:00

516 lines
15 KiB
Plaintext

/****************************************************************
Copyright (C) Lucent Technologies 1997
All Rights Reserved
Permission to use, copy, modify, and distribute this software and
its documentation for any purpose and without fee is hereby
granted, provided that the above copyright notice appear in all
copies and that both that the copyright notice and this
permission notice and warranty disclaimer appear in supporting
documentation, and that the name Lucent Technologies or any of
its entities not be used in advertising or publicity pertaining
to distribution of the software without specific, written prior
permission.
LUCENT DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE,
INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS.
IN NO EVENT SHALL LUCENT OR ANY OF ITS ENTITIES BE LIABLE FOR ANY
SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER
IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION,
ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF
THIS SOFTWARE.
****************************************************************/
%{
#include <stdio.h>
#include <string.h>
#include "awk.h"
void checkdup(Node *list, Cell *item);
int yywrap(void) { return(1); }
Node *beginloc = 0;
Node *endloc = 0;
bool infunc = false; /* = true if in arglist or body of func */
int inloop = 0; /* >= 1 if in while, for, do; can't be bool, since loops can next */
char *curfname = 0; /* current function name */
Node *arglist = 0; /* list of args for current function */
%}
%union {
Node *p;
Cell *cp;
int i;
char *s;
}
%token <i> FIRSTTOKEN /* must be first */
%token <p> PROGRAM PASTAT PASTAT2 XBEGIN XEND
%token <i> NL ',' '{' '(' '|' ';' '/' ')' '}' '[' ']'
%token <i> ARRAY
%token <i> MATCH NOTMATCH MATCHOP
%token <i> FINAL DOT ALL CCL NCCL CHAR OR STAR QUEST PLUS EMPTYRE ZERO
%token <i> AND BOR APPEND EQ GE GT LE LT NE IN
%token <i> ARG BLTIN BREAK CLOSE CONTINUE DELETE DO EXIT FOR FUNC
%token <i> GENSUB SUB GSUB IF INDEX LSUBSTR MATCHFCN NEXT NEXTFILE
%token <i> ADD MINUS MULT DIVIDE MOD
%token <i> ASSIGN ASGNOP ADDEQ SUBEQ MULTEQ DIVEQ MODEQ POWEQ
%token <i> PRINT PRINTF SPRINTF
%token <p> ELSE INTEST CONDEXPR
%token <i> POSTINCR PREINCR POSTDECR PREDECR
%token <cp> VAR IVAR VARNF CALL NUMBER STRING
%token <s> REGEXPR
%type <p> pas pattern ppattern plist pplist patlist prarg term re
%type <p> pa_pat pa_stat pa_stats
%type <s> reg_expr
%type <p> simple_stmt opt_simple_stmt stmt stmtlist
%type <p> var varname funcname varlist
%type <p> for if else while
%type <i> do st
%type <i> pst opt_pst lbrace rbrace rparen comma nl opt_nl and bor
%type <i> subop print
%type <cp> string
%right ASGNOP
%right '?'
%right ':'
%left BOR
%left AND
%left GETLINE
%nonassoc APPEND EQ GE GT LE LT NE MATCHOP IN '|'
%left ARG BLTIN BREAK CALL CLOSE CONTINUE DELETE DO EXIT FOR FUNC
%left GSUB IF INDEX LSUBSTR MATCHFCN NEXT NUMBER
%left PRINT PRINTF RETURN SPLIT SPRINTF STRING SUB SUBSTR
%left REGEXPR VAR VARNF IVAR WHILE '('
%left CAT
%left '+' '-'
%left '*' '/' '%'
%left NOT UMINUS UPLUS
%right POWER
%right DECR INCR
%left INDIRECT
%token LASTTOKEN /* must be last */
%%
program:
pas { if (errorflag==0)
winner = (Node *)stat3(PROGRAM, beginloc, $1, endloc); }
| error { yyclearin; bracecheck(); SYNTAX("bailing out"); }
;
and:
AND | and NL
;
bor:
BOR | bor NL
;
comma:
',' | comma NL
;
do:
DO | do NL
;
else:
ELSE | else NL
;
for:
FOR '(' opt_simple_stmt ';' opt_nl pattern ';' opt_nl opt_simple_stmt rparen {inloop++;} stmt
{ --inloop; $$ = stat4(FOR, $3, notnull($6), $9, $12); }
| FOR '(' opt_simple_stmt ';' ';' opt_nl opt_simple_stmt rparen {inloop++;} stmt
{ --inloop; $$ = stat4(FOR, $3, NIL, $7, $10); }
| FOR '(' varname IN varname rparen {inloop++;} stmt
{ --inloop; $$ = stat3(IN, $3, makearr($5), $8); }
;
funcname:
VAR { setfname($1); }
| CALL { setfname($1); }
;
if:
IF '(' pattern rparen { $$ = notnull($3); }
;
lbrace:
'{' | lbrace NL
;
nl:
NL | nl NL
;
opt_nl:
/* empty */ { $$ = 0; }
| nl
;
opt_pst:
/* empty */ { $$ = 0; }
| pst
;
opt_simple_stmt:
/* empty */ { $$ = 0; }
| simple_stmt
;
pas:
opt_pst { $$ = 0; }
| opt_pst pa_stats opt_pst { $$ = $2; }
;
pa_pat:
pattern { $$ = notnull($1); }
;
pa_stat:
pa_pat { $$ = stat2(PASTAT, $1, stat2(PRINT, rectonode(), NIL)); }
| pa_pat lbrace stmtlist '}' { $$ = stat2(PASTAT, $1, $3); }
| pa_pat ',' opt_nl pa_pat { $$ = pa2stat($1, $4, stat2(PRINT, rectonode(), NIL)); }
| pa_pat ',' opt_nl pa_pat lbrace stmtlist '}' { $$ = pa2stat($1, $4, $6); }
| lbrace stmtlist '}' { $$ = stat2(PASTAT, NIL, $2); }
| XBEGIN lbrace stmtlist '}'
{ beginloc = linkum(beginloc, $3); $$ = 0; }
| XEND lbrace stmtlist '}'
{ endloc = linkum(endloc, $3); $$ = 0; }
| FUNC funcname '(' varlist rparen {infunc = true;} lbrace stmtlist '}'
{ infunc = false; curfname=0; defn((Cell *)$2, $4, $8); $$ = 0; }
;
pa_stats:
pa_stat
| pa_stats opt_pst pa_stat { $$ = linkum($1, $3); }
;
patlist:
pattern
| patlist comma pattern { $$ = linkum($1, $3); }
;
ppattern:
var ASGNOP ppattern { $$ = op2($2, $1, $3); }
| ppattern '?' ppattern ':' ppattern %prec '?'
{ $$ = op3(CONDEXPR, notnull($1), $3, $5); }
| ppattern bor ppattern %prec BOR
{ $$ = op2(BOR, notnull($1), notnull($3)); }
| ppattern and ppattern %prec AND
{ $$ = op2(AND, notnull($1), notnull($3)); }
| ppattern MATCHOP reg_expr { $$ = op3($2, NIL, $1, (Node*)makedfa($3, 0)); free($3); }
| ppattern MATCHOP ppattern
{ if (constnode($3)) {
$$ = op3($2, NIL, $1, (Node*)makedfa(strnode($3), 0));
free($3);
} else
$$ = op3($2, (Node *)1, $1, $3); }
| ppattern IN varname { $$ = op2(INTEST, $1, makearr($3)); }
| '(' plist ')' IN varname { $$ = op2(INTEST, $2, makearr($5)); }
| ppattern term %prec CAT { $$ = op2(CAT, $1, $2); }
| re
| term
;
pattern:
var ASGNOP pattern { $$ = op2($2, $1, $3); }
| pattern '?' pattern ':' pattern %prec '?'
{ $$ = op3(CONDEXPR, notnull($1), $3, $5); }
| pattern bor pattern %prec BOR
{ $$ = op2(BOR, notnull($1), notnull($3)); }
| pattern and pattern %prec AND
{ $$ = op2(AND, notnull($1), notnull($3)); }
| pattern EQ pattern { $$ = op2($2, $1, $3); }
| pattern GE pattern { $$ = op2($2, $1, $3); }
| pattern GT pattern { $$ = op2($2, $1, $3); }
| pattern LE pattern { $$ = op2($2, $1, $3); }
| pattern LT pattern { $$ = op2($2, $1, $3); }
| pattern NE pattern { $$ = op2($2, $1, $3); }
| pattern MATCHOP reg_expr { $$ = op3($2, NIL, $1, (Node*)makedfa($3, 0)); free($3); }
| pattern MATCHOP pattern
{ if (constnode($3)) {
$$ = op3($2, NIL, $1, (Node*)makedfa(strnode($3), 0));
free($3);
} else
$$ = op3($2, (Node *)1, $1, $3); }
| pattern IN varname { $$ = op2(INTEST, $1, makearr($3)); }
| '(' plist ')' IN varname { $$ = op2(INTEST, $2, makearr($5)); }
| pattern '|' GETLINE var {
if (safe) SYNTAX("cmd | getline is unsafe");
else $$ = op3(GETLINE, $4, itonp($2), $1); }
| pattern '|' GETLINE {
if (safe) SYNTAX("cmd | getline is unsafe");
else $$ = op3(GETLINE, (Node*)0, itonp($2), $1); }
| pattern term %prec CAT { $$ = op2(CAT, $1, $2); }
| re
| term
;
plist:
pattern comma pattern { $$ = linkum($1, $3); }
| plist comma pattern { $$ = linkum($1, $3); }
;
pplist:
ppattern
| pplist comma ppattern { $$ = linkum($1, $3); }
;
prarg:
/* empty */ { $$ = rectonode(); }
| pplist
| '(' plist ')' { $$ = $2; }
;
print:
PRINT | PRINTF
;
pst:
NL | ';' | pst NL | pst ';'
;
rbrace:
'}' | rbrace NL
;
re:
reg_expr
{ $$ = op3(MATCH, NIL, rectonode(), (Node*)makedfa($1, 0)); free($1); }
| NOT re { $$ = op1(NOT, notnull($2)); }
;
reg_expr:
'/' {startreg();} REGEXPR '/' { $$ = $3; }
;
rparen:
')' | rparen NL
;
simple_stmt:
print prarg '|' term {
if (safe) SYNTAX("print | is unsafe");
else $$ = stat3($1, $2, itonp($3), $4); }
| print prarg APPEND term {
if (safe) SYNTAX("print >> is unsafe");
else $$ = stat3($1, $2, itonp($3), $4); }
| print prarg GT term {
if (safe) SYNTAX("print > is unsafe");
else $$ = stat3($1, $2, itonp($3), $4); }
| print prarg { $$ = stat3($1, $2, NIL, NIL); }
| DELETE varname '[' patlist ']' { $$ = stat2(DELETE, makearr($2), $4); }
| DELETE varname { $$ = stat2(DELETE, makearr($2), 0); }
| pattern { $$ = exptostat($1); }
| error { yyclearin; SYNTAX("illegal statement"); }
;
st:
nl
| ';' opt_nl
;
stmt:
BREAK st { if (!inloop) SYNTAX("break illegal outside of loops");
$$ = stat1(BREAK, NIL); }
| CONTINUE st { if (!inloop) SYNTAX("continue illegal outside of loops");
$$ = stat1(CONTINUE, NIL); }
| do {inloop++;} stmt {--inloop;} WHILE '(' pattern ')' st
{ $$ = stat2(DO, $3, notnull($7)); }
| EXIT pattern st { $$ = stat1(EXIT, $2); }
| EXIT st { $$ = stat1(EXIT, NIL); }
| for
| if stmt else stmt { $$ = stat3(IF, $1, $2, $4); }
| if stmt { $$ = stat3(IF, $1, $2, NIL); }
| lbrace stmtlist rbrace { $$ = $2; }
| NEXT st { if (infunc)
SYNTAX("next is illegal inside a function");
$$ = stat1(NEXT, NIL); }
| NEXTFILE st { if (infunc)
SYNTAX("nextfile is illegal inside a function");
$$ = stat1(NEXTFILE, NIL); }
| RETURN pattern st { $$ = stat1(RETURN, $2); }
| RETURN st { $$ = stat1(RETURN, NIL); }
| simple_stmt st
| while {inloop++;} stmt { --inloop; $$ = stat2(WHILE, $1, $3); }
| ';' opt_nl { $$ = 0; }
;
stmtlist:
stmt
| stmtlist stmt { $$ = linkum($1, $2); }
;
subop:
SUB | GSUB
;
string:
STRING
| string STRING { $$ = catstr($1, $2); }
;
term:
term '/' ASGNOP term { $$ = op2(DIVEQ, $1, $4); }
| term '+' term { $$ = op2(ADD, $1, $3); }
| term '-' term { $$ = op2(MINUS, $1, $3); }
| term '*' term { $$ = op2(MULT, $1, $3); }
| term '/' term { $$ = op2(DIVIDE, $1, $3); }
| term '%' term { $$ = op2(MOD, $1, $3); }
| term POWER term { $$ = op2(POWER, $1, $3); }
| '-' term %prec UMINUS { $$ = op1(UMINUS, $2); }
| '+' term %prec UMINUS { $$ = op1(UPLUS, $2); }
| NOT term %prec UMINUS { $$ = op1(NOT, notnull($2)); }
| BLTIN '(' ')' { $$ = op2(BLTIN, itonp($1), rectonode()); }
| BLTIN '(' patlist ')' { $$ = op2(BLTIN, itonp($1), $3); }
| BLTIN { $$ = op2(BLTIN, itonp($1), rectonode()); }
| CALL '(' ')' { $$ = op2(CALL, celltonode($1,CVAR), NIL); }
| CALL '(' patlist ')' { $$ = op2(CALL, celltonode($1,CVAR), $3); }
| CLOSE term { $$ = op1(CLOSE, $2); }
| DECR var { $$ = op1(PREDECR, $2); }
| INCR var { $$ = op1(PREINCR, $2); }
| var DECR { $$ = op1(POSTDECR, $1); }
| var INCR { $$ = op1(POSTINCR, $1); }
| GENSUB '(' reg_expr comma pattern comma pattern ')'
{ $$ = op5(GENSUB, NIL, (Node*)makedfa($3, 1), $5, $7, rectonode()); }
| GENSUB '(' pattern comma pattern comma pattern ')'
{ if (constnode($3)) {
$$ = op5(GENSUB, NIL, (Node *)makedfa(strnode($3), 1), $5, $7, rectonode());
free($3);
} else
$$ = op5(GENSUB, (Node *)1, $3, $5, $7, rectonode());
}
| GENSUB '(' reg_expr comma pattern comma pattern comma pattern ')'
{ $$ = op5(GENSUB, NIL, (Node*)makedfa($3, 1), $5, $7, $9); }
| GENSUB '(' pattern comma pattern comma pattern comma pattern ')'
{ if (constnode($3)) {
$$ = op5(GENSUB, NIL, (Node *)makedfa(strnode($3),1), $5,$7,$9);
free($3);
} else
$$ = op5(GENSUB, (Node *)1, $3, $5, $7, $9);
}
| GETLINE var LT term { $$ = op3(GETLINE, $2, itonp($3), $4); }
| GETLINE LT term { $$ = op3(GETLINE, NIL, itonp($2), $3); }
| GETLINE var { $$ = op3(GETLINE, $2, NIL, NIL); }
| GETLINE { $$ = op3(GETLINE, NIL, NIL, NIL); }
| INDEX '(' pattern comma pattern ')'
{ $$ = op2(INDEX, $3, $5); }
| INDEX '(' pattern comma reg_expr ')'
{ SYNTAX("index() doesn't permit regular expressions");
$$ = op2(INDEX, $3, (Node*)$5); }
| '(' pattern ')' { $$ = $2; }
| MATCHFCN '(' pattern comma reg_expr ')'
{ $$ = op3(MATCHFCN, NIL, $3, (Node*)makedfa($5, 1)); free($5); }
| MATCHFCN '(' pattern comma pattern ')'
{ if (constnode($5)) {
$$ = op3(MATCHFCN, NIL, $3, (Node*)makedfa(strnode($5), 1));
free($5);
} else
$$ = op3(MATCHFCN, (Node *)1, $3, $5); }
| NUMBER { $$ = celltonode($1, CCON); }
| SPLIT '(' pattern comma varname comma pattern ')' /* string */
{ $$ = op4(SPLIT, $3, makearr($5), $7, (Node*)STRING); }
| SPLIT '(' pattern comma varname comma reg_expr ')' /* const /regexp/ */
{ $$ = op4(SPLIT, $3, makearr($5), (Node*)makedfa($7, 1), (Node *)REGEXPR); free($7); }
| SPLIT '(' pattern comma varname ')'
{ $$ = op4(SPLIT, $3, makearr($5), NIL, (Node*)STRING); } /* default */
| SPRINTF '(' patlist ')' { $$ = op1($1, $3); }
| string { $$ = celltonode($1, CCON); }
| subop '(' reg_expr comma pattern ')'
{ $$ = op4($1, NIL, (Node*)makedfa($3, 1), $5, rectonode()); free($3); }
| subop '(' pattern comma pattern ')'
{ if (constnode($3)) {
$$ = op4($1, NIL, (Node*)makedfa(strnode($3), 1), $5, rectonode());
free($3);
} else
$$ = op4($1, (Node *)1, $3, $5, rectonode()); }
| subop '(' reg_expr comma pattern comma var ')'
{ $$ = op4($1, NIL, (Node*)makedfa($3, 1), $5, $7); free($3); }
| subop '(' pattern comma pattern comma var ')'
{ if (constnode($3)) {
$$ = op4($1, NIL, (Node*)makedfa(strnode($3), 1), $5, $7);
free($3);
} else
$$ = op4($1, (Node *)1, $3, $5, $7); }
| SUBSTR '(' pattern comma pattern comma pattern ')'
{ $$ = op3(SUBSTR, $3, $5, $7); }
| SUBSTR '(' pattern comma pattern ')'
{ $$ = op3(SUBSTR, $3, $5, NIL); }
| var
;
var:
varname
| varname '[' patlist ']' { $$ = op2(ARRAY, makearr($1), $3); }
| IVAR { $$ = op1(INDIRECT, celltonode($1, CVAR)); }
| INDIRECT term { $$ = op1(INDIRECT, $2); }
;
varlist:
/* nothing */ { arglist = $$ = 0; }
| VAR { arglist = $$ = celltonode($1,CVAR); }
| varlist comma VAR {
checkdup($1, $3);
arglist = $$ = linkum($1,celltonode($3,CVAR)); }
;
varname:
VAR { $$ = celltonode($1, CVAR); }
| ARG { $$ = op1(ARG, itonp($1)); }
| VARNF { $$ = op1(VARNF, (Node *) $1); }
;
while:
WHILE '(' pattern rparen { $$ = notnull($3); }
;
%%
void setfname(Cell *p)
{
if (isarr(p))
SYNTAX("%s is an array, not a function", p->nval);
else if (isfcn(p))
SYNTAX("you can't define function %s more than once", p->nval);
curfname = p->nval;
}
int constnode(Node *p)
{
return isvalue(p) && ((Cell *) (p->narg[0]))->csub == CCON;
}
char *strnode(Node *p)
{
return ((Cell *)(p->narg[0]))->sval;
}
Node *notnull(Node *n)
{
switch (n->nobj) {
case LE: case LT: case EQ: case NE: case GT: case GE:
case BOR: case AND: case NOT:
return n;
default:
return op2(NE, n, nullnode);
}
}
void checkdup(Node *vl, Cell *cp) /* check if name already in list */
{
char *s = cp->nval;
for ( ; vl; vl = vl->nnext) {
if (strcmp(s, ((Cell *)(vl->narg[0]))->nval) == 0) {
SYNTAX("duplicate argument %s", s);
break;
}
}
}