mirror of
https://github.com/python/cpython
synced 2024-09-18 20:51:39 +00:00
71876fa438
* Refactor Parser/pgen and add documentation and explanations
To improve the readability and maintainability of the parser
generator perform the following transformations:
* Separate the metagrammar parser in its own class to simplify
the parser generator logic.
* Create separate classes for DFAs and NFAs and move methods that
act exclusively on them from the parser generator to these
classes.
* Add docstrings and comment documenting the process to go from
the grammar file into NFAs and then DFAs. Detail some of the
algorithms and give some background explanations of some concepts
that will helps readers not familiar with the parser generation
process.
* Select more descriptive names for some variables and variables.
* PEP8 formatting and quote-style homogenization.
The output of the parser generator remains the same (Include/graminit.h
and Python/graminit.c remain untouched by running the new parser generator).
60 lines
1.6 KiB
Python
60 lines
1.6 KiB
Python
"""Generate Lib/keyword.py from the Grammar and Tokens files using pgen"""
|
|
|
|
import argparse
|
|
|
|
from .pgen import ParserGenerator
|
|
|
|
TEMPLATE = r'''
|
|
"""Keywords (from "Grammar/Grammar")
|
|
|
|
This file is automatically generated; please don't muck it up!
|
|
|
|
To update the symbols in this file, 'cd' to the top directory of
|
|
the python source tree and run:
|
|
|
|
python3 -m Parser.pgen.keywordgen Grammar/Grammar \
|
|
Grammar/Tokens \
|
|
Lib/keyword.py
|
|
|
|
Alternatively, you can run 'make regen-keyword'.
|
|
"""
|
|
|
|
__all__ = ["iskeyword", "kwlist"]
|
|
|
|
kwlist = [
|
|
{keywords}
|
|
]
|
|
|
|
iskeyword = frozenset(kwlist).__contains__
|
|
'''.lstrip()
|
|
|
|
EXTRA_KEYWORDS = ["async", "await"]
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(
|
|
description="Generate the Lib/keywords.py " "file from the grammar."
|
|
)
|
|
parser.add_argument(
|
|
"grammar", type=str, help="The file with the grammar definition in EBNF format"
|
|
)
|
|
parser.add_argument("tokens", type=str, help="The file with the token definitions")
|
|
parser.add_argument(
|
|
"keyword_file",
|
|
type=argparse.FileType("w"),
|
|
help="The path to write the keyword definitions",
|
|
)
|
|
args = parser.parse_args()
|
|
p = ParserGenerator(args.grammar, args.tokens)
|
|
grammar = p.make_grammar()
|
|
|
|
with args.keyword_file as thefile:
|
|
all_keywords = sorted(list(grammar.keywords) + EXTRA_KEYWORDS)
|
|
|
|
keywords = ",\n ".join(map(repr, all_keywords))
|
|
thefile.write(TEMPLATE.format(keywords=keywords))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|