From f28ec34c5c69f45dc0db8bc26df69bc7defe9c9d Mon Sep 17 00:00:00 2001 From: Ezio Melotti Date: Tue, 21 Jun 2022 22:03:12 +0200 Subject: [PATCH] gh-82927: Update files related to HTML entities. (GH-92504) --- .github/CODEOWNERS | 1 + Doc/library/html.entities.rst | 4 +- Lib/html/entities.py | 9 ++- ...2-06-19-14-56-33.gh-issue-86087.R8MkRy.rst | 2 + Tools/scripts/parse_html5_entities.py | 27 +++++--- Tools/scripts/parseentities.py | 64 ------------------- 6 files changed, 29 insertions(+), 78 deletions(-) create mode 100644 Misc/NEWS.d/next/Tools-Demos/2022-06-19-14-56-33.gh-issue-86087.R8MkRy.rst delete mode 100755 Tools/scripts/parseentities.py diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 58023b63833..013e1cbd724 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -53,6 +53,7 @@ Python/pythonrun.c @iritkatriel /Lib/html/ @ezio-melotti /Lib/_markupbase.py @ezio-melotti /Lib/test/test_html*.py @ezio-melotti +/Tools/scripts/*html5* @ezio-melotti # Import (including importlib). # Ignoring importlib.h so as to not get flagged on diff --git a/Doc/library/html.entities.rst b/Doc/library/html.entities.rst index cb8b623781c..10529561a92 100644 --- a/Doc/library/html.entities.rst +++ b/Doc/library/html.entities.rst @@ -34,12 +34,12 @@ This module defines four dictionaries, :data:`html5`, .. data:: name2codepoint - A dictionary that maps HTML entity names to the Unicode code points. + A dictionary that maps HTML4 entity names to the Unicode code points. .. data:: codepoint2name - A dictionary that maps Unicode code points to HTML entity names. + A dictionary that maps Unicode code points to HTML4 entity names. .. rubric:: Footnotes diff --git a/Lib/html/entities.py b/Lib/html/entities.py index dc508631ac4..cc59bc31449 100644 --- a/Lib/html/entities.py +++ b/Lib/html/entities.py @@ -3,8 +3,7 @@ __all__ = ['html5', 'name2codepoint', 'codepoint2name', 'entitydefs'] -# maps the HTML entity name to the Unicode code point -# from https://html.spec.whatwg.org/multipage/named-characters.html +# maps HTML4 entity name to the Unicode code point name2codepoint = { 'AElig': 0x00c6, # latin capital letter AE = latin capital ligature AE, U+00C6 ISOlat1 'Aacute': 0x00c1, # latin capital letter A with acute, U+00C1 ISOlat1 @@ -261,7 +260,11 @@ } -# maps the HTML5 named character references to the equivalent Unicode character(s) +# HTML5 named character references +# Generated by 'Tools/scripts/parse_html5_entities.py' +# from https://html.spec.whatwg.org/entities.json and +# https://html.spec.whatwg.org/multipage/named-characters.html. +# Map HTML5 named character references to the equivalent Unicode character(s). html5 = { 'Aacute': '\xc1', 'aacute': '\xe1', diff --git a/Misc/NEWS.d/next/Tools-Demos/2022-06-19-14-56-33.gh-issue-86087.R8MkRy.rst b/Misc/NEWS.d/next/Tools-Demos/2022-06-19-14-56-33.gh-issue-86087.R8MkRy.rst new file mode 100644 index 00000000000..a10d4c76884 --- /dev/null +++ b/Misc/NEWS.d/next/Tools-Demos/2022-06-19-14-56-33.gh-issue-86087.R8MkRy.rst @@ -0,0 +1,2 @@ +The ``Tools/scripts/parseentities.py`` script used to parse HTML4 entities +has been removed. diff --git a/Tools/scripts/parse_html5_entities.py b/Tools/scripts/parse_html5_entities.py index c011328b010..1e5bdad2165 100755 --- a/Tools/scripts/parse_html5_entities.py +++ b/Tools/scripts/parse_html5_entities.py @@ -2,10 +2,14 @@ """ Utility for parsing HTML5 entity definitions available from: - http://dev.w3.org/html5/spec/entities.json + https://html.spec.whatwg.org/entities.json + https://html.spec.whatwg.org/multipage/named-characters.html + +The page now contains the following note: + + "This list is static and will not be expanded or changed in the future." Written by Ezio Melotti and Iuliia Proskurnia. - """ import os @@ -14,7 +18,9 @@ from urllib.request import urlopen from html.entities import html5 -entities_url = 'http://dev.w3.org/html5/spec/entities.json' +PAGE_URL = 'https://html.spec.whatwg.org/multipage/named-characters.html' +ENTITIES_URL = 'https://html.spec.whatwg.org/entities.json' +HTML5_SECTION_START = '# HTML5 named character references' def get_json(url): """Download the json file from the url and returns a decoded object.""" @@ -62,9 +68,15 @@ def write_items(entities, file=sys.stdout): # be before their equivalent lowercase version. keys = sorted(entities.keys()) keys = sorted(keys, key=str.lower) + print(HTML5_SECTION_START, file=file) + print(f'# Generated by {sys.argv[0]!r}\n' + f'# from {ENTITIES_URL} and\n' + f'# {PAGE_URL}.\n' + f'# Map HTML5 named character references to the ' + f'equivalent Unicode character(s).', file=file) print('html5 = {', file=file) for name in keys: - print(' {!r}: {!a},'.format(name, entities[name]), file=file) + print(f' {name!r}: {entities[name]!a},', file=file) print('}', file=file) @@ -72,11 +84,8 @@ def write_items(entities, file=sys.stdout): # without args print a diff between html.entities.html5 and new_html5 # with --create print the new html5 dict # with --patch patch the Lib/html/entities.py file - new_html5 = create_dict(get_json(entities_url)) + new_html5 = create_dict(get_json(ENTITIES_URL)) if '--create' in sys.argv: - print('# map the HTML5 named character references to the ' - 'equivalent Unicode character(s)') - print('# Generated by {}. Do not edit manually.'.format(__file__)) write_items(new_html5) elif '--patch' in sys.argv: fname = 'Lib/html/entities.py' @@ -84,7 +93,7 @@ def write_items(entities, file=sys.stdout): with open(fname) as f1, open(temp_fname, 'w') as f2: skip = False for line in f1: - if line.startswith('html5 = {'): + if line.startswith(HTML5_SECTION_START): write_items(new_html5, file=f2) skip = True continue diff --git a/Tools/scripts/parseentities.py b/Tools/scripts/parseentities.py deleted file mode 100755 index 0229d3af86b..00000000000 --- a/Tools/scripts/parseentities.py +++ /dev/null @@ -1,64 +0,0 @@ -#!/usr/bin/env python3 -""" Utility for parsing HTML entity definitions available from: - - http://www.w3.org/ as e.g. - http://www.w3.org/TR/REC-html40/HTMLlat1.ent - - Input is read from stdin, output is written to stdout in form of a - Python snippet defining a dictionary "entitydefs" mapping literal - entity name to character or numeric entity. - - Marc-Andre Lemburg, mal@lemburg.com, 1999. - Use as you like. NO WARRANTIES. - -""" -import re,sys - -entityRE = re.compile(r'') - -def parse(text,pos=0,endpos=None): - - pos = 0 - if endpos is None: - endpos = len(text) - d = {} - while 1: - m = entityRE.search(text,pos,endpos) - if not m: - break - name,charcode,comment = m.groups() - d[name] = charcode,comment - pos = m.end() - return d - -def writefile(f,defs): - - f.write("entitydefs = {\n") - items = sorted(defs.items()) - for name, (charcode,comment) in items: - if charcode[:2] == '&#': - code = int(charcode[2:-1]) - if code < 256: - charcode = r"'\%o'" % code - else: - charcode = repr(charcode) - else: - charcode = repr(charcode) - comment = ' '.join(comment.split()) - f.write(" '%s':\t%s, \t# %s\n" % (name,charcode,comment)) - f.write('\n}\n') - -if __name__ == '__main__': - if len(sys.argv) > 1: - with open(sys.argv[1]) as infile: - text = infile.read() - else: - text = sys.stdin.read() - - defs = parse(text) - - if len(sys.argv) > 2: - with open(sys.argv[2],'w') as outfile: - writefile(outfile, defs) - else: - writefile(sys.stdout, defs)