mirror of
https://github.com/python/cpython
synced 2024-10-18 15:24:11 +00:00
0de65807e6
also modified check_all function to suppress all warnings since they aren't relevant to what this test is doing (allows quiet checking of regsub, for instance)
655 lines
24 KiB
Python
655 lines
24 KiB
Python
# module 're' -- A collection of regular expression operations
|
|
|
|
"""Support for regular expressions (RE).
|
|
|
|
This module provides regular expression matching operations similar to
|
|
those found in Perl. It's 8-bit clean: the strings being processed may
|
|
contain both null bytes and characters whose high bit is set. Regular
|
|
expression pattern strings may not contain null bytes, but can specify
|
|
the null byte using the \\number notation. Characters with the high
|
|
bit set may be included.
|
|
|
|
Regular expressions can contain both special and ordinary
|
|
characters. Most ordinary characters, like "A", "a", or "0", are the
|
|
simplest regular expressions; they simply match themselves. You can
|
|
concatenate ordinary characters, so last matches the string 'last'.
|
|
|
|
The special characters are:
|
|
"." Matches any character except a newline.
|
|
"^" Matches the start of the string.
|
|
"$" Matches the end of the string.
|
|
"*" Matches 0 or more (greedy) repetitions of the preceding RE.
|
|
Greedy means that it will match as many repetitions as possible.
|
|
"+" Matches 1 or more (greedy) repetitions of the preceding RE.
|
|
"?" Matches 0 or 1 (greedy) of the preceding RE.
|
|
*?,+?,?? Non-greedy versions of the previous three special characters.
|
|
{m,n} Matches from m to n repetitions of the preceding RE.
|
|
{m,n}? Non-greedy version of the above.
|
|
"\\" Either escapes special characters or signals a special sequence.
|
|
[] Indicates a set of characters.
|
|
A "^" as the first character indicates a complementing set.
|
|
"|" A|B, creates an RE that will match either A or B.
|
|
(...) Matches the RE inside the parentheses.
|
|
The contents can be retrieved or matched later in the string.
|
|
(?iLmsx) Set the I, L, M, S, or X flag for the RE.
|
|
(?:...) Non-grouping version of regular parentheses.
|
|
(?P<name>...) The substring matched by the group is accessible by name.
|
|
(?P=name) Matches the text matched earlier by the group named name.
|
|
(?#...) A comment; ignored.
|
|
(?=...) Matches if ... matches next, but doesn't consume the string.
|
|
(?!...) Matches if ... doesn't match next.
|
|
|
|
The special sequences consist of "\\" and a character from the list
|
|
below. If the ordinary character is not on the list, then the
|
|
resulting RE will match the second character.
|
|
\\number Matches the contents of the group of the same number.
|
|
\\A Matches only at the start of the string.
|
|
\\Z Matches only at the end of the string.
|
|
\\b Matches the empty string, but only at the start or end of a word.
|
|
\\B Matches the empty string, but not at the start or end of a word.
|
|
\\d Matches any decimal digit; equivalent to the set [0-9].
|
|
\\D Matches any non-digit character; equivalent to the set [^0-9].
|
|
\\s Matches any whitespace character; equivalent to [ \\t\\n\\r\\f\\v].
|
|
\\S Matches any non-whitespace character; equiv. to [^ \\t\\n\\r\\f\\v].
|
|
\\w Matches any alphanumeric character; equivalent to [a-zA-Z0-9_].
|
|
With LOCALE, it will match the set [0-9_] plus characters defined
|
|
as letters for the current locale.
|
|
\\W Matches the complement of \\w.
|
|
\\\\ Matches a literal backslash.
|
|
|
|
This module exports the following functions:
|
|
match Match a regular expression pattern to the beginning of a string.
|
|
search Search a string for the presence of a pattern.
|
|
sub Substitute occurrences of a pattern found in a string.
|
|
subn Same as sub, but also return the number of substitutions made.
|
|
split Split a string by the occurrences of a pattern.
|
|
findall Find all occurrences of a pattern in a string.
|
|
compile Compile a pattern into a RegexObject.
|
|
escape Backslash all non-alphanumerics in a string.
|
|
|
|
This module exports the following classes:
|
|
RegexObject Holds a compiled regular expression pattern.
|
|
MatchObject Contains information about pattern matches.
|
|
|
|
Some of the functions in this module takes flags as optional parameters:
|
|
I IGNORECASE Perform case-insensitive matching.
|
|
L LOCALE Make \w, \W, \b, \B, dependent on the current locale.
|
|
M MULTILINE "^" matches the beginning of lines as well as the string.
|
|
"$" matches the end of lines as well as the string.
|
|
S DOTALL "." matches any character at all, including the newline.
|
|
X VERBOSE Ignore whitespace and comments for nicer looking RE's.
|
|
|
|
This module also defines an exception 'error'.
|
|
|
|
"""
|
|
|
|
|
|
import sys
|
|
from pcre import *
|
|
|
|
__all__ = ["match","search","sub","subn","split","findall","escape","compile",
|
|
"I","L","M","S","X","IGNORECASE","LOCALE","MULTILINE","DOTALL",
|
|
"VERBOSE","error"]
|
|
|
|
#
|
|
# First, the public part of the interface:
|
|
#
|
|
|
|
# pcre.error and re.error should be the same, since exceptions can be
|
|
# raised from either module.
|
|
|
|
# compilation flags
|
|
|
|
I = IGNORECASE
|
|
L = LOCALE
|
|
M = MULTILINE
|
|
S = DOTALL
|
|
X = VERBOSE
|
|
|
|
|
|
#
|
|
#
|
|
#
|
|
|
|
_cache = {}
|
|
_MAXCACHE = 20
|
|
|
|
def _cachecompile(pattern, flags=0):
|
|
key = (pattern, flags)
|
|
try:
|
|
return _cache[key]
|
|
except KeyError:
|
|
pass
|
|
value = compile(pattern, flags)
|
|
if len(_cache) >= _MAXCACHE:
|
|
_cache.clear()
|
|
_cache[key] = value
|
|
return value
|
|
|
|
def match(pattern, string, flags=0):
|
|
"""match (pattern, string[, flags]) -> MatchObject or None
|
|
|
|
If zero or more characters at the beginning of string match the
|
|
regular expression pattern, return a corresponding MatchObject
|
|
instance. Return None if the string does not match the pattern;
|
|
note that this is different from a zero-length match.
|
|
|
|
Note: If you want to locate a match anywhere in string, use
|
|
search() instead.
|
|
|
|
"""
|
|
|
|
return _cachecompile(pattern, flags).match(string)
|
|
|
|
def search(pattern, string, flags=0):
|
|
"""search (pattern, string[, flags]) -> MatchObject or None
|
|
|
|
Scan through string looking for a location where the regular
|
|
expression pattern produces a match, and return a corresponding
|
|
MatchObject instance. Return None if no position in the string
|
|
matches the pattern; note that this is different from finding a
|
|
zero-length match at some point in the string.
|
|
|
|
"""
|
|
return _cachecompile(pattern, flags).search(string)
|
|
|
|
def sub(pattern, repl, string, count=0):
|
|
"""sub(pattern, repl, string[, count=0]) -> string
|
|
|
|
Return the string obtained by replacing the leftmost
|
|
non-overlapping occurrences of pattern in string by the
|
|
replacement repl. If the pattern isn't found, string is returned
|
|
unchanged. repl can be a string or a function; if a function, it
|
|
is called for every non-overlapping occurrence of pattern. The
|
|
function takes a single match object argument, and returns the
|
|
replacement string.
|
|
|
|
The pattern may be a string or a regex object; if you need to
|
|
specify regular expression flags, you must use a regex object, or
|
|
use embedded modifiers in a pattern; e.g.
|
|
sub("(?i)b+", "x", "bbbb BBBB") returns 'x x'.
|
|
|
|
The optional argument count is the maximum number of pattern
|
|
occurrences to be replaced; count must be a non-negative integer,
|
|
and the default value of 0 means to replace all occurrences.
|
|
|
|
"""
|
|
if type(pattern) == type(''):
|
|
pattern = _cachecompile(pattern)
|
|
return pattern.sub(repl, string, count)
|
|
|
|
def subn(pattern, repl, string, count=0):
|
|
"""subn(pattern, repl, string[, count=0]) -> (string, num substitutions)
|
|
|
|
Perform the same operation as sub(), but return a tuple
|
|
(new_string, number_of_subs_made).
|
|
|
|
"""
|
|
if type(pattern) == type(''):
|
|
pattern = _cachecompile(pattern)
|
|
return pattern.subn(repl, string, count)
|
|
|
|
def split(pattern, string, maxsplit=0):
|
|
"""split(pattern, string[, maxsplit=0]) -> list of strings
|
|
|
|
Split string by the occurrences of pattern. If capturing
|
|
parentheses are used in pattern, then the text of all groups in
|
|
the pattern are also returned as part of the resulting list. If
|
|
maxsplit is nonzero, at most maxsplit splits occur, and the
|
|
remainder of the string is returned as the final element of the
|
|
list.
|
|
|
|
"""
|
|
if type(pattern) == type(''):
|
|
pattern = _cachecompile(pattern)
|
|
return pattern.split(string, maxsplit)
|
|
|
|
def findall(pattern, string):
|
|
"""findall(pattern, string) -> list
|
|
|
|
Return a list of all non-overlapping matches of pattern in
|
|
string. If one or more groups are present in the pattern, return a
|
|
list of groups; this will be a list of tuples if the pattern has
|
|
more than one group. Empty matches are included in the result.
|
|
|
|
"""
|
|
if type(pattern) == type(''):
|
|
pattern = _cachecompile(pattern)
|
|
return pattern.findall(string)
|
|
|
|
def escape(pattern):
|
|
"""escape(string) -> string
|
|
|
|
Return string with all non-alphanumerics backslashed; this is
|
|
useful if you want to match an arbitrary literal string that may
|
|
have regular expression metacharacters in it.
|
|
|
|
"""
|
|
result = list(pattern)
|
|
for i in range(len(pattern)):
|
|
char = pattern[i]
|
|
if not char.isalnum():
|
|
if char=='\000': result[i] = '\\000'
|
|
else: result[i] = '\\'+char
|
|
return ''.join(result)
|
|
|
|
def compile(pattern, flags=0):
|
|
"""compile(pattern[, flags]) -> RegexObject
|
|
|
|
Compile a regular expression pattern into a regular expression
|
|
object, which can be used for matching using its match() and
|
|
search() methods.
|
|
|
|
"""
|
|
groupindex={}
|
|
code=pcre_compile(pattern, flags, groupindex)
|
|
return RegexObject(pattern, flags, code, groupindex)
|
|
|
|
|
|
#
|
|
# Class definitions
|
|
#
|
|
|
|
class RegexObject:
|
|
"""Holds a compiled regular expression pattern.
|
|
|
|
Methods:
|
|
match Match the pattern to the beginning of a string.
|
|
search Search a string for the presence of the pattern.
|
|
sub Substitute occurrences of the pattern found in a string.
|
|
subn Same as sub, but also return the number of substitutions made.
|
|
split Split a string by the occurrences of the pattern.
|
|
findall Find all occurrences of the pattern in a string.
|
|
|
|
"""
|
|
|
|
def __init__(self, pattern, flags, code, groupindex):
|
|
self.code = code
|
|
self.flags = flags
|
|
self.pattern = pattern
|
|
self.groupindex = groupindex
|
|
|
|
def search(self, string, pos=0, endpos=None):
|
|
"""search(string[, pos][, endpos]) -> MatchObject or None
|
|
|
|
Scan through string looking for a location where this regular
|
|
expression produces a match, and return a corresponding
|
|
MatchObject instance. Return None if no position in the string
|
|
matches the pattern; note that this is different from finding
|
|
a zero-length match at some point in the string. The optional
|
|
pos and endpos parameters have the same meaning as for the
|
|
match() method.
|
|
|
|
"""
|
|
if endpos is None or endpos>len(string):
|
|
endpos=len(string)
|
|
if endpos<pos: endpos=pos
|
|
regs = self.code.match(string, pos, endpos, 0)
|
|
if regs is None:
|
|
return None
|
|
self._num_regs=len(regs)
|
|
|
|
return MatchObject(self,
|
|
string,
|
|
pos, endpos,
|
|
regs)
|
|
|
|
def match(self, string, pos=0, endpos=None):
|
|
"""match(string[, pos][, endpos]) -> MatchObject or None
|
|
|
|
If zero or more characters at the beginning of string match
|
|
this regular expression, return a corresponding MatchObject
|
|
instance. Return None if the string does not match the
|
|
pattern; note that this is different from a zero-length match.
|
|
|
|
Note: If you want to locate a match anywhere in string, use
|
|
search() instead.
|
|
|
|
The optional second parameter pos gives an index in the string
|
|
where the search is to start; it defaults to 0. This is not
|
|
completely equivalent to slicing the string; the '' pattern
|
|
character matches at the real beginning of the string and at
|
|
positions just after a newline, but not necessarily at the
|
|
index where the search is to start.
|
|
|
|
The optional parameter endpos limits how far the string will
|
|
be searched; it will be as if the string is endpos characters
|
|
long, so only the characters from pos to endpos will be
|
|
searched for a match.
|
|
|
|
"""
|
|
if endpos is None or endpos>len(string):
|
|
endpos=len(string)
|
|
if endpos<pos: endpos=pos
|
|
regs = self.code.match(string, pos, endpos, ANCHORED)
|
|
if regs is None:
|
|
return None
|
|
self._num_regs=len(regs)
|
|
return MatchObject(self,
|
|
string,
|
|
pos, endpos,
|
|
regs)
|
|
|
|
def sub(self, repl, string, count=0):
|
|
"""sub(repl, string[, count=0]) -> string
|
|
|
|
Return the string obtained by replacing the leftmost
|
|
non-overlapping occurrences of the compiled pattern in string
|
|
by the replacement repl. If the pattern isn't found, string is
|
|
returned unchanged.
|
|
|
|
Identical to the sub() function, using the compiled pattern.
|
|
|
|
"""
|
|
return self.subn(repl, string, count)[0]
|
|
|
|
def subn(self, repl, source, count=0):
|
|
"""subn(repl, string[, count=0]) -> tuple
|
|
|
|
Perform the same operation as sub(), but return a tuple
|
|
(new_string, number_of_subs_made).
|
|
|
|
"""
|
|
if count < 0:
|
|
raise error, "negative substitution count"
|
|
if count == 0:
|
|
count = sys.maxint
|
|
n = 0 # Number of matches
|
|
pos = 0 # Where to start searching
|
|
lastmatch = -1 # End of last match
|
|
results = [] # Substrings making up the result
|
|
end = len(source)
|
|
|
|
if type(repl) is type(''):
|
|
# See if repl contains group references
|
|
try:
|
|
repl = pcre_expand(_Dummy, repl)
|
|
except:
|
|
m = MatchObject(self, source, 0, end, [])
|
|
repl = lambda m, repl=repl, expand=pcre_expand: expand(m, repl)
|
|
else:
|
|
m = None
|
|
else:
|
|
m = MatchObject(self, source, 0, end, [])
|
|
|
|
match = self.code.match
|
|
append = results.append
|
|
while n < count and pos <= end:
|
|
regs = match(source, pos, end, 0)
|
|
if not regs:
|
|
break
|
|
self._num_regs = len(regs)
|
|
i, j = regs[0]
|
|
if i == j == lastmatch:
|
|
# Empty match adjacent to previous match
|
|
pos = pos + 1
|
|
append(source[lastmatch:pos])
|
|
continue
|
|
if pos < i:
|
|
append(source[pos:i])
|
|
if m:
|
|
m.pos = pos
|
|
m.regs = regs
|
|
append(repl(m))
|
|
else:
|
|
append(repl)
|
|
pos = lastmatch = j
|
|
if i == j:
|
|
# Last match was empty; don't try here again
|
|
pos = pos + 1
|
|
append(source[lastmatch:pos])
|
|
n = n + 1
|
|
append(source[pos:])
|
|
return (''.join(results), n)
|
|
|
|
def split(self, source, maxsplit=0):
|
|
"""split(source[, maxsplit=0]) -> list of strings
|
|
|
|
Split string by the occurrences of the compiled pattern. If
|
|
capturing parentheses are used in the pattern, then the text
|
|
of all groups in the pattern are also returned as part of the
|
|
resulting list. If maxsplit is nonzero, at most maxsplit
|
|
splits occur, and the remainder of the string is returned as
|
|
the final element of the list.
|
|
|
|
"""
|
|
if maxsplit < 0:
|
|
raise error, "negative split count"
|
|
if maxsplit == 0:
|
|
maxsplit = sys.maxint
|
|
n = 0
|
|
pos = 0
|
|
lastmatch = 0
|
|
results = []
|
|
end = len(source)
|
|
match = self.code.match
|
|
append = results.append
|
|
while n < maxsplit:
|
|
regs = match(source, pos, end, 0)
|
|
if not regs:
|
|
break
|
|
i, j = regs[0]
|
|
if i == j:
|
|
# Empty match
|
|
if pos >= end:
|
|
break
|
|
pos = pos+1
|
|
continue
|
|
append(source[lastmatch:i])
|
|
rest = regs[1:]
|
|
if rest:
|
|
for a, b in rest:
|
|
if a == -1 or b == -1:
|
|
group = None
|
|
else:
|
|
group = source[a:b]
|
|
append(group)
|
|
pos = lastmatch = j
|
|
n = n + 1
|
|
append(source[lastmatch:])
|
|
return results
|
|
|
|
def findall(self, source):
|
|
"""findall(source) -> list
|
|
|
|
Return a list of all non-overlapping matches of the compiled
|
|
pattern in string. If one or more groups are present in the
|
|
pattern, return a list of groups; this will be a list of
|
|
tuples if the pattern has more than one group. Empty matches
|
|
are included in the result.
|
|
|
|
"""
|
|
pos = 0
|
|
end = len(source)
|
|
results = []
|
|
match = self.code.match
|
|
append = results.append
|
|
while pos <= end:
|
|
regs = match(source, pos, end, 0)
|
|
if not regs:
|
|
break
|
|
i, j = regs[0]
|
|
rest = regs[1:]
|
|
if not rest:
|
|
gr = source[i:j]
|
|
elif len(rest) == 1:
|
|
a, b = rest[0]
|
|
gr = source[a:b]
|
|
else:
|
|
gr = []
|
|
for (a, b) in rest:
|
|
gr.append(source[a:b])
|
|
gr = tuple(gr)
|
|
append(gr)
|
|
pos = max(j, pos+1)
|
|
return results
|
|
|
|
# The following 3 functions were contributed by Mike Fletcher, and
|
|
# allow pickling and unpickling of RegexObject instances.
|
|
def __getinitargs__(self):
|
|
return (None,None,None,None) # any 4 elements, to work around
|
|
# problems with the
|
|
# pickle/cPickle modules not yet
|
|
# ignoring the __init__ function
|
|
def __getstate__(self):
|
|
return self.pattern, self.flags, self.groupindex
|
|
def __setstate__(self, statetuple):
|
|
self.pattern = statetuple[0]
|
|
self.flags = statetuple[1]
|
|
self.groupindex = statetuple[2]
|
|
self.code = apply(pcre_compile, statetuple)
|
|
|
|
class _Dummy:
|
|
# Dummy class used by _subn_string(). Has 'group' to avoid core dump.
|
|
group = None
|
|
|
|
class MatchObject:
|
|
"""Holds a compiled regular expression pattern.
|
|
|
|
Methods:
|
|
start Return the index of the start of a matched substring.
|
|
end Return the index of the end of a matched substring.
|
|
span Return a tuple of (start, end) of a matched substring.
|
|
groups Return a tuple of all the subgroups of the match.
|
|
group Return one or more subgroups of the match.
|
|
groupdict Return a dictionary of all the named subgroups of the match.
|
|
|
|
"""
|
|
|
|
def __init__(self, re, string, pos, endpos, regs):
|
|
self.re = re
|
|
self.string = string
|
|
self.pos = pos
|
|
self.endpos = endpos
|
|
self.regs = regs
|
|
|
|
def start(self, g = 0):
|
|
"""start([group=0]) -> int or None
|
|
|
|
Return the index of the start of the substring matched by
|
|
group; group defaults to zero (meaning the whole matched
|
|
substring). Return -1 if group exists but did not contribute
|
|
to the match.
|
|
|
|
"""
|
|
if type(g) == type(''):
|
|
try:
|
|
g = self.re.groupindex[g]
|
|
except (KeyError, TypeError):
|
|
raise IndexError, 'group %s is undefined' % `g`
|
|
return self.regs[g][0]
|
|
|
|
def end(self, g = 0):
|
|
"""end([group=0]) -> int or None
|
|
|
|
Return the indices of the end of the substring matched by
|
|
group; group defaults to zero (meaning the whole matched
|
|
substring). Return -1 if group exists but did not contribute
|
|
to the match.
|
|
|
|
"""
|
|
if type(g) == type(''):
|
|
try:
|
|
g = self.re.groupindex[g]
|
|
except (KeyError, TypeError):
|
|
raise IndexError, 'group %s is undefined' % `g`
|
|
return self.regs[g][1]
|
|
|
|
def span(self, g = 0):
|
|
"""span([group=0]) -> tuple
|
|
|
|
Return the 2-tuple (m.start(group), m.end(group)). Note that
|
|
if group did not contribute to the match, this is (-1,
|
|
-1). Group defaults to zero (meaning the whole matched
|
|
substring).
|
|
|
|
"""
|
|
if type(g) == type(''):
|
|
try:
|
|
g = self.re.groupindex[g]
|
|
except (KeyError, TypeError):
|
|
raise IndexError, 'group %s is undefined' % `g`
|
|
return self.regs[g]
|
|
|
|
def groups(self, default=None):
|
|
"""groups([default=None]) -> tuple
|
|
|
|
Return a tuple containing all the subgroups of the match, from
|
|
1 up to however many groups are in the pattern. The default
|
|
argument is used for groups that did not participate in the
|
|
match.
|
|
|
|
"""
|
|
result = []
|
|
for g in range(1, self.re._num_regs):
|
|
a, b = self.regs[g]
|
|
if a == -1 or b == -1:
|
|
result.append(default)
|
|
else:
|
|
result.append(self.string[a:b])
|
|
return tuple(result)
|
|
|
|
def group(self, *groups):
|
|
"""group([group1, group2, ...]) -> string or tuple
|
|
|
|
Return one or more subgroups of the match. If there is a
|
|
single argument, the result is a single string; if there are
|
|
multiple arguments, the result is a tuple with one item per
|
|
argument. Without arguments, group1 defaults to zero (i.e. the
|
|
whole match is returned). If a groupN argument is zero, the
|
|
corresponding return value is the entire matching string; if
|
|
it is in the inclusive range [1..99], it is the string
|
|
matching the the corresponding parenthesized group. If a group
|
|
number is negative or larger than the number of groups defined
|
|
in the pattern, an IndexError exception is raised. If a group
|
|
is contained in a part of the pattern that did not match, the
|
|
corresponding result is None. If a group is contained in a
|
|
part of the pattern that matched multiple times, the last
|
|
match is returned.
|
|
|
|
If the regular expression uses the (?P<name>...) syntax, the
|
|
groupN arguments may also be strings identifying groups by
|
|
their group name. If a string argument is not used as a group
|
|
name in the pattern, an IndexError exception is raised.
|
|
|
|
"""
|
|
if len(groups) == 0:
|
|
groups = (0,)
|
|
result = []
|
|
for g in groups:
|
|
if type(g) == type(''):
|
|
try:
|
|
g = self.re.groupindex[g]
|
|
except (KeyError, TypeError):
|
|
raise IndexError, 'group %s is undefined' % `g`
|
|
if g >= len(self.regs):
|
|
raise IndexError, 'group %s is undefined' % `g`
|
|
a, b = self.regs[g]
|
|
if a == -1 or b == -1:
|
|
result.append(None)
|
|
else:
|
|
result.append(self.string[a:b])
|
|
if len(result) > 1:
|
|
return tuple(result)
|
|
elif len(result) == 1:
|
|
return result[0]
|
|
else:
|
|
return ()
|
|
|
|
def groupdict(self, default=None):
|
|
"""groupdict([default=None]) -> dictionary
|
|
|
|
Return a dictionary containing all the named subgroups of the
|
|
match, keyed by the subgroup name. The default argument is
|
|
used for groups that did not participate in the match.
|
|
|
|
"""
|
|
dict = {}
|
|
for name, index in self.re.groupindex.items():
|
|
a, b = self.regs[index]
|
|
if a == -1 or b == -1:
|
|
dict[name] = default
|
|
else:
|
|
dict[name] = self.string[a:b]
|
|
return dict
|