bpo-28806: Continue work: improve the netrc library (GH-26330)

Continue with the improvement of the library netrc

Original work and report Xiang Zhang <angwerzx@126.com>

* 📜🤖 Added by blurb_it.

Co-authored-by: blurb-it[bot] <43283697+blurb-it[bot]@users.noreply.github.com>
This commit is contained in:
Emmanuel Arias 2021-11-17 06:07:54 -03:00 committed by GitHub
parent da20d7401d
commit 15409c720b
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
4 changed files with 319 additions and 129 deletions

View file

@ -41,6 +41,10 @@ the Unix :program:`ftp` program and other FTP clients.
.. versionchanged:: 3.10
:class:`netrc` try UTF-8 encoding before using locale specific
encoding.
The entry in the netrc file no longer needs to contain all tokens. The missing
tokens' value default to an empty string. All the tokens and their values now
can contain arbitrary characters, like whitespace and non-ASCII characters.
If the login name is anonymous, it won't trigger the security check.
.. exception:: NetrcParseError
@ -85,10 +89,3 @@ Instances of :class:`~netrc.netrc` have public instance variables:
.. attribute:: netrc.macros
Dictionary mapping macro names to string lists.
.. note::
Passwords are limited to a subset of the ASCII character set. All ASCII
punctuation is allowed in passwords, however, note that whitespace and
non-printable characters are not allowed in passwords. This is a limitation
of the way the .netrc file is parsed and may be removed in the future.

View file

@ -19,6 +19,50 @@ def __str__(self):
return "%s (%s, line %s)" % (self.msg, self.filename, self.lineno)
class _netrclex:
def __init__(self, fp):
self.lineno = 1
self.instream = fp
self.whitespace = "\n\t\r "
self.pushback = []
def _read_char(self):
ch = self.instream.read(1)
if ch == "\n":
self.lineno += 1
return ch
def get_token(self):
if self.pushback:
return self.pushback.pop(0)
token = ""
fiter = iter(self._read_char, "")
for ch in fiter:
if ch in self.whitespace:
continue
if ch == '"':
for ch in fiter:
if ch == '"':
return token
elif ch == "\\":
ch = self._read_char()
token += ch
else:
if ch == "\\":
ch = self._read_char()
token += ch
for ch in fiter:
if ch in self.whitespace:
return token
elif ch == "\\":
ch = self._read_char()
token += ch
return token
def push_token(self, token):
self.pushback.append(token)
class netrc:
def __init__(self, file=None):
default_netrc = file is None
@ -34,9 +78,7 @@ def __init__(self, file=None):
self._parse(file, fp, default_netrc)
def _parse(self, file, fp, default_netrc):
lexer = shlex.shlex(fp)
lexer.wordchars += r"""!"#$%&'()*+,-./:;<=>?@[\]^_`{|}~"""
lexer.commenters = lexer.commenters.replace('#', '')
lexer = _netrclex(fp)
while 1:
# Look for a machine, default, or macdef top-level keyword
saved_lineno = lexer.lineno
@ -51,14 +93,19 @@ def _parse(self, file, fp, default_netrc):
entryname = lexer.get_token()
elif tt == 'default':
entryname = 'default'
elif tt == 'macdef': # Just skip to end of macdefs
elif tt == 'macdef':
entryname = lexer.get_token()
self.macros[entryname] = []
lexer.whitespace = ' \t'
while 1:
line = lexer.instream.readline()
if not line or line == '\012':
lexer.whitespace = ' \t\r\n'
if not line:
raise NetrcParseError(
"Macro definition missing null line terminator.",
file, lexer.lineno)
if line == '\n':
# a macro definition finished with consecutive new-line
# characters. The first \n is encountered by the
# readline() method and this is the second \n.
break
self.macros[entryname].append(line)
continue
@ -66,53 +113,55 @@ def _parse(self, file, fp, default_netrc):
raise NetrcParseError(
"bad toplevel token %r" % tt, file, lexer.lineno)
if not entryname:
raise NetrcParseError("missing %r name" % tt, file, lexer.lineno)
# We're looking at start of an entry for a named machine or default.
login = ''
account = password = None
login = account = password = ''
self.hosts[entryname] = {}
while 1:
prev_lineno = lexer.lineno
tt = lexer.get_token()
if (tt.startswith('#') or
tt in {'', 'machine', 'default', 'macdef'}):
if password:
self.hosts[entryname] = (login, account, password)
lexer.push_token(tt)
break
else:
raise NetrcParseError(
"malformed %s entry %s terminated by %s"
% (toplevel, entryname, repr(tt)),
file, lexer.lineno)
if tt.startswith('#'):
if lexer.lineno == prev_lineno:
lexer.instream.readline()
continue
if tt in {'', 'machine', 'default', 'macdef'}:
self.hosts[entryname] = (login, account, password)
lexer.push_token(tt)
break
elif tt == 'login' or tt == 'user':
login = lexer.get_token()
elif tt == 'account':
account = lexer.get_token()
elif tt == 'password':
if os.name == 'posix' and default_netrc:
prop = os.fstat(fp.fileno())
if prop.st_uid != os.getuid():
import pwd
try:
fowner = pwd.getpwuid(prop.st_uid)[0]
except KeyError:
fowner = 'uid %s' % prop.st_uid
try:
user = pwd.getpwuid(os.getuid())[0]
except KeyError:
user = 'uid %s' % os.getuid()
raise NetrcParseError(
("~/.netrc file owner (%s) does not match"
" current user (%s)") % (fowner, user),
file, lexer.lineno)
if (prop.st_mode & (stat.S_IRWXG | stat.S_IRWXO)):
raise NetrcParseError(
"~/.netrc access too permissive: access"
" permissions must restrict access to only"
" the owner", file, lexer.lineno)
password = lexer.get_token()
else:
raise NetrcParseError("bad follower token %r" % tt,
file, lexer.lineno)
self._security_check(fp, default_netrc, self.hosts[entryname][0])
def _security_check(self, fp, default_netrc, login):
if os.name == 'posix' and default_netrc and login != "anonymous":
prop = os.fstat(fp.fileno())
if prop.st_uid != os.getuid():
import pwd
try:
fowner = pwd.getpwuid(prop.st_uid)[0]
except KeyError:
fowner = 'uid %s' % prop.st_uid
try:
user = pwd.getpwuid(os.getuid())[0]
except KeyError:
user = 'uid %s' % os.getuid()
raise NetrcParseError(
(f"~/.netrc file owner ({fowner}, {user}) does not match"
" current user"))
if (prop.st_mode & (stat.S_IRWXG | stat.S_IRWXO)):
raise NetrcParseError(
"~/.netrc access too permissive: access"
" permissions must restrict access to only"
" the owner")
def authenticators(self, host):
"""Return a (user, account, password) tuple for given host."""

View file

@ -1,7 +1,7 @@
import netrc, os, unittest, sys, tempfile, textwrap
from test import support
from test.support import os_helper
import netrc, os, unittest, sys, textwrap
from test.support import os_helper, run_unittest
temp_filename = os_helper.TESTFN
class NetrcTestCase(unittest.TestCase):
@ -10,26 +10,32 @@ def make_nrc(self, test_data):
mode = 'w'
if sys.platform != 'cygwin':
mode += 't'
temp_fd, temp_filename = tempfile.mkstemp()
with os.fdopen(temp_fd, mode=mode) as fp:
with open(temp_filename, mode) as fp:
fp.write(test_data)
self.addCleanup(os.unlink, temp_filename)
return netrc.netrc(temp_filename)
try:
nrc = netrc.netrc(temp_filename)
finally:
os.unlink(temp_filename)
return nrc
def test_default(self):
def test_toplevel_non_ordered_tokens(self):
nrc = self.make_nrc("""\
machine host1.domain.com login log1 password pass1 account acct1
default login log2 password pass2
machine host.domain.com password pass1 login log1 account acct1
default login log2 password pass2 account acct2
""")
self.assertEqual(nrc.hosts['host1.domain.com'],
('log1', 'acct1', 'pass1'))
self.assertEqual(nrc.hosts['default'], ('log2', None, 'pass2'))
self.assertEqual(nrc.hosts['host.domain.com'], ('log1', 'acct1', 'pass1'))
self.assertEqual(nrc.hosts['default'], ('log2', 'acct2', 'pass2'))
nrc2 = self.make_nrc(nrc.__repr__())
self.assertEqual(nrc.hosts, nrc2.hosts)
def test_toplevel_tokens(self):
nrc = self.make_nrc("""\
machine host.domain.com login log1 password pass1 account acct1
default login log2 password pass2 account acct2
""")
self.assertEqual(nrc.hosts['host.domain.com'], ('log1', 'acct1', 'pass1'))
self.assertEqual(nrc.hosts['default'], ('log2', 'acct2', 'pass2'))
def test_macros(self):
nrc = self.make_nrc("""\
data = """\
macdef macro1
line1
line2
@ -37,33 +43,151 @@ def test_macros(self):
macdef macro2
line3
line4
""")
"""
nrc = self.make_nrc(data)
self.assertEqual(nrc.macros, {'macro1': ['line1\n', 'line2\n'],
'macro2': ['line3\n', 'line4\n']})
# strip the last \n
self.assertRaises(netrc.NetrcParseError, self.make_nrc,
data.rstrip(' ')[:-1])
def _test_passwords(self, nrc, passwd):
def test_optional_tokens(self):
data = (
"machine host.domain.com",
"machine host.domain.com login",
"machine host.domain.com account",
"machine host.domain.com password",
"machine host.domain.com login \"\" account",
"machine host.domain.com login \"\" password",
"machine host.domain.com account \"\" password"
)
for item in data:
nrc = self.make_nrc(item)
self.assertEqual(nrc.hosts['host.domain.com'], ('', '', ''))
data = (
"default",
"default login",
"default account",
"default password",
"default login \"\" account",
"default login \"\" password",
"default account \"\" password"
)
for item in data:
nrc = self.make_nrc(item)
self.assertEqual(nrc.hosts['default'], ('', '', ''))
def test_invalid_tokens(self):
data = (
"invalid host.domain.com",
"machine host.domain.com invalid",
"machine host.domain.com login log password pass account acct invalid",
"default host.domain.com invalid",
"default host.domain.com login log password pass account acct invalid"
)
for item in data:
self.assertRaises(netrc.NetrcParseError, self.make_nrc, item)
def _test_token_x(self, nrc, token, value):
nrc = self.make_nrc(nrc)
self.assertEqual(nrc.hosts['host.domain.com'], ('log', 'acct', passwd))
if token == 'login':
self.assertEqual(nrc.hosts['host.domain.com'], (value, 'acct', 'pass'))
elif token == 'account':
self.assertEqual(nrc.hosts['host.domain.com'], ('log', value, 'pass'))
elif token == 'password':
self.assertEqual(nrc.hosts['host.domain.com'], ('log', 'acct', value))
def test_password_with_leading_hash(self):
self._test_passwords("""\
def test_token_value_quotes(self):
self._test_token_x("""\
machine host.domain.com login "log" password pass account acct
""", 'login', 'log')
self._test_token_x("""\
machine host.domain.com login log password pass account "acct"
""", 'account', 'acct')
self._test_token_x("""\
machine host.domain.com login log password "pass" account acct
""", 'password', 'pass')
def test_token_value_escape(self):
self._test_token_x("""\
machine host.domain.com login \\"log password pass account acct
""", 'login', '"log')
self._test_token_x("""\
machine host.domain.com login "\\"log" password pass account acct
""", 'login', '"log')
self._test_token_x("""\
machine host.domain.com login log password pass account \\"acct
""", 'account', '"acct')
self._test_token_x("""\
machine host.domain.com login log password pass account "\\"acct"
""", 'account', '"acct')
self._test_token_x("""\
machine host.domain.com login log password \\"pass account acct
""", 'password', '"pass')
self._test_token_x("""\
machine host.domain.com login log password "\\"pass" account acct
""", 'password', '"pass')
def test_token_value_whitespace(self):
self._test_token_x("""\
machine host.domain.com login "lo g" password pass account acct
""", 'login', 'lo g')
self._test_token_x("""\
machine host.domain.com login log password "pas s" account acct
""", 'password', 'pas s')
self._test_token_x("""\
machine host.domain.com login log password pass account "acc t"
""", 'account', 'acc t')
def test_token_value_non_ascii(self):
self._test_token_x("""\
machine host.domain.com login \xa1\xa2 password pass account acct
""", 'login', '\xa1\xa2')
self._test_token_x("""\
machine host.domain.com login log password pass account \xa1\xa2
""", 'account', '\xa1\xa2')
self._test_token_x("""\
machine host.domain.com login log password \xa1\xa2 account acct
""", 'password', '\xa1\xa2')
def test_token_value_leading_hash(self):
self._test_token_x("""\
machine host.domain.com login #log password pass account acct
""", 'login', '#log')
self._test_token_x("""\
machine host.domain.com login log password pass account #acct
""", 'account', '#acct')
self._test_token_x("""\
machine host.domain.com login log password #pass account acct
""", '#pass')
""", 'password', '#pass')
def test_password_with_trailing_hash(self):
self._test_passwords("""\
def test_token_value_trailing_hash(self):
self._test_token_x("""\
machine host.domain.com login log# password pass account acct
""", 'login', 'log#')
self._test_token_x("""\
machine host.domain.com login log password pass account acct#
""", 'account', 'acct#')
self._test_token_x("""\
machine host.domain.com login log password pass# account acct
""", 'pass#')
""", 'password', 'pass#')
def test_password_with_internal_hash(self):
self._test_passwords("""\
def test_token_value_internal_hash(self):
self._test_token_x("""\
machine host.domain.com login lo#g password pass account acct
""", 'login', 'lo#g')
self._test_token_x("""\
machine host.domain.com login log password pass account ac#ct
""", 'account', 'ac#ct')
self._test_token_x("""\
machine host.domain.com login log password pa#ss account acct
""", 'pa#ss')
""", 'password', 'pa#ss')
def _test_comment(self, nrc, passwd='pass'):
nrc = self.make_nrc(nrc)
self.assertEqual(nrc.hosts['foo.domain.com'], ('bar', None, passwd))
self.assertEqual(nrc.hosts['bar.domain.com'], ('foo', None, 'pass'))
self.assertEqual(nrc.hosts['foo.domain.com'], ('bar', '', passwd))
self.assertEqual(nrc.hosts['bar.domain.com'], ('foo', '', 'pass'))
def test_comment_before_machine_line(self):
self._test_comment("""\
@ -86,6 +210,42 @@ def test_comment_before_machine_line_hash_only(self):
machine bar.domain.com login foo password pass
""")
def test_comment_after_machine_line(self):
self._test_comment("""\
machine foo.domain.com login bar password pass
# comment
machine bar.domain.com login foo password pass
""")
self._test_comment("""\
machine foo.domain.com login bar password pass
machine bar.domain.com login foo password pass
# comment
""")
def test_comment_after_machine_line_no_space(self):
self._test_comment("""\
machine foo.domain.com login bar password pass
#comment
machine bar.domain.com login foo password pass
""")
self._test_comment("""\
machine foo.domain.com login bar password pass
machine bar.domain.com login foo password pass
#comment
""")
def test_comment_after_machine_line_hash_only(self):
self._test_comment("""\
machine foo.domain.com login bar password pass
#
machine bar.domain.com login foo password pass
""")
self._test_comment("""\
machine foo.domain.com login bar password pass
machine bar.domain.com login foo password pass
#
""")
def test_comment_at_end_of_machine_line(self):
self._test_comment("""\
machine foo.domain.com login bar password pass # comment
@ -109,57 +269,40 @@ def test_comment_at_end_of_machine_line_pass_has_hash(self):
def test_security(self):
# This test is incomplete since we are normally not run as root and
# therefore can't test the file ownership being wrong.
with os_helper.temp_cwd(None) as d:
fn = os.path.join(d, '.netrc')
with open(fn, 'wt') as f:
f.write("""\
machine foo.domain.com login bar password pass
default login foo password pass
""")
with os_helper.EnvironmentVarGuard() as environ:
environ.set('HOME', d)
os.chmod(fn, 0o600)
nrc = netrc.netrc()
self.assertEqual(nrc.hosts['foo.domain.com'],
('bar', None, 'pass'))
os.chmod(fn, 0o622)
self.assertRaises(netrc.NetrcParseError, netrc.netrc)
def test_file_not_found_in_home(self):
with os_helper.temp_cwd(None) as d:
with os_helper.EnvironmentVarGuard() as environ:
environ.set('HOME', d)
self.assertRaises(FileNotFoundError, netrc.netrc)
def test_file_not_found_explicit(self):
self.assertRaises(FileNotFoundError, netrc.netrc,
file='unlikely_netrc')
def test_home_not_set(self):
with os_helper.temp_cwd(None) as fake_home:
fake_netrc_path = os.path.join(fake_home, '.netrc')
with open(fake_netrc_path, 'w') as f:
f.write('machine foo.domain.com login bar password pass')
os.chmod(fake_netrc_path, 0o600)
orig_expanduser = os.path.expanduser
called = []
def fake_expanduser(s):
called.append(s)
with os_helper.EnvironmentVarGuard() as environ:
environ.set('HOME', fake_home)
environ.set('USERPROFILE', fake_home)
result = orig_expanduser(s)
return result
with support.swap_attr(os.path, 'expanduser', fake_expanduser):
nrc = netrc.netrc()
login, account, password = nrc.authenticators('foo.domain.com')
self.assertEqual(login, 'bar')
self.assertTrue(called)
d = os_helper.TESTFN
os.mkdir(d)
self.addCleanup(os_helper.rmtree, d)
fn = os.path.join(d, '.netrc')
with open(fn, 'wt') as f:
f.write("""\
machine foo.domain.com login bar password pass
default login foo password pass
""")
with os_helper.EnvironmentVarGuard() as environ:
environ.set('HOME', d)
os.chmod(fn, 0o600)
nrc = netrc.netrc()
self.assertEqual(nrc.hosts['foo.domain.com'],
('bar', '', 'pass'))
os.chmod(fn, 0o622)
self.assertRaises(netrc.NetrcParseError, netrc.netrc)
with open(fn, 'wt') as f:
f.write("""\
machine foo.domain.com login anonymous password pass
default login foo password pass
""")
with os_helper.EnvironmentVarGuard() as environ:
environ.set('HOME', d)
os.chmod(fn, 0o600)
nrc = netrc.netrc()
self.assertEqual(nrc.hosts['foo.domain.com'],
('anonymous', '', 'pass'))
os.chmod(fn, 0o622)
self.assertEqual(nrc.hosts['foo.domain.com'],
('anonymous', '', 'pass'))
def test_main():
run_unittest(NetrcTestCase)
if __name__ == "__main__":
unittest.main()
test_main()

View file

@ -0,0 +1 @@
Improve netrc library. netrc file no longer needs to contain all tokens. And if the login name is anonymous, security check is no longer need.