",
' "; }',
+ 'a::before { content: "\u2603"; }',
+ ])
+ def test_style_content(self, content):
+ s = f''
+ self._run_check(s, [("starttag", "style", []),
+ ("data", content),
+ ("endtag", "style")])
+
+ @support.subTests('tag', ['title', 'textarea'])
+ def test_rcdata_content(self, tag):
+ source = f"<{tag}>{SAMPLE_RCDATA}"
+ self._run_check(source, [
+ ("starttag", tag, []),
+ ("data", SAMPLE_RCDATA),
+ ("endtag", tag),
+ ])
+ source = f"<{tag}>&"
+ self._run_check(source, [
+ ("starttag", tag, []),
+ ('entityref', 'amp'),
+ ("endtag", tag),
+ ])
+
+ @support.subTests('tag',
+ ['style', 'xmp', 'iframe', 'noembed', 'noframes', 'script'])
+ def test_rawtext_content(self, tag):
+ source = f"<{tag}>{SAMPLE_RAWTEXT}"
+ self._run_check(source, [
+ ("starttag", tag, []),
+ ("data", SAMPLE_RAWTEXT),
+ ("endtag", tag),
+ ])
+
+ def test_noscript_content(self):
+ source = f"{SAMPLE_RAWTEXT} "
+ # scripting=False -- normal mode
+ self._run_check(source, [
+ ('starttag', 'noscript', []),
+ ('comment', ' not a comment '),
+ ('starttag', 'not', [('a', 'start tag')]),
+ ('unknown decl', 'CDATA[not a cdata'),
+ ('comment', 'not a bogus comment'),
+ ('endtag', 'not'),
+ ('data', '☃'),
+ ('entityref', 'amp'),
+ ('charref', '9786'),
+ ('endtag', 'noscript'),
+ ])
+ # scripting=True -- RAWTEXT mode
+ self._run_check(source, [
+ ("starttag", "noscript", []),
+ ("data", SAMPLE_RAWTEXT),
+ ("endtag", "noscript"),
+ ], collector=EventCollector(scripting=True))
+
+ def test_plaintext_content(self):
+ content = SAMPLE_RAWTEXT + '' # not closing
+ source = f"<![CDATA[{content}"
+ self._run_check(source, [
+ ("starttag", "plaintext", []),
+ ("data", content),
+ ])
+
+ @support.subTests('endtag', ['script', 'SCRIPT', 'script ', 'script\n',
+ 'script/', 'script foo=bar', 'script foo=">"'])
+ def test_script_closing_tag(self, endtag):
# see issue #13358
# make sure that HTMLParser calls handle_data only once for each CDATA.
- # The normal event collector normalizes the events in get_events,
- # so we override it to return the original list of events.
- class Collector(EventCollector):
- def get_events(self):
- return self.events
-
content = """<!-- not a comment --> ¬-an-entity-ref;
<a href="" /> </p><p> <span></span></style>
'</script' + '>'"""
- for element in [' script', 'script ', ' script ',
- '\nscript', 'script\n', '\nscript\n']:
- element_lower = element.lower().strip()
- s = '<script>{content}</{element}>'.format(element=element,
- content=content)
- self._run_check(s, [("starttag", element_lower, []),
- ("data", content),
- ("endtag", element_lower)],
- collector=Collector(convert_charrefs=False))
+ s = f'<ScrIPt>{content}</{endtag}>'
+ self._run_check(s, [("starttag", "script", []),
+ ("data", content),
+ ("endtag", "script")],
+ collector=EventCollectorNoNormalize(convert_charrefs=False))
+
+ @support.subTests('tag', [
+ 'script', 'style', 'xmp', 'iframe', 'noembed', 'noframes',
+ 'textarea', 'title', 'noscript',
+ ])
+ def test_closing_tag(self, tag):
+ for endtag in [tag, tag.upper(), f'{tag} ', f'{tag}\n',
+ f'{tag}/', f'{tag} foo=bar', f'{tag} foo=">"']:
+ content = "<!-- not a comment --><i>Spam</i>"
+ s = f'<{tag.upper()}>{content}</{endtag}>'
+ self._run_check(s, [
+ ("starttag", tag, []),
+ ('data', content),
+ ("endtag", tag),
+ ], collector=EventCollectorNoNormalize(convert_charrefs=False, scripting=True))
+
+ @support.subTests('tag', [
+ 'script', 'style', 'xmp', 'iframe', 'noembed', 'noframes',
+ 'textarea', 'title', 'noscript',
+ ])
+ def test_invalid_closing_tag(self, tag):
+ content = (
+ f'< /{tag}>'
+ f'</ {tag}>'
+ f'</{tag}x>'
+ f'</{tag}\v>'
+ f'</{tag}\xa0>'
+ )
+ source = f"<{tag}>{content}</{tag}>"
+ self._run_check(source, [
+ ("starttag", tag, []),
+ ("data", content),
+ ("endtag", tag),
+ ], collector=EventCollector(convert_charrefs=False, scripting=True))
+
+ @support.subTests('tag,endtag', [
+ ('title', 'tıtle'),
+ ('style', 'ſtyle'),
+ ('style', 'ſtyle'),
+ ('style', 'style'),
+ ('iframe', 'ıframe'),
+ ('noframes', 'noframeſ'),
+ ('noscript', 'noſcript'),
+ ('noscript', 'noscrıpt'),
+ ('script', 'ſcript'),
+ ('script', 'scrıpt'),
+ ])
+ def test_invalid_nonascii_closing_tag(self, tag, endtag):
+ content = f"<br></{endtag}>"
+ source = f"<{tag}>{content}"
+ self._run_check(source, [
+ ("starttag", tag, []),
+ ("data", content),
+ ], collector=EventCollector(convert_charrefs=False, scripting=True))
+ source = f"<{tag}>{content}</{tag}>"
+ self._run_check(source, [
+ ("starttag", tag, []),
+ ("data", content),
+ ("endtag", tag),
+ ], collector=EventCollector(convert_charrefs=False, scripting=True))
+
+ @support.subTests('tail,end', [
+ ('', False),
+ ('<', False),
+ ('</', False),
+ ('</s', False),
+ ('</script', False),
+ ('</script ', True),
+ ('</script foo=bar', True),
+ ('</script foo=">', True),
+ ])
+ def test_eof_in_script(self, tail, end):
+ content = "a = 123"
+ s = f'<ScrIPt>{content}{tail}'
+ self._run_check(s, [("starttag", "script", []),
+ ("data", content if end else content + tail)],
+ collector=EventCollectorNoNormalize(convert_charrefs=False))
+
+ @support.subTests('tail,end', [
+ ('', False),
+ ('<', False),
+ ('</', False),
+ ('</t', False),
+ ('</title', False),
+ ('</title ', True),
+ ('</title foo=bar', True),
+ ('</title foo=">', True),
+ ])
+ def test_eof_in_title(self, tail, end):
+ s = f'<TitLe>Egg & Spam{tail}'
+ self._run_check(s, [("starttag", "title", []),
+ ("data", "Egg & Spam" + ('' if end else tail))],
+ collector=EventCollectorNoNormalize(convert_charrefs=True))
+ self._run_check(s, [("starttag", "title", []),
+ ('data', 'Egg '),
+ ('entityref', 'amp'),
+ ('data', ' Spam' + ('' if end else tail))],
+ collector=EventCollectorNoNormalize(convert_charrefs=False))
def test_comments(self):
html = ("<!-- I'm a valid comment -->"
'<!--me too!-->'
'<!------>'
+ '<!----->'
'<!---->'
+ # abrupt-closing-of-empty-comment
+ '<!--->'
+ '<!-->'
'<!----I have many hyphens---->'
'<!-- I have a > in the middle -->'
- '<!-- and I have -- in the middle! -->')
+ '<!-- and I have -- in the middle! -->'
+ '<!--incorrectly-closed-comment--!>'
+ '<!----!>'
+ '<!----!-->'
+ '<!---- >-->'
+ '<!---!>-->'
+ '<!--!>-->'
+ # nested-comment
+ '<!-- <!-- nested --> -->'
+ '<!--<!-->'
+ '<!--<!--!>'
+ )
expected = [('comment', " I'm a valid comment "),
('comment', 'me too!'),
('comment', '--'),
+ ('comment', '-'),
+ ('comment', ''),
+ ('comment', ''),
('comment', ''),
('comment', '--I have many hyphens--'),
('comment', ' I have a > in the middle '),
- ('comment', ' and I have -- in the middle! ')]
+ ('comment', ' and I have -- in the middle! '),
+ ('comment', 'incorrectly-closed-comment'),
+ ('comment', ''),
+ ('comment', '--!'),
+ ('comment', '-- >'),
+ ('comment', '-!>'),
+ ('comment', '!>'),
+ ('comment', ' <!-- nested '), ('data', ' -->'),
+ ('comment', '<!'),
+ ('comment', '<!'),
+ ]
self._run_check(html, expected)
def test_condcoms(self):
@@ -391,28 +596,34 @@ def test_tolerant_parsing(self):
('data', '<'),
('starttag', 'bc<', [('a', None)]),
('endtag', 'html'),
- ('data', '\n<img src="URL>'),
- ('comment', '/img'),
- ('endtag', 'html<')])
+ ('data', '\n')])
def test_starttag_junk_chars(self):
+ self._run_check("<", [('data', '<')])
+ self._run_check("<>", [('data', '<>')])
+ self._run_check("< >", [('data', '< >')])
+ self._run_check("< ", [('data', '< ')])
self._run_check("</>", [])
+ self._run_check("<$>", [('data', '<$>')])
self._run_check("</$>", [('comment', '$')])
self._run_check("</", [('data', '</')])
- self._run_check("</a", [('data', '</a')])
+ self._run_check("</a", [])
+ self._run_check("</ a>", [('comment', ' a')])
+ self._run_check("</ a", [('comment', ' a')])
self._run_check("<a<a>", [('starttag', 'a<a', [])])
self._run_check("</a<a>", [('endtag', 'a<a')])
- self._run_check("<!", [('data', '<!')])
- self._run_check("<a", [('data', '<a')])
- self._run_check("<a foo='bar'", [('data', "<a foo='bar'")])
- self._run_check("<a foo='bar", [('data', "<a foo='bar")])
- self._run_check("<a foo='>'", [('data', "<a foo='>'")])
- self._run_check("<a foo='>", [('data', "<a foo='>")])
+ self._run_check("<!", [('comment', '')])
+ self._run_check("<a", [])
+ self._run_check("<a foo='bar'", [])
+ self._run_check("<a foo='bar", [])
+ self._run_check("<a foo='>'", [])
+ self._run_check("<a foo='>", [])
self._run_check("<a$>", [('starttag', 'a$', [])])
self._run_check("<a$b>", [('starttag', 'a$b', [])])
self._run_check("<a$b/>", [('startendtag', 'a$b', [])])
self._run_check("<a$b >", [('starttag', 'a$b', [])])
self._run_check("<a$b />", [('startendtag', 'a$b', [])])
+ self._run_check("</a$b>", [('endtag', 'a$b')])
def test_slashes_in_starttag(self):
self._run_check('<a foo="var"/>', [('startendtag', 'a', [('foo', 'var')])])
@@ -445,6 +656,10 @@ def test_slashes_in_starttag(self):
]
self._run_check(html, expected)
+ def test_slashes_in_endtag(self):
+ self._run_check('</a/>', [('endtag', 'a')])
+ self._run_check('</a foo="var"/>', [('endtag', 'a')])
+
def test_declaration_junk_chars(self):
self._run_check("<!DOCTYPE foo $ >", [('decl', 'DOCTYPE foo $ ')])
@@ -515,15 +730,11 @@ def test_invalid_end_tags(self):
self._run_check(html, expected)
def test_broken_invalid_end_tag(self):
- # This is technically wrong (the "> shouldn't be included in the 'data')
- # but is probably not worth fixing it (in addition to all the cases of
- # the previous test, it would require a full attribute parsing).
- # see #13993
html = '<b>This</b attr=">"> confuses the parser'
expected = [('starttag', 'b', []),
('data', 'This'),
('endtag', 'b'),
- ('data', '"> confuses the parser')]
+ ('data', ' confuses the parser')]
self._run_check(html, expected)
def test_correct_detection_of_start_tags(self):
@@ -580,13 +791,64 @@ def test_unescape_method(self):
s = '""""""&#bad;'
self.assertEqual(p.unescape(s), unescape(s))
- def test_broken_comments(self):
- html = ('<! not really a comment >'
+ def test_eof_in_comments(self):
+ data = [
+ ('<!--', [('comment', '')]),
+ ('<!---', [('comment', '')]),
+ ('<!----', [('comment', '')]),
+ ('<!-----', [('comment', '-')]),
+ ('<!------', [('comment', '--')]),
+ ('<!----!', [('comment', '')]),
+ ('<!---!', [('comment', '-!')]),
+ ('<!---!>', [('comment', '-!>')]),
+ ('<!--foo', [('comment', 'foo')]),
+ ('<!--foo-', [('comment', 'foo')]),
+ ('<!--foo--', [('comment', 'foo')]),
+ ('<!--foo--!', [('comment', 'foo')]),
+ ('<!--<!--', [('comment', '<!')]),
+ ('<!--<!--!', [('comment', '<!')]),
+ ]
+ for html, expected in data:
+ self._run_check(html, expected)
+
+ def test_eof_in_declarations(self):
+ data = [
+ ('<!', [('comment', '')]),
+ ('<!-', [('comment', '-')]),
+ ('<![', [('comment', '[')]),
+ ('<!DOCTYPE', [('decl', 'DOCTYPE')]),
+ ('<!DOCTYPE ', [('decl', 'DOCTYPE ')]),
+ ('<!DOCTYPE html', [('decl', 'DOCTYPE html')]),
+ ('<!DOCTYPE html ', [('decl', 'DOCTYPE html ')]),
+ ('<!DOCTYPE html PUBLIC', [('decl', 'DOCTYPE html PUBLIC')]),
+ ('<!DOCTYPE html PUBLIC "foo', [('decl', 'DOCTYPE html PUBLIC "foo')]),
+ ('<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01//EN" "foo',
+ [('decl', 'DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01//EN" "foo')]),
+ ]
+ for html, expected in data:
+ self._run_check(html, expected)
+
+ @support.subTests('content', ['', 'x', 'x]', 'x]]'])
+ def test_eof_in_cdata(self, content):
+ self._run_check('<![CDATA[' + content,
+ [('unknown decl', 'CDATA[' + content)])
+ self._run_check('<![CDATA[' + content,
+ [('comment', '[CDATA[' + content)],
+ collector=EventCollector(autocdata=True))
+ self._run_check('<svg><text y="100"><![CDATA[' + content,
+ [('starttag', 'svg', []),
+ ('starttag', 'text', [('y', '100')]),
+ ('unknown decl', 'CDATA[' + content)])
+
+ def test_bogus_comments(self):
+ html = ('<!ELEMENT br EMPTY>'
+ '<! not really a comment >'
'<! not a comment either -->'
'<! -- close enough -->'
'<!><!<-- this was an empty comment>'
'<!!! another bogus comment !!!>')
expected = [
+ ('comment', 'ELEMENT br EMPTY'),
('comment', ' not really a comment '),
('comment', ' not a comment either --'),
('comment', ' -- close enough --'),
@@ -627,8 +889,57 @@ def test_broken_condcoms(self):
('startendtag', 'img', [('src', 'mammoth.bmp')]),
('unknown decl', 'endif')
]
+
self._run_check(html, expected)
+ @support.subTests('content', [
+ 'just some plain text',
+ '<!-- not a comment -->',
+ '¬-an-entity-ref;',
+ "<not a='start tag'>",
+ '',
+ '[[I have many brackets]]',
+ 'I have a > in the middle',
+ 'I have a ]] in the middle',
+ '] ]>',
+ ']] >',
+ ('\n'
+ ' if (a < b && a > b) {\n'
+ ' printf("[<marquee>How?</marquee>]");\n'
+ ' }\n'),
+ ])
+ def test_cdata_section_content(self, content):
+ # See "13.2.5.42 Markup declaration open state",
+ # "13.2.5.69 CDATA section state", and issue bpo-32876.
+ html = f'<svg><text y="100"><![CDATA[{content}]]]]><![CDATA[></text></svg>'
+ expected = [
+ ('starttag', 'svg', []),
+ ('starttag', 'text', [('y', '100')]),
+ ('unknown decl', 'CDATA[' + content),
+ ('endtag', 'text'),
+ ('endtag', 'svg'),
+ ]
+ self._run_check(html, expected)
+ self._run_check(html, expected, collector=EventCollector(autocdata=True))
+
+ def test_cdata_section(self):
+ # See "13.2.5.42 Markup declaration open state".
+ html = ('<![CDATA[foo<br>bar]]]]><![CDATA[>'
+ '<svg><text y="100"><![CDATA[foo<br>bar]]]]><![CDATA[></text></svg>'
+ '<![CDATA[foo<br>bar]]]]><![CDATA[>')
+ expected = [
+ ('comment', '[CDATA[foo<br'),
+ ('data', 'bar]]]]><![CDATA[>'),
+ ('starttag', 'svg', []),
+ ('starttag', 'text', [('y', '100')]),
+ ('unknown decl', 'CDATA[foo<br>bar'),
+ ('endtag', 'text'),
+ ('endtag', 'svg'),
+ ('comment', '[CDATA[foo<br'),
+ ('data', 'bar]]]]><![CDATA[>'),
+ ]
+ self._run_check(html, expected, collector=EventCollector(autocdata=True))
+
def test_convert_charrefs_dropped_text(self):
# #23144: make sure that all the events are triggered when
# convert_charrefs is True, even if we don't call .close()
@@ -641,6 +952,46 @@ def test_convert_charrefs_dropped_text(self):
('endtag', 'a'), ('data', ' bar & baz')]
)
+ @support.requires_resource('cpu')
+ def test_eof_no_quadratic_complexity(self):
+ # Each of these examples used to take about an hour.
+ # Now they take a fraction of a second.
+ def check(source):
+ parser = html.parser.HTMLParser()
+ parser.feed(source)
+ parser.close()
+ n = 120_000
+ check("<a " * n)
+ check("<a a=" * n)
+ check("</a " * 14 * n)
+ check("</a a=" * 11 * n)
+ check("<!--" * 4 * n)
+ check("<!" * 60 * n)
+ check("<?" * 19 * n)
+ check("</$" * 15 * n)
+ check("<![CDATA[" * 9 * n)
+ check("<!doctype" * 35 * n)
+
+ @support.requires_resource('cpu')
+ def test_incremental_no_quadratic_complexity(self):
+ # An unterminated construct fed in many small chunks used to take
+ # quadratic time, both to rescan and to concatenate the buffer.
+ # Now it takes a fraction of a second.
+ def check(prefix, chunk, suffix):
+ parser = html.parser.HTMLParser()
+ parser.feed(prefix)
+ for _ in range(200_000):
+ parser.feed(chunk)
+ parser.feed(suffix)
+ parser.close()
+ chunk = "a" * 64
+ check("<!--", chunk, "-->") # comment
+ check("<?", chunk, ">") # processing instruction
+ check("<!doctype ", chunk, ">") # doctype
+ check("<![CDATA[", chunk, "]]]]><![CDATA[>") # CDATA section
+ check("<a href='", chunk, "'>") # start tag
+ check("<script>", chunk, "</script>") # RAWTEXT element
+
class AttributesTestCase(TestCaseBase):
@@ -649,9 +1000,15 @@ def test_attr_syntax(self):
("starttag", "a", [("b", "v"), ("c", "v"), ("d", "v"), ("e", None)])
]
self._run_check("""<a b='v' c="v" d=v e>""", output)
- self._run_check("""<a b = 'v' c = "v" d = v e>""", output)
- self._run_check("""<a\nb\n=\n'v'\nc\n=\n"v"\nd\n=\nv\ne>""", output)
- self._run_check("""<a\tb\t=\t'v'\tc\t=\t"v"\td\t=\tv\te>""", output)
+ self._run_check("<a foo==bar>", [('starttag', 'a', [('foo', '=bar')])])
+ self._run_check("<a foo =bar>", [('starttag', 'a', [('foo', 'bar')])])
+ self._run_check("<a foo\t=bar>", [('starttag', 'a', [('foo', 'bar')])])
+ self._run_check("<a foo\v=bar>", [('starttag', 'a', [('foo\v', 'bar')])])
+ self._run_check("<a foo\xa0=bar>", [('starttag', 'a', [('foo\xa0', 'bar')])])
+ self._run_check("<a foo= bar>", [('starttag', 'a', [('foo', 'bar')])])
+ self._run_check("<a foo=\tbar>", [('starttag', 'a', [('foo', 'bar')])])
+ self._run_check("<a foo=\vbar>", [('starttag', 'a', [('foo', '\vbar')])])
+ self._run_check("<a foo=\xa0bar>", [('starttag', 'a', [('foo', '\xa0bar')])])
def test_attr_values(self):
self._run_check("""<a b='xxx\n\txxx' c="yyy\t\nyyy" d='\txyz\n'>""",
@@ -660,6 +1017,10 @@ def test_attr_values(self):
("d", "\txyz\n")])])
self._run_check("""<a b='' c="">""",
[("starttag", "a", [("b", ""), ("c", "")])])
+ self._run_check("<a b=\tx c=\ny>",
+ [('starttag', 'a', [('b', 'x'), ('c', 'y')])])
+ self._run_check("<a b=\v c=\xa0>",
+ [("starttag", "a", [("b", "\v"), ("c", "\xa0")])])
# Regression test for SF patch #669683.
self._run_check("<e a=rgb(1,2,3)>",
[("starttag", "e", [("a", "rgb(1,2,3)")])])
@@ -726,13 +1087,17 @@ def test_malformed_attributes(self):
)
expected = [
('starttag', 'a', [('href', "test'style='color:red;bad1'")]),
- ('data', 'test - bad1'), ('endtag', 'a'),
+ ('data', 'test - bad1'),
+ ('endtag', 'a'),
('starttag', 'a', [('href', "test'+style='color:red;ba2'")]),
- ('data', 'test - bad2'), ('endtag', 'a'),
+ ('data', 'test - bad2'),
+ ('endtag', 'a'),
('starttag', 'a', [('href', "test'\xa0style='color:red;bad3'")]),
- ('data', 'test - bad3'), ('endtag', 'a'),
+ ('data', 'test - bad3'),
+ ('endtag', 'a'),
('starttag', 'a', [('href', "test'\xa0style='color:red;bad4'")]),
- ('data', 'test - bad4'), ('endtag', 'a')
+ ('data', 'test - bad4'),
+ ('endtag', 'a'),
]
self._run_check(html, expected)
diff --git a/Lib/test/test_http_cookiejar.py b/Lib/test/test_http_cookiejar.py
index ff86ffa98c3eeab..371dad6f2d66915 100644
--- a/Lib/test/test_http_cookiejar.py
+++ b/Lib/test/test_http_cookiejar.py
@@ -829,10 +829,37 @@ def test_wrong_domain(self):
# XXX far from complete
c = CookieJar()
- interact_2965(c, "http://www.nasty.com/",
- 'foo=bar; domain=friendly.org; Version="1"')
+ interact_2965(c, "http://www.example.com/",
+ 'foo=bar; domain=python.org; Version="1"')
self.assertEqual(len(c), 0)
+ def test_negative_version(self):
+ # A negative version is neither Netscape (0) nor RFC 2965 (> 0), so
+ # such cookies must be neither set nor returned.
+ for rfc2965 in False, True:
+ for interact in interact_netscape, interact_2965:
+ with self.subTest(rfc2965=rfc2965, interact=interact.__name__):
+ c = CookieJar(DefaultCookiePolicy(rfc2965=rfc2965))
+ interact(c, "http://www.example.com/",
+ 'foo=bar; domain=python.org; Version=-1')
+ self.assertEqual(len(c), 0)
+ interact(c, "http://www.example.com/", 'foo=bar; Version=-1')
+ self.assertEqual(len(c), 0)
+
+ # Such a header is discarded before the policy is consulted.
+ c = CookieJar()
+ req = urllib.request.Request("http://www.example.com/")
+ res = FakeResponse(["Set-Cookie: foo=bar; Version=-1"],
+ "http://www.example.com/")
+ self.assertEqual(c.make_cookies(res, req), [])
+
+ # Not set from a response, for example loaded from an LWP cookie file.
+ c = CookieJar()
+ c.set_cookie(Cookie(-1, "foo", "bar", None, False,
+ ".python.org", True, True, "/", True,
+ False, None, True, None, None, {}))
+ self.assertEqual(interact_netscape(c, "http://www.python.org/"), "")
+
def test_strict_domain(self):
# Cookies whose domain is a country-code tld like .co.uk should
# not be set if CookiePolicy.strict_domain is true.
@@ -1235,6 +1262,7 @@ def cookiejar_from_cookie_headers(headers):
["Set-Cookie: b=foo; max-age=oops"],
# bad version
["Set-Cookie: b=foo; version=spam"],
+ ["Set-Cookie: b=foo; version=-1"],
["Set-Cookie:; Expires=%s" % future],
]:
c = cookiejar_from_cookie_headers(headers)
diff --git a/Lib/test/test_http_cookies.py b/Lib/test/test_http_cookies.py
index fb39f9cea129dbc..08107a0655c4989 100644
--- a/Lib/test/test_http_cookies.py
+++ b/Lib/test/test_http_cookies.py
@@ -17,10 +17,10 @@ def test_basic(self):
'repr': "<SimpleCookie: chips='ahoy' vienna='finger'>",
'output': 'Set-Cookie: chips=ahoy\nSet-Cookie: vienna=finger'},
- {'data': 'keebler="E=mc2; L=\\"Loves\\"; fudge=\\012;"',
- 'dict': {'keebler' : 'E=mc2; L="Loves"; fudge=\012;'},
- 'repr': '''<SimpleCookie: keebler='E=mc2; L="Loves"; fudge=\\n;'>''',
- 'output': 'Set-Cookie: keebler="E=mc2; L=\\"Loves\\"; fudge=\\012;"'},
+ {'data': 'keebler="E=mc2; L=\\"Loves\\"; fudge=;"',
+ 'dict': {'keebler' : 'E=mc2; L="Loves"; fudge=;'},
+ 'repr': '''<SimpleCookie: keebler='E=mc2; L="Loves"; fudge=;'>''',
+ 'output': 'Set-Cookie: keebler="E=mc2; L=\\"Loves\\"; fudge=;"'},
# Check illegal cookies that have an '=' char in an unquoted value
{'data': 'keebler=E=mc2',
@@ -504,6 +504,89 @@ def test_repr(self):
r'Set-Cookie: key=coded_val; '
r'expires=\w+, \d+ \w+ \d+ \d+:\d+:\d+ \w+')
+ def test_control_characters(self):
+ for c0 in support.control_characters_c0():
+ morsel = cookies.Morsel()
+
+ # .__setitem__()
+ with self.assertRaises(cookies.CookieError):
+ morsel[c0] = "val"
+ with self.assertRaises(cookies.CookieError):
+ morsel["path"] = c0
+
+ # .__setstate__()
+ with self.assertRaises(cookies.CookieError):
+ morsel.__setstate__({'key': c0, 'value': 'val', 'coded_value': 'coded'})
+ with self.assertRaises(cookies.CookieError):
+ morsel.__setstate__({'key': 'key', 'value': c0, 'coded_value': 'coded'})
+ with self.assertRaises(cookies.CookieError):
+ morsel.__setstate__({'key': 'key', 'value': 'val', 'coded_value': c0})
+
+ # .setdefault()
+ with self.assertRaises(cookies.CookieError):
+ morsel.setdefault("path", c0)
+ with self.assertRaises(cookies.CookieError):
+ morsel.setdefault(c0, "val")
+
+ # .set()
+ with self.assertRaises(cookies.CookieError):
+ morsel.set(c0, "val", "coded-value")
+ with self.assertRaises(cookies.CookieError):
+ morsel.set("path", c0, "coded-value")
+ with self.assertRaises(cookies.CookieError):
+ morsel.set("path", "val", c0)
+
+ # .update()
+ with self.assertRaises(cookies.CookieError):
+ morsel.update({"path": c0})
+ with self.assertRaises(cookies.CookieError):
+ morsel.update({c0: "val"})
+
+ # .__ior__()
+ with self.assertRaises(cookies.CookieError):
+ morsel |= {"path": c0}
+ with self.assertRaises(cookies.CookieError):
+ morsel |= {c0: "val"}
+
+ def test_control_characters_output(self):
+ # Tests that even if the internals of Morsel are modified
+ # that a call to .output() has control character safeguards.
+ for c0 in support.control_characters_c0():
+ morsel = cookies.Morsel()
+ morsel.set("key", "value", "coded-value")
+ morsel._key = c0 # Override private variable.
+ cookie = cookies.SimpleCookie()
+ cookie["cookie"] = morsel
+ with self.assertRaises(cookies.CookieError):
+ cookie.output()
+
+ morsel = cookies.Morsel()
+ morsel.set("key", "value", "coded-value")
+ morsel._coded_value = c0 # Override private variable.
+ cookie = cookies.SimpleCookie()
+ cookie["cookie"] = morsel
+ with self.assertRaises(cookies.CookieError):
+ cookie.output()
+
+ # Tests that .js_output() also has control character safeguards.
+ for c0 in support.control_characters_c0():
+ morsel = cookies.Morsel()
+ morsel.set("key", "value", "coded-value")
+ morsel._key = c0 # Override private variable.
+ cookie = cookies.SimpleCookie()
+ cookie["cookie"] = morsel
+ with self.assertRaises(cookies.CookieError):
+ cookie.js_output()
+
+ morsel = cookies.Morsel()
+ morsel.set("key", "value", "coded-value")
+ morsel._coded_value = c0 # Override private variable.
+ cookie = cookies.SimpleCookie()
+ cookie["cookie"] = morsel
+ with self.assertRaises(cookies.CookieError):
+ cookie.js_output()
+
+
def test_main():
run_unittest(CookieTests, MorselTests)
run_doctest(cookies)
diff --git a/Lib/test/test_httplib.py b/Lib/test/test_httplib.py
index a65044eea60dc05..0772296ea04e455 100644
--- a/Lib/test/test_httplib.py
+++ b/Lib/test/test_httplib.py
@@ -345,6 +345,51 @@ def test_invalid_headers(self):
with self.assertRaisesRegex(ValueError, 'Invalid header'):
conn.putheader(name, value)
+ def test_invalid_tunnel_headers(self):
+ cases = (
+ ('Invalid\r\nName', 'ValidValue'),
+ ('Invalid\rName', 'ValidValue'),
+ ('Invalid\nName', 'ValidValue'),
+ ('\r\nInvalidName', 'ValidValue'),
+ ('\rInvalidName', 'ValidValue'),
+ ('\nInvalidName', 'ValidValue'),
+ (' InvalidName', 'ValidValue'),
+ ('\tInvalidName', 'ValidValue'),
+ ('Invalid:Name', 'ValidValue'),
+ (':InvalidName', 'ValidValue'),
+ ('ValidName', 'Invalid\r\nValue'),
+ ('ValidName', 'Invalid\rValue'),
+ ('ValidName', 'Invalid\nValue'),
+ ('ValidName', 'InvalidValue\r\n'),
+ ('ValidName', 'InvalidValue\r'),
+ ('ValidName', 'InvalidValue\n'),
+ )
+ for name, value in cases:
+ with self.subTest((name, value)):
+ conn = client.HTTPConnection('example.com')
+ conn.set_tunnel('tunnel', headers={
+ name: value
+ })
+ conn.sock = FakeSocket('')
+ with self.assertRaisesRegex(ValueError, 'Invalid header'):
+ conn._tunnel() # Called in .connect()
+
+ def test_invalid_tunnel_host(self):
+ cases = (
+ 'invalid\r.host',
+ '\ninvalid.host',
+ 'invalid.host\r\n',
+ 'invalid.host\x00',
+ 'invalid host',
+ )
+ for tunnel_host in cases:
+ with self.subTest(tunnel_host):
+ conn = client.HTTPConnection('example.com')
+ conn.set_tunnel(tunnel_host)
+ conn.sock = FakeSocket('')
+ with self.assertRaisesRegex(ValueError, 'Tunnel host can\'t contain control characters'):
+ conn._tunnel() # Called in .connect()
+
def test_headers_debuglevel(self):
body = (
b'HTTP/1.1 200 OK\r\n'
@@ -952,6 +997,49 @@ def test_negative_content_length(self):
self.assertEqual(resp.read(), b'Hello\r\n')
self.assertTrue(resp.isclosed())
+ def test_malformed_content_length(self):
+ # RFC 9112: Content-Length = 1*DIGIT. Values that int() accepts but
+ # the grammar forbids must not be used to frame the body.
+ for value in ('+5', '5_0', '5 0'):
+ with self.subTest(value=value):
+ sock = FakeSocket(
+ 'HTTP/1.1 200 OK\r\nContent-Length: %s\r\n\r\nHello\r\n' % value)
+ resp = client.HTTPResponse(sock, method="GET")
+ resp.begin()
+ self.assertIsNone(resp.length)
+ self.assertEqual(resp.read(), b'Hello\r\n')
+ resp.close()
+
+ def test_content_length_with_whitespace(self):
+ # RFC 9112, section 5.1: optional whitespace (SP / HTAB) around
+ # the field value is not part of the value, so it must not
+ # prevent the Content-Length from being used to frame the body.
+ for value in ('7 ', '7\t', '7 \t ', ' 7 '):
+ with self.subTest(value=value):
+ sock = FakeSocket(
+ 'HTTP/1.1 200 OK\r\nContent-Length: %s\r\n\r\n'
+ 'Hello\r\nextra' % value)
+ resp = client.HTTPResponse(sock, method="GET")
+ resp.begin()
+ self.assertEqual(resp.length, 7)
+ self.assertEqual(resp.read(), b'Hello\r\n')
+ self.assertTrue(resp.isclosed())
+ resp.close()
+
+ def test_malformed_chunk_size(self):
+ # RFC 9112: chunk-size = 1*HEXDIG. Reject sizes that int(_, 16) accepts
+ # but the grammar forbids (a sign, an "0x" prefix, underscores or
+ # leading whitespace).
+ start = 'HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n'
+ for size in ('-5', '+5', '0x5', '1_f', ' 5'):
+ with self.subTest(size=size):
+ sock = FakeSocket(start + '%s\r\nHELLO\r\n0\r\n\r\n' % size)
+ resp = client.HTTPResponse(sock, method="GET")
+ resp.begin()
+ self.assertRaises(client.IncompleteRead, resp.read)
+ self.assertTrue(resp.isclosed())
+ resp.close()
+
def test_incomplete_read(self):
sock = FakeSocket('HTTP/1.1 200 OK\r\nContent-Length: 10\r\n\r\nHello\r\n')
resp = client.HTTPResponse(sock, method="GET")
@@ -1011,6 +1099,35 @@ def test_overflowing_header_limit_after_100(self):
self.assertIn('got more than ', str(cm.exception))
self.assertIn('headers', str(cm.exception))
+ def test_too_many_interim_responses(self):
+ # A server streaming "100 Continue" responses forever must not
+ # hang getresponse().
+ body = (
+ 'HTTP/1.1 100 Continue\r\n\r\n'
+ * (client._MAXINTERIMRESPONSES + 1)
+ )
+ resp = client.HTTPResponse(FakeSocket(body))
+ with self.assertRaises(client.HTTPException) as cm:
+ resp.begin()
+ self.assertIn('got more than ', str(cm.exception))
+ self.assertIn('interim responses', str(cm.exception))
+
+ def test_multiple_interim_responses(self):
+ # A reasonable number of interim responses before the final
+ # response is skipped as before.
+ body = (
+ 'HTTP/1.1 100 Continue\r\n\r\n' * 3 +
+ 'HTTP/1.1 200 OK\r\n'
+ 'Content-Length: 5\r\n'
+ '\r\n'
+ 'hello'
+ )
+ resp = client.HTTPResponse(FakeSocket(body), method="GET")
+ resp.begin()
+ self.assertEqual(resp.status, 200)
+ self.assertEqual(resp.read(), b'hello')
+ resp.close()
+
def test_overflowing_chunked_line(self):
body = (
'HTTP/1.1 200 OK\r\n'
@@ -1082,6 +1199,35 @@ def test_chunked_trailers(self):
self.assertEqual(sock.file.read(), b"") #we read to the end
resp.close()
+ def test_chunked_too_many_trailers(self):
+ """A response streaming endless trailer lines must raise, not hang"""
+ too_many_trailers = "".join(
+ f"X-Trailer{i}: {i}\r\n" for i in range(client._MAXHEADERS + 1)
+ )
+ # An unbounded read() reaches the trailers via the final 0 chunk.
+ sock = FakeSocket(
+ chunked_start + last_chunk + too_many_trailers + chunked_end)
+ resp = client.HTTPResponse(sock, method="GET")
+ resp.begin()
+ with self.assertRaisesRegex(
+ client.HTTPException,
+ f"got more than {client._MAXHEADERS} trailers",
+ ):
+ resp.read()
+ resp.close()
+
+ # A bounded read(amt) larger than the body hits the same limit.
+ sock = FakeSocket(
+ chunked_start + last_chunk + too_many_trailers + chunked_end)
+ resp = client.HTTPResponse(sock, method="GET")
+ resp.begin()
+ with self.assertRaisesRegex(
+ client.HTTPException,
+ f"got more than {client._MAXHEADERS} trailers",
+ ):
+ resp.read(len(chunked_expected) + 1)
+ resp.close()
+
def test_chunked_sync(self):
"""Check that we don't read past the end of the chunked-encoding stream"""
expected = chunked_expected
@@ -1192,6 +1338,76 @@ def run_server():
thread.join()
self.assertEqual(result, b"proxied data\n")
+ def test_large_content_length(self):
+ serv = socket.socket()
+ serv.bind((HOST, 0))
+ serv.listen()
+ self.addCleanup(serv.close)
+
+ def run_server():
+ [conn, address] = serv.accept()
+ with conn:
+ while conn.recv(1024):
+ conn.sendall(
+ b"HTTP/1.1 200 Ok\r\n"
+ b"Content-Length: %d\r\n"
+ b"\r\n" % size)
+ conn.sendall(b'A' * (size//3))
+ conn.sendall(b'B' * (size - size//3))
+
+ thread = threading.Thread(target=run_server)
+ thread.start()
+ self.addCleanup(thread.join, 1.0)
+
+ conn = client.HTTPConnection(*serv.getsockname())
+ try:
+ for w in range(15, 27):
+ size = 1 << w
+ conn.request("GET", "/")
+ with conn.getresponse() as response:
+ self.assertEqual(len(response.read()), size)
+ finally:
+ conn.close()
+ thread.join(1.0)
+
+ def test_large_content_length_truncated(self):
+ serv = socket.socket()
+ serv.bind((HOST, 0))
+ serv.listen()
+ self.addCleanup(serv.close)
+
+ def run_server():
+ while True:
+ [conn, address] = serv.accept()
+ with conn:
+ conn.recv(1024)
+ if not size:
+ break
+ conn.sendall(
+ b"HTTP/1.1 200 Ok\r\n"
+ b"Content-Length: %d\r\n"
+ b"\r\n"
+ b"Text" % size)
+
+ thread = threading.Thread(target=run_server)
+ thread.start()
+ self.addCleanup(thread.join, 1.0)
+
+ conn = client.HTTPConnection(*serv.getsockname())
+ try:
+ for w in range(18, 65):
+ size = 1 << w
+ conn.request("GET", "/")
+ with conn.getresponse() as response:
+ self.assertRaises(client.IncompleteRead, response.read)
+ conn.close()
+ finally:
+ conn.close()
+ size = 0
+ conn.request("GET", "/")
+ conn.close()
+ thread.join(1.0)
+
def test_putrequest_override_domain_validation(self):
"""
It should be possible to override the default validation
diff --git a/Lib/test/test_httpservers.py b/Lib/test/test_httpservers.py
index 8c9be6890039234..f2be621e7cc29d4 100644
--- a/Lib/test/test_httpservers.py
+++ b/Lib/test/test_httpservers.py
@@ -641,6 +641,20 @@ def test_html_escape_filename(self):
print(os.environ["%s"])
"""
+cgi_file7 = """\
+#!%s
+import os
+import sys
+
+print("Content-type: text/plain")
+print()
+
+content_length = int(os.environ["CONTENT_LENGTH"])
+body = sys.stdin.buffer.read(content_length)
+
+print(f"{content_length} {len(body)}")
+"""
+
@unittest.skipIf(hasattr(os, 'geteuid') and os.geteuid() == 0,
"This test can't be run reliably as root (issue #13308).")
@@ -663,6 +677,7 @@ def setUp(self):
self.file2_path = None
self.file3_path = None
self.file4_path = None
+ self.file7_path = None
# The shebang line should be pure ASCII: use symlink if possible.
# See issue #7668.
@@ -706,6 +721,11 @@ def setUp(self):
file4.write(cgi_file4 % (self.pythonexe, 'QUERY_STRING'))
os.chmod(self.file4_path, 0o777)
+ self.file7_path = os.path.join(self.cgi_dir, 'file7.py')
+ with open(self.file7_path, 'w', encoding='utf-8') as file7:
+ file7.write(cgi_file7 % self.pythonexe)
+ os.chmod(self.file7_path, 0o777)
+
os.chdir(self.parent_dir)
def tearDown(self):
@@ -723,6 +743,8 @@ def tearDown(self):
os.remove(self.file3_path)
if self.file4_path:
os.remove(self.file4_path)
+ if self.file7_path:
+ os.remove(self.file7_path)
os.rmdir(self.cgi_child_dir)
os.rmdir(self.cgi_dir)
os.rmdir(self.parent_dir)
@@ -789,6 +811,22 @@ def test_post(self):
self.assertEqual(res.read(), b'1, python, 123456' + self.linesep)
+ def test_large_content_length(self):
+ for w in range(15, 25):
+ size = 1 << w
+ body = b'X' * size
+ headers = {'Content-Length' : str(size)}
+ res = self.request('/cgi-bin/file7.py', 'POST', body, headers)
+ self.assertEqual(res.read(), b'%d %d' % (size, size) + self.linesep)
+
+ def test_large_content_length_truncated(self):
+ with support.swap_attr(self.request_handler, 'timeout', 0.001):
+ for w in range(18, 65):
+ size = 1 << w
+ headers = {'Content-Length' : str(size)}
+ res = self.request('/cgi-bin/file1.py', 'POST', b'x', headers)
+ self.assertEqual(res.read(), b'Hello World' + self.linesep)
+
def test_invaliduri(self):
res = self.request('/cgi-bin/invalid')
res.read()
diff --git a/Lib/test/test_imaplib.py b/Lib/test/test_imaplib.py
index 300a6d7b652b28b..0e95a442a5c8ecb 100644
--- a/Lib/test/test_imaplib.py
+++ b/Lib/test/test_imaplib.py
@@ -189,6 +189,10 @@ def cmd_LOGIN(self, tag, args):
self.server.logged = args[0]
self._send_tagged(tag, 'OK', 'LOGIN completed')
+ def cmd_SELECT(self, tag, args):
+ self._send_line(b'* 2 EXISTS')
+ self._send_tagged(tag, 'OK', '[READ-WRITE] SELECT completed.')
+
class NewIMAPTestsMixin():
client = None
@@ -485,6 +489,21 @@ def cmd_LSUB(self, tag, args):
self.assertEqual(typ, 'OK')
self.assertEqual(data[0], b'() "." directoryA')
+ def test_control_characters(self):
+ client, server = self._setup(SimpleIMAPHandler)
+ client.login('user', 'pass')
+ for c in '\0\r\n':
+ with self.assertRaises(ValueError):
+ client.select(f'a{c}b')
+ # Other control characters are valid in a quoted string and can
+ # occur in mailbox names returned by the server, so they must
+ # still be accepted.
+ for c in support.control_characters_c0():
+ if c in '\0\r\n':
+ continue
+ typ, _ = client.select(f'a{c}b')
+ self.assertEqual(typ, 'OK')
+
class NewIMAPTests(NewIMAPTestsMixin, unittest.TestCase):
imap_class = imaplib.IMAP4
@@ -854,6 +873,21 @@ def handle(self):
self.assertRaises(imaplib.IMAP4.error,
self.imap_class, *server.server_address)
+ @reap_threads
+ def test_truncated_large_literal(self):
+ size = 0
+ class BadHandler(SimpleIMAPHandler):
+ def handle(self):
+ self._send_textline('* OK {%d}' % size)
+ self._send_textline('IMAP4rev1')
+
+ for exponent in range(15, 64):
+ size = 1 << exponent
+ with self.subTest(f"size=2e{size}"):
+ with self.reaped_server(BadHandler) as server:
+ with self.assertRaises(imaplib.IMAP4.abort):
+ self.imap_class(*server.server_address)
+
@reap_threads
def test_simple_with_statement(self):
# simplest call
diff --git a/Lib/test/test_ipaddress.py b/Lib/test/test_ipaddress.py
index 65b9168fe0ead76..1e1ae4d46226ec9 100644
--- a/Lib/test/test_ipaddress.py
+++ b/Lib/test/test_ipaddress.py
@@ -96,8 +96,23 @@ def pickle_test(self, addr):
class CommonTestMixin_v4(CommonTestMixin):
def test_leading_zeros(self):
- self.assertInstancesEqual("000.000.000.000", "0.0.0.0")
- self.assertInstancesEqual("192.168.000.001", "192.168.0.1")
+ # bpo-36384: no leading zeros to avoid ambiguity with octal notation
+ msg = "Leading zeros are not permitted in '\d+'"
+ addresses = [
+ "000.000.000.000",
+ "192.168.000.001",
+ "016.016.016.016",
+ "192.168.000.001",
+ "001.000.008.016",
+ "01.2.3.40",
+ "1.02.3.40",
+ "1.2.03.40",
+ "1.2.3.040",
+ ]
+ for address in addresses:
+ with self.subTest(address=address):
+ with self.assertAddressError(msg):
+ self.factory(address)
def test_int(self):
self.assertInstancesEqual(0, "0.0.0.0")
@@ -230,15 +245,6 @@ def assertBadOctet(addr, octet):
assertBadOctet("1.2.3.4::", "4::")
assertBadOctet("1.a.2.3", "a")
- def test_octal_decimal_ambiguity(self):
- def assertBadOctet(addr, octet):
- msg = "Ambiguous (octal/decimal) value in %r not permitted in %r"
- with self.assertAddressError(re.escape(msg % (octet, addr))):
- ipaddress.IPv4Address(addr)
-
- assertBadOctet("016.016.016.016", "016")
- assertBadOctet("001.000.008.016", "008")
-
def test_octet_length(self):
def assertBadOctet(addr, octet):
msg = "At most 3 characters permitted in %r in %r"
@@ -295,6 +301,18 @@ def assertBadSplit(addr):
# A trailing IPv4 address is two parts
assertBadSplit("10:9:8:7:6:5:4:3:42.42.42.42")
+ def test_bad_address_split_v6_too_long(self):
+ def assertBadSplit(addr):
+ msg = r"At most 45 characters expected in '%s"
+ with self.assertAddressError(msg, re.escape(addr[:45])):
+ ipaddress.IPv6Address(addr)
+
+ # Long IPv6 address
+ long_addr = ("0:" * 10000) + "0"
+ assertBadSplit(long_addr)
+ assertBadSplit(long_addr + ":255.255.255.255")
+ assertBadSplit(long_addr + ":ffff:255.255.255.255")
+
def test_bad_address_split_v6_too_many_parts(self):
def assertBadSplit(addr):
msg = "Exactly 8 parts expected without '::' in %r"
@@ -1676,6 +1694,11 @@ def testIPv6AddressTooLarge(self):
self.assertEqual(ipaddress.ip_address('FFFF::192.0.2.1'),
ipaddress.ip_address('FFFF::c000:201'))
+ self.assertEqual(ipaddress.ip_address('0000:0000:0000:0000:0000:FFFF:192.168.255.255'),
+ ipaddress.ip_address('::ffff:c0a8:ffff'))
+ self.assertEqual(ipaddress.ip_address('FFFF:0000:0000:0000:0000:0000:192.168.255.255'),
+ ipaddress.ip_address('ffff::c0a8:ffff'))
+
def testIPVersion(self):
self.assertEqual(self.ipv4_address.version, 4)
self.assertEqual(self.ipv6_address.version, 6)
@@ -1786,7 +1809,7 @@ def testReservedIpv4(self):
self.assertEqual(True, ipaddress.ip_network('0.0.0.0').is_unspecified)
def testPrivateNetworks(self):
- self.assertEqual(False, ipaddress.ip_network("0.0.0.0/0").is_private)
+ self.assertEqual(True, ipaddress.ip_network("0.0.0.0/0").is_private)
self.assertEqual(False, ipaddress.ip_network("1.0.0.0/8").is_private)
self.assertEqual(True, ipaddress.ip_network("0.0.0.0/8").is_private)
@@ -1918,6 +1941,36 @@ def testIpv4Mapped(self):
self.assertEqual(ipaddress.ip_address('::ffff:c0a8:101').ipv4_mapped,
ipaddress.ip_address('192.168.1.1'))
+ def testIpv4MappedProperties(self):
+ # Test that an IPv4 mapped IPv6 address has
+ # the same properties as an IPv4 address.
+ for addr4 in (
+ "178.62.3.251", # global
+ "169.254.169.254", # link local
+ "127.0.0.1", # loopback
+ "224.0.0.1", # multicast
+ "192.168.0.1", # private
+ "0.0.0.0", # unspecified
+ "100.64.0.1", # public and not global
+ ):
+ with self.subTest(addr4):
+ ipv4 = ipaddress.IPv4Address(addr4)
+ ipv6 = ipaddress.IPv6Address(f"::ffff:{addr4}")
+
+ self.assertEqual(ipv4.is_global, ipv6.is_global)
+ self.assertEqual(ipv4.is_private, ipv6.is_private)
+ self.assertEqual(ipv4.is_reserved, ipv6.is_reserved)
+ self.assertEqual(ipv4.is_multicast, ipv6.is_multicast)
+ self.assertEqual(ipv4.is_unspecified, ipv6.is_unspecified)
+ self.assertEqual(ipv4.is_link_local, ipv6.is_link_local)
+ self.assertEqual(ipv4.is_loopback, ipv6.is_loopback)
+
+ def testIpv4MappedPrivateCheck(self):
+ self.assertEqual(
+ True, ipaddress.ip_address('::ffff:192.168.1.1').is_private)
+ self.assertEqual(
+ False, ipaddress.ip_address('::ffff:172.32.0.0').is_private)
+
def testAddrExclude(self):
addr1 = ipaddress.ip_network('10.1.1.0/24')
addr2 = ipaddress.ip_network('10.1.1.0/26')
@@ -2019,6 +2072,11 @@ def testCompressIPv6Address(self):
'::7:6:5:4:3:2:0': '0:7:6:5:4:3:2:0/128',
'7:6:5:4:3:2:1::': '7:6:5:4:3:2:1:0/128',
'0:6:5:4:3:2:1::': '0:6:5:4:3:2:1:0/128',
+ '0000:0000:0000:0000:0000:0000:255.255.255.255': '::ffff:ffff/128',
+ # 3.7 prints IPv4-mapped addresses in hex (gh-87799 is not backported).
+ '0000:0000:0000:0000:0000:ffff:255.255.255.255': '::ffff:ffff:ffff/128',
+ 'ffff:ffff:ffff:ffff:ffff:ffff:255.255.255.255':
+ 'ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff/128',
}
for uncompressed, compressed in list(test_addresses.items()):
self.assertEqual(compressed, str(ipaddress.IPv6Interface(
@@ -2155,5 +2213,34 @@ def testV6HashIsNotConstant(self):
ipv6_address2 = ipaddress.IPv6Interface("2001:658:22a:cafe:200:0:0:2")
self.assertNotEqual(ipv6_address1.__hash__(), ipv6_address2.__hash__())
+ # issue 134062 Hash collisions in IPv4Network and IPv6Network
+ def testNetworkV4HashCollisions(self):
+ self.assertNotEqual(
+ ipaddress.IPv4Network("192.168.1.255/32").__hash__(),
+ ipaddress.IPv4Network("192.168.1.0/24").__hash__()
+ )
+ self.assertNotEqual(
+ ipaddress.IPv4Network("172.24.255.0/24").__hash__(),
+ ipaddress.IPv4Network("172.24.0.0/16").__hash__()
+ )
+ self.assertNotEqual(
+ ipaddress.IPv4Network("192.168.1.87/32").__hash__(),
+ ipaddress.IPv4Network("192.168.1.86/31").__hash__()
+ )
+
+ # issue 134062 Hash collisions in IPv4Network and IPv6Network
+ def testNetworkV6HashCollisions(self):
+ self.assertNotEqual(
+ ipaddress.IPv6Network("fe80::/64").__hash__(),
+ ipaddress.IPv6Network("fe80::ffff:ffff:ffff:0/112").__hash__()
+ )
+ self.assertNotEqual(
+ ipaddress.IPv4Network("10.0.0.0/8").__hash__(),
+ ipaddress.IPv6Network(
+ "ffff:ffff:ffff:ffff:ffff:ffff:aff:0/112"
+ ).__hash__()
+ )
+
+
if __name__ == '__main__':
unittest.main()
diff --git a/Lib/test/test_list.py b/Lib/test/test_list.py
index c9cdc1e384a645e..bdc9b9f81712a94 100644
--- a/Lib/test/test_list.py
+++ b/Lib/test/test_list.py
@@ -200,6 +200,31 @@ def __eq__(self, other):
list4 = [1]
self.assertFalse(list3 == list4)
+ def test_lt_operator_modifying_operand(self):
+ # See gh-120298
+ class evil:
+ def __lt__(self, other):
+ other.clear()
+ return NotImplemented
+
+ a = [[evil()]]
+ with self.assertRaises(TypeError):
+ a[0] < a
+
+ def test_list_index_modifing_operand(self):
+ # See gh-120384
+ class evil:
+ def __init__(self, lst):
+ self.lst = lst
+ def __iter__(self):
+ yield from self.lst
+ self.lst.clear()
+
+ lst = list(range(5))
+ operand = evil(lst)
+ with self.assertRaises(ValueError):
+ lst[::-1] = operand
+
def test_count_index_remove_crashes(self):
# bpo-38610: The count(), index(), and remove() methods were not
# holding strong references to list elements while calling
diff --git a/Lib/test/test_mimetypes.py b/Lib/test/test_mimetypes.py
index f29de8c3a1a2832..ac53e5b98196d28 100644
--- a/Lib/test/test_mimetypes.py
+++ b/Lib/test/test_mimetypes.py
@@ -7,6 +7,9 @@
from test import support
+# The default list, for test_default_knownfiles
+knownfiles = mimetypes.knownfiles
+
# Tell it we don't know about external files:
mimetypes.knownfiles = []
mimetypes.inited = False
@@ -26,6 +29,10 @@ def test_default_data(self):
eq(self.db.guess_type("foo.tar.bz2"), ("application/x-tar", "bzip2"))
eq(self.db.guess_type("foo.tar.xz"), ("application/x-tar", "xz"))
+ @unittest.skipIf(sys.platform == "win32", "non-Windows only")
+ def test_default_knownfiles(self):
+ self.assertIn("/etc/mime.types", knownfiles)
+
def test_data_urls(self):
eq = self.assertEqual
guess_type = self.db.guess_type
@@ -191,6 +198,9 @@ def test_registry_parsing(self):
eq(self.db.guess_type("image.jpg"), ("image/jpeg", None))
eq(self.db.guess_type("image.png"), ("image/png", None))
+ def test_default_knownfiles(self):
+ self.assertEqual(knownfiles, [])
+
class MiscTestCase(unittest.TestCase):
def test__all__(self):
diff --git a/Lib/test/test_minidom.py b/Lib/test/test_minidom.py
index 0b76f6e87bc50b5..2cde54d4187275f 100644
--- a/Lib/test/test_minidom.py
+++ b/Lib/test/test_minidom.py
@@ -2,13 +2,15 @@
import copy
import pickle
+import time
+import io
from test import support
import unittest
import pyexpat
import xml.dom.minidom
-from xml.dom.minidom import parse, Node, Document, parseString
+from xml.dom.minidom import parse, Attr, Node, Document, Element, parseString
from xml.dom.minidom import getDOMImplementation
from xml.parsers.expat import ExpatError
@@ -162,6 +164,36 @@ def testAppendChild(self):
self.confirm(dom.documentElement.childNodes[-1].data == "Hello")
dom.unlink()
+ @support.requires_resource('cpu')
+ def testAppendChildNoQuadraticComplexity(self):
+ impl = getDOMImplementation()
+
+ newdoc = impl.createDocument(None, "some_tag", None)
+ top_element = newdoc.documentElement
+ children = [newdoc.createElement(f"child-{i}") for i in range(1, 2 ** 15 + 1)]
+ element = top_element
+
+ start = time.monotonic()
+ for child in children:
+ element.appendChild(child)
+ element = child
+ end = time.monotonic()
+
+ # This example used to take at least 30 seconds.
+ # Conservative assertion due to the wide variety of systems and
+ # build configs timing based tests wind up run under.
+ # A --with-address-sanitizer --with-pydebug build on a rpi5 still
+ # completes this loop in <0.5 seconds.
+ self.assertLess(end - start, 4)
+
+ def testSetAttributeNodeWithoutOwnerDocument(self):
+ # regression test for gh-142754
+ elem = Element("test")
+ attr = Attr("id")
+ attr.value = "test-id"
+ elem.setAttributeNode(attr)
+ self.assertEqual(elem.getAttribute("id"), "test-id")
+
def testAppendChildFragment(self):
dom, orig, c1, c2, c3, frag = self._create_fragment_test_nodes()
dom.documentElement.appendChild(frag)
diff --git a/Lib/test/test_ntpath.py b/Lib/test/test_ntpath.py
index 89b0d5303db5a3e..0a07aec39bea862 100644
--- a/Lib/test/test_ntpath.py
+++ b/Lib/test/test_ntpath.py
@@ -3,7 +3,8 @@
import sys
import unittest
import warnings
-from test.support import TestFailed, FakePath
+from ntpath import ALLOW_MISSING
+from test.support import TestFailed, FakePath, EnvironmentVarGuard
from test import support, test_genericpath
from tempfile import TemporaryFile
@@ -14,6 +15,28 @@
# but for those that require it we import here.
nt = None
+# 3.7's ntpath defines an XP-compatible _getfinalpathname() fallback on
+# every platform, so check the nt module for the real one.
+HAVE_GETFINALPATHNAME = hasattr(nt, '_getfinalpathname')
+
+try:
+ import ctypes
+except ImportError:
+ HAVE_GETSHORTPATHNAME = False
+else:
+ HAVE_GETSHORTPATHNAME = True
+ def _getshortpathname(path):
+ GSPN = ctypes.WinDLL("kernel32", use_last_error=True).GetShortPathNameW
+ GSPN.argtypes = [ctypes.c_wchar_p, ctypes.c_wchar_p, ctypes.c_uint32]
+ GSPN.restype = ctypes.c_uint32
+ result_len = GSPN(path, None, 0)
+ if not result_len:
+ raise OSError("failed to get short path name 0x{:08X}"
+ .format(ctypes.get_last_error()))
+ result = ctypes.create_unicode_buffer(result_len)
+ result_len = GSPN(path, result, result_len)
+ return result[:result_len]
+
def _norm(path):
if isinstance(path, (bytes, str, os.PathLike)):
@@ -47,6 +70,27 @@ def tester(fn, wantResult):
%(str(fn), str(wantResult), repr(gotResult)))
+def _parameterize(*parameters):
+ """Simplistic decorator to parametrize a test
+
+ Runs the decorated test multiple times in subTest, with a value from
+ 'parameters' passed as an extra positional argument.
+ Calls doCleanups() after each run.
+
+ Not for general use. Intended to avoid indenting for easier backports.
+
+ See https://discuss.python.org/t/91827 for discussing generalizations.
+ """
+ def _parametrize_decorator(func):
+ def _parameterized(self, *args, **kwargs):
+ for parameter in parameters:
+ with self.subTest(parameter):
+ func(self, *args, parameter, **kwargs)
+ self.doCleanups()
+ return _parameterized
+ return _parametrize_decorator
+
+
class NtpathTestCase(unittest.TestCase):
def assertPathEqual(self, path1, path2):
if path1 == path2 or _norm(path1) == _norm(path2):
@@ -209,8 +253,392 @@ def test_normpath(self):
tester("ntpath.normpath('\\\\.\\NUL')", r'\\.\NUL')
tester("ntpath.normpath('\\\\?\\D:/XY\\Z')", r'\\?\D:/XY\Z')
+ def test_realpath_curdir(self):
+ expected = ntpath.normpath(os.getcwd())
+ tester("ntpath.realpath('.')", expected)
+ tester("ntpath.realpath('./.')", expected)
+ tester("ntpath.realpath('/'.join(['.'] * 100))", expected)
+ tester("ntpath.realpath('.\\.')", expected)
+ tester("ntpath.realpath('\\'.join(['.'] * 100))", expected)
+
+ def test_realpath_curdir_strict(self):
+ expected = ntpath.normpath(os.getcwd())
+ tester("ntpath.realpath('.', strict=True)", expected)
+ tester("ntpath.realpath('./.', strict=True)", expected)
+ tester("ntpath.realpath('/'.join(['.'] * 100), strict=True)", expected)
+ tester("ntpath.realpath('.\\.', strict=True)", expected)
+ tester("ntpath.realpath('\\'.join(['.'] * 100), strict=True)", expected)
+
+ def test_realpath_curdir_missing_ok(self):
+ expected = ntpath.normpath(os.getcwd())
+ tester("ntpath.realpath('.', strict=ALLOW_MISSING)",
+ expected)
+ tester("ntpath.realpath('./.', strict=ALLOW_MISSING)",
+ expected)
+ tester("ntpath.realpath('/'.join(['.'] * 100), strict=ALLOW_MISSING)",
+ expected)
+ tester("ntpath.realpath('.\\.', strict=ALLOW_MISSING)",
+ expected)
+ tester("ntpath.realpath('\\'.join(['.'] * 100), strict=ALLOW_MISSING)",
+ expected)
+
+ def test_realpath_pardir(self):
+ expected = ntpath.normpath(os.getcwd())
+ tester("ntpath.realpath('..')", ntpath.dirname(expected))
+ tester("ntpath.realpath('../..')",
+ ntpath.dirname(ntpath.dirname(expected)))
+ tester("ntpath.realpath('/'.join(['..'] * 50))",
+ ntpath.splitdrive(expected)[0] + '\\')
+ tester("ntpath.realpath('..\\..')",
+ ntpath.dirname(ntpath.dirname(expected)))
+ tester("ntpath.realpath('\\'.join(['..'] * 50))",
+ ntpath.splitdrive(expected)[0] + '\\')
+
+ def test_realpath_pardir_strict(self):
+ expected = ntpath.normpath(os.getcwd())
+ tester("ntpath.realpath('..', strict=True)", ntpath.dirname(expected))
+ tester("ntpath.realpath('../..', strict=True)",
+ ntpath.dirname(ntpath.dirname(expected)))
+ tester("ntpath.realpath('/'.join(['..'] * 50), strict=True)",
+ ntpath.splitdrive(expected)[0] + '\\')
+ tester("ntpath.realpath('..\\..', strict=True)",
+ ntpath.dirname(ntpath.dirname(expected)))
+ tester("ntpath.realpath('\\'.join(['..'] * 50), strict=True)",
+ ntpath.splitdrive(expected)[0] + '\\')
+
+ def test_realpath_pardir_missing_ok(self):
+ expected = ntpath.normpath(os.getcwd())
+ tester("ntpath.realpath('..', strict=ALLOW_MISSING)",
+ ntpath.dirname(expected))
+ tester("ntpath.realpath('../..', strict=ALLOW_MISSING)",
+ ntpath.dirname(ntpath.dirname(expected)))
+ tester("ntpath.realpath('/'.join(['..'] * 50), strict=ALLOW_MISSING)",
+ ntpath.splitdrive(expected)[0] + '\\')
+ tester("ntpath.realpath('..\\..', strict=ALLOW_MISSING)",
+ ntpath.dirname(ntpath.dirname(expected)))
+ tester("ntpath.realpath('\\'.join(['..'] * 50), strict=ALLOW_MISSING)",
+ ntpath.splitdrive(expected)[0] + '\\')
+
+ @support.skip_unless_symlink
+ @unittest.skipUnless(HAVE_GETFINALPATHNAME, 'need _getfinalpathname')
+ @_parameterize({}, {'strict': True}, {'strict': ALLOW_MISSING})
+ def test_realpath_basic(self, kwargs):
+ ABSTFN = ntpath.abspath(support.TESTFN)
+ open(ABSTFN, "wb").close()
+ self.addCleanup(support.unlink, ABSTFN)
+ self.addCleanup(support.unlink, ABSTFN + "1")
+
+ os.symlink(ABSTFN, ABSTFN + "1")
+ self.assertPathEqual(ntpath.realpath(ABSTFN + "1", **kwargs), ABSTFN)
+ self.assertPathEqual(ntpath.realpath(os.fsencode(ABSTFN + "1"), **kwargs),
+ os.fsencode(ABSTFN))
+
+ @support.skip_unless_symlink
+ @unittest.skipUnless(HAVE_GETFINALPATHNAME, 'need _getfinalpathname')
+ def test_realpath_strict(self):
+ # Bug #43757: raise FileNotFoundError in strict mode if we encounter
+ # a path that does not exist.
+ ABSTFN = ntpath.abspath(support.TESTFN)
+ os.symlink(ABSTFN + "1", ABSTFN)
+ self.addCleanup(support.unlink, ABSTFN)
+ self.assertRaises(FileNotFoundError, ntpath.realpath, ABSTFN, strict=True)
+ self.assertRaises(FileNotFoundError, ntpath.realpath, ABSTFN + "2", strict=True)
+
+ @support.skip_unless_symlink
+ @unittest.skipUnless(HAVE_GETFINALPATHNAME, 'need _getfinalpathname')
+ @_parameterize({}, {'strict': True}, {'strict': ALLOW_MISSING})
+ def test_realpath_relative(self, kwargs):
+ ABSTFN = ntpath.abspath(support.TESTFN)
+ open(ABSTFN, "wb").close()
+ self.addCleanup(support.unlink, ABSTFN)
+ self.addCleanup(support.unlink, ABSTFN + "1")
+
+ os.symlink(ABSTFN, ntpath.relpath(ABSTFN + "1"))
+ self.assertPathEqual(ntpath.realpath(ABSTFN + "1", **kwargs), ABSTFN)
+
+ @support.skip_unless_symlink
+ @unittest.skipUnless(HAVE_GETFINALPATHNAME, 'need _getfinalpathname')
+ def test_realpath_broken_symlinks(self):
+ ABSTFN = ntpath.abspath(support.TESTFN)
+ os.mkdir(ABSTFN)
+ self.addCleanup(support.rmtree, ABSTFN)
+
+ with support.change_cwd(ABSTFN):
+ os.mkdir("subdir")
+ os.chdir("subdir")
+ os.symlink(".", "recursive")
+ os.symlink("..", "parent")
+ os.chdir("..")
+ os.symlink(".", "self")
+ os.symlink("missing", "broken")
+ os.symlink(r"broken\bar", "broken1")
+ os.symlink(r"self\self\broken", "broken2")
+ os.symlink(r"subdir\parent\subdir\parent\broken", "broken3")
+ os.symlink(ABSTFN + r"\broken", "broken4")
+ os.symlink(r"recursive\..\broken", "broken5")
+
+ self.assertPathEqual(ntpath.realpath("broken"),
+ ABSTFN + r"\missing")
+ self.assertPathEqual(ntpath.realpath(r"broken\foo"),
+ ABSTFN + r"\missing\foo")
+ # bpo-38453: We no longer recursively resolve segments of relative
+ # symlinks that the OS cannot resolve.
+ self.assertPathEqual(ntpath.realpath(r"broken1"),
+ ABSTFN + r"\broken\bar")
+ self.assertPathEqual(ntpath.realpath(r"broken1\baz"),
+ ABSTFN + r"\broken\bar\baz")
+ self.assertPathEqual(ntpath.realpath("broken2"),
+ ABSTFN + r"\self\self\missing")
+ self.assertPathEqual(ntpath.realpath("broken3"),
+ ABSTFN + r"\subdir\parent\subdir\parent\missing")
+ self.assertPathEqual(ntpath.realpath("broken4"),
+ ABSTFN + r"\missing")
+ self.assertPathEqual(ntpath.realpath("broken5"),
+ ABSTFN + r"\missing")
+
+ self.assertPathEqual(ntpath.realpath(b"broken"),
+ os.fsencode(ABSTFN + r"\missing"))
+ self.assertPathEqual(ntpath.realpath(rb"broken\foo"),
+ os.fsencode(ABSTFN + r"\missing\foo"))
+ self.assertPathEqual(ntpath.realpath(rb"broken1"),
+ os.fsencode(ABSTFN + r"\broken\bar"))
+ self.assertPathEqual(ntpath.realpath(rb"broken1\baz"),
+ os.fsencode(ABSTFN + r"\broken\bar\baz"))
+ self.assertPathEqual(ntpath.realpath(b"broken2"),
+ os.fsencode(ABSTFN + r"\self\self\missing"))
+ self.assertPathEqual(ntpath.realpath(rb"broken3"),
+ os.fsencode(ABSTFN + r"\subdir\parent\subdir\parent\missing"))
+ self.assertPathEqual(ntpath.realpath(b"broken4"),
+ os.fsencode(ABSTFN + r"\missing"))
+ self.assertPathEqual(ntpath.realpath(b"broken5"),
+ os.fsencode(ABSTFN + r"\missing"))
+
+ @support.skip_unless_symlink
+ @unittest.skipUnless(HAVE_GETFINALPATHNAME, 'need _getfinalpathname')
+ def test_realpath_symlink_loops(self):
+ # Symlink loops in non-strict mode are non-deterministic as to which
+ # path is returned, but it will always be the fully resolved path of
+ # one member of the cycle
+ ABSTFN = ntpath.abspath(support.TESTFN)
+ self.addCleanup(support.unlink, ABSTFN)
+ self.addCleanup(support.unlink, ABSTFN + "1")
+ self.addCleanup(support.unlink, ABSTFN + "2")
+ self.addCleanup(support.unlink, ABSTFN + "y")
+ self.addCleanup(support.unlink, ABSTFN + "c")
+ self.addCleanup(support.unlink, ABSTFN + "a")
+
+ os.symlink(ABSTFN, ABSTFN)
+ self.assertPathEqual(ntpath.realpath(ABSTFN), ABSTFN)
+
+ os.symlink(ABSTFN + "1", ABSTFN + "2")
+ os.symlink(ABSTFN + "2", ABSTFN + "1")
+ expected = (ABSTFN + "1", ABSTFN + "2")
+ self.assertPathIn(ntpath.realpath(ABSTFN + "1"), expected)
+ self.assertPathIn(ntpath.realpath(ABSTFN + "2"), expected)
+
+ self.assertPathIn(ntpath.realpath(ABSTFN + "1\\x"),
+ (ntpath.join(r, "x") for r in expected))
+ self.assertPathEqual(ntpath.realpath(ABSTFN + "1\\.."),
+ ntpath.dirname(ABSTFN))
+ self.assertPathEqual(ntpath.realpath(ABSTFN + "1\\..\\x"),
+ ntpath.dirname(ABSTFN) + "\\x")
+ os.symlink(ABSTFN + "x", ABSTFN + "y")
+ self.assertPathEqual(ntpath.realpath(ABSTFN + "1\\..\\"
+ + ntpath.basename(ABSTFN) + "y"),
+ ABSTFN + "x")
+ self.assertPathIn(ntpath.realpath(ABSTFN + "1\\..\\"
+ + ntpath.basename(ABSTFN) + "1"),
+ expected)
+
+ os.symlink(ntpath.basename(ABSTFN) + "a\\b", ABSTFN + "a")
+ self.assertPathEqual(ntpath.realpath(ABSTFN + "a"), ABSTFN + "a")
+
+ os.symlink("..\\" + ntpath.basename(ntpath.dirname(ABSTFN))
+ + "\\" + ntpath.basename(ABSTFN) + "c", ABSTFN + "c")
+ self.assertPathEqual(ntpath.realpath(ABSTFN + "c"), ABSTFN + "c")
+
+ # Test using relative path as well.
+ self.assertPathEqual(ntpath.realpath(ntpath.basename(ABSTFN)), ABSTFN)
+
+ @support.skip_unless_symlink
+ @unittest.skipUnless(HAVE_GETFINALPATHNAME, 'need _getfinalpathname')
+ def test_realpath_symlink_loops_strict(self):
+ # Symlink loops raise OSError in strict mode
+ ABSTFN = ntpath.abspath(support.TESTFN)
+ self.addCleanup(support.unlink, ABSTFN)
+ self.addCleanup(support.unlink, ABSTFN + "1")
+ self.addCleanup(support.unlink, ABSTFN + "2")
+ self.addCleanup(support.unlink, ABSTFN + "y")
+ self.addCleanup(support.unlink, ABSTFN + "c")
+ self.addCleanup(support.unlink, ABSTFN + "a")
+
+ os.symlink(ABSTFN, ABSTFN)
+ self.assertRaises(OSError, ntpath.realpath, ABSTFN, strict=True)
+
+ os.symlink(ABSTFN + "1", ABSTFN + "2")
+ os.symlink(ABSTFN + "2", ABSTFN + "1")
+ self.assertRaises(OSError, ntpath.realpath, ABSTFN + "1", strict=True)
+ self.assertRaises(OSError, ntpath.realpath, ABSTFN + "2", strict=True)
+ self.assertRaises(OSError, ntpath.realpath, ABSTFN + "1\\x", strict=True)
+ # Windows eliminates '..' components before resolving links, so the
+ # following call is not expected to raise.
+ self.assertPathEqual(ntpath.realpath(ABSTFN + "1\\..", strict=True),
+ ntpath.dirname(ABSTFN))
+ self.assertRaises(OSError, ntpath.realpath, ABSTFN + "1\\..\\x", strict=True)
+ os.symlink(ABSTFN + "x", ABSTFN + "y")
+ self.assertRaises(OSError, ntpath.realpath, ABSTFN + "1\\..\\"
+ + ntpath.basename(ABSTFN) + "y",
+ strict=True)
+ self.assertRaises(OSError, ntpath.realpath,
+ ABSTFN + "1\\..\\" + ntpath.basename(ABSTFN) + "1",
+ strict=True)
+
+ os.symlink(ntpath.basename(ABSTFN) + "a\\b", ABSTFN + "a")
+ self.assertRaises(OSError, ntpath.realpath, ABSTFN + "a", strict=True)
+
+ os.symlink("..\\" + ntpath.basename(ntpath.dirname(ABSTFN))
+ + "\\" + ntpath.basename(ABSTFN) + "c", ABSTFN + "c")
+ self.assertRaises(OSError, ntpath.realpath, ABSTFN + "c", strict=True)
+
+ # Test using relative path as well.
+ self.assertRaises(OSError, ntpath.realpath, ntpath.basename(ABSTFN),
+ strict=True)
+
+ @support.skip_unless_symlink
+ @unittest.skipUnless(HAVE_GETFINALPATHNAME, 'need _getfinalpathname')
+ def test_realpath_symlink_loops_raise(self):
+ # Symlink loops raise OSError in ALLOW_MISSING mode
+ ABSTFN = ntpath.abspath(support.TESTFN)
+ self.addCleanup(support.unlink, ABSTFN)
+ self.addCleanup(support.unlink, ABSTFN + "1")
+ self.addCleanup(support.unlink, ABSTFN + "2")
+ self.addCleanup(support.unlink, ABSTFN + "y")
+ self.addCleanup(support.unlink, ABSTFN + "c")
+ self.addCleanup(support.unlink, ABSTFN + "a")
+ self.addCleanup(support.unlink, ABSTFN + "x")
+
+ os.symlink(ABSTFN, ABSTFN)
+ self.assertRaises(OSError, ntpath.realpath, ABSTFN, strict=ALLOW_MISSING)
+
+ os.symlink(ABSTFN + "1", ABSTFN + "2")
+ os.symlink(ABSTFN + "2", ABSTFN + "1")
+ self.assertRaises(OSError, ntpath.realpath, ABSTFN + "1",
+ strict=ALLOW_MISSING)
+ self.assertRaises(OSError, ntpath.realpath, ABSTFN + "2",
+ strict=ALLOW_MISSING)
+ self.assertRaises(OSError, ntpath.realpath, ABSTFN + "1\\x",
+ strict=ALLOW_MISSING)
+
+ # Windows eliminates '..' components before resolving links;
+ # realpath is not expected to raise if this removes the loop.
+ self.assertPathEqual(ntpath.realpath(ABSTFN + "1\\.."),
+ ntpath.dirname(ABSTFN))
+ self.assertPathEqual(ntpath.realpath(ABSTFN + "1\\..\\x"),
+ ntpath.dirname(ABSTFN) + "\\x")
+
+ os.symlink(ABSTFN + "x", ABSTFN + "y")
+ self.assertPathEqual(ntpath.realpath(ABSTFN + "1\\..\\"
+ + ntpath.basename(ABSTFN) + "y"),
+ ABSTFN + "x")
+ self.assertRaises(
+ OSError, ntpath.realpath,
+ ABSTFN + "1\\..\\" + ntpath.basename(ABSTFN) + "1",
+ strict=ALLOW_MISSING)
+
+ os.symlink(ntpath.basename(ABSTFN) + "a\\b", ABSTFN + "a")
+ self.assertRaises(OSError, ntpath.realpath, ABSTFN + "a",
+ strict=ALLOW_MISSING)
+
+ os.symlink("..\\" + ntpath.basename(ntpath.dirname(ABSTFN))
+ + "\\" + ntpath.basename(ABSTFN) + "c", ABSTFN + "c")
+ self.assertRaises(OSError, ntpath.realpath, ABSTFN + "c",
+ strict=ALLOW_MISSING)
+
+ # Test using relative path as well.
+ self.assertRaises(OSError, ntpath.realpath, ntpath.basename(ABSTFN),
+ strict=ALLOW_MISSING)
+
+ @support.skip_unless_symlink
+ @unittest.skipUnless(HAVE_GETFINALPATHNAME, 'need _getfinalpathname')
+ @_parameterize({}, {'strict': True}, {'strict': ALLOW_MISSING})
+ def test_realpath_symlink_prefix(self, kwargs):
+ ABSTFN = ntpath.abspath(support.TESTFN)
+ self.addCleanup(support.unlink, ABSTFN + "3")
+ self.addCleanup(support.unlink, "\\\\?\\" + ABSTFN + "3.")
+ self.addCleanup(support.unlink, ABSTFN + "3link")
+ self.addCleanup(support.unlink, ABSTFN + "3.link")
+
+ with open(ABSTFN + "3", "wb") as f:
+ f.write(b'0')
+ os.symlink(ABSTFN + "3", ABSTFN + "3link")
+
+ with open("\\\\?\\" + ABSTFN + "3.", "wb") as f:
+ f.write(b'1')
+ os.symlink("\\\\?\\" + ABSTFN + "3.", ABSTFN + "3.link")
+
+ self.assertPathEqual(ntpath.realpath(ABSTFN + "3link", **kwargs),
+ ABSTFN + "3")
+ self.assertPathEqual(ntpath.realpath(ABSTFN + "3.link", **kwargs),
+ "\\\\?\\" + ABSTFN + "3.")
+
+ # Resolved paths should be usable to open target files
+ with open(ntpath.realpath(ABSTFN + "3link"), "rb") as f:
+ self.assertEqual(f.read(), b'0')
+ with open(ntpath.realpath(ABSTFN + "3.link"), "rb") as f:
+ self.assertEqual(f.read(), b'1')
+
+ # When the prefix is included, it is not stripped
+ self.assertPathEqual(ntpath.realpath("\\\\?\\" + ABSTFN + "3link", **kwargs),
+ "\\\\?\\" + ABSTFN + "3")
+ self.assertPathEqual(ntpath.realpath("\\\\?\\" + ABSTFN + "3.link", **kwargs),
+ "\\\\?\\" + ABSTFN + "3.")
+
+ @unittest.skipUnless(HAVE_GETFINALPATHNAME, 'need _getfinalpathname')
+ def test_realpath_nul(self):
+ tester("ntpath.realpath('NUL')", r'\\.\NUL')
+ tester("ntpath.realpath('NUL', strict=False)", r'\\.\NUL')
+ tester("ntpath.realpath('NUL', strict=True)", r'\\.\NUL')
+ tester("ntpath.realpath('NUL', strict=ALLOW_MISSING)", r'\\.\NUL')
+
+ @unittest.skipUnless(HAVE_GETFINALPATHNAME, 'need _getfinalpathname')
+ @unittest.skipUnless(HAVE_GETSHORTPATHNAME, 'need _getshortpathname')
+ def test_realpath_cwd(self):
+ ABSTFN = ntpath.abspath(support.TESTFN)
+
+ support.unlink(ABSTFN)
+ support.rmtree(ABSTFN)
+ os.mkdir(ABSTFN)
+ self.addCleanup(support.rmtree, ABSTFN)
+
+ test_dir_long = ntpath.join(ABSTFN, "MyVeryLongDirectoryName")
+ os.mkdir(test_dir_long)
+
+ test_dir_short = _getshortpathname(test_dir_long)
+ test_file_long = ntpath.join(test_dir_long, "file.txt")
+ test_file_short = ntpath.join(test_dir_short, "file.txt")
+
+ with open(test_file_long, "wb") as f:
+ f.write(b"content")
+
+ self.assertPathEqual(test_file_long, ntpath.realpath(test_file_short))
+
+ for kwargs in {}, {'strict': True}, {'strict': ALLOW_MISSING}:
+ with self.subTest(**kwargs):
+ with support.change_cwd(test_dir_long):
+ self.assertPathEqual(
+ test_file_long,
+ ntpath.realpath("file.txt", **kwargs))
+ with support.change_cwd(test_dir_long.lower()):
+ self.assertPathEqual(
+ test_file_long,
+ ntpath.realpath("file.txt", **kwargs))
+ with support.change_cwd(test_dir_short):
+ self.assertPathEqual(
+ test_file_long,
+ ntpath.realpath("file.txt", **kwargs))
+
def test_expandvars(self):
- with support.EnvironmentVarGuard() as env:
+ with EnvironmentVarGuard() as env:
env.clear()
env["foo"] = "bar"
env["{foo"] = "baz1"
@@ -239,7 +667,7 @@ def test_expandvars(self):
def test_expandvars_nonascii(self):
def check(value, expected):
tester('ntpath.expandvars(%r)' % value, expected)
- with support.EnvironmentVarGuard() as env:
+ with EnvironmentVarGuard() as env:
env.clear()
nonascii = support.FS_NONASCII
env['spam'] = nonascii
@@ -255,10 +683,23 @@ def check(value, expected):
check('%spam%bar', '%sbar' % nonascii)
check('%{}%bar'.format(nonascii), 'ham%sbar' % nonascii)
+ @support.requires_resource('cpu')
+ def test_expandvars_large(self):
+ expandvars = ntpath.expandvars
+ with EnvironmentVarGuard() as env:
+ env.clear()
+ env["A"] = "B"
+ n = 100_000
+ self.assertEqual(expandvars('%A%'*n), 'B'*n)
+ self.assertEqual(expandvars('%A%A'*n), 'BA'*n)
+ self.assertEqual(expandvars("''"*n + '%%'), "''"*n + '%')
+ self.assertEqual(expandvars("%%"*n), "%"*n)
+ self.assertEqual(expandvars("$$"*n), "$"*n)
+
def test_expanduser(self):
tester('ntpath.expanduser("test")', 'test')
- with support.EnvironmentVarGuard() as env:
+ with EnvironmentVarGuard() as env:
env.clear()
tester('ntpath.expanduser("~test")', '~test')
@@ -469,6 +910,7 @@ def test_nt_helpers(self):
self.assertIsInstance(b_final_path, bytes)
self.assertGreater(len(b_final_path), 0)
+
class NtCommonTest(test_genericpath.CommonTest, unittest.TestCase):
pathmodule = ntpath
attributes = ['relpath']
diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py
index 3a76940a089475b..58aa9d7f18a17e1 100644
--- a/Lib/test/test_os.py
+++ b/Lib/test/test_os.py
@@ -1200,6 +1200,18 @@ def test_exist_ok_existing_regular_file(self):
self.assertRaises(OSError, os.makedirs, path, exist_ok=True)
os.remove(path)
+ @unittest.skipUnless(os.name == 'nt', "requires Windows")
+ def test_win32_mkdir_700(self):
+ base = support.TESTFN
+ path = os.path.abspath(os.path.join(support.TESTFN, 'dir'))
+ os.mkdir(path, mode=0o700)
+ out = subprocess.check_output(["cacls.exe", path, "/s"], encoding="oem")
+ os.rmdir(path)
+ self.assertEqual(
+ out.strip(),
+ f'{path} "D:P(A;OICI;FA;;;SY)(A;OICI;FA;;;BA)(A;OICI;FA;;;OW)"',
+ )
+
def tearDown(self):
path = os.path.join(support.TESTFN, 'dir1', 'dir2', 'dir3',
'dir4', 'dir5', 'dir6')
diff --git a/Lib/test/test_plistlib.py b/Lib/test/test_plistlib.py
index c6c95fe67ff71ab..1e7dc8a9a8c3a41 100644
--- a/Lib/test/test_plistlib.py
+++ b/Lib/test/test_plistlib.py
@@ -744,8 +744,7 @@ def test_xml_plist_with_entity_decl(self):
class TestBinaryPlistlib(unittest.TestCase):
- @staticmethod
- def decode(*objects, offset_size=1, ref_size=1):
+ def build(self, *objects, offset_size=1, ref_size=1):
data = [b'bplist00']
offset = 8
offsets = []
@@ -757,7 +756,11 @@ def decode(*objects, offset_size=1, ref_size=1):
len(objects), 0, offset)
data.extend(offsets)
data.append(tail)
- return plistlib.loads(b''.join(data), fmt=plistlib.FMT_BINARY)
+ return b''.join(data)
+
+ def decode(self, *objects, offset_size=1, ref_size=1):
+ data = self.build(*objects, offset_size=offset_size, ref_size=ref_size)
+ return plistlib.loads(data, fmt=plistlib.FMT_BINARY)
def test_nonstandard_refs_size(self):
# Issue #21538: Refs and offsets are 24-bit integers
@@ -867,6 +870,34 @@ def test_invalid_binary(self):
with self.assertRaises(plistlib.InvalidFileException):
plistlib.loads(b'bplist00' + data, fmt=plistlib.FMT_BINARY)
+ def test_truncated_large_data(self):
+ self.addCleanup(support.unlink, support.TESTFN)
+ def check(data):
+ with open(support.TESTFN, 'wb') as f:
+ f.write(data)
+ # buffered file
+ with open(support.TESTFN, 'rb') as f:
+ with self.assertRaises(plistlib.InvalidFileException):
+ plistlib.load(f, fmt=plistlib.FMT_BINARY)
+ # unbuffered file
+ with open(support.TESTFN, 'rb', buffering=0) as f:
+ with self.assertRaises(plistlib.InvalidFileException):
+ plistlib.load(f, fmt=plistlib.FMT_BINARY)
+ for w in range(20, 64):
+ s = 1 << w
+ # data
+ check(self.build(b'\x4f\x13' + s.to_bytes(8, 'big')))
+ # ascii string
+ check(self.build(b'\x5f\x13' + s.to_bytes(8, 'big')))
+ # unicode string
+ check(self.build(b'\x6f\x13' + s.to_bytes(8, 'big')))
+ # array
+ check(self.build(b'\xaf\x13' + s.to_bytes(8, 'big')))
+ # dict
+ check(self.build(b'\xdf\x13' + s.to_bytes(8, 'big')))
+ # number of objects
+ check(b'bplist00' + struct.pack('>6xBBQQQ', 1, 1, s, 0, 8))
+
class TestPlistlibDeprecated(unittest.TestCase):
def test_io_deprecated(self):
diff --git a/Lib/test/test_poplib.py b/Lib/test/test_poplib.py
index 20d4eeac12d4b2f..12f628a89e39494 100644
--- a/Lib/test/test_poplib.py
+++ b/Lib/test/test_poplib.py
@@ -13,6 +13,7 @@
from unittest import TestCase, skipUnless
from test import support as test_support
+from test.support import control_characters_c0
HOST = test_support.HOST
PORT = 0
@@ -354,6 +355,13 @@ def test_quit(self):
self.assertIsNone(self.client.sock)
self.assertIsNone(self.client.file)
+ def test_control_characters(self):
+ for c0 in control_characters_c0():
+ with self.assertRaises(ValueError):
+ self.client.user(f'user{c0}')
+ with self.assertRaises(ValueError):
+ self.client.pass_(f'{c0}pass')
+
@requires_ssl
def test_stls_capa(self):
capa = self.client.capa()
diff --git a/Lib/test/test_posixpath.py b/Lib/test/test_posixpath.py
index e73b31cb648b227..dc615e0e8edc5ca 100644
--- a/Lib/test/test_posixpath.py
+++ b/Lib/test/test_posixpath.py
@@ -1,8 +1,10 @@
import os
+import sys
import posixpath
import unittest
import warnings
-from posixpath import realpath, abspath, dirname, basename
+from functools import partial
+from posixpath import realpath, abspath, dirname, basename, ALLOW_MISSING
from test import support, test_genericpath
from test.support import FakePath
from unittest import mock
@@ -33,6 +35,26 @@ def safe_rmdir(dirname):
except OSError:
pass
+def _parameterize(*parameters):
+ """Simplistic decorator to parametrize a test
+
+ Runs the decorated test multiple times in subTest, with a value from
+ 'parameters' passed as an extra positional argument.
+ Does *not* call doCleanups() after each run.
+
+ Not for general use. Intended to avoid indenting for easier backports.
+
+ See https://discuss.python.org/t/91827 for discussing generalizations.
+ """
+ def _parametrize_decorator(func):
+ def _parameterized(self, *args, **kwargs):
+ for parameter in parameters:
+ with self.subTest(parameter):
+ func(self, *args, parameter, **kwargs)
+ return _parameterized
+ return _parametrize_decorator
+
+
class PosixPathTest(unittest.TestCase):
def setUp(self):
@@ -308,52 +330,167 @@ def test_normpath(self):
b"/foo/bar")
@skip_if_ABSTFN_contains_backslash
- def test_realpath_curdir(self):
- self.assertEqual(realpath('.'), os.getcwd())
- self.assertEqual(realpath('./.'), os.getcwd())
- self.assertEqual(realpath('/'.join(['.'] * 100)), os.getcwd())
+ @_parameterize({}, {'strict': True}, {'strict': ALLOW_MISSING})
+ def test_realpath_curdir(self, kwargs):
+ self.assertEqual(realpath('.', **kwargs), os.getcwd())
+ self.assertEqual(realpath('./.', **kwargs), os.getcwd())
+ self.assertEqual(realpath('/'.join(['.'] * 100), **kwargs), os.getcwd())
- self.assertEqual(realpath(b'.'), os.getcwdb())
- self.assertEqual(realpath(b'./.'), os.getcwdb())
- self.assertEqual(realpath(b'/'.join([b'.'] * 100)), os.getcwdb())
+ self.assertEqual(realpath(b'.', **kwargs), os.getcwdb())
+ self.assertEqual(realpath(b'./.', **kwargs), os.getcwdb())
+ self.assertEqual(realpath(b'/'.join([b'.'] * 100), **kwargs), os.getcwdb())
@skip_if_ABSTFN_contains_backslash
- def test_realpath_pardir(self):
- self.assertEqual(realpath('..'), dirname(os.getcwd()))
- self.assertEqual(realpath('../..'), dirname(dirname(os.getcwd())))
- self.assertEqual(realpath('/'.join(['..'] * 100)), '/')
+ @_parameterize({}, {'strict': True}, {'strict': ALLOW_MISSING})
+ def test_realpath_pardir(self, kwargs):
+ self.assertEqual(realpath('..', **kwargs), dirname(os.getcwd()))
+ self.assertEqual(realpath('../..', **kwargs), dirname(dirname(os.getcwd())))
+ self.assertEqual(realpath('/'.join(['..'] * 100), **kwargs), '/')
- self.assertEqual(realpath(b'..'), dirname(os.getcwdb()))
- self.assertEqual(realpath(b'../..'), dirname(dirname(os.getcwdb())))
- self.assertEqual(realpath(b'/'.join([b'..'] * 100)), b'/')
+ self.assertEqual(realpath(b'..', **kwargs), dirname(os.getcwdb()))
+ self.assertEqual(realpath(b'../..', **kwargs), dirname(dirname(os.getcwdb())))
+ self.assertEqual(realpath(b'/'.join([b'..'] * 100), **kwargs), b'/')
@unittest.skipUnless(hasattr(os, "symlink"),
"Missing symlink implementation")
@skip_if_ABSTFN_contains_backslash
- def test_realpath_basic(self):
+ @_parameterize({}, {'strict': ALLOW_MISSING})
+ def test_realpath_basic(self, kwargs):
# Basic operation.
try:
os.symlink(ABSTFN+"1", ABSTFN)
- self.assertEqual(realpath(ABSTFN), ABSTFN+"1")
+ self.assertEqual(realpath(ABSTFN, **kwargs), ABSTFN+"1")
finally:
support.unlink(ABSTFN)
@unittest.skipUnless(hasattr(os, "symlink"),
"Missing symlink implementation")
@skip_if_ABSTFN_contains_backslash
- def test_realpath_relative(self):
+ def test_realpath_strict(self):
+ # Bug #43757: raise FileNotFoundError in strict mode if we encounter
+ # a path that does not exist.
+ try:
+ os.symlink(ABSTFN+"1", ABSTFN)
+ self.assertRaises(FileNotFoundError, realpath, ABSTFN, strict=True)
+ self.assertRaises(FileNotFoundError, realpath, ABSTFN + "2", strict=True)
+ finally:
+ support.unlink(ABSTFN)
+
+ def test_realpath_invalid_paths(self):
+ path = '/\x00'
+ self.assertRaises(ValueError, realpath, path, strict=False)
+ self.assertRaises(ValueError, realpath, path, strict=True)
+ self.assertRaises(ValueError, realpath, path, strict=ALLOW_MISSING)
+ path = b'/\x00'
+ self.assertRaises(ValueError, realpath, path, strict=False)
+ self.assertRaises(ValueError, realpath, path, strict=True)
+ self.assertRaises(ValueError, realpath, path, strict=ALLOW_MISSING)
+ path = '/nonexistent/x\x00'
+ self.assertRaises(ValueError, realpath, path, strict=False)
+ self.assertRaises(FileNotFoundError, realpath, path, strict=True)
+ self.assertRaises(ValueError, realpath, path, strict=ALLOW_MISSING)
+ path = b'/nonexistent/x\x00'
+ self.assertRaises(ValueError, realpath, path, strict=False)
+ self.assertRaises(FileNotFoundError, realpath, path, strict=True)
+ self.assertRaises(ValueError, realpath, path, strict=ALLOW_MISSING)
+ path = '/\x00/..'
+ self.assertRaises(ValueError, realpath, path, strict=False)
+ self.assertRaises(ValueError, realpath, path, strict=True)
+ self.assertRaises(ValueError, realpath, path, strict=ALLOW_MISSING)
+ path = b'/\x00/..'
+ self.assertRaises(ValueError, realpath, path, strict=False)
+ self.assertRaises(ValueError, realpath, path, strict=True)
+ self.assertRaises(ValueError, realpath, path, strict=ALLOW_MISSING)
+
+ path = '/nonexistent/x\x00/..'
+ self.assertRaises(ValueError, realpath, path, strict=False)
+ self.assertRaises(FileNotFoundError, realpath, path, strict=True)
+ self.assertRaises(ValueError, realpath, path, strict=ALLOW_MISSING)
+ path = b'/nonexistent/x\x00/..'
+ self.assertRaises(ValueError, realpath, path, strict=False)
+ self.assertRaises(FileNotFoundError, realpath, path, strict=True)
+ self.assertRaises(ValueError, realpath, path, strict=ALLOW_MISSING)
+
+ path = '/\udfff'
+ if sys.platform == 'win32':
+ self.assertEqual(realpath(path, strict=False), path)
+ self.assertRaises(FileNotFoundError, realpath, path, strict=True)
+ self.assertEqual(realpath(path, strict=ALLOW_MISSING), path)
+ else:
+ self.assertRaises(UnicodeEncodeError, realpath, path, strict=False)
+ self.assertRaises(UnicodeEncodeError, realpath, path, strict=True)
+ self.assertRaises(UnicodeEncodeError, realpath, path, strict=ALLOW_MISSING)
+ path = '/nonexistent/\udfff'
+ if sys.platform == 'win32':
+ self.assertEqual(realpath(path, strict=False), path)
+ self.assertEqual(realpath(path, strict=ALLOW_MISSING), path)
+ else:
+ self.assertRaises(UnicodeEncodeError, realpath, path, strict=False)
+ self.assertRaises(UnicodeEncodeError, realpath, path, strict=ALLOW_MISSING)
+ self.assertRaises(FileNotFoundError, realpath, path, strict=True)
+ path = '/\udfff/..'
+ if sys.platform == 'win32':
+ self.assertEqual(realpath(path, strict=False), '/')
+ self.assertRaises(FileNotFoundError, realpath, path, strict=True)
+ self.assertEqual(realpath(path, strict=ALLOW_MISSING), '/')
+ else:
+ self.assertRaises(UnicodeEncodeError, realpath, path, strict=False)
+ self.assertRaises(UnicodeEncodeError, realpath, path, strict=True)
+ self.assertRaises(UnicodeEncodeError, realpath, path, strict=ALLOW_MISSING)
+ path = '/nonexistent/\udfff/..'
+ if sys.platform == 'win32':
+ self.assertEqual(realpath(path, strict=False), '/nonexistent')
+ self.assertEqual(realpath(path, strict=ALLOW_MISSING), '/nonexistent')
+ else:
+ self.assertRaises(UnicodeEncodeError, realpath, path, strict=False)
+ self.assertRaises(UnicodeEncodeError, realpath, path, strict=ALLOW_MISSING)
+ self.assertRaises(FileNotFoundError, realpath, path, strict=True)
+
+ path = b'/\xff'
+ if sys.platform == 'win32':
+ self.assertRaises(UnicodeDecodeError, realpath, path, strict=False)
+ self.assertRaises(UnicodeDecodeError, realpath, path, strict=True)
+ self.assertRaises(UnicodeDecodeError, realpath, path, strict=ALLOW_MISSING)
+ else:
+ self.assertEqual(realpath(path, strict=False), path)
+ self.assertRaises(FileNotFoundError, realpath, path, strict=True)
+ self.assertEqual(realpath(path, strict=ALLOW_MISSING), path)
+ path = b'/nonexistent/\xff'
+ if sys.platform == 'win32':
+ self.assertRaises(UnicodeDecodeError, realpath, path, strict=False)
+ self.assertRaises(UnicodeDecodeError, realpath, path, strict=ALLOW_MISSING)
+ else:
+ self.assertEqual(realpath(path, strict=False), path)
+ self.assertRaises(FileNotFoundError, realpath, path, strict=True)
+
+ @unittest.skipUnless(hasattr(os, "symlink"),
+ "Missing symlink implementation")
+ @skip_if_ABSTFN_contains_backslash
+ @_parameterize({}, {'strict': ALLOW_MISSING})
+ def test_realpath_relative(self, kwargs):
try:
os.symlink(posixpath.relpath(ABSTFN+"1"), ABSTFN)
- self.assertEqual(realpath(ABSTFN), ABSTFN+"1")
+ self.assertEqual(realpath(ABSTFN, **kwargs), ABSTFN+"1")
finally:
support.unlink(ABSTFN)
@unittest.skipUnless(hasattr(os, "symlink"),
"Missing symlink implementation")
@skip_if_ABSTFN_contains_backslash
+ @_parameterize({}, {'strict': ALLOW_MISSING})
+ def test_realpath_missing_pardir(self, kwargs):
+ try:
+ os.symlink(support.TESTFN + "1", support.TESTFN)
+ self.assertEqual(
+ realpath("nonexistent/../" + support.TESTFN, **kwargs), ABSTFN + "1")
+ finally:
+ support.unlink(support.TESTFN)
+
+ @support.skip_unless_symlink
+ @skip_if_ABSTFN_contains_backslash
def test_realpath_symlink_loops(self):
# Bug #930024, return the path unchanged if we get into an infinite
- # symlink loop.
+ # symlink loop in non-strict mode (default).
try:
os.symlink(ABSTFN, ABSTFN)
self.assertEqual(realpath(ABSTFN), ABSTFN)
@@ -393,13 +530,57 @@ def test_realpath_symlink_loops(self):
@unittest.skipUnless(hasattr(os, "symlink"),
"Missing symlink implementation")
@skip_if_ABSTFN_contains_backslash
- def test_realpath_repeated_indirect_symlinks(self):
+ @_parameterize({'strict': True}, {'strict': ALLOW_MISSING})
+ def test_realpath_symlink_loops_strict(self, kwargs):
+ # Bug #43757, raise OSError if we get into an infinite symlink loop in
+ # the strict modes.
+ try:
+ os.symlink(ABSTFN, ABSTFN)
+ self.assertRaises(OSError, realpath, ABSTFN, **kwargs)
+
+ os.symlink(ABSTFN+"1", ABSTFN+"2")
+ os.symlink(ABSTFN+"2", ABSTFN+"1")
+ self.assertRaises(OSError, realpath, ABSTFN+"1", **kwargs)
+ self.assertRaises(OSError, realpath, ABSTFN+"2", **kwargs)
+
+ self.assertRaises(OSError, realpath, ABSTFN+"1/x", **kwargs)
+ self.assertRaises(OSError, realpath, ABSTFN+"1/..", **kwargs)
+ self.assertRaises(OSError, realpath, ABSTFN+"1/../x", **kwargs)
+ os.symlink(ABSTFN+"x", ABSTFN+"y")
+ self.assertRaises(OSError, realpath,
+ ABSTFN+"1/../" + basename(ABSTFN) + "y", **kwargs)
+ self.assertRaises(OSError, realpath,
+ ABSTFN+"1/../" + basename(ABSTFN) + "1", **kwargs)
+
+ os.symlink(basename(ABSTFN) + "a/b", ABSTFN+"a")
+ self.assertRaises(OSError, realpath, ABSTFN+"a", **kwargs)
+
+ os.symlink("../" + basename(dirname(ABSTFN)) + "/" +
+ basename(ABSTFN) + "c", ABSTFN+"c")
+ self.assertRaises(OSError, realpath, ABSTFN+"c", **kwargs)
+
+ # Test using relative path as well.
+ with support.change_cwd(dirname(ABSTFN)):
+ self.assertRaises(OSError, realpath, basename(ABSTFN), **kwargs)
+ finally:
+ support.unlink(ABSTFN)
+ support.unlink(ABSTFN+"1")
+ support.unlink(ABSTFN+"2")
+ support.unlink(ABSTFN+"y")
+ support.unlink(ABSTFN+"c")
+ support.unlink(ABSTFN+"a")
+
+ @unittest.skipUnless(hasattr(os, "symlink"),
+ "Missing symlink implementation")
+ @skip_if_ABSTFN_contains_backslash
+ @_parameterize({}, {'strict': True}, {'strict': ALLOW_MISSING})
+ def test_realpath_repeated_indirect_symlinks(self, kwargs):
# Issue #6975.
try:
os.mkdir(ABSTFN)
os.symlink('../' + basename(ABSTFN), ABSTFN + '/self')
os.symlink('self/self/self', ABSTFN + '/link')
- self.assertEqual(realpath(ABSTFN + '/link'), ABSTFN)
+ self.assertEqual(realpath(ABSTFN + '/link', **kwargs), ABSTFN)
finally:
support.unlink(ABSTFN + '/self')
support.unlink(ABSTFN + '/link')
@@ -408,14 +589,15 @@ def test_realpath_repeated_indirect_symlinks(self):
@unittest.skipUnless(hasattr(os, "symlink"),
"Missing symlink implementation")
@skip_if_ABSTFN_contains_backslash
- def test_realpath_deep_recursion(self):
+ @_parameterize({}, {'strict': True}, {'strict': ALLOW_MISSING})
+ def test_realpath_deep_recursion(self, kwargs):
depth = 10
try:
os.mkdir(ABSTFN)
for i in range(depth):
os.symlink('/'.join(['%d' % i] * 10), ABSTFN + '/%d' % (i + 1))
os.symlink('.', ABSTFN + '/0')
- self.assertEqual(realpath(ABSTFN + '/%d' % depth), ABSTFN)
+ self.assertEqual(realpath(ABSTFN + '/%d' % depth, **kwargs), ABSTFN)
# Test using relative path as well.
with support.change_cwd(ABSTFN):
@@ -428,7 +610,8 @@ def test_realpath_deep_recursion(self):
@unittest.skipUnless(hasattr(os, "symlink"),
"Missing symlink implementation")
@skip_if_ABSTFN_contains_backslash
- def test_realpath_resolve_parents(self):
+ @_parameterize({}, {'strict': ALLOW_MISSING})
+ def test_realpath_resolve_parents(self, kwargs):
# We also need to resolve any symlinks in the parents of a relative
# path passed to realpath. E.g.: current working directory is
# /usr/doc with 'doc' being a symlink to /usr/share/doc. We call
@@ -439,7 +622,8 @@ def test_realpath_resolve_parents(self):
os.symlink(ABSTFN + "/y", ABSTFN + "/k")
with support.change_cwd(ABSTFN + "/k"):
- self.assertEqual(realpath("a"), ABSTFN + "/y/a")
+ self.assertEqual(realpath("a", **kwargs),
+ ABSTFN + "/y/a")
finally:
support.unlink(ABSTFN + "/k")
safe_rmdir(ABSTFN + "/y")
@@ -448,7 +632,8 @@ def test_realpath_resolve_parents(self):
@unittest.skipUnless(hasattr(os, "symlink"),
"Missing symlink implementation")
@skip_if_ABSTFN_contains_backslash
- def test_realpath_resolve_before_normalizing(self):
+ @_parameterize({}, {'strict': True}, {'strict': ALLOW_MISSING})
+ def test_realpath_resolve_before_normalizing(self, kwargs):
# Bug #990669: Symbolic links should be resolved before we
# normalize the path. E.g.: if we have directories 'a', 'k' and 'y'
# in the following hierarchy:
@@ -463,10 +648,10 @@ def test_realpath_resolve_before_normalizing(self):
os.symlink(ABSTFN + "/k/y", ABSTFN + "/link-y")
# Absolute path.
- self.assertEqual(realpath(ABSTFN + "/link-y/.."), ABSTFN + "/k")
+ self.assertEqual(realpath(ABSTFN + "/link-y/..", **kwargs), ABSTFN + "/k")
# Relative path.
with support.change_cwd(dirname(ABSTFN)):
- self.assertEqual(realpath(basename(ABSTFN) + "/link-y/.."),
+ self.assertEqual(realpath(basename(ABSTFN) + "/link-y/..", **kwargs),
ABSTFN + "/k")
finally:
support.unlink(ABSTFN + "/link-y")
@@ -477,7 +662,8 @@ def test_realpath_resolve_before_normalizing(self):
@unittest.skipUnless(hasattr(os, "symlink"),
"Missing symlink implementation")
@skip_if_ABSTFN_contains_backslash
- def test_realpath_resolve_first(self):
+ @_parameterize({}, {'strict': True}, {'strict': ALLOW_MISSING})
+ def test_realpath_resolve_first(self, kwargs):
# Bug #1213894: The first component of the path, if not absolute,
# must be resolved too.
@@ -487,13 +673,70 @@ def test_realpath_resolve_first(self):
os.symlink(ABSTFN, ABSTFN + "link")
with support.change_cwd(dirname(ABSTFN)):
base = basename(ABSTFN)
- self.assertEqual(realpath(base + "link"), ABSTFN)
- self.assertEqual(realpath(base + "link/k"), ABSTFN + "/k")
+ self.assertEqual(realpath(base + "link", **kwargs), ABSTFN)
+ self.assertEqual(realpath(base + "link/k", **kwargs), ABSTFN + "/k")
finally:
support.unlink(ABSTFN + "link")
safe_rmdir(ABSTFN + "/k")
safe_rmdir(ABSTFN)
+ @support.skip_unless_symlink
+ @skip_if_ABSTFN_contains_backslash
+ @unittest.skipIf(os.chmod not in os.supports_follow_symlinks, "Can't set symlink permissions")
+ @unittest.skipIf(sys.platform != "darwin", "only macOS requires read permission to readlink()")
+ @_parameterize({'strict': True}, {'strict': ALLOW_MISSING})
+ def test_realpath_unreadable_symlink_strict(self, kwargs):
+ try:
+ os.symlink(ABSTFN+"1", ABSTFN)
+ os.chmod(ABSTFN, 0o000, follow_symlinks=False)
+ with self.assertRaises(PermissionError):
+ realpath(ABSTFN, **kwargs)
+ with self.assertRaises(PermissionError):
+ realpath(ABSTFN + '/foo', **kwargs),
+ with self.assertRaises(PermissionError):
+ realpath(ABSTFN + '/../foo', **kwargs)
+ with self.assertRaises(PermissionError):
+ realpath(ABSTFN + '/foo/..', **kwargs)
+ finally:
+ os.chmod(ABSTFN, 0o755, follow_symlinks=False)
+ os.unlink(ABSTFN)
+
+ @skip_if_ABSTFN_contains_backslash
+ @support.skip_unless_symlink
+ def test_realpath_unreadable_directory(self):
+ try:
+ os.mkdir(ABSTFN)
+ os.mkdir(ABSTFN + '/k')
+ os.chmod(ABSTFN, 0o000)
+ self.assertEqual(realpath(ABSTFN, strict=False), ABSTFN)
+ self.assertEqual(realpath(ABSTFN, strict=True), ABSTFN)
+ self.assertEqual(realpath(ABSTFN, strict=ALLOW_MISSING), ABSTFN)
+
+ try:
+ os.stat(ABSTFN)
+ except PermissionError:
+ pass
+ else:
+ self.skipTest('Cannot block permissions')
+
+ self.assertEqual(realpath(ABSTFN + '/k', strict=False),
+ ABSTFN + '/k')
+ self.assertRaises(PermissionError, realpath, ABSTFN + '/k',
+ strict=True)
+ self.assertRaises(PermissionError, realpath, ABSTFN + '/k',
+ strict=ALLOW_MISSING)
+
+ self.assertEqual(realpath(ABSTFN + '/missing', strict=False),
+ ABSTFN + '/missing')
+ self.assertRaises(PermissionError, realpath, ABSTFN + '/missing',
+ strict=True)
+ self.assertRaises(PermissionError, realpath, ABSTFN + '/missing',
+ strict=ALLOW_MISSING)
+ finally:
+ os.chmod(ABSTFN, 0o755)
+ safe_rmdir(ABSTFN + '/k')
+ safe_rmdir(ABSTFN)
+
def test_relpath(self):
(real_getcwd, os.getcwd) = (os.getcwd, lambda: r"/home/user/bar")
try:
@@ -670,9 +913,12 @@ def test_path_normpath(self):
def test_path_abspath(self):
self.assertPathEqual(self.path.abspath)
- def test_path_realpath(self):
+ @_parameterize({}, {'strict': True}, {'strict': ALLOW_MISSING})
+ def test_path_realpath(self, kwargs):
self.assertPathEqual(self.path.realpath)
+ self.assertPathEqual(partial(self.path.realpath, **kwargs))
+
def test_path_relpath(self):
self.assertPathEqual(self.path.relpath)
diff --git a/Lib/test/test_pyexpat.py b/Lib/test/test_pyexpat.py
index 92fffc45f30a964..bbc2f9b4213e034 100644
--- a/Lib/test/test_pyexpat.py
+++ b/Lib/test/test_pyexpat.py
@@ -11,6 +11,7 @@
from xml.parsers import expat
from xml.parsers.expat import errors
+from test.support import infinite_recursion
from test.support import sortdict
@@ -642,6 +643,24 @@ def test_change_size_2(self):
parser.Parse(xml2, 1)
self.assertEqual(self.n, 4)
+class ElementDeclHandlerTest(unittest.TestCase):
+ def test_deeply_nested_content_model(self):
+ # This should raise a RecursionError and not crash.
+ # See https://github.com/python/cpython/issues/145986.
+ N = 500_000
+ data = (
+ b'<!DOCTYPE root [\n<!ELEMENT root '
+ + b'(a, ' * N + b'a' + b')' * N
+ + b'>\n]>\n<root/>\n'
+ )
+
+ parser = expat.ParserCreate()
+ parser.ElementDeclHandler = lambda _1, _2: None
+ with infinite_recursion():
+ with self.assertRaises(RecursionError):
+ parser.Parse(data)
+
+
class MalformedInputTest(unittest.TestCase):
def test1(self):
xml = b"\0\r\n"
@@ -729,5 +748,94 @@ def resolve_entity(context, base, system_id, public_id):
self.assertEqual(handler_call_args, [("bar", "baz")])
+class ParentParserLifetimeTest(unittest.TestCase):
+ """
+ Subparsers make use of their parent XML_Parser inside of Expat.
+ As a result, parent parsers need to outlive subparsers.
+
+ See https://github.com/python/cpython/issues/139400.
+ """
+
+ def test_parent_parser_outlives_its_subparsers__single(self):
+ parser = expat.ParserCreate()
+ subparser = parser.ExternalEntityParserCreate(None)
+
+ # Now try to cause garbage collection of the parent parser
+ # while it's still being referenced by a related subparser.
+ del parser
+
+ def test_parent_parser_outlives_its_subparsers__multiple(self):
+ parser = expat.ParserCreate()
+ subparser_one = parser.ExternalEntityParserCreate(None)
+ subparser_two = parser.ExternalEntityParserCreate(None)
+
+ # Now try to cause garbage collection of the parent parser
+ # while it's still being referenced by a related subparser.
+ del parser
+
+ def test_parent_parser_outlives_its_subparsers__chain(self):
+ parser = expat.ParserCreate()
+ subparser = parser.ExternalEntityParserCreate(None)
+ subsubparser = subparser.ExternalEntityParserCreate(None)
+
+ # Now try to cause garbage collection of the parent parsers
+ # while they are still being referenced by a related subparser.
+ del parser
+ del subparser
+
+class ReparseDeferralTest(unittest.TestCase):
+ def test_getter_setter_round_trip(self):
+ parser = expat.ParserCreate()
+ enabled = (expat.version_info >= (2, 6, 0))
+
+ self.assertIs(parser.GetReparseDeferralEnabled(), enabled)
+ parser.SetReparseDeferralEnabled(False)
+ self.assertIs(parser.GetReparseDeferralEnabled(), False)
+ parser.SetReparseDeferralEnabled(True)
+ self.assertIs(parser.GetReparseDeferralEnabled(), enabled)
+
+ def test_reparse_deferral_enabled(self):
+ if expat.version_info < (2, 6, 0):
+ self.skipTest(f'Expat {expat.version_info} does not '
+ 'support reparse deferral')
+
+ started = []
+
+ def start_element(name, _):
+ started.append(name)
+
+ parser = expat.ParserCreate()
+ parser.StartElementHandler = start_element
+ self.assertTrue(parser.GetReparseDeferralEnabled())
+
+ for chunk in (b'<doc', b'/>'):
+ parser.Parse(chunk, False)
+
+ # The key test: Have handlers already fired? Expecting: no.
+ self.assertEqual(started, [])
+
+ parser.Parse(b'', True)
+
+ self.assertEqual(started, ['doc'])
+
+ def test_reparse_deferral_disabled(self):
+ started = []
+
+ def start_element(name, _):
+ started.append(name)
+
+ parser = expat.ParserCreate()
+ parser.StartElementHandler = start_element
+ if expat.version_info >= (2, 6, 0):
+ parser.SetReparseDeferralEnabled(False)
+ self.assertFalse(parser.GetReparseDeferralEnabled())
+
+ for chunk in (b'<doc', b'/>'):
+ parser.Parse(chunk, False)
+
+ # The key test: Have handlers already fired? Expecting: yes.
+ self.assertEqual(started, ['doc'])
+
+
if __name__ == "__main__":
unittest.main()
diff --git a/Lib/test/test_sax.py b/Lib/test/test_sax.py
index 3044960a0ed1654..13ebb4a2c62b9a8 100644
--- a/Lib/test/test_sax.py
+++ b/Lib/test/test_sax.py
@@ -19,6 +19,7 @@
import codecs
import gc
import os.path
+import pyexpat
import shutil
from urllib.error import URLError
from test import support
@@ -1168,6 +1169,56 @@ def test_expat_incremental_reset(self):
self.assertEqual(result.getvalue(), start + b"<doc>text</doc>")
+ @unittest.skipIf(pyexpat.version_info < (2, 6, 0),
+ f'Expat {pyexpat.version_info} does not '
+ 'support reparse deferral')
+ def test_flush_reparse_deferral_enabled(self):
+ result = BytesIO()
+ xmlgen = XMLGenerator(result)
+ parser = create_parser()
+ parser.setContentHandler(xmlgen)
+
+ for chunk in ("<doc", ">"):
+ parser.feed(chunk)
+
+ self.assertEqual(result.getvalue(), start) # i.e. no elements started
+ self.assertTrue(parser._parser.GetReparseDeferralEnabled())
+
+ parser.flush()
+
+ self.assertTrue(parser._parser.GetReparseDeferralEnabled())
+ self.assertEqual(result.getvalue(), start + b"<doc>")
+
+ parser.feed("</doc>")
+ parser.close()
+
+ self.assertEqual(result.getvalue(), start + b"<doc></doc>")
+
+ def test_flush_reparse_deferral_disabled(self):
+ result = BytesIO()
+ xmlgen = XMLGenerator(result)
+ parser = create_parser()
+ parser.setContentHandler(xmlgen)
+
+ for chunk in ("<doc", ">"):
+ parser.feed(chunk)
+
+ if pyexpat.version_info >= (2, 6, 0):
+ parser._parser.SetReparseDeferralEnabled(False)
+ self.assertEqual(result.getvalue(), start) # i.e. no elements started
+
+ self.assertFalse(parser._parser.GetReparseDeferralEnabled())
+
+ parser.flush()
+
+ self.assertFalse(parser._parser.GetReparseDeferralEnabled())
+ self.assertEqual(result.getvalue(), start + b"<doc>")
+
+ parser.feed("</doc>")
+ parser.close()
+
+ self.assertEqual(result.getvalue(), start + b"<doc></doc>")
+
# ===== Locator support
def test_expat_locator_noinfo(self):
diff --git a/Lib/test/test_shutil.py b/Lib/test/test_shutil.py
index 433c55fd9a8b775..a8c6f8e86e3c153 100644
--- a/Lib/test/test_shutil.py
+++ b/Lib/test/test_shutil.py
@@ -225,6 +225,37 @@ def test_on_error(self):
self.assertEqual(self.errorState, 3,
"Expected call to onerror function did not happen.")
+ @unittest.skipIf(hasattr(os, 'geteuid') and os.geteuid() == 0,
+ "This test can't be run reliably as root (issue #1076467).")
+ @unittest.skipUnless(shutil.rmtree.avoids_symlink_attacks,
+ 'requires the fd based implementation of rmtree()')
+ def test_on_error_kwargs(self):
+ os.mkdir(TESTFN)
+ self.addCleanup(shutil.rmtree, TESTFN)
+
+ child_dir_path = os.path.join(TESTFN, 'b')
+ child_file_path = os.path.join(child_dir_path, 'a')
+ os.mkdir(child_dir_path)
+ support.create_empty_file(child_file_path)
+ old_child_dir_mode = os.stat(child_dir_path).st_mode
+ # Make unwritable.
+ new_mode = stat.S_IREAD|stat.S_IEXEC
+ os.chmod(child_dir_path, new_mode)
+
+ self.addCleanup(os.chmod, child_dir_path, old_child_dir_mode)
+
+ calls = []
+ def onerror(func, path, exc_info, direntry=None, dir_fd=None):
+ calls.append((func, path, exc_info))
+ if func is os.unlink:
+ self.assertEqual(direntry.name, os.path.basename(path))
+ self.assertTrue(os.path.samestat(
+ os.stat(path), os.stat(direntry.name, dir_fd=dir_fd)))
+
+ shutil.rmtree(TESTFN, onerror=onerror, _onerror_kwargs=True)
+ self.assertIn((os.unlink, child_file_path),
+ [(func, path) for func, path, exc_info in calls])
+
def check_args_to_onerror(self, func, arg, exc):
# test_rmtree_errors deliberately runs rmtree
# on a directory that is chmod 500, which will fail.
@@ -1299,6 +1330,71 @@ def test_unpack_archive_zip(self):
with self.assertRaises(TypeError):
self.check_unpack_archive('zip', filter='data')
+ def test_unpack_archive_zip_badpaths(self):
+ srcdir = self.mkdtemp()
+ zipname = os.path.join(srcdir, 'test.zip')
+ abspath = os.path.join(srcdir, 'abspath')
+ with zipfile.ZipFile(zipname, 'w') as zf:
+ zf.writestr(abspath, 'badfile')
+ zf.writestr(os.sep + abspath, 'badfile')
+ zf.writestr('/abspath', 'badfile')
+ zf.writestr('C:/abspath', 'badfile')
+ zf.writestr('D:\\abspath', 'badfile')
+ zf.writestr('E:abspath', 'badfile')
+ zf.writestr('F:/G:/abspath', 'badfile')
+ zf.writestr('//server/share/abspath', 'badfile')
+ zf.writestr('\\\\server2\\share\\abspath', 'badfile')
+ zf.writestr('../relpath', 'badfile')
+ zf.writestr(os.pardir + os.sep + 'relpath2', 'badfile')
+ zf.writestr('good/file', 'goodfile')
+ zf.writestr('good..file', 'goodfile')
+
+ dstdir = os.path.join(self.mkdtemp(), 'dst')
+ unpack_archive(zipname, dstdir)
+ self.assertTrue(os.path.isfile(os.path.join(dstdir, 'good', 'file')))
+ self.assertTrue(os.path.isfile(os.path.join(dstdir, 'good..file')))
+ self.assertFalse(os.path.exists(abspath))
+ self.assertFalse(os.path.exists(os.path.join(dstdir, 'abspath')))
+ self.assertFalse(os.path.exists(os.path.join(dstdir, 'G_')))
+ self.assertFalse(os.path.exists(os.path.join(dstdir, 'server')))
+ if os.name != 'nt':
+ self.assertTrue(os.path.isfile(os.path.join(dstdir, 'C:', 'abspath')))
+ self.assertTrue(os.path.isfile(os.path.join(dstdir, 'D:\\abspath')))
+ self.assertTrue(os.path.isfile(os.path.join(dstdir, 'E:abspath')))
+ self.assertTrue(os.path.isfile(os.path.join(dstdir, 'F:', 'G:', 'abspath')))
+ self.assertTrue(os.path.isfile(os.path.join(dstdir, '\\\\server2\\share\\abspath')))
+ if os.pardir == '..':
+ self.assertFalse(os.path.exists(os.path.join(dstdir, '..', 'relpath')))
+ self.assertFalse(os.path.exists(os.path.join(dstdir, 'relpath')))
+ else:
+ self.assertTrue(os.path.isfile(os.path.join(dstdir, '..', 'relpath')))
+ self.assertFalse(os.path.exists(os.path.join(dstdir, os.pardir, 'relpath2')))
+ self.assertFalse(os.path.exists(os.path.join(dstdir, 'relpath2')))
+
+ dstdir2 = os.path.join(self.mkdtemp(), 'dst')
+ os.mkdir(dstdir2)
+ with support.change_cwd(dstdir2):
+ unpack_archive(zipname, '')
+ self.assertTrue(os.path.isfile(os.path.join('good', 'file')))
+ self.assertTrue(os.path.isfile('good..file'))
+ self.assertFalse(os.path.exists(abspath))
+ self.assertFalse(os.path.exists('abspath'))
+ self.assertFalse(os.path.exists('C_'))
+ self.assertFalse(os.path.exists('server'))
+ if os.name != 'nt':
+ self.assertTrue(os.path.isfile(os.path.join('C:', 'abspath')))
+ self.assertTrue(os.path.isfile('D:\\abspath'))
+ self.assertTrue(os.path.isfile('E:abspath'))
+ self.assertTrue(os.path.isfile(os.path.join('F:', 'G:', 'abspath')))
+ self.assertTrue(os.path.isfile('\\\\server2\\share\\abspath'))
+ if os.pardir == '..':
+ self.assertFalse(os.path.exists(os.path.join('..', 'relpath')))
+ self.assertFalse(os.path.exists('relpath'))
+ else:
+ self.assertTrue(os.path.isfile(os.path.join('..', 'relpath')))
+ self.assertFalse(os.path.exists(os.path.join(os.pardir, 'relpath2')))
+ self.assertFalse(os.path.exists('relpath2'))
+
def test_unpack_registry(self):
formats = get_unpack_formats()
@@ -1756,11 +1852,7 @@ def test_move_dangling_symlink(self):
dst_link = os.path.join(self.dst_dir, 'quux')
shutil.move(dst, dst_link)
self.assertTrue(os.path.islink(dst_link))
- # On Windows, os.path.realpath does not follow symlinks (issue #9949)
- if os.name == 'nt':
- self.assertEqual(os.path.realpath(src), os.readlink(dst_link))
- else:
- self.assertEqual(os.path.realpath(src), os.path.realpath(dst_link))
+ self.assertEqual(os.path.realpath(src), os.path.realpath(dst_link))
@support.skip_unless_symlink
@mock_rename
diff --git a/Lib/test/test_site.py b/Lib/test/test_site.py
index 8815c839984e918..c5af2cf9046c302 100644
--- a/Lib/test/test_site.py
+++ b/Lib/test/test_site.py
@@ -18,6 +18,7 @@
import urllib.request
import urllib.error
import shutil
+import stat
import subprocess
import sysconfig
import tempfile
@@ -184,6 +185,44 @@ def test_addsitedir(self):
finally:
pth_file.cleanup()
+ def test_addsitedir_dotfile(self):
+ pth_file = PthFile('.dotfile')
+ pth_file.cleanup(prep=True)
+ try:
+ pth_file.create()
+ site.addsitedir(pth_file.base_dir, set())
+ self.assertNotIn(site.makepath(pth_file.good_dir_path)[0], sys.path)
+ self.assertIn(pth_file.base_dir, sys.path)
+ finally:
+ pth_file.cleanup()
+
+ @unittest.skipUnless(hasattr(os, 'chflags'), 'test needs os.chflags()')
+ def test_addsitedir_hidden_flags(self):
+ pth_file = PthFile()
+ pth_file.cleanup(prep=True)
+ try:
+ pth_file.create()
+ st = os.stat(pth_file.file_path)
+ os.chflags(pth_file.file_path, st.st_flags | stat.UF_HIDDEN)
+ site.addsitedir(pth_file.base_dir, set())
+ self.assertNotIn(site.makepath(pth_file.good_dir_path)[0], sys.path)
+ self.assertIn(pth_file.base_dir, sys.path)
+ finally:
+ pth_file.cleanup()
+
+ @unittest.skipUnless(sys.platform == 'win32', 'test needs Windows')
+ def test_addsitedir_hidden_file_attribute(self):
+ pth_file = PthFile()
+ pth_file.cleanup(prep=True)
+ try:
+ pth_file.create()
+ subprocess.check_call(['attrib', '+H', pth_file.file_path])
+ site.addsitedir(pth_file.base_dir, set())
+ self.assertNotIn(site.makepath(pth_file.good_dir_path)[0], sys.path)
+ self.assertIn(pth_file.base_dir, sys.path)
+ finally:
+ pth_file.cleanup()
+
# This tests _getuserbase, hence the double underline
# to distinguish from a test for getuserbase
def test__getuserbase(self):
diff --git a/Lib/test/test_socket.py b/Lib/test/test_socket.py
index 5f4b019c048d9cd..addf0f1de029e98 100644
--- a/Lib/test/test_socket.py
+++ b/Lib/test/test_socket.py
@@ -488,19 +488,27 @@ class SocketPairTest(unittest.TestCase, ThreadableTest):
def __init__(self, methodName='runTest'):
unittest.TestCase.__init__(self, methodName=methodName)
ThreadableTest.__init__(self)
+ self.cli = None
+ self.serv = None
+
+ def socketpair(self):
+ # To be overridden by some child classes.
+ return socket.socketpair()
def setUp(self):
- self.serv, self.cli = socket.socketpair()
+ self.serv, self.cli = self.socketpair()
def tearDown(self):
- self.serv.close()
+ if self.serv:
+ self.serv.close()
self.serv = None
def clientSetUp(self):
pass
def clientTearDown(self):
- self.cli.close()
+ if self.cli:
+ self.cli.close()
self.cli = None
ThreadableTest.clientTearDown(self)
@@ -4176,6 +4184,112 @@ def _testSend(self):
self.assertEqual(msg, MSG)
+class PurePythonSocketPairTest(SocketPairTest):
+ # Explicitly use socketpair AF_INET or AF_INET6 to ensure that is the
+ # code path we're using regardless platform is the pure python one where
+ # `_socket.socketpair` does not exist. (AF_INET does not work with
+ # _socket.socketpair on many platforms).
+ def socketpair(self):
+ # called by super().setUp().
+ try:
+ return socket.socketpair(socket.AF_INET6)
+ except OSError:
+ return socket.socketpair(socket.AF_INET)
+
+ # Local imports in this class make for easy security fix backporting.
+
+ def setUp(self):
+ if hasattr(_socket, "socketpair"):
+ self._orig_sp = socket.socketpair
+ # This forces the version using the non-OS provided socketpair
+ # emulation via an AF_INET socket in Lib/socket.py.
+ socket.socketpair = socket._fallback_socketpair
+ else:
+ # This platform already uses the non-OS provided version.
+ self._orig_sp = None
+ super().setUp()
+
+ def tearDown(self):
+ super().tearDown()
+ if self._orig_sp is not None:
+ # Restore the default socket.socketpair definition.
+ socket.socketpair = self._orig_sp
+
+ def test_recv(self):
+ msg = self.serv.recv(1024)
+ self.assertEqual(msg, MSG)
+
+ def _test_recv(self):
+ self.cli.send(MSG)
+
+ def test_send(self):
+ self.serv.send(MSG)
+
+ def _test_send(self):
+ msg = self.cli.recv(1024)
+ self.assertEqual(msg, MSG)
+
+ def test_ipv4(self):
+ cli, srv = socket.socketpair(socket.AF_INET)
+ cli.close()
+ srv.close()
+
+ def _test_ipv4(self):
+ pass
+
+ @unittest.skipIf(not hasattr(_socket, 'IPPROTO_IPV6') or
+ not hasattr(_socket, 'IPV6_V6ONLY'),
+ "IPV6_V6ONLY option not supported")
+ @unittest.skipUnless(support.IPV6_ENABLED, 'IPv6 required for this test')
+ def test_ipv6(self):
+ cli, srv = socket.socketpair(socket.AF_INET6)
+ cli.close()
+ srv.close()
+
+ def _test_ipv6(self):
+ pass
+
+ def test_injected_authentication_failure(self):
+ orig_getsockname = socket.socket.getsockname
+ inject_sock = None
+
+ def inject_getsocketname(self):
+ nonlocal inject_sock
+ sockname = orig_getsockname(self)
+ # Connect to the listening socket ahead of the
+ # client socket.
+ if inject_sock is None:
+ inject_sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+ inject_sock.setblocking(False)
+ try:
+ inject_sock.connect(sockname[:2])
+ except (BlockingIOError, InterruptedError):
+ pass
+ inject_sock.setblocking(True)
+ return sockname
+
+ sock1 = sock2 = None
+ try:
+ socket.socket.getsockname = inject_getsocketname
+ with self.assertRaises(OSError):
+ sock1, sock2 = socket.socketpair()
+ finally:
+ socket.socket.getsockname = orig_getsockname
+ if inject_sock:
+ inject_sock.close()
+ if sock1: # This cleanup isn't needed on a successful test.
+ sock1.close()
+ if sock2:
+ sock2.close()
+
+ def _test_injected_authentication_failure(self):
+ # No-op. Exists for base class threading infrastructure to call.
+ # We could refactor this test into its own lesser class along with the
+ # setUp and tearDown code to construct an ideal; it is simpler to keep
+ # it here and live with extra overhead one this _one_ failure test.
+ pass
+
+
class NonBlockingTCPTests(ThreadedTCPSocketTest):
def __init__(self, methodName='runTest'):
diff --git a/Lib/test/test_ssl.py b/Lib/test/test_ssl.py
index 920a11d96de487d..d3bf96f5438b17e 100644
--- a/Lib/test/test_ssl.py
+++ b/Lib/test/test_ssl.py
@@ -3,12 +3,15 @@
import sys
import unittest
from test import support
+import contextlib
import re
import socket
import select
import struct
+import tempfile
import time
import datetime
+import zipfile
import gc
import http.client
import os
@@ -119,6 +122,19 @@ def data_file(*name):
SIGNED_CERTFILE_ECC = data_file("keycertecc.pem")
SIGNED_CERTFILE_ECC_HOSTNAME = 'localhost-ecc'
+# gh-160040: a leaf signed by a compromised eMudhra intermediate whose public
+# key is on the _ssl distrusted list. The chain to distrusted_root.pem is
+# otherwise valid.
+# The PEMs live in a password-protected zip so the compromised eMudhra certs
+# don't trip security scanners that cannot tell internal test fixtures from
+# production secrets and flag any certificate or private key they find. It is
+# encrypted, not merely zipped, because such scanners read inside plain archives.
+# The password is fixed, public, and not a secret; it only keeps the archive
+# from storing the PEMs in the clear.
+DISTRUSTED_SPKI_ZIP = data_file("distrusted.zip")
+DISTRUSTED_SPKI_CHOSEN_KNIGHT = b"Brave Sir Robin" # not a secret, see comment
+DISTRUSTED_SPKI_HOSTNAME = "localhost"
+
# Same certificate as pycacert.pem, but without extra text in file
SIGNING_CA = data_file("capath", "ceff1710.0")
# cert with all kinds of subject alt names
@@ -362,6 +378,34 @@ def testing_context(server_cert=SIGNED_CERTFILE):
return client_context, server_context, hostname
+def connected_bio_pair(client_context, server_context, hostname, max_retry=5):
+ """Handshake a client and a server SSLObject against each other.
+
+ Everything happens in memory, so this needs no socket and no thread.
+ Returns the two objects followed by their four BIOs, in the order
+ client, server, c_in, c_out, s_in, s_out.
+ """
+ c_in, c_out = ssl.MemoryBIO(), ssl.MemoryBIO()
+ s_in, s_out = ssl.MemoryBIO(), ssl.MemoryBIO()
+ client = client_context.wrap_bio(c_in, c_out, server_hostname=hostname)
+ server = server_context.wrap_bio(s_in, s_out, server_side=True)
+
+ # Loop on the handshake for a bit to get it settled
+ for _ in range(max_retry):
+ with contextlib.suppress(ssl.SSLWantReadError):
+ client.do_handshake()
+ if c_out.pending:
+ s_in.write(c_out.read())
+ with contextlib.suppress(ssl.SSLWantReadError):
+ server.do_handshake()
+ if s_out.pending:
+ c_in.write(s_out.read())
+ # Now the handshakes should be complete (don't raise WantReadError)
+ client.do_handshake()
+ server.do_handshake()
+ return client, server, c_in, c_out, s_in, s_out
+
+
class BasicSocketTests(unittest.TestCase):
def test_constants(self):
@@ -1495,6 +1539,8 @@ def test_load_verify_cadata(self):
ctx.load_verify_locations(cadata="broken")
with self.assertRaisesRegex(ssl.SSLError, "not enough data"):
ctx.load_verify_locations(cadata=b"broken")
+ with self.assertRaises(ssl.SSLError):
+ ctx.load_verify_locations(cadata=cacert_der + b"A")
def test_load_dh_params(self):
@@ -1833,6 +1879,10 @@ def test_subclass(self):
def test_bad_server_hostname(self):
ctx = ssl.create_default_context()
+ # Omitting the name entirely is bad too: this context checks it.
+ with self.assertWarns(DeprecationWarning):
+ ctx.wrap_bio(ssl.MemoryBIO(), ssl.MemoryBIO(),
+ server_hostname=None)
with self.assertRaises(ValueError):
ctx.wrap_bio(ssl.MemoryBIO(), ssl.MemoryBIO(),
server_hostname="")
@@ -1913,6 +1963,66 @@ def test_private_init(self):
with self.assertRaisesRegex(TypeError, "public constructor"):
ssl.SSLObject(bio, bio)
+ def test_check_hostname_requires_server_hostname(self):
+ # wrap_bio() used to accept a context asking for hostname checking
+ # without a name to check against, and then verify the certificate
+ # chain but never the peer's identity without a warning. Now
+ # a warning is emitted in this scenario.
+ client_context, _, hostname = testing_context()
+ self.assertTrue(client_context.check_hostname)
+
+ server_hostname = None
+ with self.assertWarnsRegex(
+ DeprecationWarning,
+ "check_hostname requires server_hostname"):
+ client_context.wrap_bio(ssl.MemoryBIO(), ssl.MemoryBIO(),
+ server_hostname=server_hostname)
+ # The sibling constructor refuses the very same call, but with
+ # a ValueError instead of DeprecationWarning.
+ with socket.socket() as sock:
+ with self.assertRaisesRegex(
+ ValueError,
+ "check_hostname requires server_hostname"):
+ client_context.wrap_socket(
+ sock, server_hostname=server_hostname)
+
+ # A name was all that was missing.
+ with support.check_no_warnings(self):
+ client_context.wrap_bio(ssl.MemoryBIO(), ssl.MemoryBIO(),
+ server_hostname=hostname)
+
+ # Asking for no hostname check remains a way to say so explicitly.
+ context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT)
+ context.check_hostname = False
+ self.assertFalse(context.check_hostname)
+ with support.check_no_warnings(self):
+ context.wrap_bio(ssl.MemoryBIO(), ssl.MemoryBIO())
+
+ def test_server_side_bad_params(self):
+ # A server neither sends a hostname nor resumes a client's session,
+ # so wrap_bio() rejects both in server mode like wrap_socket()
+ client_context, server_context, hostname = testing_context()
+
+ with self.assertRaisesRegex(
+ ValueError,
+ "server_hostname can only be specified in client mode"):
+ server_context.wrap_bio(ssl.MemoryBIO(), ssl.MemoryBIO(),
+ server_side=True,
+ server_hostname=hostname)
+
+ client, server, *_ = connected_bio_pair(
+ client_context, server_context, hostname)
+ session = client.session
+ self.assertIsNotNone(session)
+ with self.assertRaisesRegex(
+ ValueError, "session can only be specified in client mode"):
+ server_context.wrap_bio(ssl.MemoryBIO(), ssl.MemoryBIO(),
+ server_side=True, session=session)
+
+ # Neither argument is what a server passes, so this still works.
+ server_context.wrap_bio(ssl.MemoryBIO(), ssl.MemoryBIO(),
+ server_side=True)
+
def test_unwrap(self):
client_ctx, server_ctx, hostname = testing_context()
c_in = ssl.MemoryBIO()
@@ -1955,6 +2065,110 @@ def test_unwrap(self):
c_in.write(s_out.read())
client.unwrap()
+ def test_distrusted_spki(self):
+ # gh-160040: the server chain is otherwise valid, but the eMudhra
+ # intermediate's public key is on the _ssl distrusted list, so
+ # verification rejects it.
+ client_context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT)
+ server_context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER)
+ # the contexts load the PEMs now, so the files are only needed here
+ with tempfile.TemporaryDirectory() as tmpdir:
+ with zipfile.ZipFile(DISTRUSTED_SPKI_ZIP) as z:
+ z.extractall(tmpdir, pwd=DISTRUSTED_SPKI_CHOSEN_KNIGHT)
+ certdir = os.path.join(tmpdir, "distrusted")
+ client_context.load_verify_locations(
+ os.path.join(certdir, "distrusted_root.pem"))
+ server_context.load_cert_chain(
+ os.path.join(certdir, "distrusted_keycert.pem"))
+
+ with self.assertRaises(ssl.SSLCertVerificationError) as cm:
+ connected_bio_pair(client_context, server_context,
+ DISTRUSTED_SPKI_HOSTNAME)
+ # X509_V_ERR_CERT_REJECTED
+ self.assertEqual(cm.exception.verify_code, 28)
+
+ def test_sni_callback_context_released_and_callback_raises(self):
+ # Variant of the test below without a HelloRetryRequest: the callback
+ # switches the connection to another context, drops the last
+ # references to the context that carries it, and raises. The C
+ # callback must not touch that context after the Python callback
+ # returned.
+ client_ctx, server_ctx, hostname = testing_context()
+ leaf_ctx = server_ctx
+
+ def sni_cb(sslobj, server_name, ctx):
+ sslobj.context = leaf_ctx
+ del ctx
+ raise LookupError("no certificate for " + repr(server_name))
+
+ def make_server():
+ dispatch_ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER)
+ dispatch_ctx.load_cert_chain(SIGNED_CERTFILE)
+ dispatch_ctx.sni_callback = sni_cb
+ s_in, s_out = ssl.MemoryBIO(), ssl.MemoryBIO()
+ server = dispatch_ctx.wrap_bio(s_in, s_out, server_side=True)
+ return server, s_in, s_out
+
+ server, s_in, s_out = make_server()
+ c_in, c_out = ssl.MemoryBIO(), ssl.MemoryBIO()
+ client = client_ctx.wrap_bio(c_in, c_out, server_hostname=hostname)
+ with self.assertRaises(ssl.SSLWantReadError):
+ client.do_handshake()
+ s_in.write(c_out.read())
+ # 3.7 has no sys.unraisablehook; the callback's exception is
+ # reported to stderr by PyErr_WriteUnraisable().
+ with support.captured_stderr() as stderr:
+ with self.assertRaises(ssl.SSLError):
+ server.do_handshake()
+ self.assertIn('LookupError', stderr.getvalue())
+ self.assertIs(server.context, leaf_ctx)
+
+ def test_sni_callback_context_released_before_second_client_hello(self):
+ # The SSLContext carrying sni_callback may be released by the
+ # application once the callback has switched the connection over to
+ # another context. If the server then sends a HelloRetryRequest, the
+ # second ClientHello makes OpenSSL consult the original SSL_CTX's
+ # servername callback again; that must not use the deallocated
+ # SSLContext object.
+ client_ctx, leaf_ctx, hostname = testing_context()
+ calls = []
+
+ def make_server():
+ dispatch_ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER)
+ dispatch_ctx.load_cert_chain(SIGNED_CERTFILE)
+ # Force a HelloRetryRequest: the client offers an X25519 key
+ # share first, the server only accepts P-384.
+ dispatch_ctx.set_ecdh_curve("secp384r1")
+ def sni_cb(sslobj, server_name, ctx):
+ calls.append(server_name)
+ sslobj.context = leaf_ctx
+ dispatch_ctx.sni_callback = sni_cb
+ s_in, s_out = ssl.MemoryBIO(), ssl.MemoryBIO()
+ server = dispatch_ctx.wrap_bio(s_in, s_out, server_side=True)
+ return server, s_in, s_out, weakref.ref(dispatch_ctx)
+
+ # After this only the C-level SSL object references dispatch_ctx.
+ server, s_in, s_out, dispatch_ref = make_server()
+ c_in, c_out = ssl.MemoryBIO(), ssl.MemoryBIO()
+ client = client_ctx.wrap_bio(c_in, c_out, server_hostname=hostname)
+ for _ in range(10):
+ for obj, out, peer_in in ((client, c_out, s_in),
+ (server, s_out, c_in)):
+ try:
+ obj.do_handshake()
+ except ssl.SSLWantReadError:
+ pass
+ if out.pending:
+ peer_in.write(out.read())
+ client.do_handshake()
+ server.do_handshake()
+ support.gc_collect()
+ self.assertIsNone(dispatch_ref())
+ self.assertGreaterEqual(len(calls), 1)
+ self.assertEqual(calls[0], hostname)
+ self.assertIs(server.context, leaf_ctx)
+ self.assertIsNotNone(client.cipher())
+
class SimpleBackgroundTests(unittest.TestCase):
"""Tests that connect to a simple server running in the background"""
@@ -4151,6 +4365,12 @@ def test_npn_protocols(self):
if len(stats['server_npn_protocols']) else 'nothing'
self.assertEqual(server_result, expected, msg % (server_result, "server"))
+ def test_empty_npn_protocols(self):
+ """npn_protocols cannot be empty, see CVE-2024-5642 & gh-121227"""
+ client_context, server_context, hostname = testing_context()
+ with self.assertRaises(ssl.SSLError):
+ server_context.set_npn_protocols([])
+
def sni_contexts(self):
server_context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER)
server_context.load_cert_chain(SIGNED_CERTFILE)
diff --git a/Lib/test/test_tarfile.py b/Lib/test/test_tarfile.py
index 64ab69796d1d386..d2d4b8f7f79aa14 100644
--- a/Lib/test/test_tarfile.py
+++ b/Lib/test/test_tarfile.py
@@ -1,4 +1,5 @@
import sys
+import errno
import os
import io
from hashlib import md5
@@ -42,6 +43,7 @@ def md5sum(data):
xzname = os.path.join(TEMPDIR, "testtar.tar.xz")
tmpname = os.path.join(TEMPDIR, "tmp.tar")
dotlessname = os.path.join(TEMPDIR, "testtar")
+SPACE = b" "
md5_regtype = "65f477c818ad9e15f7feab0c6d37742f"
md5_sparse = "a54fbc4ca4f4399a90e1b27164012fc6"
@@ -944,10 +946,30 @@ def test_header_offset(self):
self.assertEqual(tarinfo.type, self.longnametype)
+ def test_longname_file_not_directory(self):
+ # Test reading a longname file and ensure it is not handled as a directory
+ # Issue #141707
+ buf = io.BytesIO()
+ with tarfile.open(mode='w', fileobj=buf, format=self.format) as tar:
+ ti = tarfile.TarInfo()
+ ti.type = tarfile.AREGTYPE
+ ti.name = ('a' * 99) + '/' + ('b' * 3)
+ tar.addfile(ti)
+
+ expected = {t.name: t.type for t in tar.getmembers()}
+
+ buf.seek(0)
+ with tarfile.open(mode='r', fileobj=buf) as tar:
+ actual = {t.name: t.type for t in tar.getmembers()}
+
+ self.assertEqual(expected, actual)
+
+
class GNUReadTest(LongnameTest, ReadTest, unittest.TestCase):
subdir = "gnu"
longnametype = tarfile.GNUTYPE_LONGNAME
+ format = tarfile.GNU_FORMAT
# Since 3.2 tarfile is supposed to accurately restore sparse members and
# produce files with holes. This is what we actually want to test here.
@@ -1007,6 +1029,7 @@ class PaxReadTest(LongnameTest, ReadTest, unittest.TestCase):
subdir = "pax"
longnametype = tarfile.XHDTYPE
+ format = tarfile.PAX_FORMAT
def test_pax_global_headers(self):
tar = tarfile.open(tarname, encoding="iso8859-1")
@@ -1359,19 +1382,16 @@ def test_extractall_symlinks(self):
with open(source_file,'w') as f:
f.write('something\n')
os.symlink(source_file, target_file)
- tar = tarfile.open(temparchive,'w')
- tar.add(source_file)
- tar.add(target_file)
- tar.close()
+ with tarfile.open(temparchive, 'w') as tar:
+ tar.add(source_file, arcname="source")
+ tar.add(target_file, arcname="symlink")
# Let's extract it to the location which contains the symlink
- tar = tarfile.open(temparchive,'r')
- # this should not raise OSError: [Errno 17] File exists
- try:
- tar.extractall(path=tempdir)
- except OSError:
- self.fail("extractall failed with symlinked files")
- finally:
- tar.close()
+ with tarfile.open(temparchive, errorlevel=2) as tar:
+ # this should not raise OSError: [Errno 17] File exists
+ try:
+ tar.extractall(path=tempdir)
+ except OSError:
+ self.fail("extractall failed with symlinked files")
finally:
support.unlink(temparchive)
support.rmtree(tempdir)
@@ -2245,9 +2265,35 @@ def test__all__(self):
'tar_filter', 'FilterError', 'AbsoluteLinkError',
'OutsideDestinationError', 'SpecialFileError',
'AbsolutePathError', 'LinkOutsideDestinationError',
+ 'LinkFallbackError',
}
support.check__all__(self, tarfile, blacklist=blacklist)
+ @unittest.skipUnless(support.can_symlink(), 'requires symlink support')
+ @unittest.skipUnless(hasattr(os, 'chmod'), "missing os.chmod")
+ @unittest.mock.patch('os.chmod')
+ def test_deferred_directory_attributes_update(self, mock_chmod):
+ # Regression test for gh-127987: setting attributes on arbitrary files
+ tempdir = os.path.join(TEMPDIR, 'test127987')
+ def mock_chmod_side_effect(path, mode, **kwargs):
+ target_path = os.path.realpath(path)
+ if os.path.commonpath([target_path, tempdir]) != tempdir:
+ raise Exception("should not try to chmod anything outside the destination", target_path)
+ mock_chmod.side_effect = mock_chmod_side_effect
+
+ outside_tree_dir = os.path.join(TEMPDIR, 'outside_tree_dir')
+ with ArchiveMaker() as arc:
+ arc.add('x', symlink_to='.')
+ arc.add('x', type=tarfile.DIRTYPE, mode='?rwsrwsrwt')
+ arc.add('x', symlink_to=outside_tree_dir)
+
+ os.makedirs(outside_tree_dir)
+ try:
+ arc.open().extractall(path=tempdir, filter='tar')
+ finally:
+ support.rmtree(outside_tree_dir)
+ support.rmtree(tempdir)
+
class CommandLineTest(unittest.TestCase):
@@ -2797,9 +2843,21 @@ def tearDownClass(cls):
shutil.rmtree(cls.control_dir)
def check_files_present(self, directory):
- got_paths = set(
- p.relative_to(directory)
- for p in pathlib.Path(directory).glob('**/*'))
+ try:
+ got_paths = set(
+ p.relative_to(directory)
+ for p in pathlib.Path(directory).glob('**/*'))
+ except OSError as err:
+ if getattr(err, 'winerror', None) == 123:
+ # 123 is Windows error ERROR_INVALID_NAME
+ # glob() fails this way when symlink targets are too long
+ # Fixed in 3.8+: https://github.com/python/cpython/issues/79487
+ raise unittest.SkipTest('Path.glob failed')
+ raise
+ if self.extraction_filter == 'data':
+ # The 'data' filter is expected to reject special files
+ for path in 'ustar/fifotype', 'ustar/blktype', 'ustar/chrtype':
+ got_paths.discard(pathlib.Path(path))
self.assertEqual(self.control_paths, got_paths)
@contextmanager
@@ -3026,10 +3084,28 @@ def __exit__(self, *exc):
self.bio = None
def add(self, name, *, type=None, symlink_to=None, hardlink_to=None,
- mode=None, **kwargs):
- """Add a member to the test archive. Call within `with`."""
+ mode=None, size=None, content=None, **kwargs):
+ """Add a member to the test archive. Call within `with`.
+
+ Provides many shortcuts:
+ - default `type` is based on symlink_to, hardlink_to, and trailing `/`
+ in name (which is stripped)
+ - size & content defaults are based on each other
+ - content can be str or bytes
+ - mode should be textual ('-rwxrwxrwx')
+
+ (add more! this is unstable internal test-only API)
+ """
name = str(name)
tarinfo = tarfile.TarInfo(name).replace(**kwargs)
+ if content is not None:
+ if isinstance(content, str):
+ content = content.encode()
+ size = len(content)
+ if size is not None:
+ tarinfo.size = size
+ if content is None:
+ content = bytes(tarinfo.size)
if mode:
tarinfo.mode = _filemode_to_int(mode)
if symlink_to is not None:
@@ -3043,7 +3119,7 @@ def add(self, name, *, type=None, symlink_to=None, hardlink_to=None,
if type is not None:
tarinfo.type = type
if tarinfo.isreg():
- fileobj = io.BytesIO(bytes(tarinfo.size))
+ fileobj = io.BytesIO(content)
else:
fileobj = None
self.tar_w.addfile(tarinfo, fileobj)
@@ -3064,8 +3140,41 @@ class TestExtractionFilters(unittest.TestCase):
# The destination for the extraction, within `outerdir`
destdir = outerdir / 'dest'
+ @classmethod
+ def setUpClass(cls):
+ # Posix and Windows have different pathname resolution:
+ # either symlink or a '..' component resolve first.
+ # Let's see which we are on.
+ if support.can_symlink():
+ testpath = os.path.join(TEMPDIR, 'resolution_test')
+ os.mkdir(testpath)
+
+ # testpath/current links to `.` which is all of:
+ # - `testpath`
+ # - `testpath/current`
+ # - `testpath/current/current`
+ # - etc.
+ os.symlink('.', os.path.join(testpath, 'current'))
+
+ # we'll test where `testpath/current/../file` ends up
+ with open(os.path.join(testpath, 'current', '..', 'file'), 'w'):
+ pass
+
+ if os.path.exists(os.path.join(testpath, 'file')):
+ # Windows collapses 'current\..' to '.' first, leaving
+ # 'testpath\file'
+ cls.dotdot_resolves_early = True
+ elif os.path.exists(os.path.join(testpath, '..', 'file')):
+ # Posix resolves 'current' to '.' first, leaving
+ # 'testpath/../file'
+ cls.dotdot_resolves_early = False
+ else:
+ raise AssertionError('Could not determine link resolution')
+ else:
+ cls.dotdot_resolves_early = False
+
@contextmanager
- def check_context(self, tar, filter):
+ def check_context(self, tar, filter, *, check_flag=True, ignored_trees=()):
"""Extracts `tar` to `self.destdir` and allows checking the result
If an error occurs, it must be checked using `expect_exception`
@@ -3074,15 +3183,29 @@ def check_context(self, tar, filter):
except the destination directory itself and parent directories of
other files.
When checking directories, do so before their contents.
+
+ A file called 'flag' is made in outerdir (i.e. outside destdir)
+ before extraction; it should not be altered nor should its contents
+ be read/copied.
+
+ *ignored_trees* is a set of directories to remove (including their
+ contents) right after the archive is extracted. It is a workaround
+ for Path.glob() failing to get all files in Python 3.10 and below.
"""
with support.temp_dir(self.outerdir):
+ flag_path = self.outerdir / 'flag'
+ flag_path.write_text('capture me')
try:
tar.extractall(self.destdir, filter=filter)
except Exception as exc:
self.raised_exception = exc
+ self.reraise_exception = True
self.expected_paths = set()
else:
+ for ignored_tree in ignored_trees:
+ support.rmtree((self.destdir / ignored_tree).resolve())
self.raised_exception = None
+ self.reraise_exception = False
try:
self.expected_paths = set(self.outerdir.glob('**/*'))
except OSError as err:
@@ -3094,15 +3217,21 @@ def check_context(self, tar, filter):
else:
raise
self.expected_paths.discard(self.destdir)
+ self.expected_paths.discard(flag_path)
try:
- yield
+ yield self
finally:
tar.close()
- if self.raised_exception:
+ if self.reraise_exception:
raise self.raised_exception
self.assertEqual(self.expected_paths, set())
+ if check_flag:
+ self.assertEqual(flag_path.read_text(), 'capture me')
+ else:
+ assert filter == 'fully_trusted'
- def expect_file(self, name, type=None, symlink_to=None, mode=None):
+ def expect_file(self, name, type=None, symlink_to=None, mode=None,
+ size=None, content=None):
"""Check a single file. See check_context."""
if self.raised_exception:
raise self.raised_exception
@@ -3112,7 +3241,7 @@ def expect_file(self, name, type=None, symlink_to=None, mode=None):
self.expected_paths.remove(path)
# When checking mode, ignore Windows (which can only set user read and
- # user write bits). Newer versions of Python use `os_helper.can_chmod()`
+ # user write bits). Newer versions of Python use `support.can_chmod()`
# instead of hardcoding Windows.
if mode is not None and sys.platform != 'win32':
got = stat.filemode(stat.S_IMODE(path.stat().st_mode))
@@ -3126,24 +3255,46 @@ def expect_file(self, name, type=None, symlink_to=None, mode=None):
# The symlink might be the same (textually) as what we expect,
# but some systems change the link to an equivalent path, so
# we fall back to samefile().
- if expected != got:
- self.assertTrue(got.samefile(expected))
+ try:
+ if expected != got:
+ self.assertTrue(got.samefile(expected))
+ except Exception as e:
+ # show both paths, even if `samefile` fails
+ # (no BaseException.add_note() before 3.11)
+ raise AssertionError(
+ 'expected=%r, got=%r' % (expected, got)) from e
elif type == tarfile.REGTYPE or type is None:
self.assertTrue(path.is_file())
elif type == tarfile.DIRTYPE:
self.assertTrue(path.is_dir())
elif type == tarfile.FIFOTYPE:
self.assertTrue(path.is_fifo())
+ elif type == tarfile.SYMTYPE:
+ self.assertTrue(path.is_symlink())
else:
raise NotImplementedError(type)
+ if size is not None:
+ self.assertEqual(path.stat().st_size, size)
+ if content is not None:
+ self.assertEqual(path.read_text(), content)
for parent in path.parents:
self.expected_paths.discard(parent)
+ def expect_any_tree(self, name):
+ """Check a directory; forget about its contents."""
+ tree_path = (self.destdir / name).resolve()
+ self.expect_file(tree_path, type=tarfile.DIRTYPE)
+ self.expected_paths = {
+ p for p in self.expected_paths
+ if tree_path not in p.parents
+ }
+
def expect_exception(self, exc_type, message_re='.'):
with self.assertRaisesRegex(exc_type, message_re):
if self.raised_exception is not None:
raise self.raised_exception
- self.raised_exception = None
+ self.reraise_exception = False
+ return self.raised_exception
def test_benign_file(self):
with ArchiveMaker() as arc:
@@ -3177,12 +3328,33 @@ def test_absolute(self):
tarfile.AbsolutePathError,
"""['"].*escaped.evil['"] has an absolute path""")
+ def test_parent_dir_out_and_back(self):
+ # Test a member that leaves the destination and comes back.
+ # The containment check looks at the resolved path, which stays
+ # inside, but the intermediate directories are created from the
+ # name as given, which does not.
+ with ArchiveMaker() as arc:
+ arc.add(f'../escaped.evil/../{self.destdir.name}/sub/file',
+ content='content')
+
+ for filter in 'tar', 'data':
+ with self.subTest(filter):
+ with self.check_context(arc.open(), filter):
+ self.expect_file('sub/file', content='content')
+
def test_parent_symlink(self):
# Test interplaying symlinks
# Inspired by 'dirsymlink2a' in jwilk/traversal-archives
with ArchiveMaker() as arc:
+
+ # `current` links to `.` which is both:
+ # - the destination directory
+ # - `current` itself
arc.add('current', symlink_to='.')
+
+ # effectively points to ./../
arc.add('parent', symlink_to='current/..')
+
arc.add('parent/evil')
if support.can_symlink():
@@ -3206,10 +3378,19 @@ def test_parent_symlink(self):
+ "which is outside the destination")
with self.check_context(arc.open(), 'data'):
- self.expect_exception(
- tarfile.LinkOutsideDestinationError,
- """'parent' would link to ['"].*outerdir['"], """
- + "which is outside the destination")
+ if self.dotdot_resolves_early:
+ # 'current/../..' normalises to '..', which is rejected.
+ self.expect_exception(
+ tarfile.LinkOutsideDestinationError,
+ """'parent' would link to ['"].*outerdir['"], """
+ + "which is outside the destination")
+ else:
+ # 'current/..' normalises to '.'; the rewritten link is
+ # created and 'parent/evil' lands harmlessly inside the
+ # destination.
+ self.expect_file('current', symlink_to='.')
+ self.expect_file('parent', symlink_to='.')
+ self.expect_file('evil')
else:
# No symlink support. The symlinks are ignored.
@@ -3220,12 +3401,92 @@ def test_parent_symlink(self):
with self.check_context(arc.open(), 'data'):
self.expect_file('parent/evil')
+ @support.skip_unless_symlink
+ def test_realpath_limit_attack(self):
+ # (CVE-2025-4517)
+
+ with ArchiveMaker() as arc:
+ # populate the symlinks and dirs that expand in os.path.realpath()
+ # The component length is chosen so that in common cases, the unexpanded
+ # path fits in PATH_MAX, but it overflows when the final symlink
+ # is expanded
+ steps = "abcdefghijklmnop"
+ if sys.platform == 'win32':
+ component = 'd' * 25
+ elif 'PC_PATH_MAX' in os.pathconf_names:
+ max_path_len = os.pathconf(self.outerdir.parent, "PC_PATH_MAX")
+ path_sep_len = 1
+ dest_len = len(str(self.destdir)) + path_sep_len
+ component_len = (max_path_len - dest_len) // (len(steps) + path_sep_len)
+ component = 'd' * component_len
+ else:
+ raise NotImplementedError("Need to guess component length for {sys.platform}")
+ path = ""
+ step_path = ""
+ for i in steps:
+ arc.add(os.path.join(path, component), type=tarfile.DIRTYPE,
+ mode='drwxrwxrwx')
+ arc.add(os.path.join(path, i), symlink_to=component)
+ path = os.path.join(path, component)
+ step_path = os.path.join(step_path, i)
+ # create the final symlink that exceeds PATH_MAX and simply points
+ # to the top dir.
+ # this link will never be expanded by
+ # os.path.realpath(strict=False), nor anything after it.
+ linkpath = os.path.join(*steps, "l"*254)
+ parent_segments = [".."] * len(steps)
+ arc.add(linkpath, symlink_to=os.path.join(*parent_segments))
+ # make a symlink outside to keep the tar command happy
+ arc.add("escape", symlink_to=os.path.join(linkpath, ".."))
+ # use the symlinks above, that are not checked, to create a hardlink
+ # to a file outside of the destination path
+ arc.add("flaglink", hardlink_to=os.path.join("escape", "flag"))
+ # now that we have the hardlink we can overwrite the file
+ arc.add("flaglink", content='overwrite')
+ # we can also create new files as well!
+ arc.add("escape/newfile", content='new')
+
+ with self.subTest('fully_trusted'), \
+ self.check_context(arc.open(), filter='fully_trusted',
+ check_flag=False, ignored_trees={component}):
+ if sys.platform == 'win32':
+ self.expect_exception((FileNotFoundError, FileExistsError))
+ elif self.raised_exception:
+ # Cannot symlink/hardlink: tarfile falls back to getmember()
+ self.expect_exception(KeyError)
+ # Otherwise, this block should never enter.
+ else:
+ self.expect_file('flaglink', content='overwrite')
+ self.expect_file('../newfile', content='new')
+ self.expect_file('escape', type=tarfile.SYMTYPE)
+ self.expect_file('a', symlink_to=component)
+
+ for filter in 'tar', 'data':
+ with self.subTest(filter), self.check_context(arc.open(), filter=filter):
+ exc = self.expect_exception((OSError, KeyError))
+ if isinstance(exc, OSError):
+ if sys.platform == 'win32':
+ # 3: ERROR_PATH_NOT_FOUND
+ # 5: ERROR_ACCESS_DENIED
+ # 206: ERROR_FILENAME_EXCED_RANGE
+ self.assertIn(exc.winerror, (3, 5, 206))
+ else:
+ self.assertEqual(exc.errno, errno.ENAMETOOLONG)
+
def test_parent_symlink2(self):
# Test interplaying symlinks
# Inspired by 'dirsymlink2b' in jwilk/traversal-archives
+
with ArchiveMaker() as arc:
+
+ # `current` links to `.` which is both the destination directory
+ # and `current` itself
arc.add('current', symlink_to='.')
+
+ # `current/parent` is also available as `./parent`,
+ # and effectively points to `./../`
arc.add('current/parent', symlink_to='..')
+
arc.add('parent/evil')
with self.check_context(arc.open(), 'fully_trusted'):
@@ -3239,6 +3500,7 @@ def test_parent_symlink2(self):
with self.check_context(arc.open(), 'tar'):
if support.can_symlink():
+ # Fail when extracting a file outside destination
self.expect_exception(
tarfile.OutsideDestinationError,
"'parent/evil' would be extracted to "
@@ -3249,10 +3511,24 @@ def test_parent_symlink2(self):
self.expect_file('parent/evil')
with self.check_context(arc.open(), 'data'):
- self.expect_exception(
- tarfile.LinkOutsideDestinationError,
- """'current/parent' would link to ['"].*['"], """
- + "which is outside the destination")
+ if support.can_symlink():
+ if self.dotdot_resolves_early:
+ # Fail when extracting a file outside destination
+ self.expect_exception(
+ tarfile.OutsideDestinationError,
+ "'parent/evil' would be extracted to "
+ + """['"].*evil['"], which is outside """
+ + "the destination")
+ else:
+ # Fail as soon as we have a symlink outside the destination
+ self.expect_exception(
+ tarfile.LinkOutsideDestinationError,
+ "'current/parent' would link to "
+ + """['"].*outerdir['"], which is outside """
+ + "the destination")
+ else:
+ self.expect_file('current/')
+ self.expect_file('parent/evil')
def test_absolute_symlink(self):
# Test symlink to an absolute path
@@ -3281,11 +3557,29 @@ def test_absolute_symlink(self):
with self.check_context(arc.open(), 'data'):
self.expect_exception(
tarfile.AbsoluteLinkError,
- "'parent' is a symlink to an absolute path")
+ "'parent' is a link to an absolute path")
+
+ def test_absolute_hardlink(self):
+ # Test hardlink to an absolute path
+ # Inspired by 'dirsymlink' in https://github.com/jwilk/traversal-archives
+ with ArchiveMaker() as arc:
+ arc.add('parent', hardlink_to=self.outerdir / 'foo')
+
+ with self.check_context(arc.open(), 'fully_trusted'):
+ self.expect_exception(KeyError, ".*foo. not found")
+
+ with self.check_context(arc.open(), 'tar'):
+ self.expect_exception(KeyError, ".*foo. not found")
+
+ with self.check_context(arc.open(), 'data'):
+ self.expect_exception(
+ tarfile.AbsoluteLinkError,
+ "'parent' is a link to an absolute path")
def test_sly_relative0(self):
# Inspired by 'relative0' in jwilk/traversal-archives
with ArchiveMaker() as arc:
+ # points to `../../tmp/moo`
arc.add('../moo', symlink_to='..//tmp/moo')
try:
@@ -3335,6 +3629,392 @@ def test_sly_relative2(self):
+ """['"].*moo['"], which is outside the """
+ "destination")
+ @support.skip_unless_symlink
+ def test_normpath_realpath_mismatch(self):
+ # The link-target check must validate the value that will actually
+ # be written to disk (the normalised linkname), not the original.
+ # Here 'a' is a symlink to a deep nonexistent path, so realpath()
+ # of 'a/../../...' stays inside the destination while normpath()
+ # collapses 'a/..' lexically and escapes.
+ depth = len(self.destdir.parts) + 5
+ deep = '/'.join(f'p{i}' for i in range(depth))
+ sneaky = 'a/' + '../' * depth + 'flag'
+ for kind in 'symlink_to', 'hardlink_to':
+ with self.subTest(kind):
+ with ArchiveMaker() as arc:
+ arc.add('a', symlink_to=deep)
+ arc.add('escape', **{kind: sneaky})
+ with self.check_context(arc.open(), 'data'):
+ self.expect_exception(
+ tarfile.LinkOutsideDestinationError)
+
+ @support.skip_unless_symlink
+ def test_symlink_trailing_slash(self):
+ # A trailing slash on a symlink member's name must not cause the
+ # link target to be resolved relative to the wrong directory.
+ with ArchiveMaker() as arc:
+ t = tarfile.TarInfo('x/')
+ t.type = tarfile.SYMTYPE
+ t.linkname = '..'
+ arc.tar_w.addfile(t)
+ arc.add('x/escaped', content='hi')
+
+ with self.check_context(arc.open(), 'data'):
+ self.expect_exception(tarfile.LinkOutsideDestinationError)
+
+ @support.skip_unless_symlink
+ def test_link_at_destination(self):
+ # A link member whose name resolves to the destination directory
+ # itself must be rejected: otherwise the destination is replaced
+ # by a symlink and later members can be redirected through it.
+ for name in '', '.', './':
+ with ArchiveMaker() as arc:
+ t = tarfile.TarInfo(name)
+ t.type = tarfile.SYMTYPE
+ t.linkname = '.'
+ arc.tar_w.addfile(t)
+
+ for filter in 'tar', 'data':
+ with self.subTest(name=name, filter=filter):
+ with self.check_context(arc.open(), filter):
+ self.expect_exception(tarfile.OutsideDestinationError)
+
+ @support.skip_unless_symlink
+ def test_empty_name_symlink_chain(self):
+ # Regression test for a chain of empty-named symlinks that
+ # incrementally redirects the destination outwards.
+ with ArchiveMaker() as arc:
+ for name, target in [('', ''), ('a/', '..'),
+ ('', 'dummy'), ('', 'a'),
+ ('b/', '..'),
+ ('', 'dummy'), ('', 'a/b')]:
+ t = tarfile.TarInfo(name)
+ t.type = tarfile.SYMTYPE
+ t.linkname = target
+ arc.tar_w.addfile(t)
+ arc.add('escaped', content='hi')
+
+ with self.check_context(arc.open(), 'data'):
+ self.expect_exception(tarfile.FilterError)
+
+ def test_deep_symlink(self):
+ # Test that symlinks and hardlinks inside a directory
+ # point to the correct file (`target` of size 3).
+ # If links aren't supported we get a copy of the file.
+ with ArchiveMaker() as arc:
+ arc.add('targetdir/target', size=3)
+ # a hardlink's linkname is relative to the archive
+ arc.add('linkdir/hardlink', hardlink_to=os.path.join(
+ 'targetdir', 'target'))
+ # a symlink's linkname is relative to the link's directory
+ arc.add('linkdir/symlink', symlink_to=os.path.join(
+ '..', 'targetdir', 'target'))
+
+ for filter in 'tar', 'data', 'fully_trusted':
+ with self.check_context(arc.open(), filter):
+ self.expect_file('targetdir/target', size=3)
+ self.expect_file('linkdir/hardlink', size=3)
+ if support.can_symlink():
+ self.expect_file('linkdir/symlink', size=3,
+ symlink_to='../targetdir/target')
+ else:
+ self.expect_file('linkdir/symlink', size=3)
+
+ def test_chains(self):
+ # Test chaining of symlinks/hardlinks.
+ # Symlinks are created before the files they point to.
+ with ArchiveMaker() as arc:
+ arc.add('linkdir/symlink', symlink_to='hardlink')
+ arc.add('symlink2', symlink_to=os.path.join(
+ 'linkdir', 'hardlink2'))
+ arc.add('targetdir/target', size=3)
+ arc.add('linkdir/hardlink', hardlink_to=os.path.join('targetdir', 'target'))
+ arc.add('linkdir/hardlink2', hardlink_to=os.path.join('linkdir', 'symlink'))
+
+ for filter in 'tar', 'data', 'fully_trusted':
+ with self.check_context(arc.open(), filter):
+ self.expect_file('targetdir/target', size=3)
+ self.expect_file('linkdir/hardlink', size=3)
+ self.expect_file('linkdir/hardlink2', size=3)
+ if support.can_symlink():
+ self.expect_file('linkdir/symlink', size=3,
+ symlink_to='hardlink')
+ self.expect_file('symlink2', size=3,
+ symlink_to='linkdir/hardlink2')
+ else:
+ self.expect_file('linkdir/symlink', size=3)
+ self.expect_file('symlink2', size=3)
+
+ def test_sneaky_hardlink_fallback(self):
+ # (CVE-2025-4330)
+ # Test that when hardlink extraction falls back to extracting members
+ # from the archive, the extracted member is (re-)filtered.
+ with ArchiveMaker() as arc:
+ # Create a directory structure so the c/escape symlink stays
+ # inside the path
+ arc.add("a/t/dummy")
+ # Create b/ directory
+ arc.add("b/")
+ # Point "c" to the bottom of the tree in "a"
+ arc.add("c", symlink_to=os.path.join("a", "t"))
+ # link to non-existant location under "a"
+ arc.add("c/escape", symlink_to=os.path.join("..", "..",
+ "link_here"))
+ # Move "c" to point to "b" ("c/escape" no longer exists)
+ arc.add("c", symlink_to="b")
+ # Attempt to create a hard link to "c/escape". Since it doesn't
+ # exist it will attempt to extract "cescape" but at "boom".
+ arc.add("boom", hardlink_to=os.path.join("c", "escape"))
+
+ with self.check_context(arc.open(), 'data'):
+ if not support.can_symlink():
+ # When 'c/escape' is extracted, 'c' is a regular
+ # directory, and 'c/escape' *would* point outside
+ # the destination if symlinks were allowed.
+ self.expect_exception(
+ tarfile.LinkOutsideDestinationError)
+ elif sys.platform == "win32":
+ # On Windows, 'c/escape' points outside the destination
+ self.expect_exception(tarfile.LinkOutsideDestinationError)
+ else:
+ e = self.expect_exception(
+ tarfile.LinkFallbackError,
+ "link 'boom' would be extracted as a copy of "
+ + "'c/escape', which was rejected")
+ self.assertIsInstance(e.__cause__,
+ tarfile.LinkOutsideDestinationError)
+ for filter in 'tar', 'fully_trusted':
+ with self.subTest(filter), self.check_context(arc.open(), filter):
+ if not support.can_symlink():
+ if filter == 'fully_trusted' or sys.platform == "win32":
+ self.expect_file("a/t/dummy")
+ self.expect_file("b/")
+ self.expect_file("c/")
+ else:
+ self.expect_exception(
+ tarfile.LinkFallbackError,
+ "link 'boom' would be extracted as a copy of "
+ + "'c/escape', which was rejected")
+ else:
+ self.expect_file("a/t/dummy")
+ self.expect_file("b/")
+ self.expect_file("a/t/escape", symlink_to='../../link_here')
+ self.expect_file("boom", symlink_to='../../link_here')
+ self.expect_file("c", symlink_to='b')
+
+ def test_sneaky_hardlink_fallback_deep(self):
+ # (CVE-2026-11940)
+ with ArchiveMaker() as arc:
+ arc.add("a/b/s", symlink_to=os.path.join("..", "escape"))
+ arc.add("s", hardlink_to=os.path.join("a", "b", "s"))
+
+ with self.check_context(arc.open(), 'data'):
+ e = self.expect_exception(
+ tarfile.LinkFallbackError,
+ "link 's' would be extracted as a copy of "
+ + "'a/b/s', which was rejected")
+ self.assertIsInstance(e.__cause__,
+ tarfile.LinkOutsideDestinationError)
+
+ for filter in 'tar', 'fully_trusted':
+ with self.subTest(filter), self.check_context(arc.open(), filter):
+ if not support.can_symlink():
+ self.expect_file("a/")
+ self.expect_file("a/b/")
+ else:
+ self.expect_file("a/b/s", symlink_to=os.path.join('..', 'escape'))
+ self.expect_file("s", symlink_to=os.path.join('..', 'escape'))
+
+ @support.skip_unless_hardlink
+ def test_sneaky_hardlink_relocation(self):
+ with ArchiveMaker() as arc:
+ arc.add("a/escape", content="decoy")
+ arc.add("a/b/s", symlink_to=os.path.join("..", "escape"))
+ arc.add("s", hardlink_to=os.path.join("a", "b", "s"))
+
+ for filter in 'data', 'tar':
+ with self.subTest(filter), self.check_context(arc.open(), filter):
+ self.expect_file("a/escape", content="decoy")
+ if support.can_symlink():
+ self.expect_file("a/b/s", symlink_to=os.path.join('..', 'escape'))
+ else:
+ self.expect_file("a/b/s", content="decoy")
+ self.expect_file("s", content="decoy")
+ self.assertFalse((self.destdir / "s").is_symlink())
+
+ def test_exfiltration_via_symlink(self):
+ # (CVE-2025-4138)
+ # Test changing symlinks that result in a symlink pointing outside
+ # the extraction directory, unless prevented by 'data' filter's
+ # normalization.
+ with ArchiveMaker() as arc:
+ arc.add("escape", symlink_to=os.path.join('link', 'link', '..', '..', 'link-here'))
+ arc.add("link", symlink_to='./')
+
+ for filter in 'tar', 'data', 'fully_trusted':
+ with self.check_context(arc.open(), filter):
+ if support.can_symlink():
+ self.expect_file("link", symlink_to='./')
+ if filter == 'data':
+ self.expect_file("escape", symlink_to='link-here')
+ else:
+ self.expect_file("escape",
+ symlink_to='link/link/../../link-here')
+ else:
+ # Nothing is extracted.
+ pass
+
+ def test_chmod_outside_dir(self):
+ # (CVE-2024-12718)
+ # Test that members used for delayed updates of directory metadata
+ # are (re-)filtered.
+ with ArchiveMaker() as arc:
+ # "pwn" is a veeeery innocent symlink:
+ arc.add("a/pwn", symlink_to='.')
+ # But now "pwn" is also a directory, so it's scheduled to have its
+ # metadata updated later:
+ arc.add("a/pwn/", mode='drwxrwxrwx')
+ # Oops, "pwn" is not so innocent any more:
+ arc.add("a/pwn", symlink_to='x/../')
+ # Newly created symlink points to the dest dir,
+ # so it's OK for the "data" filter.
+ arc.add('a/x', symlink_to=('../'))
+ # But now "pwn" points outside the dest dir
+
+ for filter in 'tar', 'data', 'fully_trusted':
+ with self.check_context(arc.open(), filter) as cc:
+ if not support.can_symlink():
+ self.expect_file("a/pwn/")
+ elif filter == 'data':
+ self.expect_file("a/x", symlink_to='../')
+ self.expect_file("a/pwn", symlink_to='.')
+ else:
+ self.expect_file("a/x", symlink_to='../')
+ self.expect_file("a/pwn", symlink_to='x/../')
+ if sys.platform != "win32":
+ st_mode = cc.outerdir.stat().st_mode
+ self.assertNotEqual(st_mode & 0o777, 0o777)
+
+ @unittest.skipUnless(hasattr(os, 'chown'), "missing os.chown")
+ @unittest.skipUnless(hasattr(os, 'lchown'), "missing os.lchown")
+ @unittest.skipUnless(hasattr(os, 'geteuid'), "missing os.geteuid")
+ @support.subTests('link_type', (tarfile.SYMTYPE, tarfile.LNKTYPE))
+ def test_chown_links_on_extract(self, link_type):
+ with ArchiveMaker() as arc:
+ arc.add("test.txt",
+ uid=1337, gid=1337, uname="", gname="", mode='-rwxr-xr-x')
+ arc.add("link",
+ type=link_type,
+ linkname='test.txt',
+ uid=1337, gid=1337, uname="", gname="", mode='-rwxr-xr-x')
+
+ with support.temp_dir() as tmpdir, \
+ arc.open() as tar, \
+ unittest.mock.patch("os.chown") as mock_chown, \
+ unittest.mock.patch("os.lchown") as mock_lchown, \
+ unittest.mock.patch("os.geteuid") as mock_geteuid:
+ # Set UID to 0 so chown() is attempted.
+ mock_geteuid.return_value = 0
+ tar.extract("link", path=tmpdir, filter='data')
+ extract_path = os.path.join(tmpdir, "link")
+
+ if link_type == tarfile.SYMTYPE:
+ mock_chown.assert_not_called()
+ mock_lchown.assert_called_once_with(extract_path, -1, -1)
+ else:
+ mock_chown.assert_has_calls([
+ unittest.mock.call(extract_path, -1, -1),
+ unittest.mock.call(extract_path, -1, -1)
+ ])
+ mock_lchown.assert_not_called()
+
+ @unittest.skipUnless(hasattr(os, 'chown'), "missing os.chown")
+ @unittest.skipUnless(hasattr(os, 'lchown'), "missing os.lchown")
+ @unittest.skipUnless(hasattr(os, 'geteuid'), "missing os.geteuid")
+ @support.subTests('link_type', (tarfile.SYMTYPE, tarfile.LNKTYPE))
+ def test_chown_links_on_extractall(self, link_type):
+ with ArchiveMaker() as arc:
+ arc.add("test.txt",
+ uid=1337, gid=1337, uname="", gname="", mode='-rwxr-xr-x')
+ arc.add("link",
+ type=link_type,
+ linkname='test.txt',
+ uid=1337, gid=1337, uname="", gname="", mode='-rwxr-xr-x')
+
+ with support.temp_dir() as tmpdir, \
+ arc.open() as tar, \
+ unittest.mock.patch("os.chown") as mock_chown, \
+ unittest.mock.patch("os.lchown") as mock_lchown, \
+ unittest.mock.patch("os.geteuid") as mock_geteuid:
+ # Set UID to 0 so chown() is attempted.
+ mock_geteuid.return_value = 0
+ tar.extractall(path=tmpdir, filter='data')
+ extract_link_path = os.path.join(tmpdir, "link")
+ extract_file_path = os.path.join(tmpdir, "test.txt")
+
+ if link_type == tarfile.SYMTYPE:
+ mock_chown.assert_called_once_with(extract_file_path, -1, -1)
+ mock_lchown.assert_called_once_with(extract_link_path, -1, -1)
+ else:
+ mock_chown.assert_has_calls([
+ unittest.mock.call(extract_file_path, -1, -1),
+ unittest.mock.call(extract_link_path, -1, -1)
+ ])
+ mock_lchown.assert_not_called()
+
+ def test_extract_filters_target(self):
+ # Test that when extract() falls back to extracting (rather than
+ # linking) a hardlink target, it filters the target.
+ with ArchiveMaker() as arc:
+ arc.add("target")
+ arc.add("link", hardlink_to="target")
+ def testing_filter(member, path):
+ if member.name == 'target':
+ # target: set read-only
+ return member.replace(mode=stat.S_IRUSR)
+ # link: don't overwrite the mode
+ return member.replace(mode=None)
+ tempdir = pathlib.Path(TEMPDIR) / 'extract'
+ with support.temp_dir(tempdir), arc.open() as tar:
+ tar.extract("link", path=tempdir, filter=testing_filter)
+ path = tempdir / 'link'
+ if sys.platform != 'win32':
+ self.assertFalse(path.stat().st_mode & stat.S_IWUSR)
+
+ def test_extract_filters_target_none(self):
+ # Test that when extract() falls back to extracting (rather than
+ # linking) a hardlink target, the member is skipped if the filter
+ # returns None.
+ with ArchiveMaker() as arc:
+ arc.add('a/b/s', symlink_to='../escape')
+ arc.add('q', hardlink_to='a/b/s')
+ def filter_unsafe_members(member, path):
+ try:
+ return tarfile.data_filter(member, path)
+ except tarfile.FilterError as error:
+ return None
+ with self.check_context(arc.open(), filter_unsafe_members):
+ if support.can_symlink():
+ self.expect_file('a/b/s', symlink_to='../escape')
+ else:
+ self.expect_file('a/b/') # symlink is not extracted
+
+ def test_link_fallback_normalizes(self):
+ # Make sure hardlink fallbacks work for non-normalized paths for all
+ # filters
+ with ArchiveMaker() as arc:
+ arc.add("dir/")
+ arc.add("dir/../afile")
+ arc.add("link1", hardlink_to='dir/../afile')
+ arc.add("link2", hardlink_to='dir/../dir/../afile')
+
+ for filter in 'tar', 'data', 'fully_trusted':
+ with self.check_context(arc.open(), filter) as cc:
+ self.expect_file("dir/")
+ self.expect_file("afile")
+ self.expect_file("link1")
+ self.expect_file("link2")
+
def test_modes(self):
# Test how file modes are extracted
# (Note that the modes are ignored on platforms without working chmod)
@@ -3436,7 +4116,7 @@ def test_tar_filter(self):
# The 'tar' filter returns TarInfo objects with the same name/type.
# (It can also fail for particularly "evil" input, but we don't have
# that in the test archive.)
- with tarfile.TarFile.open(tarname) as tar:
+ with tarfile.TarFile.open(tarname, encoding="iso8859-1") as tar:
for tarinfo in tar.getmembers():
filtered = tarfile.tar_filter(tarinfo, '')
self.assertIs(filtered.name, tarinfo.name)
@@ -3445,7 +4125,7 @@ def test_tar_filter(self):
def test_data_filter(self):
# The 'data' filter either raises, or returns TarInfo with the same
# name/type.
- with tarfile.TarFile.open(tarname) as tar:
+ with tarfile.TarFile.open(tarname, encoding="iso8859-1") as tar:
for tarinfo in tar.getmembers():
try:
filtered = tarfile.data_filter(tarinfo, '')
@@ -3574,13 +4254,13 @@ def valueerror_filter(tarinfo, path):
# If errorlevel is 0, errors affected by errorlevel are ignored
with self.check_context(arc.open(errorlevel=0), extracterror_filter):
- self.expect_file('file')
+ pass
with self.check_context(arc.open(errorlevel=0), filtererror_filter):
- self.expect_file('file')
+ pass
with self.check_context(arc.open(errorlevel=0), oserror_filter):
- self.expect_file('file')
+ pass
with self.check_context(arc.open(errorlevel=0), tarerror_filter):
self.expect_exception(tarfile.TarError)
@@ -3591,7 +4271,7 @@ def valueerror_filter(tarinfo, path):
# If 1, all fatal errors are raised
with self.check_context(arc.open(errorlevel=1), extracterror_filter):
- self.expect_file('file')
+ pass
with self.check_context(arc.open(errorlevel=1), filtererror_filter):
self.expect_exception(tarfile.FilterError)
@@ -3627,6 +4307,177 @@ def valueerror_filter(tarinfo, path):
with self.check_context(arc.open(errorlevel='boo!'), filtererror_filter):
self.expect_exception(TypeError) # errorlevel is not int
+ @support.subTests('format', [tarfile.GNU_FORMAT, tarfile.PAX_FORMAT])
+ def test_getmembers_big_size(self, format):
+ # gh-151981: A loop in seek() for streaming files tried to read the
+ # declared number of blocks even at EOF
+ tinfo = tarfile.TarInfo("huge-file")
+ tinfo.size = 1 << 64
+ bio = io.BytesIO()
+ # Write header without data
+ bio.write(tinfo.tobuf(format))
+
+ # Reset & try to get contents
+ bio.seek(0)
+ with tarfile.open(fileobj=bio, mode="r|") as tar:
+ with self.assertRaises(tarfile.ReadError):
+ tar.getmembers()
+
+
+class OffsetValidationTests(unittest.TestCase):
+ tarname = tmpname
+ invalid_posix_header = (
+ # name: 100 bytes
+ tarfile.NUL * tarfile.LENGTH_NAME
+ # mode, space, null terminator: 8 bytes
+ + b"000755" + SPACE + tarfile.NUL
+ # uid, space, null terminator: 8 bytes
+ + b"000001" + SPACE + tarfile.NUL
+ # gid, space, null terminator: 8 bytes
+ + b"000001" + SPACE + tarfile.NUL
+ # size, space: 12 bytes
+ + b"\xff" * 11 + SPACE
+ # mtime, space: 12 bytes
+ + tarfile.NUL * 11 + SPACE
+ # chksum: 8 bytes
+ + b"0011407" + tarfile.NUL
+ # type: 1 byte
+ + tarfile.REGTYPE
+ # linkname: 100 bytes
+ + tarfile.NUL * tarfile.LENGTH_LINK
+ # magic: 6 bytes, version: 2 bytes
+ + tarfile.POSIX_MAGIC
+ # uname: 32 bytes
+ + tarfile.NUL * 32
+ # gname: 32 bytes
+ + tarfile.NUL * 32
+ # devmajor, space, null terminator: 8 bytes
+ + tarfile.NUL * 6 + SPACE + tarfile.NUL
+ # devminor, space, null terminator: 8 bytes
+ + tarfile.NUL * 6 + SPACE + tarfile.NUL
+ # prefix: 155 bytes
+ + tarfile.NUL * tarfile.LENGTH_PREFIX
+ # padding: 12 bytes
+ + tarfile.NUL * 12
+ )
+ invalid_gnu_header = (
+ # name: 100 bytes
+ tarfile.NUL * tarfile.LENGTH_NAME
+ # mode, null terminator: 8 bytes
+ + b"0000755" + tarfile.NUL
+ # uid, null terminator: 8 bytes
+ + b"0000001" + tarfile.NUL
+ # gid, space, null terminator: 8 bytes
+ + b"0000001" + tarfile.NUL
+ # size, space: 12 bytes
+ + b"\xff" * 11 + SPACE
+ # mtime, space: 12 bytes
+ + tarfile.NUL * 11 + SPACE
+ # chksum: 8 bytes
+ + b"0011327" + tarfile.NUL
+ # type: 1 byte
+ + tarfile.REGTYPE
+ # linkname: 100 bytes
+ + tarfile.NUL * tarfile.LENGTH_LINK
+ # magic: 8 bytes
+ + tarfile.GNU_MAGIC
+ # uname: 32 bytes
+ + tarfile.NUL * 32
+ # gname: 32 bytes
+ + tarfile.NUL * 32
+ # devmajor, null terminator: 8 bytes
+ + tarfile.NUL * 8
+ # devminor, null terminator: 8 bytes
+ + tarfile.NUL * 8
+ # padding: 167 bytes
+ + tarfile.NUL * 167
+ )
+ invalid_v7_header = (
+ # name: 100 bytes
+ tarfile.NUL * tarfile.LENGTH_NAME
+ # mode, space, null terminator: 8 bytes
+ + b"000755" + SPACE + tarfile.NUL
+ # uid, space, null terminator: 8 bytes
+ + b"000001" + SPACE + tarfile.NUL
+ # gid, space, null terminator: 8 bytes
+ + b"000001" + SPACE + tarfile.NUL
+ # size, space: 12 bytes
+ + b"\xff" * 11 + SPACE
+ # mtime, space: 12 bytes
+ + tarfile.NUL * 11 + SPACE
+ # chksum: 8 bytes
+ + b"0010070" + tarfile.NUL
+ # type: 1 byte
+ + tarfile.REGTYPE
+ # linkname: 100 bytes
+ + tarfile.NUL * tarfile.LENGTH_LINK
+ # padding: 255 bytes
+ + tarfile.NUL * 255
+ )
+ valid_gnu_header = tarfile.TarInfo("filename").tobuf(tarfile.GNU_FORMAT)
+ data_block = b"\xff" * tarfile.BLOCKSIZE
+
+ def _write_buffer(self, buffer):
+ with open(self.tarname, "wb") as f:
+ f.write(buffer)
+
+ def _get_members(self, ignore_zeros=None):
+ with open(self.tarname, "rb") as f:
+ with tarfile.open(
+ mode="r", fileobj=f, ignore_zeros=ignore_zeros
+ ) as tar:
+ return tar.getmembers()
+
+ def _assert_raises_read_error_exception(self):
+ with self.assertRaisesRegex(
+ tarfile.ReadError, "file could not be opened successfully"
+ ):
+ self._get_members()
+
+ def test_invalid_offset_header_validations(self):
+ for tar_format, invalid_header in (
+ ("posix", self.invalid_posix_header),
+ ("gnu", self.invalid_gnu_header),
+ ("v7", self.invalid_v7_header),
+ ):
+ with self.subTest(format=tar_format):
+ self._write_buffer(invalid_header)
+ self._assert_raises_read_error_exception()
+
+ def test_early_stop_at_invalid_offset_header(self):
+ buffer = self.valid_gnu_header + self.invalid_gnu_header + self.valid_gnu_header
+ self._write_buffer(buffer)
+ members = self._get_members()
+ self.assertEqual(len(members), 1)
+ self.assertEqual(members[0].name, "filename")
+ self.assertEqual(members[0].offset, 0)
+
+ def test_ignore_invalid_archive(self):
+ # 3 invalid headers with their respective data
+ buffer = (self.invalid_gnu_header + self.data_block) * 3
+ self._write_buffer(buffer)
+ members = self._get_members(ignore_zeros=True)
+ self.assertEqual(len(members), 0)
+
+ def test_ignore_invalid_offset_headers(self):
+ for first_block, second_block, expected_offset in (
+ (
+ (self.valid_gnu_header),
+ (self.invalid_gnu_header + self.data_block),
+ 0,
+ ),
+ (
+ (self.invalid_gnu_header + self.data_block),
+ (self.valid_gnu_header),
+ 1024,
+ ),
+ ):
+ self._write_buffer(first_block + second_block)
+ members = self._get_members(ignore_zeros=True)
+ self.assertEqual(len(members), 1)
+ self.assertEqual(members[0].name, "filename")
+ self.assertEqual(members[0].offset, expected_offset)
+
def setUpModule():
support.unlink(TEMPDIR)
diff --git a/Lib/test/test_tempfile.py b/Lib/test/test_tempfile.py
index 8171aa6e0ae52af..f31e31402fe732e 100644
--- a/Lib/test/test_tempfile.py
+++ b/Lib/test/test_tempfile.py
@@ -5,11 +5,14 @@
import os
import pathlib
import signal
+import stat
import sys
import re
import warnings
import contextlib
import weakref
+import subprocess
+import shutil
from unittest import mock
import unittest
@@ -768,6 +771,33 @@ def test_mode(self):
finally:
os.rmdir(dir)
+ @unittest.skipUnless(os.name == "nt", "Only on Windows.")
+ def test_mode_win32(self):
+ # Use icacls.exe to extract the users with some level of access
+ # Main thing we are testing is that the BUILTIN\Users group has
+ # no access. The exact ACL is going to vary based on which user
+ # is running the test.
+ dir = self.do_create()
+ try:
+ out = subprocess.check_output(["icacls.exe", dir], encoding="oem").casefold()
+ finally:
+ os.rmdir(dir)
+
+ dir = dir.casefold()
+ users = set()
+ found_user = False
+ for line in out.strip().splitlines():
+ acl = None
+ # First line of result includes our directory
+ if line.startswith(dir):
+ acl = line[len(dir):].strip()
+ elif line and line[:1].isspace():
+ acl = line.strip()
+ if acl:
+ users.add(acl.partition(":")[0])
+
+ self.assertNotIn(r"BUILTIN\Users".casefold(), users)
+
def test_collision_with_existing_file(self):
# mkdtemp tries another name when a file with
# the chosen name already exists
@@ -1379,7 +1409,29 @@ def test_explict_cleanup_ignore_errors(self):
temp_path.exists(),
f"TemporaryDirectory {temp_path!s} exists after cleanup")
- @os_helper.skip_unless_symlink
+ @unittest.skipUnless(os.name == "nt", "Only on Windows.")
+ def test_explicit_cleanup_correct_error(self):
+ with tempfile.TemporaryDirectory() as working_dir:
+ temp_dir = self.do_create(dir=working_dir)
+ with open(os.path.join(temp_dir.name, "example.txt"), 'wb'):
+ # Previously raised NotADirectoryError on some OSes
+ # (e.g. Windows). See bpo-43153.
+ with self.assertRaises(PermissionError):
+ temp_dir.cleanup()
+
+ @unittest.skipUnless(os.name == "nt", "Only on Windows.")
+ def test_cleanup_with_used_directory(self):
+ with tempfile.TemporaryDirectory() as working_dir:
+ temp_dir = self.do_create(dir=working_dir)
+ subdir = os.path.join(temp_dir.name, "subdir")
+ os.mkdir(subdir)
+ with support.change_cwd(subdir):
+ # Previously raised RecursionError on some OSes
+ # (e.g. Windows). See bpo-35144.
+ with self.assertRaises(PermissionError):
+ temp_dir.cleanup()
+
+ @support.skip_unless_symlink
def test_cleanup_with_symlink_to_a_directory(self):
# cleanup() should not follow symlinks to directories (issue #12464)
d1 = self.do_create()
@@ -1400,6 +1452,180 @@ def test_cleanup_with_symlink_to_a_directory(self):
"were deleted")
d2.cleanup()
+ @support.skip_unless_symlink
+ def test_cleanup_with_symlink_modes(self):
+ # cleanup() should not follow symlinks when fixing mode bits (#91133)
+ with self.do_create(recurse=0) as d2:
+ file1 = os.path.join(d2, 'file1')
+ open(file1, 'wb').close()
+ dir1 = os.path.join(d2, 'dir1')
+ os.mkdir(dir1)
+ for mode in range(8):
+ mode <<= 6
+ with self.subTest(mode=format(mode, '03o')):
+ def test(target, target_is_directory):
+ d1 = self.do_create(recurse=0)
+ symlink = os.path.join(d1.name, 'symlink')
+ os.symlink(target, symlink,
+ target_is_directory=target_is_directory)
+ try:
+ os.chmod(symlink, mode, follow_symlinks=False)
+ except NotImplementedError:
+ pass
+ try:
+ os.chmod(symlink, mode)
+ except FileNotFoundError:
+ pass
+ os.chmod(d1.name, mode)
+ d1.cleanup()
+ self.assertFalse(os.path.exists(d1.name))
+
+ with self.subTest('nonexisting file'):
+ test('nonexisting', target_is_directory=False)
+ with self.subTest('nonexisting dir'):
+ test('nonexisting', target_is_directory=True)
+
+ with self.subTest('existing file'):
+ os.chmod(file1, mode)
+ old_mode = os.stat(file1).st_mode
+ test(file1, target_is_directory=False)
+ new_mode = os.stat(file1).st_mode
+ self.assertEqual(new_mode, old_mode,
+ '%03o != %03o' % (new_mode, old_mode))
+
+ with self.subTest('existing dir'):
+ os.chmod(dir1, mode)
+ old_mode = os.stat(dir1).st_mode
+ test(dir1, target_is_directory=True)
+ new_mode = os.stat(dir1).st_mode
+ self.assertEqual(new_mode, old_mode,
+ '%03o != %03o' % (new_mode, old_mode))
+
+ @unittest.skipUnless(hasattr(os, 'chflags'), 'requires os.chflags')
+ @support.skip_unless_symlink
+ def test_cleanup_with_symlink_flags(self):
+ # cleanup() should not follow symlinks when fixing flags (#91133)
+ flags = stat.UF_IMMUTABLE | stat.UF_NOUNLINK
+ self.check_flags(flags)
+
+ with self.do_create(recurse=0) as d2:
+ file1 = os.path.join(d2, 'file1')
+ open(file1, 'wb').close()
+ dir1 = os.path.join(d2, 'dir1')
+ os.mkdir(dir1)
+ def test(target, target_is_directory):
+ d1 = self.do_create(recurse=0)
+ symlink = os.path.join(d1.name, 'symlink')
+ os.symlink(target, symlink,
+ target_is_directory=target_is_directory)
+ try:
+ os.chflags(symlink, flags, follow_symlinks=False)
+ except NotImplementedError:
+ pass
+ try:
+ os.chflags(symlink, flags)
+ except FileNotFoundError:
+ pass
+ os.chflags(d1.name, flags)
+ d1.cleanup()
+ self.assertFalse(os.path.exists(d1.name))
+
+ with self.subTest('nonexisting file'):
+ test('nonexisting', target_is_directory=False)
+ with self.subTest('nonexisting dir'):
+ test('nonexisting', target_is_directory=True)
+
+ with self.subTest('existing file'):
+ os.chflags(file1, flags)
+ old_flags = os.stat(file1).st_flags
+ test(file1, target_is_directory=False)
+ new_flags = os.stat(file1).st_flags
+ self.assertEqual(new_flags, old_flags)
+
+ with self.subTest('existing dir'):
+ os.chflags(dir1, flags)
+ old_flags = os.stat(dir1).st_flags
+ test(dir1, target_is_directory=True)
+ new_flags = os.stat(dir1).st_flags
+ self.assertEqual(new_flags, old_flags)
+
+ @support.skip_unless_symlink
+ @unittest.skipIf(hasattr(os, 'geteuid') and os.geteuid() == 0,
+ 'test needs non-root account')
+ @unittest.skipUnless(shutil.rmtree.avoids_symlink_attacks,
+ 'requires the fd based implementation of rmtree()')
+ def test_cleanup_with_symlink_race(self):
+ # cleanup() should not operate on files outside of the temporary
+ # directory when a directory is replaced with a symlink while it
+ # recovers from a PermissionError (CVE-2026-12345).
+ with self.do_create(recurse=0) as target:
+ target_file = os.path.join(target, 'file1')
+ open(target_file, 'wb').close()
+ target_mode = os.stat(target_file).st_mode
+
+ d1 = self.do_create(recurse=0)
+ dir1 = os.path.join(d1.name, 'dir1')
+ os.mkdir(dir1)
+ open(os.path.join(dir1, 'file1'), 'wb').close()
+ # Removing contents of dir1 fails with a PermissionError, and
+ # dir1 is replaced with a symlink to target at the very moment
+ # cleanup() starts to recover from that error.
+ os.chmod(dir1, 0o500)
+ unlink = os.unlink
+ def hook(path, *, dir_fd=None):
+ try:
+ return unlink(path, dir_fd=dir_fd)
+ except PermissionError:
+ if not os.path.islink(dir1):
+ os.chmod(dir1, 0o700)
+ os.rename(dir1, dir1 + '_moved')
+ os.symlink(target, dir1)
+ raise
+ try:
+ with mock.patch('os.unlink', hook):
+ with contextlib.suppress(OSError):
+ d1.cleanup()
+ finally:
+ if os.path.islink(dir1):
+ os.unlink(dir1)
+ os.rename(dir1 + '_moved', dir1)
+ os.chmod(dir1, 0o700)
+ d1.cleanup()
+
+ self.assertTrue(os.path.exists(target_file))
+ self.assertEqual(os.stat(target_file).st_mode, target_mode)
+
+ @support.subTests('ignore_errors', (True, False))
+ def test_parent_mode_preserved(self, ignore_errors):
+ # Test that cleanup does not touch the parent directory,
+ # even if that prevents removal.
+ for mode in range(8):
+ mode <<= 6
+ with self.subTest(mode=format(mode, '03o')):
+ outer = self.do_create()
+ with outer:
+ d = self.do_create(dir=outer.name, dirs=2, files=2,
+ ignore_cleanup_errors=ignore_errors)
+ with d:
+ os.chmod(outer.name, mode)
+ orig_mode = os.stat(outer.name).st_mode
+ try:
+ d.cleanup()
+ except PermissionError:
+ if ignore_errors:
+ raise
+ self.assertEqual(os.stat(outer.name).st_mode, orig_mode)
+ outer.cleanup()
+ self.assertFalse(os.path.exists(outer.name))
+
+ @unittest.skipUnless(sys.platform.startswith(('linux', 'darwin', 'freebsd')),
+ 'regression test for supported platforms')
+ def test_cleanup_safe(self):
+ """Verify that cleanup uses the safer code path"""
+ # This is a regression test. Feel free to add exceptions for new
+ # platforms, but don't forget to update the docs.
+ self.assertTrue(tempfile._rmtree_use_dir_fd)
+
@support.cpython_only
def test_del_on_collection(self):
# A TemporaryDirectory is deleted when garbage collected
@@ -1556,6 +1782,51 @@ def test_context_manager(self):
self.assertEqual(name, d.name)
self.assertFalse(os.path.exists(name))
+ def test_modes(self):
+ for mode in range(8):
+ mode <<= 6
+ with self.subTest(mode=format(mode, '03o')):
+ d = self.do_create(recurse=3, dirs=2, files=2)
+ with d:
+ # Change files and directories mode recursively.
+ for root, dirs, files in os.walk(d.name, topdown=False):
+ for name in files:
+ os.chmod(os.path.join(root, name), mode)
+ os.chmod(root, mode)
+ d.cleanup()
+ self.assertFalse(os.path.exists(d.name))
+
+ def check_flags(self, flags):
+ # skip the test if these flags are not supported (ex: FreeBSD 13)
+ filename = support.TESTFN
+ try:
+ open(filename, "w").close()
+ try:
+ os.chflags(filename, flags)
+ except OSError as exc:
+ # "OSError: [Errno 45] Operation not supported"
+ self.skipTest(f"chflags() doesn't support flags "
+ f"{flags:#b}: {exc}")
+ else:
+ os.chflags(filename, 0)
+ finally:
+ support.unlink(filename)
+
+ @unittest.skipUnless(hasattr(os, 'chflags'), 'requires os.chflags')
+ def test_flags(self):
+ flags = stat.UF_IMMUTABLE | stat.UF_NOUNLINK
+ self.check_flags(flags)
+
+ d = self.do_create(recurse=3, dirs=2, files=2)
+ with d:
+ # Change files and directories flags recursively.
+ for root, dirs, files in os.walk(d.name, topdown=False):
+ for name in files:
+ os.chflags(os.path.join(root, name), flags)
+ os.chflags(root, flags)
+ d.cleanup()
+ self.assertFalse(os.path.exists(d.name))
+
if __name__ == "__main__":
unittest.main()
diff --git a/Lib/test/test_unicodedata.py b/Lib/test/test_unicodedata.py
index 170778fa977db1b..cb14bb8876e32e1 100644
--- a/Lib/test/test_unicodedata.py
+++ b/Lib/test/test_unicodedata.py
@@ -77,6 +77,7 @@ def tearDown(self):
del self.db
class UnicodeFunctionsTest(UnicodeDatabaseTest):
+ old = False
# Update this if the database changes. Make sure to do a full rebuild
# (e.g. 'make distclean && make') to get the correct checksum.
@@ -100,7 +101,8 @@ def test_function_checksum(self):
]
h.update(''.join(data).encode("ascii"))
result = h.hexdigest()
- self.assertEqual(result, self.expectedchecksum)
+ if not self.old:
+ self.assertEqual(result, self.expectedchecksum)
def test_digit(self):
self.assertEqual(self.db.digit('A', None), None)
@@ -119,9 +121,13 @@ def test_numeric(self):
self.assertEqual(self.db.numeric('9'), 9)
self.assertEqual(self.db.numeric('\u215b'), 0.125)
self.assertEqual(self.db.numeric('\u2468'), 9.0)
- self.assertEqual(self.db.numeric('\ua627'), 7.0)
+ # New in 5.1.0
+ self.assertEqual(self.db.numeric('\ua627', None),
+ None if self.old else 7.0)
self.assertEqual(self.db.numeric('\U00020000', None), None)
- self.assertEqual(self.db.numeric('\U0001012A'), 9000)
+ # New in 4.1.0
+ self.assertEqual(self.db.numeric('\U0001012A', None),
+ None if self.old else 9000)
self.assertRaises(TypeError, self.db.numeric)
self.assertRaises(TypeError, self.db.numeric, 'xx')
@@ -144,7 +150,8 @@ def test_category(self):
self.assertEqual(self.db.category('a'), 'Ll')
self.assertEqual(self.db.category('A'), 'Lu')
self.assertEqual(self.db.category('\U00020000'), 'Lo')
- self.assertEqual(self.db.category('\U0001012A'), 'No')
+ self.assertEqual(self.db.category('\U0001012A'),
+ 'Cn' if self.old else 'No')
self.assertRaises(TypeError, self.db.category)
self.assertRaises(TypeError, self.db.category, 'xx')
@@ -158,6 +165,15 @@ def test_bidirectional(self):
self.assertRaises(TypeError, self.db.bidirectional)
self.assertRaises(TypeError, self.db.bidirectional, 'xx')
+ def test_bidirectional_unassigned(self):
+ self.assertEqual(self.db.bidirectional('\u0378'), '')
+ self.assertEqual(self.db.bidirectional('\u077F'), '' if self.old else 'AL')
+ self.assertEqual(self.db.bidirectional('\u20CF'), '')
+ self.assertEqual(self.db.bidirectional('\u0590'), '')
+ self.assertEqual(self.db.bidirectional('\uFFFF'), '')
+ self.assertEqual(self.db.bidirectional('\U0001FFFE'), '')
+ self.assertEqual(self.db.bidirectional('\U00010D01'), '' if self.old else 'AL')
+
def test_decomposition(self):
self.assertEqual(self.db.decomposition('\uFFFE'),'')
self.assertEqual(self.db.decomposition('\u00bc'), '<fraction> 0031 2044 0034')
@@ -208,6 +224,34 @@ def test_issue10254(self):
b = 'C\u0338' * 20 + '\xC7'
self.assertEqual(self.db.normalize('NFC', a), b)
+ def test_long_combining_mark_run(self):
+ # gh-149079: avoid quadratic canonical ordering.
+ payload = "a" + ("\u0300\u0327" * 32)
+ nfd = "a" + ("\u0327" * 32) + ("\u0300" * 32)
+ nfc = "\u00e0" + ("\u0327" * 32) + ("\u0300" * 31)
+
+ self.assertEqual(self.db.normalize("NFD", payload), nfd)
+ self.assertEqual(self.db.normalize("NFKD", payload), nfd)
+ self.assertEqual(self.db.normalize("NFC", payload), nfc)
+ self.assertEqual(self.db.normalize("NFKC", payload), nfc)
+
+ def test_combining_mark_run_fast_paths(self):
+ # gh-149079: cover short runs and already-sorted long runs.
+ short_payload = "a" + ("\u0300\u0327" * 9) + "\u0300"
+ short_nfd = "a" + ("\u0327" * 9) + ("\u0300" * 10)
+ short_nfc = "\u00e0" + ("\u0327" * 9) + ("\u0300" * 9)
+ long_sorted = "a" + ("\u0327" * 30) + ("\u0300" * 30)
+ long_sorted_nfc = "\u00e0" + ("\u0327" * 30) + ("\u0300" * 29)
+
+ self.assertEqual(self.db.normalize("NFD", short_payload), short_nfd)
+ self.assertEqual(self.db.normalize("NFKD", short_payload), short_nfd)
+ self.assertEqual(self.db.normalize("NFC", short_payload), short_nfc)
+ self.assertEqual(self.db.normalize("NFKC", short_payload), short_nfc)
+ self.assertEqual(self.db.normalize("NFD", long_sorted), long_sorted)
+ self.assertEqual(self.db.normalize("NFKD", long_sorted), long_sorted)
+ self.assertEqual(self.db.normalize("NFC", long_sorted), long_sorted_nfc)
+ self.assertEqual(self.db.normalize("NFKC", long_sorted), long_sorted_nfc)
+
def test_issue29456(self):
# Fix #29456
u1176_str_a = '\u1100\u1176\u11a8'
@@ -236,8 +280,18 @@ def test_east_asian_width(self):
self.assertEqual(eaw('\U00020000'), 'W')
def test_east_asian_width_9_0_changes(self):
- self.assertEqual(self.db.ucd_3_2_0.east_asian_width('\u231a'), 'N')
- self.assertEqual(self.db.east_asian_width('\u231a'), 'W')
+ self.assertEqual(self.db.east_asian_width('\u231a'),
+ 'N' if self.old else 'W')
+
+
+class Unicode_3_2_0_FunctionsTest(UnicodeFunctionsTest):
+ old = True
+
+ def setUp(self):
+ # 3.7's UnicodeDatabaseTest sets self.db in setUp().
+ import unicodedata
+ self.db = unicodedata.ucd_3_2_0
+
class UnicodeMiscTest(UnicodeDatabaseTest):
diff --git a/Lib/test/test_urllib.py b/Lib/test/test_urllib.py
index 0305e7aa1884960..733ec21864ecfda 100644
--- a/Lib/test/test_urllib.py
+++ b/Lib/test/test_urllib.py
@@ -9,6 +9,7 @@
import unittest
from unittest.mock import patch
from test import support
+from test.support import control_characters_c0
import os
try:
import ssl
@@ -660,6 +661,13 @@ def test_invalid_base64_data(self):
# missing padding character
self.assertRaises(ValueError,urllib.request.urlopen,'data:;base64,Cg=')
+ def test_invalid_mediatype(self):
+ for c0 in control_characters_c0():
+ self.assertRaises(ValueError,urllib.request.urlopen,
+ f'data:text/html;{c0},data')
+ for c0 in control_characters_c0():
+ self.assertRaises(ValueError,urllib.request.urlopen,
+ f'data:text/html{c0};base64,ZGF0YQ==')
class urlretrieve_FileTests(unittest.TestCase):
"""Test urllib.urlretrieve() on local files"""
diff --git a/Lib/test/test_urllib2.py b/Lib/test/test_urllib2.py
index 3bb4a12e86d5005..04a3988cb2f5559 100644
--- a/Lib/test/test_urllib2.py
+++ b/Lib/test/test_urllib2.py
@@ -264,6 +264,50 @@ def test_password_manager_default_port(self):
self.assertEqual(find_user_pass("i", "http://j.example.com:80"),
(None, None))
+ def test_password_manager_scheme(self):
+ mgr = urllib.request.HTTPPasswordMgr()
+ mgr.add_password(
+ "realm", "https://example.com/", "user", "password")
+
+ self.assertEqual(
+ mgr.find_user_password("realm", "https://example.com/"),
+ ("user", "password"))
+ self.assertEqual(
+ mgr.find_user_password("realm", "http://example.com/"),
+ (None, None))
+ # Support an authority without a scheme.
+ self.assertEqual(
+ mgr.find_user_password("realm", "example.com"),
+ ("user", "password"))
+ # An authority without a scheme continues to match any scheme.
+ mgr.add_password(
+ "realm", "schemeless.example.com", "user", "password")
+ for scheme in "http", "https":
+ with self.subTest(scheme=scheme):
+ self.assertEqual(
+ mgr.find_user_password(
+ "realm", f"{scheme}://schemeless.example.com/"),
+ ("user", "password"))
+
+ # A network-path reference also has no scheme.
+ mgr.add_password(
+ "realm", "//network-path.example.com/", "user", "password")
+ self.assertEqual(
+ mgr.find_user_password(
+ "realm", "https://network-path.example.com/"),
+ ("user", "password"))
+
+ def test_password_manager_reduced_uri(self):
+ mgr = urllib.request.HTTPPasswordMgr()
+
+ self.assertEqual(
+ mgr.reduce_uri("http://example.com/path"),
+ ("example.com:80", "/path"))
+ self.assertTrue(
+ mgr.is_suburi(
+ ("example.com", "/path"),
+ ("example.com", "/path/subpath")))
+
class MockOpener:
addheaders = []
@@ -1671,6 +1715,18 @@ def test_basic_prior_auth_auto_send(self):
# expect request to be sent with auth header
self.assertTrue(http_handler.has_auth_header)
+ def test_basic_prior_auth_different_scheme(self):
+ pwd_manager = HTTPPasswordMgrWithPriorAuth()
+ auth_handler = HTTPBasicAuthHandler(pwd_manager)
+ auth_handler.add_password(
+ None, "https://example.com/", "user", "password",
+ is_authenticated=True)
+
+ request = Request("http://example.com/")
+ auth_handler.http_request(request)
+
+ self.assertFalse(request.has_header("Authorization"))
+
def test_basic_prior_auth_send_after_first_success(self):
# Auto send auth header after authentication is successful once
diff --git a/Lib/test/test_urlparse.py b/Lib/test/test_urlparse.py
index bedf37455fee00a..ad0c300846132ea 100644
--- a/Lib/test/test_urlparse.py
+++ b/Lib/test/test_urlparse.py
@@ -1135,6 +1135,70 @@ def test_issue14072(self):
self.assertEqual(p2.scheme, 'tel')
self.assertEqual(p2.path, '+31641044153')
+ def test_invalid_bracketed_hosts(self):
+ self.assertRaises(ValueError, urllib.parse.urlsplit, 'Scheme://user@[192.0.2.146]/Path?Query')
+ self.assertRaises(ValueError, urllib.parse.urlsplit, 'Scheme://user@[important.com:8000]/Path?Query')
+ self.assertRaises(ValueError, urllib.parse.urlsplit, 'Scheme://user@[v123r.IP]/Path?Query')
+ self.assertRaises(ValueError, urllib.parse.urlsplit, 'Scheme://user@[v12ae]/Path?Query')
+ self.assertRaises(ValueError, urllib.parse.urlsplit, 'Scheme://user@[v.IP]/Path?Query')
+ self.assertRaises(ValueError, urllib.parse.urlsplit, 'Scheme://user@[v123.]/Path?Query')
+ self.assertRaises(ValueError, urllib.parse.urlsplit, 'Scheme://user@[v]/Path?Query')
+ self.assertRaises(ValueError, urllib.parse.urlsplit, 'Scheme://user@[0439:23af::2309::fae7:1234]/Path?Query')
+ self.assertRaises(ValueError, urllib.parse.urlsplit, 'Scheme://user@[0439:23af:2309::fae7:1234:2342:438e:192.0.2.146]/Path?Query')
+ self.assertRaises(ValueError, urllib.parse.urlsplit, 'Scheme://user@]v6a.ip[/Path')
+ # 3.7 parses http: URLs on a separate fast path; check it too.
+ self.assertRaises(ValueError, urllib.parse.urlsplit, 'http://user@[192.0.2.146]/Path?Query')
+ self.assertRaises(ValueError, urllib.parse.urlsplit, 'http://[important.com:8000]/Path')
+ self.assertRaises(ValueError, urllib.parse.urlsplit, 'scheme://prefix.[v6a.ip]')
+ self.assertRaises(ValueError, urllib.parse.urlsplit, 'scheme://[v6a.ip].suffix')
+ self.assertRaises(ValueError, urllib.parse.urlsplit, 'scheme://prefix.[v6a.ip]/')
+ self.assertRaises(ValueError, urllib.parse.urlsplit, 'scheme://[v6a.ip].suffix/')
+ self.assertRaises(ValueError, urllib.parse.urlsplit, 'scheme://prefix.[v6a.ip]?')
+ self.assertRaises(ValueError, urllib.parse.urlsplit, 'scheme://[v6a.ip].suffix?')
+ self.assertRaises(ValueError, urllib.parse.urlsplit, 'scheme://prefix.[::1]')
+ self.assertRaises(ValueError, urllib.parse.urlsplit, 'scheme://[::1].suffix')
+ self.assertRaises(ValueError, urllib.parse.urlsplit, 'scheme://prefix.[::1]/')
+ self.assertRaises(ValueError, urllib.parse.urlsplit, 'scheme://[::1].suffix/')
+ self.assertRaises(ValueError, urllib.parse.urlsplit, 'scheme://prefix.[::1]?')
+ self.assertRaises(ValueError, urllib.parse.urlsplit, 'scheme://[::1].suffix?')
+ self.assertRaises(ValueError, urllib.parse.urlsplit, 'scheme://prefix.[::1]:a')
+ self.assertRaises(ValueError, urllib.parse.urlsplit, 'scheme://[::1].suffix:a')
+ self.assertRaises(ValueError, urllib.parse.urlsplit, 'scheme://prefix.[::1]:a1')
+ self.assertRaises(ValueError, urllib.parse.urlsplit, 'scheme://[::1].suffix:a1')
+ self.assertRaises(ValueError, urllib.parse.urlsplit, 'scheme://prefix.[::1]:1a')
+ self.assertRaises(ValueError, urllib.parse.urlsplit, 'scheme://[::1].suffix:1a')
+ self.assertRaises(ValueError, urllib.parse.urlsplit, 'scheme://prefix.[::1]:')
+ self.assertRaises(ValueError, urllib.parse.urlsplit, 'scheme://[::1].suffix:/')
+ self.assertRaises(ValueError, urllib.parse.urlsplit, 'scheme://prefix.[::1]:?')
+ self.assertRaises(ValueError, urllib.parse.urlsplit, 'scheme://user@prefix.[v6a.ip]')
+ self.assertRaises(ValueError, urllib.parse.urlsplit, 'scheme://user@[v6a.ip].suffix')
+ self.assertRaises(ValueError, urllib.parse.urlsplit, 'scheme://[v6a.ip')
+ self.assertRaises(ValueError, urllib.parse.urlsplit, 'scheme://v6a.ip]')
+ self.assertRaises(ValueError, urllib.parse.urlsplit, 'scheme://]v6a.ip[')
+ self.assertRaises(ValueError, urllib.parse.urlsplit, 'scheme://]v6a.ip')
+ self.assertRaises(ValueError, urllib.parse.urlsplit, 'scheme://v6a.ip[')
+ self.assertRaises(ValueError, urllib.parse.urlsplit, 'scheme://prefix.[v6a.ip')
+ self.assertRaises(ValueError, urllib.parse.urlsplit, 'scheme://v6a.ip].suffix')
+ self.assertRaises(ValueError, urllib.parse.urlsplit, 'scheme://prefix]v6a.ip[suffix')
+ self.assertRaises(ValueError, urllib.parse.urlsplit, 'scheme://prefix]v6a.ip')
+ self.assertRaises(ValueError, urllib.parse.urlsplit, 'scheme://v6a.ip[suffix')
+
+ def test_splitting_bracketed_hosts(self):
+ p1 = urllib.parse.urlsplit('scheme://user@[v6a.ip]:1234/path?query')
+ self.assertEqual(p1.hostname, 'v6a.ip')
+ self.assertEqual(p1.username, 'user')
+ self.assertEqual(p1.path, '/path')
+ self.assertEqual(p1.port, 1234)
+ p2 = urllib.parse.urlsplit('scheme://user@[0439:23af:2309::fae7%test]/path?query')
+ self.assertEqual(p2.hostname, '0439:23af:2309::fae7%test')
+ self.assertEqual(p2.username, 'user')
+ self.assertEqual(p2.path, '/path')
+ self.assertIs(p2.port, None)
+ p3 = urllib.parse.urlsplit('scheme://user@[0439:23af:2309::fae7:1234:192.0.2.146%test]/path?query')
+ self.assertEqual(p3.hostname, '0439:23af:2309::fae7:1234:192.0.2.146%test')
+ self.assertEqual(p3.username, 'user')
+ self.assertEqual(p3.path, '/path')
+
def test_telurl_params(self):
p1 = urllib.parse.urlparse('tel:123-4;phone-context=+1-650-516')
self.assertEqual(p1.scheme, 'tel')
diff --git a/Lib/test/test_webbrowser.py b/Lib/test/test_webbrowser.py
index 519a9432abe012d..123b474318a36ae 100644
--- a/Lib/test/test_webbrowser.py
+++ b/Lib/test/test_webbrowser.py
@@ -1,6 +1,7 @@
+import io
+import os
import webbrowser
import unittest
-import os
import sys
import subprocess
from unittest import mock
@@ -45,6 +46,14 @@ def _test(self, meth, *, args=[URL], kw={}, options, arguments):
popen_args.pop(popen_args.index(option))
self.assertEqual(popen_args, arguments)
+ def test_reject_dash_prefixes(self):
+ browser = self.browser_class(name=CMD_NAME)
+ with self.assertRaisesRegex(
+ ValueError,
+ r"^Invalid URL \(leading dash disallowed\): '--key=val http.*'$"
+ ):
+ browser.open(f"--key=val {URL}")
+
class GenericBrowserCommandTest(CommandTestMixin, unittest.TestCase):
@@ -90,6 +99,14 @@ def test_open_new_tab(self):
options=[],
arguments=[URL])
+ def test_reject_action_dash_prefixes(self):
+ browser = self.browser_class(name=CMD_NAME)
+ with self.assertRaises(ValueError):
+ browser.open('%action--incognito')
+ # new=1: action is "--new-window", so "%action" itself expands to
+ # a dash-prefixed flag even with no dash in the original URL.
+ with self.assertRaises(ValueError):
+ browser.open('%action', new=1)
class MozillaCommandTest(CommandTestMixin, unittest.TestCase):
@@ -215,6 +232,73 @@ def test_open_new_tab(self):
arguments=['openURL({},new-tab)'.format(URL)])
+class MockPopenPipe:
+ def __init__(self, cmd, mode):
+ self.cmd = cmd
+ self.mode = mode
+ self.pipe = io.StringIO()
+ self._closed = False
+
+ def write(self, buf):
+ self.pipe.write(buf)
+
+ def close(self):
+ self._closed = True
+ return None
+
+
+@unittest.skipUnless(sys.platform == "darwin", "macOS specific test")
+class MacOSXOSAScriptTest(unittest.TestCase):
+ def setUp(self):
+ # Ensure that 'BROWSER' is not set to 'open' or something else.
+ # See: https://github.com/python/cpython/issues/131254.
+ ctx = os_helper.EnvironmentVarGuard()
+ env = ctx.__enter__()
+ self.addCleanup(ctx.__exit__)
+ env.unset("BROWSER")
+
+ support.patch(self, os, "popen", self.mock_popen)
+ self.browser = webbrowser.MacOSXOSAScript("default")
+
+ def mock_popen(self, cmd, mode):
+ self.popen_pipe = MockPopenPipe(cmd, mode)
+ return self.popen_pipe
+
+ def test_default(self):
+ browser = webbrowser.get()
+ assert isinstance(browser, webbrowser.MacOSXOSAScript)
+ self.assertEqual(browser._name, "default")
+
+ def test_default_open(self):
+ url = "https://python.org"
+ self.browser.open(url)
+ self.assertTrue(self.popen_pipe._closed)
+ self.assertEqual(self.popen_pipe.cmd, "/usr/bin/osascript")
+ script = self.popen_pipe.pipe.getvalue()
+ self.assertEqual(script.strip(), f'open location "{url}"')
+
+ def test_url_quote(self):
+ self.browser.open('https://python.org/"quote"')
+ script = self.popen_pipe.pipe.getvalue()
+ self.assertEqual(
+ script.strip(), 'open location "https://python.org/%22quote%22"'
+ )
+
+ def test_explicit_browser(self):
+ browser = webbrowser.MacOSXOSAScript("safari")
+ browser.open("https://python.org")
+ script = self.popen_pipe.pipe.getvalue()
+ self.assertIn('tell application "safari"', script)
+ self.assertIn('open location "https://python.org"', script)
+
+ def test_reject_dash_prefixes(self):
+ with self.assertRaisesRegex(
+ ValueError,
+ r"^Invalid URL \(leading dash disallowed\): '--key=val http.*'$"
+ ):
+ self.browser.open(f"--key=val {URL}")
+
+
class BrowserRegistrationTest(unittest.TestCase):
def setUp(self):
diff --git a/Lib/test/test_wsgiref.py b/Lib/test/test_wsgiref.py
index da9481f839bf600..398c73acfb46fb1 100644
--- a/Lib/test/test_wsgiref.py
+++ b/Lib/test/test_wsgiref.py
@@ -1,5 +1,6 @@
from unittest import mock
from test import support
+from test.support import control_characters_c0
from test.test_httpservers import NoLogRequestHandler
from unittest import TestCase
from wsgiref.util import setup_testing_defaults
@@ -517,6 +518,22 @@ def testExtras(self):
'\r\n'
)
+ def testRaisesControlCharacters(self):
+ for c0 in control_characters_c0():
+ with self.subTest(c0):
+ headers = Headers()
+ self.assertRaises(ValueError, headers.__setitem__, f"key{c0}", "val")
+ self.assertRaises(ValueError, headers.add_header, f"key{c0}", "val", param="param")
+ # HTAB (\x09) is allowed in values, not names.
+ if c0 == "\t":
+ headers["key"] = f"val{c0}"
+ headers.add_header("key", f"val{c0}")
+ headers.setdefault(f"key", f"val{c0}")
+ else:
+ self.assertRaises(ValueError, headers.__setitem__, "key", f"val{c0}")
+ self.assertRaises(ValueError, headers.add_header, "key", f"val{c0}", param="param")
+ self.assertRaises(ValueError, headers.add_header, "key", "val", param=f"param{c0}")
+
class ErrorHandler(BaseCGIHandler):
"""Simple handler subclass for testing BaseHandler"""
@@ -853,6 +870,25 @@ def write(self, b):
self.assertIsNotNone(h.status)
self.assertIsNotNone(h.environ)
+ def testRaisesControlCharacters(self):
+ for c0 in control_characters_c0():
+ with self.subTest(c0):
+ base = BaseHandler()
+ with self.assertRaises(ValueError):
+ base.start_response(c0, [('x', 'y')])
+
+ base = BaseHandler()
+ with self.assertRaises(ValueError):
+ base.start_response('200 OK', [(c0, 'y')])
+
+ # HTAB (\x09) is allowed in header values, but not in names.
+ base = BaseHandler()
+ if c0 != "\t":
+ with self.assertRaises(ValueError):
+ base.start_response('200 OK', [('x', c0)])
+ else:
+ base.start_response('200 OK', [('x', c0)])
+
if __name__ == "__main__":
unittest.main()
diff --git a/Lib/test/test_xml_etree.py b/Lib/test/test_xml_etree.py
index 5ba0de82cdf9a75..fd229d4bdbd5925 100644
--- a/Lib/test/test_xml_etree.py
+++ b/Lib/test/test_xml_etree.py
@@ -11,6 +11,7 @@
import io
import operator
import pickle
+import pyexpat
import sys
import types
import unittest
@@ -98,6 +99,7 @@
<document>&entity;</document>
"""
+
def checkwarnings(*filters, quiet=False):
def decorator(test):
def newtest(*args, **kwargs):
@@ -1047,40 +1049,49 @@ def test_html_empty_elems_serialization(self):
class XMLPullParserTest(unittest.TestCase):
- def _feed(self, parser, data, chunk_size=None):
+ def _feed(self, parser, data, chunk_size=None, flush=False):
if chunk_size is None:
parser.feed(data)
else:
for i in range(0, len(data), chunk_size):
parser.feed(data[i:i+chunk_size])
+ if flush:
+ parser.flush()
def assert_event_tags(self, parser, expected):
events = parser.read_events()
self.assertEqual([(action, elem.tag) for action, elem in events],
expected)
- def test_simple_xml(self):
- for chunk_size in (None, 1, 5):
- with self.subTest(chunk_size=chunk_size):
- parser = ET.XMLPullParser()
- self.assert_event_tags(parser, [])
- self._feed(parser, "<!-- comment -->\n", chunk_size)
- self.assert_event_tags(parser, [])
- self._feed(parser,
- "<root>\n <element key='value'>text</element",
- chunk_size)
- self.assert_event_tags(parser, [])
- self._feed(parser, ">\n", chunk_size)
- self.assert_event_tags(parser, [('end', 'element')])
- self._feed(parser, "<element>text</element>tail\n", chunk_size)
- self._feed(parser, "<empty-element/>\n", chunk_size)
- self.assert_event_tags(parser, [
- ('end', 'element'),
- ('end', 'empty-element'),
- ])
- self._feed(parser, "</root>\n", chunk_size)
- self.assert_event_tags(parser, [('end', 'root')])
- self.assertIsNone(parser.close())
+ def test_simple_xml(self, chunk_size=None, flush=False):
+ parser = ET.XMLPullParser()
+ self.assert_event_tags(parser, [])
+ self._feed(parser, "<!-- comment -->\n", chunk_size, flush)
+ self.assert_event_tags(parser, [])
+ self._feed(parser,
+ "<root>\n <element key='value'>text</element",
+ chunk_size, flush)
+ self.assert_event_tags(parser, [])
+ self._feed(parser, ">\n", chunk_size, flush)
+ self.assert_event_tags(parser, [('end', 'element')])
+ self._feed(parser, "<element>text</element>tail\n", chunk_size, flush)
+ self._feed(parser, "<empty-element/>\n", chunk_size, flush)
+ self.assert_event_tags(parser, [
+ ('end', 'element'),
+ ('end', 'empty-element'),
+ ])
+ self._feed(parser, "</root>\n", chunk_size, flush)
+ self.assert_event_tags(parser, [('end', 'root')])
+ self.assertIsNone(parser.close())
+
+ def test_simple_xml_chunk_1(self):
+ self.test_simple_xml(chunk_size=1, flush=True)
+
+ def test_simple_xml_chunk_5(self):
+ self.test_simple_xml(chunk_size=5, flush=True)
+
+ def test_simple_xml_chunk_22(self):
+ self.test_simple_xml(chunk_size=22)
def test_feed_while_iterating(self):
parser = ET.XMLPullParser()
@@ -1198,6 +1209,56 @@ def test_unknown_event(self):
with self.assertRaises(ValueError):
ET.XMLPullParser(events=('start', 'end', 'bogus'))
+ @unittest.skipIf(pyexpat.version_info < (2, 6, 0),
+ f'Expat {pyexpat.version_info} does not '
+ 'support reparse deferral')
+ def test_flush_reparse_deferral_enabled(self):
+ parser = ET.XMLPullParser(events=('start', 'end'))
+
+ for chunk in ("<doc", ">"):
+ parser.feed(chunk)
+
+ self.assert_event_tags(parser, []) # i.e. no elements started
+ if ET is pyET:
+ self.assertTrue(parser._parser._parser.GetReparseDeferralEnabled())
+
+ parser.flush()
+
+ self.assert_event_tags(parser, [('start', 'doc')])
+ if ET is pyET:
+ self.assertTrue(parser._parser._parser.GetReparseDeferralEnabled())
+
+ parser.feed("</doc>")
+ parser.close()
+
+ self.assert_event_tags(parser, [('end', 'doc')])
+
+ def test_flush_reparse_deferral_disabled(self):
+ parser = ET.XMLPullParser(events=('start', 'end'))
+
+ for chunk in ("<doc", ">"):
+ parser.feed(chunk)
+
+ if pyexpat.version_info >= (2, 6, 0):
+ if not ET is pyET:
+ self.skipTest(f'XMLParser.(Get|Set)ReparseDeferralEnabled '
+ 'methods not available in C')
+ parser._parser._parser.SetReparseDeferralEnabled(False)
+ self.assert_event_tags(parser, []) # i.e. no elements started
+
+ if ET is pyET:
+ self.assertFalse(parser._parser._parser.GetReparseDeferralEnabled())
+
+ parser.flush()
+
+ self.assert_event_tags(parser, [('start', 'doc')])
+ if ET is pyET:
+ self.assertFalse(parser._parser._parser.GetReparseDeferralEnabled())
+
+ parser.feed("</doc>")
+ parser.close()
+
+ self.assert_event_tags(parser, [('end', 'doc')])
#
# xinclude tests (samples from appendix C of the xinclude specification)
@@ -2216,6 +2277,37 @@ def test_find_xpath(self):
self.assertRaisesRegex(SyntaxError, 'XPath', e.find, './tag[last()-0]')
self.assertRaisesRegex(SyntaxError, 'XPath', e.find, './tag[last()+1]')
+ def test_find_xpath_index_no_quadratic_complexity(self):
+ class CountingElement(ET.Element):
+ findall_calls = 0
+ def findall(self, *args, **kwargs):
+ type(self).findall_calls += 1
+ return super().findall(*args, **kwargs)
+
+ def work(n, pattern):
+ root = CountingElement("root")
+ for _ in range(n):
+ ET.SubElement(root, "a")
+ CountingElement.findall_calls = 0
+ root.findall(pattern)
+ return CountingElement.findall_calls
+
+ for pattern in [".//a[1]", ".//a[last()]"]:
+ w1 = work(1024, pattern)
+ w2 = work(2048, pattern)
+ w3 = work(4096, pattern)
+
+ self.assertGreater(w1, 0)
+ r1 = w2 / w1
+ r2 = w3 / w2
+ # Doubling N must not ~double the parent.findall calls.
+ # Linear-in-N call counts indicate the cache is missing.
+ self.assertLess(
+ max(r1, r2), 1.5,
+ msg=f"Possible quadratic behavior on {pattern!r}: "
+ f"calls={w1, w2, w3} ratios={r1, r2}",
+ )
+
def test_findall(self):
e = ET.XML(SAMPLE_XML)
e[2] = ET.XML(SAMPLE_SECTION)
diff --git a/Lib/test/test_zipfile.py b/Lib/test/test_zipfile.py
index 6a6df859856cdd8..71e67bd5dda8c25 100644
--- a/Lib/test/test_zipfile.py
+++ b/Lib/test/test_zipfile.py
@@ -9,6 +9,7 @@
import struct
import zipfile
import unittest
+from unittest import mock
from tempfile import TemporaryFile
@@ -806,6 +807,8 @@ def make_zip64_file(
self, file_size_64_set=False, file_size_extra=False,
compress_size_64_set=False, compress_size_extra=False,
header_offset_64_set=False, header_offset_extra=False,
+ extensible_data=b'',
+ end_of_central_dir_size=None, offset_to_end_of_central_dir=None,
):
"""Generate bytes sequence for a zip with (incomplete) zip64 data.
@@ -859,6 +862,12 @@ def make_zip64_file(
central_dir_size = struct.pack('<Q', 58 + 8 * len(central_zip64_fields))
offset_to_central_dir = struct.pack('<Q', 50 + 8 * len(local_zip64_fields))
+ if end_of_central_dir_size is None:
+ end_of_central_dir_size = 44 + len(extensible_data)
+ if offset_to_end_of_central_dir is None:
+ offset_to_end_of_central_dir = (108
+ + 8 * len(local_zip64_fields)
+ + 8 * len(central_zip64_fields))
local_extra_length = struct.pack("<H", 4 + 8 * len(local_zip64_fields))
central_extra_length = struct.pack("<H", 4 + 8 * len(central_zip64_fields))
@@ -887,14 +896,17 @@ def make_zip64_file(
+ filename
+ central_extra
# Zip64 end of central directory
- + b"PK\x06\x06,\x00\x00\x00\x00\x00\x00\x00-\x00-"
- + b"\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00"
+ + b"PK\x06\x06"
+ + struct.pack('<Q', end_of_central_dir_size)
+ + b"-\x00-\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00"
+ b"\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00"
+ central_dir_size
+ offset_to_central_dir
+ + extensible_data
# Zip64 end of central directory locator
- + b"PK\x06\x07\x00\x00\x00\x00l\x00\x00\x00\x00\x00\x00\x00\x01"
- + b"\x00\x00\x00"
+ + b"PK\x06\x07\x00\x00\x00\x00"
+ + struct.pack('<Q', offset_to_end_of_central_dir)
+ + b"\x01\x00\x00\x00"
# end of central directory
+ b"PK\x05\x06\x00\x00\x00\x00\x01\x00\x01\x00:\x00\x00\x002\x00"
+ b"\x00\x00\x00\x00"
@@ -925,6 +937,7 @@ def test_bad_zip64_extra(self):
with self.assertRaises(zipfile.BadZipFile) as e:
zipfile.ZipFile(io.BytesIO(missing_file_size_extra))
self.assertIn('file size', str(e.exception).lower())
+ self.assertTrue(zipfile.is_zipfile(io.BytesIO(missing_file_size_extra)))
# zip64 file size present, zip64 compress size present, one field in
# extra, expecting two, equals missing compress size.
@@ -936,6 +949,7 @@ def test_bad_zip64_extra(self):
with self.assertRaises(zipfile.BadZipFile) as e:
zipfile.ZipFile(io.BytesIO(missing_compress_size_extra))
self.assertIn('compress size', str(e.exception).lower())
+ self.assertTrue(zipfile.is_zipfile(io.BytesIO(missing_compress_size_extra)))
# zip64 compress size present, no fields in extra, expecting one,
# equals missing compress size.
@@ -945,6 +959,7 @@ def test_bad_zip64_extra(self):
with self.assertRaises(zipfile.BadZipFile) as e:
zipfile.ZipFile(io.BytesIO(missing_compress_size_extra))
self.assertIn('compress size', str(e.exception).lower())
+ self.assertTrue(zipfile.is_zipfile(io.BytesIO(missing_compress_size_extra)))
# zip64 file size present, zip64 compress size present, zip64 header
# offset present, two fields in extra, expecting three, equals missing
@@ -959,6 +974,7 @@ def test_bad_zip64_extra(self):
with self.assertRaises(zipfile.BadZipFile) as e:
zipfile.ZipFile(io.BytesIO(missing_header_offset_extra))
self.assertIn('header offset', str(e.exception).lower())
+ self.assertTrue(zipfile.is_zipfile(io.BytesIO(missing_header_offset_extra)))
# zip64 compress size present, zip64 header offset present, one field
# in extra, expecting two, equals missing header offset
@@ -971,6 +987,7 @@ def test_bad_zip64_extra(self):
with self.assertRaises(zipfile.BadZipFile) as e:
zipfile.ZipFile(io.BytesIO(missing_header_offset_extra))
self.assertIn('header offset', str(e.exception).lower())
+ self.assertTrue(zipfile.is_zipfile(io.BytesIO(missing_header_offset_extra)))
# zip64 file size present, zip64 header offset present, one field in
# extra, expecting two, equals missing header offset
@@ -983,6 +1000,7 @@ def test_bad_zip64_extra(self):
with self.assertRaises(zipfile.BadZipFile) as e:
zipfile.ZipFile(io.BytesIO(missing_header_offset_extra))
self.assertIn('header offset', str(e.exception).lower())
+ self.assertTrue(zipfile.is_zipfile(io.BytesIO(missing_header_offset_extra)))
# zip64 header offset present, no fields in extra, expecting one,
# equals missing header offset
@@ -994,6 +1012,63 @@ def test_bad_zip64_extra(self):
with self.assertRaises(zipfile.BadZipFile) as e:
zipfile.ZipFile(io.BytesIO(missing_header_offset_extra))
self.assertIn('header offset', str(e.exception).lower())
+ self.assertTrue(zipfile.is_zipfile(io.BytesIO(missing_header_offset_extra)))
+
+ def test_bad_zip64_end_of_central_dir(self):
+ zipdata = self.make_zip64_file(end_of_central_dir_size=0)
+ with self.assertRaisesRegex(zipfile.BadZipFile, 'Corrupt.*record'):
+ zipfile.ZipFile(io.BytesIO(zipdata))
+ self.assertFalse(zipfile.is_zipfile(io.BytesIO(zipdata)))
+
+ zipdata = self.make_zip64_file(end_of_central_dir_size=100)
+ with self.assertRaisesRegex(zipfile.BadZipFile, 'Corrupt.*record'):
+ zipfile.ZipFile(io.BytesIO(zipdata))
+ self.assertFalse(zipfile.is_zipfile(io.BytesIO(zipdata)))
+
+ zipdata = self.make_zip64_file(offset_to_end_of_central_dir=0)
+ with self.assertRaisesRegex(zipfile.BadZipFile, 'Corrupt.*record'):
+ zipfile.ZipFile(io.BytesIO(zipdata))
+ self.assertFalse(zipfile.is_zipfile(io.BytesIO(zipdata)))
+
+ zipdata = self.make_zip64_file(offset_to_end_of_central_dir=1000)
+ with self.assertRaisesRegex(zipfile.BadZipFile, 'Corrupt.*locator'):
+ zipfile.ZipFile(io.BytesIO(zipdata))
+ self.assertFalse(zipfile.is_zipfile(io.BytesIO(zipdata)))
+
+ def test_zip64_end_of_central_dir_record_not_found(self):
+ zipdata = self.make_zip64_file()
+ zipdata = zipdata.replace(b"PK\x06\x06", b'\x00'*4)
+ with self.assertRaisesRegex(zipfile.BadZipFile, 'record not found'):
+ zipfile.ZipFile(io.BytesIO(zipdata))
+ self.assertFalse(zipfile.is_zipfile(io.BytesIO(zipdata)))
+
+ zipdata = self.make_zip64_file(
+ extensible_data=b'\xca\xfe\x04\x00\x00\x00data')
+ zipdata = zipdata.replace(b"PK\x06\x06", b'\x00'*4)
+ with self.assertRaisesRegex(zipfile.BadZipFile, 'record not found'):
+ zipfile.ZipFile(io.BytesIO(zipdata))
+ self.assertFalse(zipfile.is_zipfile(io.BytesIO(zipdata)))
+
+ def test_zip64_extensible_data(self):
+ # These values are what is set in the make_zip64_file method.
+ expected_file_size = 8
+ expected_compress_size = 8
+ expected_header_offset = 0
+ expected_content = b"test1234"
+
+ zipdata = self.make_zip64_file(
+ extensible_data=b'\xca\xfe\x04\x00\x00\x00data')
+ with zipfile.ZipFile(io.BytesIO(zipdata)) as zf:
+ zinfo = zf.infolist()[0]
+ self.assertEqual(zinfo.file_size, expected_file_size)
+ self.assertEqual(zinfo.compress_size, expected_compress_size)
+ self.assertEqual(zinfo.header_offset, expected_header_offset)
+ self.assertEqual(zf.read(zinfo), expected_content)
+ self.assertTrue(zipfile.is_zipfile(io.BytesIO(zipdata)))
+
+ with self.assertRaisesRegex(zipfile.BadZipFile, 'record not found'):
+ zipfile.ZipFile(io.BytesIO(b'prepended' + zipdata))
+ self.assertFalse(zipfile.is_zipfile(io.BytesIO(b'prepended' + zipdata)))
def test_generated_valid_zip64_extra(self):
# These values are what is set in the make_zip64_file method.
@@ -1939,11 +2014,170 @@ def test_seek_tell(self):
fp.seek(0, os.SEEK_SET)
self.assertEqual(fp.tell(), 0)
+ @requires_zlib
+ def test_full_overlap(self):
+ data = (
+ b'PK\x03\x04\x14\x00\x00\x00\x08\x00\xa0lH\x05\xe2\x1e'
+ b'8\xbb\x10\x00\x00\x00\t\x04\x00\x00\x01\x00\x00\x00a\xed'
+ b'\xc0\x81\x08\x00\x00\x00\xc00\xd6\xfbK\\d\x0b`P'
+ b'K\x01\x02\x14\x00\x14\x00\x00\x00\x08\x00\xa0lH\x05\xe2'
+ b'\x1e8\xbb\x10\x00\x00\x00\t\x04\x00\x00\x01\x00\x00\x00\x00'
+ b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00aPK'
+ b'\x01\x02\x14\x00\x14\x00\x00\x00\x08\x00\xa0lH\x05\xe2\x1e'
+ b'8\xbb\x10\x00\x00\x00\t\x04\x00\x00\x01\x00\x00\x00\x00\x00'
+ b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00bPK\x05'
+ b'\x06\x00\x00\x00\x00\x02\x00\x02\x00^\x00\x00\x00/\x00\x00'
+ b'\x00\x00\x00'
+ )
+ with zipfile.ZipFile(io.BytesIO(data), 'r') as zipf:
+ self.assertEqual(zipf.namelist(), ['a', 'b'])
+ zi = zipf.getinfo('a')
+ self.assertEqual(zi.header_offset, 0)
+ self.assertEqual(zi.compress_size, 16)
+ self.assertEqual(zi.file_size, 1033)
+ zi = zipf.getinfo('b')
+ self.assertEqual(zi.header_offset, 0)
+ self.assertEqual(zi.compress_size, 16)
+ self.assertEqual(zi.file_size, 1033)
+ self.assertEqual(len(zipf.read('a')), 1033)
+ with self.assertRaisesRegex(zipfile.BadZipFile, 'File name.*differ'):
+ zipf.read('b')
+
+ @requires_zlib
+ def test_quoted_overlap(self):
+ data = (
+ b'PK\x03\x04\x14\x00\x00\x00\x08\x00\xa0lH\x05Y\xfc'
+ b'8\x044\x00\x00\x00(\x04\x00\x00\x01\x00\x00\x00a\x00'
+ b'\x1f\x00\xe0\xffPK\x03\x04\x14\x00\x00\x00\x08\x00\xa0l'
+ b'H\x05\xe2\x1e8\xbb\x10\x00\x00\x00\t\x04\x00\x00\x01\x00'
+ b'\x00\x00b\xed\xc0\x81\x08\x00\x00\x00\xc00\xd6\xfbK\\'
+ b'd\x0b`PK\x01\x02\x14\x00\x14\x00\x00\x00\x08\x00\xa0'
+ b'lH\x05Y\xfc8\x044\x00\x00\x00(\x04\x00\x00\x01'
+ b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
+ b'\x00aPK\x01\x02\x14\x00\x14\x00\x00\x00\x08\x00\xa0l'
+ b'H\x05\xe2\x1e8\xbb\x10\x00\x00\x00\t\x04\x00\x00\x01\x00'
+ b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00$\x00\x00\x00'
+ b'bPK\x05\x06\x00\x00\x00\x00\x02\x00\x02\x00^\x00\x00'
+ b'\x00S\x00\x00\x00\x00\x00'
+ )
+ with zipfile.ZipFile(io.BytesIO(data), 'r') as zipf:
+ self.assertEqual(zipf.namelist(), ['a', 'b'])
+ zi = zipf.getinfo('a')
+ self.assertEqual(zi.header_offset, 0)
+ self.assertEqual(zi.compress_size, 52)
+ self.assertEqual(zi.file_size, 1064)
+ zi = zipf.getinfo('b')
+ self.assertEqual(zi.header_offset, 36)
+ self.assertEqual(zi.compress_size, 16)
+ self.assertEqual(zi.file_size, 1033)
+ with self.assertRaisesRegex(zipfile.BadZipFile, 'Overlapped entries'):
+ zipf.read('a')
+ self.assertEqual(len(zipf.read('b')), 1033)
+
def tearDown(self):
unlink(TESTFN)
unlink(TESTFN2)
+class AbstractBoundedDecompressTests:
+ # ZipExtFile._read1() bounds the output of each decompress() call so that a
+ # small member declaring a large uncompressed size cannot expand into one
+ # unbounded read.
+ def test_read1_output_is_bounded(self):
+ buf = io.BytesIO()
+ with zipfile.ZipFile(buf, "w", compression=self.compression) as zf:
+ zf.writestr("big", b"\0" * (4 * 1024 * 1024))
+ with zipfile.ZipFile(io.BytesIO(buf.getvalue())) as zf:
+ with zf.open("big") as f:
+ self.assertLessEqual(len(f._read1(100)), f.MIN_READ_SIZE)
+
+
+class StoredBoundedDecompressTests(AbstractBoundedDecompressTests,
+ unittest.TestCase):
+ compression = zipfile.ZIP_STORED
+
+
+@requires_zlib
+class DeflateBoundedDecompressTests(AbstractBoundedDecompressTests,
+ unittest.TestCase):
+ compression = zipfile.ZIP_DEFLATED
+
+
+@requires_bz2
+class Bzip2BoundedDecompressTests(AbstractBoundedDecompressTests,
+ unittest.TestCase):
+ compression = zipfile.ZIP_BZIP2
+
+
+@requires_lzma
+class LzmaBoundedDecompressTests(AbstractBoundedDecompressTests,
+ unittest.TestCase):
+ compression = zipfile.ZIP_LZMA
+
+
+
+class MonkeypatchedDecompressorTests(unittest.TestCase):
+ # Some third-party projects monkey-patch _get_decompressor() to add
+ # additional compression schemes. This can break at any time as the
+ # internal compressor objects change.
+ # To protect users, we try to keep this case working.
+ # See also: GH-156002 and GH-113767.
+ COMPRESSION = 99
+
+ class Compressor:
+ """Compressor with only the original BZ2Compressor API"""
+ def compress(self, data):
+ return data.swapcase()
+
+ def flush(self):
+ return b''
+
+ class Decompressor:
+ """Decompressor with only the 3.3+ BZ2Decompressor API"""
+ eof = False
+
+ def decompress(self, data):
+ return data.swapcase()
+
+ def test_roundtrip_monkeypatched_decompressor(self):
+ orig_check_compression = zipfile._check_compression
+ orig_get_compressor = zipfile._get_compressor
+ orig_get_decompressor = zipfile._get_decompressor
+
+ def check_compression(compression):
+ if compression != self.COMPRESSION:
+ orig_check_compression(compression)
+
+ def get_compressor(compress_type, compresslevel=None):
+ if compress_type == self.COMPRESSION:
+ return self.Compressor()
+ return orig_get_compressor(compress_type, compresslevel)
+
+ def get_decompressor(compress_type):
+ if compress_type == self.COMPRESSION:
+ return self.Decompressor()
+ return orig_get_decompressor(compress_type)
+
+ with mock.patch.object(zipfile, '_check_compression', check_compression), \
+ mock.patch.object(zipfile, '_get_compressor', get_compressor), \
+ mock.patch.object(zipfile, '_get_decompressor', get_decompressor):
+ data = bytes(range(256)) * 8
+ buf = io.BytesIO()
+ with zipfile.ZipFile(buf, "w", compression=self.COMPRESSION) as zf:
+ zf.writestr("member", data)
+ self.assertIn(data.swapcase(), buf.getvalue())
+ with zipfile.ZipFile(io.BytesIO(buf.getvalue())) as zf:
+ self.assertEqual(zf.read("member"), data)
+ with zf.open("member") as f:
+ self.assertEqual(f.read(100), data[:100])
+ self.assertEqual(f.read1(100), data[100:200])
+ f.seek(-100, os.SEEK_END)
+ self.assertEqual(f.read(), data[-100:])
+ # Rewinding past the read buffer re-creates the decompressor.
+ f.seek(0)
+ self.assertEqual(f.read(), data)
+
+
class AbstractBadCrcTests:
def test_testzip_with_bad_crc(self):
"""Tests that files with bad CRCs return their name from testzip."""
diff --git a/Lib/turtledemo/__main__.py b/Lib/turtledemo/__main__.py
index 12be5098dad274a..96e7f5004e795cb 100644
--- a/Lib/turtledemo/__main__.py
+++ b/Lib/turtledemo/__main__.py
@@ -138,7 +138,7 @@ def __init__(self, filename=None):
# so that our menu bar appears.
subprocess.run(
[
- 'osascript',
+ '/usr/bin/osascript',
'-e', 'tell application "System Events"',
'-e', 'set frontmost of the first process whose '
'unix id is {} to true'.format(os.getpid()),
diff --git a/Lib/unittest/test/test_discovery.py b/Lib/unittest/test/test_discovery.py
index 204043b493b5d2f..16e081e1fb76ec1 100644
--- a/Lib/unittest/test/test_discovery.py
+++ b/Lib/unittest/test/test_discovery.py
@@ -723,11 +723,13 @@ class Module(object):
original_listdir = os.listdir
original_isfile = os.path.isfile
original_isdir = os.path.isdir
+ original_realpath = os.path.realpath
def cleanup():
os.listdir = original_listdir
os.path.isfile = original_isfile
os.path.isdir = original_isdir
+ os.path.realpath = original_realpath
del sys.modules['foo']
if full_path in sys.path:
sys.path.remove(full_path)
@@ -742,6 +744,10 @@ def isdir(_):
os.listdir = listdir
os.path.isfile = isfile
os.path.isdir = isdir
+ if os.name == 'nt':
+ # ntpath.realpath may inject path prefixes when failing to
+ # resolve real files, so we substitute abspath() here instead.
+ os.path.realpath = os.path.abspath
return full_path
def test_detect_module_clash(self):
diff --git a/Lib/urllib/parse.py b/Lib/urllib/parse.py
index 79ea635ef87db42..46a3dfc58e77dad 100644
--- a/Lib/urllib/parse.py
+++ b/Lib/urllib/parse.py
@@ -34,6 +34,7 @@
import re
import sys
import collections
+import ipaddress
__all__ = ["urlparse", "urlunparse", "urljoin", "urldefrag",
"urlsplit", "urlunsplit", "urlencode", "parse_qs",
@@ -425,6 +426,41 @@ def _remove_unsafe_bytes_from_url(url):
url = url.replace(b, "")
return url
+
+def _check_bracketed_netloc(netloc):
+ # Note that this function must mirror the splitting
+ # done in NetlocResultMixins._hostinfo().
+ hostname_and_port = netloc.rpartition('@')[2]
+ before_bracket, have_open_br, bracketed = hostname_and_port.partition('[')
+ if have_open_br:
+ # No data is allowed before a bracket.
+ if before_bracket:
+ raise ValueError("Invalid IPv6 URL")
+ hostname, _, port = bracketed.partition(']')
+ # No data is allowed after the bracket but before the port delimiter.
+ if port and not port.startswith(":"):
+ raise ValueError("Invalid IPv6 URL")
+ else:
+ hostname, _, port = hostname_and_port.partition(':')
+ _check_bracketed_host(hostname)
+
+# Valid bracketed hosts are defined in
+# https://www.rfc-editor.org/rfc/rfc3986#page-49 and https://url.spec.whatwg.org/
+def _check_bracketed_host(hostname):
+ if hostname.startswith('v'):
+ if not re.match(r"\Av[a-fA-F0-9]+\..+\Z", hostname):
+ raise ValueError(f"IPvFuture address is invalid")
+ else:
+ # The ipaddress module only accepts IPv6 scope ids ("%zone") from
+ # 3.9 on, so validate the address part here and check the scope id
+ # the way 3.9's IPv6Address does.
+ address, percent, scope_id = hostname.partition('%')
+ if percent and (not scope_id or '%' in scope_id):
+ raise ValueError("Invalid IPv6 scope id in %r" % hostname)
+ ip = ipaddress.ip_address(address) # Throws Value Error if not IPv6 or IPv4
+ if isinstance(ip, ipaddress.IPv4Address):
+ raise ValueError(f"An IPv4 address cannot be in brackets")
+
def urlsplit(url, scheme='', allow_fragments=True):
"""Parse a URL into 5 components:
<scheme>://<netloc>/<path>?<query>#<fragment>
@@ -458,6 +494,8 @@ def urlsplit(url, scheme='', allow_fragments=True):
if (('[' in netloc and ']' not in netloc) or
(']' in netloc and '[' not in netloc)):
raise ValueError("Invalid IPv6 URL")
+ if '[' in netloc and ']' in netloc:
+ _check_bracketed_netloc(netloc)
if allow_fragments and '#' in url:
url, fragment = url.split('#', 1)
if '?' in url:
@@ -482,6 +520,8 @@ def urlsplit(url, scheme='', allow_fragments=True):
if (('[' in netloc and ']' not in netloc) or
(']' in netloc and '[' not in netloc)):
raise ValueError("Invalid IPv6 URL")
+ if '[' in netloc and ']' in netloc:
+ _check_bracketed_netloc(netloc)
if allow_fragments and '#' in url:
url, fragment = url.split('#', 1)
if '?' in url:
diff --git a/Lib/urllib/request.py b/Lib/urllib/request.py
index 993fde87b9225dc..6c9b7ae9e3fcfb0 100644
--- a/Lib/urllib/request.py
+++ b/Lib/urllib/request.py
@@ -845,16 +845,17 @@ def add_password(self, realm, uri, user, passwd):
self.passwd[realm] = {}
for default_port in True, False:
reduced_uri = tuple(
- self.reduce_uri(u, default_port) for u in uri)
+ self._reduce_uri_with_scheme(u, default_port) for u in uri)
self.passwd[realm][reduced_uri] = (user, passwd)
def find_user_password(self, realm, authuri):
domains = self.passwd.get(realm, {})
for default_port in True, False:
- reduced_authuri = self.reduce_uri(authuri, default_port)
+ reduced_authuri = self._reduce_uri_with_scheme(
+ authuri, default_port)
for uris, authinfo in domains.items():
for uri in uris:
- if self.is_suburi(uri, reduced_authuri):
+ if self._is_suburi_with_scheme(uri, reduced_authuri):
return authinfo
return None, None
@@ -881,6 +882,17 @@ def reduce_uri(self, uri, default_port=True):
authority = "%s:%d" % (host, dport)
return authority, path
+ def _reduce_uri_with_scheme(self, uri, default_port=True):
+ parts = urlsplit(uri)
+ scheme = parts[0] if parts[1] else None
+ return (scheme or None, *self.reduce_uri(uri, default_port))
+
+ def _is_suburi_with_scheme(self, base, test):
+ if (base[0] is not None and test[0] is not None and
+ base[0] != test[0]):
+ return False
+ return self.is_suburi(base[1:], test[1:])
+
def is_suburi(self, base, test):
"""Check if test is below base in a URI tree
@@ -926,14 +938,15 @@ def update_authenticated(self, uri, is_authenticated=False):
for default_port in True, False:
for u in uri:
- reduced_uri = self.reduce_uri(u, default_port)
+ reduced_uri = self._reduce_uri_with_scheme(u, default_port)
self.authenticated[reduced_uri] = is_authenticated
def is_authenticated(self, authuri):
for default_port in True, False:
- reduced_authuri = self.reduce_uri(authuri, default_port)
+ reduced_authuri = self._reduce_uri_with_scheme(
+ authuri, default_port)
for uri in self.authenticated:
- if self.is_suburi(uri, reduced_authuri):
+ if self._is_suburi_with_scheme(uri, reduced_authuri):
return self.authenticated[uri]
@@ -1656,6 +1669,11 @@ def data_open(self, req):
scheme, data = url.split(":",1)
mediatype, data = data.split(",",1)
+ # Disallow control characters within mediatype.
+ if re.search(r"[\x00-\x1F\x7F]", mediatype):
+ raise ValueError(
+ "Control characters not allowed in data: mediatype")
+
# even base64 encoded data URLs might be quoted so unquote in any case:
data = unquote_to_bytes(data)
if mediatype.endswith(";base64"):
diff --git a/Lib/webbrowser.py b/Lib/webbrowser.py
index b04ec7b65ae3b65..b180644e4b59ad2 100755
--- a/Lib/webbrowser.py
+++ b/Lib/webbrowser.py
@@ -154,6 +154,12 @@ def open_new(self, url):
def open_new_tab(self, url):
return self.open(url, 2)
+ @staticmethod
+ def _check_url(url):
+ """Ensures that the URL is safe to pass to subprocesses as a parameter"""
+ if url and url.lstrip().startswith("-"):
+ raise ValueError(f"Invalid URL (leading dash disallowed): {url!r}")
+
class GenericBrowser(BaseBrowser):
"""Class for all browsers started with a command
@@ -170,6 +176,7 @@ def __init__(self, name):
self.basename = os.path.basename(self.name)
def open(self, url, new=0, autoraise=True):
+ self._check_url(url)
cmdline = [self.name] + [arg.replace("%s", url)
for arg in self.args]
try:
@@ -189,6 +196,7 @@ class BackgroundBrowser(GenericBrowser):
def open(self, url, new=0, autoraise=True):
cmdline = [self.name] + [arg.replace("%s", url)
for arg in self.args]
+ self._check_url(url)
try:
if sys.platform[:3] == 'win':
p = subprocess.Popen(cmdline)
@@ -266,7 +274,9 @@ def open(self, url, new=0, autoraise=True):
raise Error("Bad 'new' parameter to open(); " +
"expected 0, 1, or 2, got %s" % new)
- args = [arg.replace("%s", url).replace("%action", action)
+ self._check_url(url.replace("%action", action))
+
+ args = [arg.replace("%action", action).replace("%s", url)
for arg in self.remote_args]
args = [arg for arg in args if arg]
success = self._invoke(args, True, autoraise)
@@ -353,6 +363,7 @@ class Konqueror(BaseBrowser):
"""
def open(self, url, new=0, autoraise=True):
+ self._check_url(url)
# XXX Currently I know no way to prevent KFM from opening a new win.
if new == 2:
action = "newTab"
@@ -436,6 +447,7 @@ def _remote(self, action):
return 1
def open(self, url, new=0, autoraise=True):
+ self._check_url(url)
if new:
ok = self._remote("LOADNEW " + url)
else:
@@ -593,6 +605,7 @@ def register_standard_browsers():
if sys.platform[:3] == "win":
class WindowsDefault(BaseBrowser):
def open(self, url, new=0, autoraise=True):
+ self._check_url(url)
try:
os.startfile(url)
except OSError:
@@ -622,6 +635,7 @@ def __init__(self, name):
self.name = name
def open(self, url, new=0, autoraise=True):
+ self._check_url(url)
assert "'" not in url
# hack for local urls
if not ':' in url:
@@ -659,6 +673,7 @@ def __init__(self, name):
self._name = name
def open(self, url, new=0, autoraise=True):
+ self._check_url(url)
if self._name == 'default':
script = 'open location "%s"' % url.replace('"', '%22') # opens in default browser
else:
@@ -669,7 +684,7 @@ def open(self, url, new=0, autoraise=True):
end
'''%(self._name, url.replace('"', '%22'))
- osapipe = os.popen("osascript", "w")
+ osapipe = os.popen("/usr/bin/osascript", "w")
if osapipe is None:
return False
diff --git a/Lib/wsgiref/handlers.py b/Lib/wsgiref/handlers.py
index eee3f948c723060..85827d60aed79db 100644
--- a/Lib/wsgiref/handlers.py
+++ b/Lib/wsgiref/handlers.py
@@ -1,7 +1,7 @@
"""Base classes for server/gateway implementations"""
from .util import FileWrapper, guess_scheme, is_hop_by_hop
-from .headers import Headers
+from .headers import Headers, _name_disallowed_re
import sys, os, time
@@ -238,6 +238,8 @@ def start_response(self, status, headers,exc_info=None):
self.status = status
self.headers = self.headers_class(headers)
status = self._convert_string_type(status, "Status")
+ if _name_disallowed_re.search(status):
+ raise ValueError("Control characters are not allowed in status")
assert len(status)>=4,"Status must be at least 4 characters"
assert status[:3].isdigit(), "Status message must begin w/3-digit code"
assert status[3]==" ", "Status message must have a space after code"
diff --git a/Lib/wsgiref/headers.py b/Lib/wsgiref/headers.py
index fab851c5a44430d..17559b0a37bd201 100644
--- a/Lib/wsgiref/headers.py
+++ b/Lib/wsgiref/headers.py
@@ -9,6 +9,11 @@
# existence of which force quoting of the parameter value.
import re
tspecials = re.compile(r'[ \(\)<>@,;:\\"/\[\]\?=]')
+# Disallowed characters for headers and values.
+# HTAB (\x09) is allowed in header values, but
+# not in header names. (RFC 9110 Section 5.5)
+_name_disallowed_re = re.compile(r'[\x00-\x1F\x7F]')
+_value_disallowed_re = re.compile(r'[\x00-\x08\x0A-\x1F\x7F]')
def _formatparam(param, value=None, quote=1):
"""Convenience function to format and return a key=value pair.
@@ -35,12 +40,15 @@ def __init__(self, headers=None):
self._headers = headers
if __debug__:
for k, v in headers:
- self._convert_string_type(k)
- self._convert_string_type(v)
+ self._convert_string_type(k, name=True)
+ self._convert_string_type(v, name=False)
- def _convert_string_type(self, value):
+ def _convert_string_type(self, value, *, name):
"""Convert/check value type."""
if type(value) is str:
+ regex = (_name_disallowed_re if name else _value_disallowed_re)
+ if regex.search(value):
+ raise ValueError("Control characters not allowed in headers")
return value
raise AssertionError("Header names/values must be"
" of type str (got {0})".format(repr(value)))
@@ -53,14 +61,14 @@ def __setitem__(self, name, val):
"""Set the value of a header."""
del self[name]
self._headers.append(
- (self._convert_string_type(name), self._convert_string_type(val)))
+ (self._convert_string_type(name, name=True), self._convert_string_type(val, name=False)))
def __delitem__(self,name):
"""Delete all occurrences of a header, if present.
Does *not* raise an exception if the header is missing.
"""
- name = self._convert_string_type(name.lower())
+ name = self._convert_string_type(name.lower(), name=True)
self._headers[:] = [kv for kv in self._headers if kv[0].lower() != name]
def __getitem__(self,name):
@@ -87,13 +95,13 @@ def get_all(self, name):
fields deleted and re-inserted are always appended to the header list.
If no fields exist with the given name, returns an empty list.
"""
- name = self._convert_string_type(name.lower())
+ name = self._convert_string_type(name.lower(), name=True)
return [kv[1] for kv in self._headers if kv[0].lower()==name]
def get(self,name,default=None):
"""Get the first header value for 'name', or return 'default'"""
- name = self._convert_string_type(name.lower())
+ name = self._convert_string_type(name.lower(), name=True)
for k,v in self._headers:
if k.lower()==name:
return v
@@ -148,8 +156,8 @@ def setdefault(self,name,value):
and value 'value'."""
result = self.get(name)
if result is None:
- self._headers.append((self._convert_string_type(name),
- self._convert_string_type(value)))
+ self._headers.append((self._convert_string_type(name, name=True),
+ self._convert_string_type(value, name=False)))
return value
else:
return result
@@ -172,13 +180,13 @@ def add_header(self, _name, _value, **_params):
"""
parts = []
if _value is not None:
- _value = self._convert_string_type(_value)
+ _value = self._convert_string_type(_value, name=False)
parts.append(_value)
for k, v in _params.items():
- k = self._convert_string_type(k)
+ k = self._convert_string_type(k, name=True)
if v is None:
parts.append(k.replace('_', '-'))
else:
- v = self._convert_string_type(v)
+ v = self._convert_string_type(v, name=False)
parts.append(_formatparam(k.replace('_', '-'), v))
- self._headers.append((self._convert_string_type(_name), "; ".join(parts)))
+ self._headers.append((self._convert_string_type(_name, name=True), "; ".join(parts)))
diff --git a/Lib/xml/dom/minidom.py b/Lib/xml/dom/minidom.py
index 24957ea14fa73b1..600bf0b781dcdaa 100644
--- a/Lib/xml/dom/minidom.py
+++ b/Lib/xml/dom/minidom.py
@@ -291,13 +291,6 @@ def _append_child(self, node):
childNodes.append(node)
node.parentNode = self
-def _in_document(node):
- # return True iff node is part of a document tree
- while node is not None:
- if node.nodeType == Node.DOCUMENT_NODE:
- return True
- node = node.parentNode
- return False
def _write_data(writer, data):
"Writes datachars to writer."
@@ -354,6 +347,7 @@ class Attr(Node):
def __init__(self, qName, namespaceURI=EMPTY_NAMESPACE, localName=None,
prefix=None):
self.ownerElement = None
+ self.ownerDocument = None
self._name = qName
self.namespaceURI = namespaceURI
self._prefix = prefix
@@ -677,6 +671,7 @@ class Element(Node):
def __init__(self, tagName, namespaceURI=EMPTY_NAMESPACE, prefix=None,
localName=None):
+ self.ownerDocument = None
self.parentNode = None
self.tagName = self.nodeName = tagName
self.prefix = prefix
@@ -1512,7 +1507,7 @@ def _clear_id_cache(node):
if node.nodeType == Node.DOCUMENT_NODE:
node._id_cache.clear()
node._id_search_stack = None
- elif _in_document(node):
+ elif node.ownerDocument:
node.ownerDocument._id_cache.clear()
node.ownerDocument._id_search_stack= None
diff --git a/Lib/xml/etree/ElementPath.py b/Lib/xml/etree/ElementPath.py
index ef32917b14d41e2..24d78bfdd4a2965 100644
--- a/Lib/xml/etree/ElementPath.py
+++ b/Lib/xml/etree/ElementPath.py
@@ -229,15 +229,22 @@ def select(context, result):
index = -1
def select(context, result):
parent_map = get_parent_map(context)
+ cache = {}
for elem in result:
try:
parent = parent_map[elem]
+ except KeyError:
+ continue
+ key = (parent, elem.tag)
+ if key not in cache:
# FIXME: what if the selector is "*" ?
- elems = list(parent.findall(elem.tag))
- if elems[index] is elem:
- yield elem
- except (IndexError, KeyError):
- pass
+ elems = parent.findall(elem.tag)
+ try:
+ cache[key] = elems[index]
+ except IndexError:
+ cache[key] = None
+ if cache[key] is elem:
+ yield elem
return select
raise SyntaxError("invalid predicate")
diff --git a/Lib/xml/etree/ElementTree.py b/Lib/xml/etree/ElementTree.py
index 87277045b525bb1..8d1ef84a2a42936 100644
--- a/Lib/xml/etree/ElementTree.py
+++ b/Lib/xml/etree/ElementTree.py
@@ -1298,6 +1298,11 @@ def read_events(self):
else:
yield event
+ def flush(self):
+ if self._parser is None:
+ raise ValueError("flush() called after end of stream")
+ self._parser.flush()
+
def XML(text, parser=None):
"""Parse XML document from string constant.
@@ -1648,6 +1653,15 @@ def close(self):
del self.parser, self._parser
del self.target, self._target
+ def flush(self):
+ was_enabled = self.parser.GetReparseDeferralEnabled()
+ try:
+ self.parser.SetReparseDeferralEnabled(False)
+ self.parser.Parse(b"", False)
+ except self._error as v:
+ self._raiseerror(v)
+ finally:
+ self.parser.SetReparseDeferralEnabled(was_enabled)
# Import the C accelerators
try:
diff --git a/Lib/xml/sax/expatreader.py b/Lib/xml/sax/expatreader.py
index 5066ffc2fa51f02..e74b02acc68a052 100644
--- a/Lib/xml/sax/expatreader.py
+++ b/Lib/xml/sax/expatreader.py
@@ -220,6 +220,20 @@ def feed(self, data, isFinal = 0):
# FIXME: when to invoke error()?
self._err_handler.fatalError(exc)
+ def flush(self):
+ if self._parser is None:
+ return
+
+ was_enabled = self._parser.GetReparseDeferralEnabled()
+ try:
+ self._parser.SetReparseDeferralEnabled(False)
+ self._parser.Parse(b"", False)
+ except expat.error as e:
+ exc = SAXParseException(expat.ErrorString(e.code), e, self)
+ self._err_handler.fatalError(exc)
+ finally:
+ self._parser.SetReparseDeferralEnabled(was_enabled)
+
def _close_source(self):
source = self._source
try:
diff --git a/Lib/zipfile.py b/Lib/zipfile.py
index 63748d371f0e755..c58b5b8c4c99654 100644
--- a/Lib/zipfile.py
+++ b/Lib/zipfile.py
@@ -202,7 +202,7 @@ def is_zipfile(filename):
else:
with open(filename, "rb") as fp:
result = _check_zipfile(fp)
- except OSError:
+ except (OSError, BadZipFile):
pass
return result
@@ -210,16 +210,15 @@ def _EndRecData64(fpin, offset, endrec):
"""
Read the ZIP64 end-of-archive records and use that to update endrec
"""
- try:
- fpin.seek(offset - sizeEndCentDir64Locator, 2)
- except OSError:
- # If the seek fails, the file is not large enough to contain a ZIP64
+ offset -= sizeEndCentDir64Locator
+ if offset < 0:
+ # The file is not large enough to contain a ZIP64
# end-of-archive record, so just return the end record we were given.
return endrec
-
+ fpin.seek(offset)
data = fpin.read(sizeEndCentDir64Locator)
if len(data) != sizeEndCentDir64Locator:
- return endrec
+ raise OSError("Unknown I/O error")
sig, diskno, reloff, disks = struct.unpack(structEndArchive64Locator, data)
if sig != stringEndArchive64Locator:
return endrec
@@ -227,16 +226,33 @@ def _EndRecData64(fpin, offset, endrec):
if diskno != 0 or disks > 1:
raise BadZipFile("zipfiles that span multiple disks are not supported")
- # Assume no 'zip64 extensible data'
- fpin.seek(offset - sizeEndCentDir64Locator - sizeEndCentDir64, 2)
+ offset -= sizeEndCentDir64
+ if reloff > offset:
+ raise BadZipFile("Corrupt zip64 end of central directory locator")
+ # First, check the assumption that there is no prepended data.
+ fpin.seek(reloff)
+ extrasz = offset - reloff
data = fpin.read(sizeEndCentDir64)
if len(data) != sizeEndCentDir64:
- return endrec
+ raise OSError("Unknown I/O error")
+ if not data.startswith(stringEndArchive64) and reloff != offset:
+ # Since we already have seen the Zip64 EOCD Locator, it's
+ # possible we got here because there is prepended data.
+ # Assume no 'zip64 extensible data'
+ fpin.seek(offset)
+ extrasz = 0
+ data = fpin.read(sizeEndCentDir64)
+ if len(data) != sizeEndCentDir64:
+ raise OSError("Unknown I/O error")
+ if not data.startswith(stringEndArchive64):
+ raise BadZipFile("Zip64 end of central directory record not found")
+
sig, sz, create_version, read_version, disk_num, disk_dir, \
dircount, dircount2, dirsize, diroffset = \
struct.unpack(structEndArchive64, data)
- if sig != stringEndArchive64:
- return endrec
+ if (diroffset + dirsize != reloff or
+ sz + 12 != sizeEndCentDir64 + extrasz):
+ raise BadZipFile("Corrupt zip64 end of central directory record")
# Update the original endrec using data from the ZIP64 record
endrec[_ECD_SIGNATURE] = sig
@@ -246,6 +262,7 @@ def _EndRecData64(fpin, offset, endrec):
endrec[_ECD_ENTRIES_TOTAL] = dircount2
endrec[_ECD_SIZE] = dirsize
endrec[_ECD_OFFSET] = diroffset
+ endrec[_ECD_LOCATION] = offset - extrasz
return endrec
@@ -279,7 +296,7 @@ def _EndRecData(fpin):
endrec.append(filesize - sizeEndCentDir)
# Try to read the "Zip64 end of central directory" structure
- return _EndRecData64(fpin, -sizeEndCentDir, endrec)
+ return _EndRecData64(fpin, filesize - sizeEndCentDir, endrec)
# Either this is not a ZIP file, or it is a ZIP file with an archive
# comment. Search the end of the file for the "end of central directory"
@@ -303,8 +320,7 @@ def _EndRecData(fpin):
endrec.append(maxCommentStart + start)
# Try to read the "Zip64 end of central directory" structure
- return _EndRecData64(fpin, maxCommentStart + start - filesize,
- endrec)
+ return _EndRecData64(fpin, maxCommentStart + start, endrec)
# Unable to find a valid end of central directory structure
return None
@@ -334,6 +350,7 @@ class ZipInfo (object):
'compress_size',
'file_size',
'_raw_time',
+ '_end_offset',
)
def __init__(self, filename="NoName", date_time=(1980,1,1,0,0,0)):
@@ -373,6 +390,7 @@ def __init__(self, filename="NoName", date_time=(1980,1,1,0,0,0)):
self.volume = 0 # Volume number of file header
self.internal_attr = 0 # Internal attributes
self.external_attr = 0 # External file attributes
+ self._end_offset = None # Start of the next local header or central directory
# Other attributes are set by class ZipFile:
# header_offset Byte offset to the file header
# CRC CRC-32 of the uncompressed file
@@ -631,7 +649,16 @@ def __init__(self):
self._unconsumed = b''
self.eof = False
- def decompress(self, data):
+ @property
+ def needs_input(self):
+ # While the LZMA properties header is still being buffered, more input
+ # is required; afterwards defer to the wrapped decompressor so a bounded
+ # decompress() call can be drained across reads.
+ if self._decomp is None:
+ return True
+ return self._decomp.needs_input
+
+ def decompress(self, data, max_length=-1):
if self._decomp is None:
self._unconsumed += data
if len(self._unconsumed) <= 4:
@@ -647,7 +674,7 @@ def decompress(self, data):
data = self._unconsumed[4 + psize:]
del self._unconsumed
- result = self._decomp.decompress(data)
+ result = self._decomp.decompress(data, max_length)
self.eof = self._decomp.eof
return result
@@ -996,8 +1023,15 @@ def _read1(self, n):
data = self._decompressor.unconsumed_tail
if n > len(data):
data += self._read2(n - len(data))
- else:
+ elif self._compress_type == ZIP_STORED:
data = self._read2(n)
+ else:
+ # bzip2/lzma/zstd: a bounded decompress() call may leave input
+ # buffered inside the decompressor; drain that before reading more.
+ if getattr(self._decompressor, "needs_input", True):
+ data = self._read2(n)
+ else:
+ data = b''
if self._compress_type == ZIP_STORED:
self._eof = self._compress_left <= 0
@@ -1010,8 +1044,17 @@ def _read1(self, n):
if self._eof:
data += self._decompressor.flush()
else:
- data = self._decompressor.decompress(data)
- self._eof = self._decompressor.eof or self._compress_left <= 0
+ # Bound the output of a single decompress() call (mirroring the
+ # DEFLATE path above) so that a small compressed member cannot
+ # expand into one unbounded read.
+ try:
+ data = self._decompressor.decompress(data, max(n, self.MIN_READ_SIZE))
+ except TypeError:
+ # See MonkeypatchedDecompressorTests in test_core.py
+ data = self._decompressor.decompress(data)
+ self._eof = (self._decompressor.eof or
+ self._compress_left <= 0 and
+ getattr(self._decompressor, "needs_input", True))
data = data[:self._left]
self._left -= len(data)
@@ -1204,6 +1247,7 @@ class ZipFile:
fp = None # Set here since __del__ checks it
_windows_illegal_name_trans_table = None
+ _ignore_invalid_names = False
def __init__(self, file, mode="r", compression=ZIP_STORED, allowZip64=True,
compresslevel=None):
@@ -1331,9 +1375,6 @@ def _RealGetContents(self):
# "concat" is zero, unless zip was concatenated to another file
concat = endrec[_ECD_LOCATION] - size_cd - offset_cd
- if endrec[_ECD_SIGNATURE] == stringEndArchive64:
- # If Zip64 extension structures are present, account for them
- concat -= (sizeEndCentDir64 + sizeEndCentDir64Locator)
if self.debug > 2:
inferred = concat + offset_cd
@@ -1391,6 +1432,12 @@ def _RealGetContents(self):
if self.debug > 2:
print("total", total)
+ end_offset = self.start_dir
+ for zinfo in sorted(self.filelist,
+ key=lambda zinfo: zinfo.header_offset,
+ reverse=True):
+ zinfo._end_offset = end_offset
+ end_offset = zinfo.header_offset
def namelist(self):
"""Return a list of file names in the archive."""
@@ -1546,6 +1593,10 @@ def open(self, name, mode="r", pwd=None, *, force_zip64=False):
'File name in directory %r and header %r differ.'
% (zinfo.orig_filename, fname))
+ if (zinfo._end_offset is not None and
+ zef_file.tell() + zinfo.compress_size > zinfo._end_offset):
+ raise BadZipFile(f"Overlapped entries: {zinfo.orig_filename!r} (possible zip bomb)")
+
# check for encrypted flag & handle password
is_encrypted = zinfo.flag_bits & 0x1
if is_encrypted:
@@ -1659,19 +1710,27 @@ def _extract_member(self, member, targetpath, pwd):
# build the destination pathname, replacing
# forward slashes to platform specific separators.
- arcname = member.filename.replace('/', os.path.sep)
-
- if os.path.altsep:
+ arcname = member.filename
+ if os.path.sep != '/':
+ arcname = arcname.replace('/', os.path.sep)
+ if os.path.altsep and os.path.altsep != '/':
arcname = arcname.replace(os.path.altsep, os.path.sep)
# interpret absolute pathname as relative, remove drive letter or
# UNC path, redundant separators, "." and ".." components.
- arcname = os.path.splitdrive(arcname)[1]
+ drive, arcname = os.path.splitdrive(arcname)
+ if self._ignore_invalid_names and (drive or arcname.startswith(os.path.sep)):
+ return None
+ if self._ignore_invalid_names and os.path.pardir in arcname.split(os.path.sep):
+ return None
invalid_path_parts = ('', os.path.curdir, os.path.pardir)
arcname = os.path.sep.join(x for x in arcname.split(os.path.sep)
if x not in invalid_path_parts)
if os.path.sep == '\\':
# filter illegal characters on Windows
- arcname = self._sanitize_windows_name(arcname, os.path.sep)
+ arcname2 = self._sanitize_windows_name(arcname, os.path.sep)
+ if self._ignore_invalid_names and arcname2 != arcname:
+ return None
+ arcname = arcname2
targetpath = os.path.join(targetpath, arcname)
targetpath = os.path.normpath(targetpath)
@@ -1913,7 +1972,7 @@ def _write_end_record(self):
" would require ZIP64 extensions")
zip64endrec = struct.pack(
structEndArchive64, stringEndArchive64,
- 44, 45, 45, 0, 0, centDirCount, centDirCount,
+ sizeEndCentDir64 - 12, 45, 45, 0, 0, centDirCount, centDirCount,
centDirSize, centDirOffset)
self.fp.write(zip64endrec)
diff --git a/Misc/ACKS b/Misc/ACKS
index bd62c958c5d2de9..eea91613615eb54 100644
--- a/Misc/ACKS
+++ b/Misc/ACKS
@@ -911,6 +911,7 @@ Wolfgang Langner
Detlef Lannert
Rémi Lapeyre
Soren Larsen
+Seth Michael Larson
Amos Latteier
Piers Lauder
Ben Laurie
@@ -1322,6 +1323,7 @@ Dhushyanth Ramasamy
Ashwin Ramaswami
Jeff Ramnani
Bayard Randel
+Eashwar Ranganathan
Varpu Rantala
Brodie Rao
Rémi Rampin
diff --git a/Misc/NEWS.d/next/Build/2026-07-09-22-45-00.gh-issue-153438.Qr7N2p.rst b/Misc/NEWS.d/next/Build/2026-07-09-22-45-00.gh-issue-153438.Qr7N2p.rst
new file mode 100644
index 000000000000000..edab8e6ba7b2595
--- /dev/null
+++ b/Misc/NEWS.d/next/Build/2026-07-09-22-45-00.gh-issue-153438.Qr7N2p.rst
@@ -0,0 +1,2 @@
+Update Windows build and installer tooling and documentation to use the
+current download URL for ``nuget.exe``.
diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-10-27-19-38-33.gh-issue-102388.vd5YUZ.rst b/Misc/NEWS.d/next/Core and Builtins/2023-10-27-19-38-33.gh-issue-102388.vd5YUZ.rst
new file mode 100644
index 000000000000000..268a3d310f2b499
--- /dev/null
+++ b/Misc/NEWS.d/next/Core and Builtins/2023-10-27-19-38-33.gh-issue-102388.vd5YUZ.rst
@@ -0,0 +1 @@
+Fix a bug where ``iso2022_jp_3`` and ``iso2022_jp_2004`` codecs read out of bounds
diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-06-10-10-42-48.gh-issue-120298.napREA.rst b/Misc/NEWS.d/next/Core and Builtins/2024-06-10-10-42-48.gh-issue-120298.napREA.rst
new file mode 100644
index 000000000000000..531d39517ac4231
--- /dev/null
+++ b/Misc/NEWS.d/next/Core and Builtins/2024-06-10-10-42-48.gh-issue-120298.napREA.rst
@@ -0,0 +1,2 @@
+Fix use-after free in ``list_richcompare_impl`` which can be invoked via
+some specificly tailored evil input.
diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-06-13-12-17-52.gh-issue-120384.w1UBGl.rst b/Misc/NEWS.d/next/Core and Builtins/2024-06-13-12-17-52.gh-issue-120384.w1UBGl.rst
new file mode 100644
index 000000000000000..4a4db821ce29b8c
--- /dev/null
+++ b/Misc/NEWS.d/next/Core and Builtins/2024-06-13-12-17-52.gh-issue-120384.w1UBGl.rst
@@ -0,0 +1,3 @@
+Fix an array out of bounds crash in ``list_ass_subscript``, which could be
+invoked via some specificly tailored input: including concurrent modification
+of a list object, where one thread assigns a slice and another clears it.
diff --git a/Misc/NEWS.d/next/Library/2020-07-09-11-32-28.bpo-12800.fNgWwx.rst b/Misc/NEWS.d/next/Library/2020-07-09-11-32-28.bpo-12800.fNgWwx.rst
new file mode 100644
index 000000000000000..fdd7c5e74f33a33
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2020-07-09-11-32-28.bpo-12800.fNgWwx.rst
@@ -0,0 +1,4 @@
+Extracting a symlink from a tarball should succeed and overwrite the symlink
+if it already exists. The fix is to remove the existing file or symlink
+before extraction. Based on patch by Chris AtLee, Jeffrey Kintscher, and
+Senthil Kumaran.
diff --git a/Misc/NEWS.d/next/Library/2021-12-06-22-10-53.bpo-43153.J7mjSy.rst b/Misc/NEWS.d/next/Library/2021-12-06-22-10-53.bpo-43153.J7mjSy.rst
new file mode 100644
index 000000000000000..7800e0a4869adff
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2021-12-06-22-10-53.bpo-43153.J7mjSy.rst
@@ -0,0 +1,4 @@
+On Windows, ``tempfile.TemporaryDirectory`` previously masked a
+``PermissionError`` with ``NotADirectoryError`` during directory cleanup. It
+now correctly raises ``PermissionError`` if errors are not ignored. Patch by
+Andrei Kulakov and Ken Jin.
diff --git a/Misc/NEWS.d/next/Library/2022-12-01-16-57-44.gh-issue-91133.LKMVCV.rst b/Misc/NEWS.d/next/Library/2022-12-01-16-57-44.gh-issue-91133.LKMVCV.rst
new file mode 100644
index 000000000000000..7991048fc48e039
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2022-12-01-16-57-44.gh-issue-91133.LKMVCV.rst
@@ -0,0 +1,2 @@
+Fix a bug in :class:`tempfile.TemporaryDirectory` cleanup, which now no longer
+dereferences symlinks when working around file system permission errors.
diff --git a/Misc/NEWS.d/next/Library/2022-12-20-10-55-14.gh-issue-100372.utfP65.rst b/Misc/NEWS.d/next/Library/2022-12-20-10-55-14.gh-issue-100372.utfP65.rst
new file mode 100644
index 000000000000000..ec37aff5092c3af
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2022-12-20-10-55-14.gh-issue-100372.utfP65.rst
@@ -0,0 +1,2 @@
+:meth:`ssl.SSLContext.load_verify_locations` no longer incorrectly accepts
+some cases of trailing data when parsing DER.
diff --git a/Misc/NEWS.d/next/Library/2023-04-26-09-54-25.gh-issue-103848.aDSnpR.rst b/Misc/NEWS.d/next/Library/2023-04-26-09-54-25.gh-issue-103848.aDSnpR.rst
new file mode 100644
index 000000000000000..81e5904aa6cca22
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2023-04-26-09-54-25.gh-issue-103848.aDSnpR.rst
@@ -0,0 +1,2 @@
+Add checks to ensure that ``[`` bracketed ``]`` hosts found by
+:func:`urllib.parse.urlsplit` are of IPv6 or IPvFuture format.
diff --git a/Misc/NEWS.d/next/Library/2023-08-10-17-36-22.gh-issue-107845.dABiMJ.rst b/Misc/NEWS.d/next/Library/2023-08-10-17-36-22.gh-issue-107845.dABiMJ.rst
new file mode 100644
index 000000000000000..32c1fb93f4ab2ce
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2023-08-10-17-36-22.gh-issue-107845.dABiMJ.rst
@@ -0,0 +1,3 @@
+:func:`tarfile.data_filter` now takes the location of symlinks into account
+when determining their target, so it will no longer reject some valid
+tarballs with ``LinkOutsideDestinationError``.
diff --git a/Misc/NEWS.d/next/Library/2023-09-28-13-15-51.gh-issue-109858.43e2dg.rst b/Misc/NEWS.d/next/Library/2023-09-28-13-15-51.gh-issue-109858.43e2dg.rst
new file mode 100644
index 000000000000000..be279caffc46eeb
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2023-09-28-13-15-51.gh-issue-109858.43e2dg.rst
@@ -0,0 +1,3 @@
+Protect :mod:`zipfile` from "quoted-overlap" zipbomb. It now raises
+BadZipFile when try to read an entry that overlaps with other entry or
+central directory.
diff --git a/Misc/NEWS.d/next/Library/2023-12-05-18-57-53.gh-issue-79325.P2vMVK.rst b/Misc/NEWS.d/next/Library/2023-12-05-18-57-53.gh-issue-79325.P2vMVK.rst
new file mode 100644
index 000000000000000..f3c32d27b5fe661
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2023-12-05-18-57-53.gh-issue-79325.P2vMVK.rst
@@ -0,0 +1,2 @@
+Fix an infinite recursion error in :func:`tempfile.TemporaryDirectory`
+cleanup on Windows.
diff --git a/Misc/NEWS.d/next/Library/2024-07-27-16-10-41.gh-issue-121650.nf6oc9.rst b/Misc/NEWS.d/next/Library/2024-07-27-16-10-41.gh-issue-121650.nf6oc9.rst
new file mode 100644
index 000000000000000..83dd28d4ac575b8
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2024-07-27-16-10-41.gh-issue-121650.nf6oc9.rst
@@ -0,0 +1,5 @@
+:mod:`email` headers with embedded newlines are now quoted on output. The
+:mod:`~email.generator` will now refuse to serialize (write) headers that
+are unsafely folded or delimited; see
+:attr:`~email.policy.Policy.verify_generated_headers`. (Contributed by Bas
+Bloemsaat and Petr Viktorin in :gh:`121650`.)
diff --git a/Misc/NEWS.d/next/Library/2025-05-15-14-27-01.gh-issue-134062.fRbJet.rst b/Misc/NEWS.d/next/Library/2025-05-15-14-27-01.gh-issue-134062.fRbJet.rst
new file mode 100644
index 000000000000000..f62a3ec480193d5
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2025-05-15-14-27-01.gh-issue-134062.fRbJet.rst
@@ -0,0 +1,3 @@
+:mod:`ipaddress`: fix collisions in :meth:`~object.__hash__` for
+:class:`~ipaddress.IPv4Network` and :class:`~ipaddress.IPv6Network`
+objects.
diff --git a/Misc/NEWS.d/next/Library/2025-05-28-15-53-27.gh-issue-128840.Nur2pB.rst b/Misc/NEWS.d/next/Library/2025-05-28-15-53-27.gh-issue-128840.Nur2pB.rst
new file mode 100644
index 000000000000000..faff433aa4b86e3
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2025-05-28-15-53-27.gh-issue-128840.Nur2pB.rst
@@ -0,0 +1 @@
+Fix parsing long IPv6 addresses with embedded IPv4 address.
diff --git a/Misc/NEWS.d/next/Library/2025-07-23-00-35-29.gh-issue-130577.c7EITy.rst b/Misc/NEWS.d/next/Library/2025-07-23-00-35-29.gh-issue-130577.c7EITy.rst
new file mode 100644
index 000000000000000..342cabbc865dc46
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2025-07-23-00-35-29.gh-issue-130577.c7EITy.rst
@@ -0,0 +1,3 @@
+:mod:`tarfile` now validates archives to ensure member offsets are
+non-negative. (Contributed by Alexander Enrique Urieles Nieto in
+:gh:`130577`.)
diff --git a/Misc/NEWS.d/next/Library/2025-09-16-19-05-29.gh-issue-138998.URl0Y_.rst b/Misc/NEWS.d/next/Library/2025-09-16-19-05-29.gh-issue-138998.URl0Y_.rst
new file mode 100644
index 000000000000000..ce377abc93c8636
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2025-09-16-19-05-29.gh-issue-138998.URl0Y_.rst
@@ -0,0 +1 @@
+Update bundled libexpat to 2.7.2
diff --git a/Misc/NEWS.d/next/Library/2025-09-25-07-33-43.gh-issue-139312.ygE8AC.rst b/Misc/NEWS.d/next/Library/2025-09-25-07-33-43.gh-issue-139312.ygE8AC.rst
new file mode 100644
index 000000000000000..5178bda379b3e02
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2025-09-25-07-33-43.gh-issue-139312.ygE8AC.rst
@@ -0,0 +1 @@
+Upgrade bundled libexpat to 2.7.3
diff --git a/Misc/NEWS.d/next/Library/2025-11-18-06-35-53.gh-issue-141707.DBmQIy.rst b/Misc/NEWS.d/next/Library/2025-11-18-06-35-53.gh-issue-141707.DBmQIy.rst
new file mode 100644
index 000000000000000..1f5b8ed90b8a904
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2025-11-18-06-35-53.gh-issue-141707.DBmQIy.rst
@@ -0,0 +1,2 @@
+Don't change :class:`tarfile.TarInfo` type from ``AREGTYPE`` to ``DIRTYPE`` when parsing
+GNU long name or link headers.
diff --git a/Misc/NEWS.d/next/Library/2026-01-31-17-15-49.gh-issue-144363.X9f0sU.rst b/Misc/NEWS.d/next/Library/2026-01-31-17-15-49.gh-issue-144363.X9f0sU.rst
new file mode 100644
index 000000000000000..c17cea6613d06b6
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2026-01-31-17-15-49.gh-issue-144363.X9f0sU.rst
@@ -0,0 +1 @@
+Update bundled `libexpat <https://libexpat.github.io/>`_ to 2.7.4
diff --git a/Misc/NEWS.d/next/Library/2026-02-15-00-00-00.gh-issue-144833.TUelo1.rst b/Misc/NEWS.d/next/Library/2026-02-15-00-00-00.gh-issue-144833.TUelo1.rst
new file mode 100644
index 000000000000000..6d5b18f59ee7ea5
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2026-02-15-00-00-00.gh-issue-144833.TUelo1.rst
@@ -0,0 +1,3 @@
+Fixed a use-after-free in :mod:`ssl` when ``SSL_new()`` returns NULL in
+``newPySSLSocket()``. The error was reported via a dangling pointer after the
+object had already been freed.
diff --git a/Misc/NEWS.d/next/Library/2026-03-17-20-52-24.gh-issue-146083.NxZa_c.rst b/Misc/NEWS.d/next/Library/2026-03-17-20-52-24.gh-issue-146083.NxZa_c.rst
new file mode 100644
index 000000000000000..6805a40a03e7346
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2026-03-17-20-52-24.gh-issue-146083.NxZa_c.rst
@@ -0,0 +1 @@
+Update bundled `libexpat <https://libexpat.github.io/>`_ to version 2.7.5.
diff --git a/Misc/NEWS.d/next/Library/2026-06-02-13-10-00.gh-issue-150751.Hk2Vw9.rst b/Misc/NEWS.d/next/Library/2026-06-02-13-10-00.gh-issue-150751.Hk2Vw9.rst
new file mode 100644
index 000000000000000..670d677aff79ff2
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2026-06-02-13-10-00.gh-issue-150751.Hk2Vw9.rst
@@ -0,0 +1,5 @@
+:mod:`http.client` now validates the ``Content-Length`` header and the
+chunked ``chunk-size`` against the RFC 9112 grammar (``1*DIGIT`` and
+``1*HEXDIG``) before parsing them, rejecting values such as ``+5``, ``5_0``
+or a ``0x``-prefixed or negative chunk size that :func:`int` would otherwise
+accept. This avoids framing a response differently from a strict peer.
diff --git a/Misc/NEWS.d/next/Library/2026-07-29-11-20-00.gh-issue-98820.Qm7Hs4.rst b/Misc/NEWS.d/next/Library/2026-07-29-11-20-00.gh-issue-98820.Qm7Hs4.rst
new file mode 100644
index 000000000000000..aa9ae8d937004fe
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2026-07-29-11-20-00.gh-issue-98820.Qm7Hs4.rst
@@ -0,0 +1,2 @@
+Fix quadratic time in :meth:`csv.Sniffer.sniff` for a sample which contains
+quoted fields, in particular for a single column of quoted fields.
diff --git a/Misc/NEWS.d/next/Library/2026-07-29-11-25-00.gh-issue-109638.Vt2Rn9.rst b/Misc/NEWS.d/next/Library/2026-07-29-11-25-00.gh-issue-109638.Vt2Rn9.rst
new file mode 100644
index 000000000000000..f44163a20c8278e
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2026-07-29-11-25-00.gh-issue-109638.Vt2Rn9.rst
@@ -0,0 +1,3 @@
+Fix exponential time in :meth:`csv.Sniffer.sniff` for a sample which contains
+many quote characters. A doubled quote character is now also detected in
+a field which contains the delimiter or a line break.
diff --git a/Misc/NEWS.d/next/Library/2026-08-26-02-30-00.gh-issue-156353.abcdef.rst b/Misc/NEWS.d/next/Library/2026-08-26-02-30-00.gh-issue-156353.abcdef.rst
new file mode 100644
index 000000000000000..0dd925cf596277a
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2026-08-26-02-30-00.gh-issue-156353.abcdef.rst
@@ -0,0 +1 @@
+Fix :mod:`configparser` parsing when using whitespace in *delimiters*.
diff --git a/Misc/NEWS.d/next/Library/2026-09-08-13-06-29.gh-issue-156002.vmOC8T.rst b/Misc/NEWS.d/next/Library/2026-09-08-13-06-29.gh-issue-156002.vmOC8T.rst
new file mode 100644
index 000000000000000..689967bff72fe7c
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2026-09-08-13-06-29.gh-issue-156002.vmOC8T.rst
@@ -0,0 +1,5 @@
+:mod:`zipfile` again reads members through a third-party decompressor
+installed by monkey-patching the private ``_get_decompressor()`` to return an
+object that only implements old BZ2Decompressor API from Python 3.3.
+Note that decompressors without ``needs_input`` and two-argument
+``decompress()`` are vulnerable to CVE 2026-15310.
diff --git a/Misc/NEWS.d/next/Library/2026-10-08-16-18-43.gh-issue-87720.Elqdl9.rst b/Misc/NEWS.d/next/Library/2026-10-08-16-18-43.gh-issue-87720.Elqdl9.rst
new file mode 100644
index 000000000000000..a2df7654b31444b
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2026-10-08-16-18-43.gh-issue-87720.Elqdl9.rst
@@ -0,0 +1,3 @@
+Fixed error in :mod:`email` new policies where header folding failed to
+detect the need to re-quote text obtained from encoded words in certain
+cases.
diff --git a/Misc/NEWS.d/next/Library/2026-10-10-22-14-22.gh-issue-150751.JInuGL.rst b/Misc/NEWS.d/next/Library/2026-10-10-22-14-22.gh-issue-150751.JInuGL.rst
new file mode 100644
index 000000000000000..40c8d606aa67734
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2026-10-10-22-14-22.gh-issue-150751.JInuGL.rst
@@ -0,0 +1,5 @@
+:mod:`http.client` now ignores optional whitespace around the
+``Content-Length`` header value when validating it, as required by
+:rfc:`9112`. Previously a value with trailing whitespace (for example
+``Content-Length: 5`` followed by a space) was treated as missing and the
+response body was read until the connection was closed.
diff --git a/Misc/NEWS.d/next/Security/2021-03-30-16-29-51.bpo-36384.sCAmLs.rst b/Misc/NEWS.d/next/Security/2021-03-30-16-29-51.bpo-36384.sCAmLs.rst
new file mode 100644
index 000000000000000..f956cde948ec57d
--- /dev/null
+++ b/Misc/NEWS.d/next/Security/2021-03-30-16-29-51.bpo-36384.sCAmLs.rst
@@ -0,0 +1,6 @@
+:mod:`ipaddress` module no longer accepts any leading zeros in IPv4 address
+strings. Leading zeros are ambiguous and interpreted as octal notation by
+some libraries. For example the legacy function :func:`socket.inet_aton`
+treats leading zeros as octal notatation. glibc implementation of modern
+:func:`~socket.inet_pton` does not accept any leading zeros. For a while
+the :mod:`ipaddress` module used to accept ambiguous leading zeros.
diff --git a/Misc/NEWS.d/next/Security/2022-09-07-10-42-00.gh-issue-97514.Yggdsl.rst b/Misc/NEWS.d/next/Security/2022-09-07-10-42-00.gh-issue-97514.Yggdsl.rst
new file mode 100644
index 000000000000000..cb280bda7a45b94
--- /dev/null
+++ b/Misc/NEWS.d/next/Security/2022-09-07-10-42-00.gh-issue-97514.Yggdsl.rst
@@ -0,0 +1,5 @@
+CVE-2022-42919 does not affect Python 3.7. Upstream 3.9 made the
+:mod:`multiprocessing` *forkserver* prefer Linux abstract sockets, which have
+no filesystem permissions, so any user on the system could connect to it;
+3.9.16 reverted that. In 3.7, ``AF_UNIX`` listeners have always used a
+filesystem path, so no code change is needed.
diff --git a/Misc/NEWS.d/next/Security/2023-02-13-21-41-34.gh-issue-86155.ppIGSC.rst b/Misc/NEWS.d/next/Security/2023-02-13-21-41-34.gh-issue-86155.ppIGSC.rst
new file mode 100644
index 000000000000000..bb85481b2296970
--- /dev/null
+++ b/Misc/NEWS.d/next/Security/2023-02-13-21-41-34.gh-issue-86155.ppIGSC.rst
@@ -0,0 +1,2 @@
+:meth:`html.parser.HTMLParser.close` no longer loses data when the
+``<script>`` tag is not closed. Patch by Waylan Limberg.
diff --git a/Misc/NEWS.d/next/Security/2024-01-02-19-52-23.gh-issue-113659.DkmnQc.rst b/Misc/NEWS.d/next/Security/2024-01-02-19-52-23.gh-issue-113659.DkmnQc.rst
new file mode 100644
index 000000000000000..744687e72324d13
--- /dev/null
+++ b/Misc/NEWS.d/next/Security/2024-01-02-19-52-23.gh-issue-113659.DkmnQc.rst
@@ -0,0 +1 @@
+Skip ``.pth`` files with names starting with a dot or hidden file attribute.
diff --git a/Misc/NEWS.d/next/Security/2024-02-18-03-14-40.gh-issue-115398.tzvxH8.rst b/Misc/NEWS.d/next/Security/2024-02-18-03-14-40.gh-issue-115398.tzvxH8.rst
new file mode 100644
index 000000000000000..a40fcd35ef99ae4
--- /dev/null
+++ b/Misc/NEWS.d/next/Security/2024-02-18-03-14-40.gh-issue-115398.tzvxH8.rst
@@ -0,0 +1,8 @@
+Allow controlling Expat >=2.6.0 reparse deferral (CVE-2023-52425) by adding
+five new methods:
+
+* :meth:`xml.etree.ElementTree.XMLParser.flush`
+* :meth:`xml.etree.ElementTree.XMLPullParser.flush`
+* :meth:`xml.parsers.expat.xmlparser.GetReparseDeferralEnabled`
+* :meth:`xml.parsers.expat.xmlparser.SetReparseDeferralEnabled`
+* :meth:`xml.sax.expatreader.ExpatParser.flush`
diff --git a/Misc/NEWS.d/next/Security/2024-05-01-20-57-09.gh-issue-118486.K44KJG.rst b/Misc/NEWS.d/next/Security/2024-05-01-20-57-09.gh-issue-118486.K44KJG.rst
new file mode 100644
index 000000000000000..a28a4e5cdb69913
--- /dev/null
+++ b/Misc/NEWS.d/next/Security/2024-05-01-20-57-09.gh-issue-118486.K44KJG.rst
@@ -0,0 +1,4 @@
+:func:`os.mkdir` on Windows now accepts *mode* of ``0o700`` to restrict
+the new directory to the current user. This fixes CVE-2024-4030
+affecting :func:`tempfile.mkdtemp` in scenarios where the base temporary
+directory is more permissive than the default.
diff --git a/Misc/NEWS.d/next/Security/2024-05-21-22-11-31.gh-issue-119342.BTFj4Z.rst b/Misc/NEWS.d/next/Security/2024-05-21-22-11-31.gh-issue-119342.BTFj4Z.rst
new file mode 100644
index 000000000000000..04fd8faca4cf7e0
--- /dev/null
+++ b/Misc/NEWS.d/next/Security/2024-05-21-22-11-31.gh-issue-119342.BTFj4Z.rst
@@ -0,0 +1,5 @@
+Fix a potential memory denial of service in the :mod:`plistlib` module.
+When reading a Plist file received from untrusted source, it could cause
+an arbitrary amount of memory to be allocated.
+This could have led to symptoms including a :exc:`MemoryError`, swapping, out
+of memory (OOM) killed processes or containers, or even system crashes.
diff --git a/Misc/NEWS.d/next/Security/2024-05-23-11-44-41.gh-issue-119452.PRfsSv.rst b/Misc/NEWS.d/next/Security/2024-05-23-11-44-41.gh-issue-119452.PRfsSv.rst
new file mode 100644
index 000000000000000..98956627f2b30df
--- /dev/null
+++ b/Misc/NEWS.d/next/Security/2024-05-23-11-44-41.gh-issue-119452.PRfsSv.rst
@@ -0,0 +1,5 @@
+Fix a potential memory denial of service in the :mod:`http.server` module.
+When a malicious user is connected to the CGI server on Windows, it could cause
+an arbitrary amount of memory to be allocated.
+This could have led to symptoms including a :exc:`MemoryError`, swapping, out
+of memory (OOM) killed processes or containers, or even system crashes.
diff --git a/Misc/NEWS.d/next/Security/2024-05-23-11-47-48.gh-issue-119451.qkJe9-.rst b/Misc/NEWS.d/next/Security/2024-05-23-11-47-48.gh-issue-119451.qkJe9-.rst
new file mode 100644
index 000000000000000..6d6f25cd2f8bf7c
--- /dev/null
+++ b/Misc/NEWS.d/next/Security/2024-05-23-11-47-48.gh-issue-119451.qkJe9-.rst
@@ -0,0 +1,5 @@
+Fix a potential memory denial of service in the :mod:`http.client` module.
+When connecting to a malicious server, it could cause
+an arbitrary amount of memory to be allocated.
+This could have led to symptoms including a :exc:`MemoryError`, swapping, out
+of memory (OOM) killed processes or containers, or even system crashes.
diff --git a/Misc/NEWS.d/next/Security/2024-05-24-21-00-52.gh-issue-119511.jKrXQ8.rst b/Misc/NEWS.d/next/Security/2024-05-24-21-00-52.gh-issue-119511.jKrXQ8.rst
new file mode 100644
index 000000000000000..f7b4031120e643e
--- /dev/null
+++ b/Misc/NEWS.d/next/Security/2024-05-24-21-00-52.gh-issue-119511.jKrXQ8.rst
@@ -0,0 +1,7 @@
+Fix a potential denial of service in the :mod:`imaplib` module. When connecting
+to a malicious server, it could cause an arbitrary amount of memory to be
+allocated. On many systems this is harmless as unused virtual memory is only a
+mapping, but if this hit a virtual address size limit it could lead to a
+:exc:`MemoryError` or other process crash. On unusual systems or builds where
+all allocated memory is touched and backed by actual ram or storage it could've
+consumed resources doing so until similarly crashing.
diff --git a/Misc/NEWS.d/next/Security/2024-07-22-13-11-28.gh-issue-122133.0mPeta.rst b/Misc/NEWS.d/next/Security/2024-07-22-13-11-28.gh-issue-122133.0mPeta.rst
new file mode 100644
index 000000000000000..3544eb3824d0dac
--- /dev/null
+++ b/Misc/NEWS.d/next/Security/2024-07-22-13-11-28.gh-issue-122133.0mPeta.rst
@@ -0,0 +1,5 @@
+Authenticate the socket connection for the ``socket.socketpair()`` fallback
+on platforms where ``AF_UNIX`` is not available like Windows.
+
+Patch by Gregory P. Smith <greg@krypto.org> and Seth Larson <seth@python.org>. Reported by Ellie
+<el@horse64.org>
diff --git a/Misc/NEWS.d/next/Security/2024-08-06-11-43-08.gh-issue-80222.wfR4BU.rst b/Misc/NEWS.d/next/Security/2024-08-06-11-43-08.gh-issue-80222.wfR4BU.rst
new file mode 100644
index 000000000000000..0f0661d0b1cf4ac
--- /dev/null
+++ b/Misc/NEWS.d/next/Security/2024-08-06-11-43-08.gh-issue-80222.wfR4BU.rst
@@ -0,0 +1,6 @@
+Fix bug in the folding of quoted strings when flattening an email message using
+a modern email policy. Previously when a quoted string was folded so that
+it spanned more than one line, the surrounding quotes and internal escapes
+would be omitted. This could theoretically be used to spoof header lines
+using a carefully constructed quoted string if the resulting rendered email
+was transmitted or re-parsed.
diff --git a/Misc/NEWS.d/next/Security/2024-08-06-12-27-34.gh-issue-121284.8rwPxe.rst b/Misc/NEWS.d/next/Security/2024-08-06-12-27-34.gh-issue-121284.8rwPxe.rst
new file mode 100644
index 000000000000000..923e91170d355f5
--- /dev/null
+++ b/Misc/NEWS.d/next/Security/2024-08-06-12-27-34.gh-issue-121284.8rwPxe.rst
@@ -0,0 +1,7 @@
+Fix bug in the folding of rfc2047 encoded-words when flattening an email message
+using a modern email policy. Previously when an encoded-word was too long
+for a line, it would be decoded, split across lines, and re-encoded. But commas
+and other special characters in the original text could be left unencoded and
+unquoted. This could theoretically be used to spoof header lines using
+a carefully constructed encoded-word if the resulting rendered email was
+transmitted or re-parsed.
diff --git a/Misc/NEWS.d/next/Security/2024-08-07-10-42-13.gh-issue-122792.oiTMo9.rst b/Misc/NEWS.d/next/Security/2024-08-07-10-42-13.gh-issue-122792.oiTMo9.rst
new file mode 100644
index 000000000000000..18e293ba0c03b5a
--- /dev/null
+++ b/Misc/NEWS.d/next/Security/2024-08-07-10-42-13.gh-issue-122792.oiTMo9.rst
@@ -0,0 +1,3 @@
+Changed IPv4-mapped ``ipaddress.IPv6Address`` to consistently use the mapped IPv4
+address value for deciding properties. Properties which have their behavior fixed
+are ``is_multicast``, ``is_reserved``, ``is_link_local``, ``is_global``, and ``is_unspecified``.
diff --git a/Misc/NEWS.d/next/Security/2025-01-14-11-19-07.gh-issue-128840.M1doZW.rst b/Misc/NEWS.d/next/Security/2025-01-14-11-19-07.gh-issue-128840.M1doZW.rst
new file mode 100644
index 000000000000000..b57ec3e70dcc5fa
--- /dev/null
+++ b/Misc/NEWS.d/next/Security/2025-01-14-11-19-07.gh-issue-128840.M1doZW.rst
@@ -0,0 +1,2 @@
+Short-circuit the processing of long IPv6 addresses early in :mod:`ipaddress` to prevent excessive
+memory consumption and a minor denial-of-service.
diff --git a/Misc/NEWS.d/next/Security/2025-01-28-14-08-03.gh-issue-105704.EnhHxu.rst b/Misc/NEWS.d/next/Security/2025-01-28-14-08-03.gh-issue-105704.EnhHxu.rst
new file mode 100644
index 000000000000000..bff1bc6b0d609c4
--- /dev/null
+++ b/Misc/NEWS.d/next/Security/2025-01-28-14-08-03.gh-issue-105704.EnhHxu.rst
@@ -0,0 +1,4 @@
+When using :func:`urllib.parse.urlsplit` and :func:`urllib.parse.urlparse` host
+parsing would not reject domain names containing square brackets (``[`` and
+``]``). Square brackets are only valid for IPv6 and IPvFuture hosts according to
+`RFC 3986 Section 3.2.2 <https://www.rfc-editor.org/rfc/rfc3986#section-3.2.2>`__.
diff --git a/Misc/NEWS.d/next/Security/2025-03-14-23-28-39.gh-issue-131261.0aB6nM.rst b/Misc/NEWS.d/next/Security/2025-03-14-23-28-39.gh-issue-131261.0aB6nM.rst
new file mode 100644
index 000000000000000..fa56e7a452bc2ab
--- /dev/null
+++ b/Misc/NEWS.d/next/Security/2025-03-14-23-28-39.gh-issue-131261.0aB6nM.rst
@@ -0,0 +1 @@
+Upgrade to libexpat 2.7.0
diff --git a/Misc/NEWS.d/next/Security/2025-04-07-04-11-08.gh-issue-131809.4MBDuy.rst b/Misc/NEWS.d/next/Security/2025-04-07-04-11-08.gh-issue-131809.4MBDuy.rst
new file mode 100644
index 000000000000000..1421b4d851dd50b
--- /dev/null
+++ b/Misc/NEWS.d/next/Security/2025-04-07-04-11-08.gh-issue-131809.4MBDuy.rst
@@ -0,0 +1 @@
+Update bundled libexpat to 2.7.1
diff --git a/Misc/NEWS.d/next/Security/2025-05-09-20-22-54.gh-issue-133767.kN2i3Q.rst b/Misc/NEWS.d/next/Security/2025-05-09-20-22-54.gh-issue-133767.kN2i3Q.rst
new file mode 100644
index 000000000000000..39d2f1e1a892cf3
--- /dev/null
+++ b/Misc/NEWS.d/next/Security/2025-05-09-20-22-54.gh-issue-133767.kN2i3Q.rst
@@ -0,0 +1,2 @@
+Fix use-after-free in the "unicode-escape" decoder with a non-"strict" error
+handler.
diff --git a/Misc/NEWS.d/next/Security/2025-05-30-22-33-27.gh-issue-136065.bu337o.rst b/Misc/NEWS.d/next/Security/2025-05-30-22-33-27.gh-issue-136065.bu337o.rst
new file mode 100644
index 000000000000000..1d152bb5318380b
--- /dev/null
+++ b/Misc/NEWS.d/next/Security/2025-05-30-22-33-27.gh-issue-136065.bu337o.rst
@@ -0,0 +1 @@
+Fix quadratic complexity in :func:`os.path.expandvars`.
diff --git a/Misc/NEWS.d/next/Security/2025-06-02-11-32-23.gh-issue-135034.RLGjbp.rst b/Misc/NEWS.d/next/Security/2025-06-02-11-32-23.gh-issue-135034.RLGjbp.rst
new file mode 100644
index 000000000000000..e3f984afc215784
--- /dev/null
+++ b/Misc/NEWS.d/next/Security/2025-06-02-11-32-23.gh-issue-135034.RLGjbp.rst
@@ -0,0 +1,6 @@
+Fixes multiple issues that allowed ``tarfile`` extraction filters
+(``filter="data"`` and ``filter="tar"``) to be bypassed using crafted
+symlinks and hard links.
+
+Addresses CVE 2024-12718, CVE 2025-4138, CVE 2025-4330, and CVE 2025-4517.
+
diff --git a/Misc/NEWS.d/next/Security/2025-06-09-20-38-25.gh-issue-118350.KgWCcP.rst b/Misc/NEWS.d/next/Security/2025-06-09-20-38-25.gh-issue-118350.KgWCcP.rst
new file mode 100644
index 000000000000000..6ad3caf33b22012
--- /dev/null
+++ b/Misc/NEWS.d/next/Security/2025-06-09-20-38-25.gh-issue-118350.KgWCcP.rst
@@ -0,0 +1,2 @@
+Fix support of escapable raw text mode (elements "textarea" and "title")
+in :class:`html.parser.HTMLParser`.
diff --git a/Misc/NEWS.d/next/Security/2025-06-13-15-55-22.gh-issue-135462.KBeJpc.rst b/Misc/NEWS.d/next/Security/2025-06-13-15-55-22.gh-issue-135462.KBeJpc.rst
new file mode 100644
index 000000000000000..cf9aa8dbdf2efed
--- /dev/null
+++ b/Misc/NEWS.d/next/Security/2025-06-13-15-55-22.gh-issue-135462.KBeJpc.rst
@@ -0,0 +1,4 @@
+Fix quadratic complexity in processing specially crafted input in
+:class:`html.parser.HTMLParser`. End-of-file errors are now handled according
+to the HTML5 specs -- comments and declarations are automatically closed,
+tags are ignored.
diff --git a/Misc/NEWS.d/next/Security/2025-06-18-13-28-08.gh-issue-102555.nADrzJ.rst b/Misc/NEWS.d/next/Security/2025-06-18-13-28-08.gh-issue-102555.nADrzJ.rst
new file mode 100644
index 000000000000000..71d15ee0852ebdc
--- /dev/null
+++ b/Misc/NEWS.d/next/Security/2025-06-18-13-28-08.gh-issue-102555.nADrzJ.rst
@@ -0,0 +1,3 @@
+Fix comment parsing in :class:`html.parser.HTMLParser` according to the
+HTML5 standard. ``--!>`` now ends the comment. ``-- >`` no longer ends the
+comment. Support abnormally ended empty comments ``<-->`` and ``<--->``.
diff --git a/Misc/NEWS.d/next/Security/2025-06-18-13-34-55.gh-issue-135661.NZlpWf.rst b/Misc/NEWS.d/next/Security/2025-06-18-13-34-55.gh-issue-135661.NZlpWf.rst
new file mode 100644
index 000000000000000..fe000d936aae9d1
--- /dev/null
+++ b/Misc/NEWS.d/next/Security/2025-06-18-13-34-55.gh-issue-135661.NZlpWf.rst
@@ -0,0 +1,5 @@
+Fix CDATA section parsing in :class:`html.parser.HTMLParser` according to
+the HTML5 standard: ``] ]>`` and ``]] >`` no longer end the CDATA section.
+Add private method ``_set_support_cdata()`` which can be used to specify
+how to parse ``<[CDATA[`` --- as a CDATA section in foreign content
+(SVG or MathML) or as a bogus comment in the HTML namespace.
diff --git a/Misc/NEWS.d/next/Security/2025-06-25-14-13-39.gh-issue-135661.idjQ0B.rst b/Misc/NEWS.d/next/Security/2025-06-25-14-13-39.gh-issue-135661.idjQ0B.rst
new file mode 100644
index 000000000000000..27e886abdb58e56
--- /dev/null
+++ b/Misc/NEWS.d/next/Security/2025-06-25-14-13-39.gh-issue-135661.idjQ0B.rst
@@ -0,0 +1,20 @@
+Fix parsing start and end tags in :class:`html.parser.HTMLParser`
+according to the HTML5 standard.
+
+* Whitespaces no longer accepted between ``</`` and the tag name.
+ E.g. ``</ script>`` does not end the script section.
+
+* Vertical tabulation (``\v``) and non-ASCII whitespaces no longer recognized
+ as whitespaces. The only whitespaces are ``\t\n\r\f`` and space.
+
+* Null character (U+0000) no longer ends the tag name.
+
+* Attributes and slashes after the tag name in end tags are now ignored,
+ instead of terminating after the first ``>`` in quoted attribute value.
+ E.g. ``</script/foo=">"/>``.
+
+* Multiple slashes and whitespaces between the last attribute and closing ``>``
+ are now ignored in both start and end tags. E.g. ``<a foo=bar/ //>``.
+
+* Multiple ``=`` between attribute name and value are no longer collapsed.
+ E.g. ``<a foo==bar>`` produces attribute "foo" with value "=bar".
diff --git a/Misc/NEWS.d/next/Security/2025-06-28-13-23-53.gh-issue-136063.aGk0Jv.rst b/Misc/NEWS.d/next/Security/2025-06-28-13-23-53.gh-issue-136063.aGk0Jv.rst
new file mode 100644
index 000000000000000..940a3ad5a72f68a
--- /dev/null
+++ b/Misc/NEWS.d/next/Security/2025-06-28-13-23-53.gh-issue-136063.aGk0Jv.rst
@@ -0,0 +1,2 @@
+:mod:`email.message`: ensure linear complexity for legacy HTTP parameters
+parsing. Patch by Bénédikt Tran.
diff --git a/Misc/NEWS.d/next/Security/2025-07-28-10-35-59.gh-issue-121227.Orp1wf.rst b/Misc/NEWS.d/next/Security/2025-07-28-10-35-59.gh-issue-121227.Orp1wf.rst
new file mode 100644
index 000000000000000..6350f74a396f213
--- /dev/null
+++ b/Misc/NEWS.d/next/Security/2025-07-28-10-35-59.gh-issue-121227.Orp1wf.rst
@@ -0,0 +1,2 @@
+Raise an :exc:`SSL.SSLError` if an empty *protocols* argument is passed to
+:meth:`ssl.SSLContext.set_npn_protocols` to fix ``CVE-2024-5642``.
diff --git a/Misc/NEWS.d/next/Security/2025-08-15-23-08-44.gh-issue-137836.b55rhh.rst b/Misc/NEWS.d/next/Security/2025-08-15-23-08-44.gh-issue-137836.b55rhh.rst
new file mode 100644
index 000000000000000..c30c9439a76a19d
--- /dev/null
+++ b/Misc/NEWS.d/next/Security/2025-08-15-23-08-44.gh-issue-137836.b55rhh.rst
@@ -0,0 +1,3 @@
+Add support of the "plaintext" element, RAWTEXT elements "xmp", "iframe",
+"noembed" and "noframes", and optionally RAWTEXT element "noscript" in
+:class:`html.parser.HTMLParser`.
diff --git a/Misc/NEWS.d/next/Security/2025-09-29-00-01-28.gh-issue-139400.X2T-jO.rst b/Misc/NEWS.d/next/Security/2025-09-29-00-01-28.gh-issue-139400.X2T-jO.rst
new file mode 100644
index 000000000000000..a5dea3b5f8147ab
--- /dev/null
+++ b/Misc/NEWS.d/next/Security/2025-09-29-00-01-28.gh-issue-139400.X2T-jO.rst
@@ -0,0 +1,4 @@
+:mod:`xml.parsers.expat`: Make sure that parent Expat parsers are only
+garbage-collected once they are no longer referenced by subparsers created
+by :meth:`~xml.parsers.expat.xmlparser.ExternalEntityParserCreate`.
+Patch by Sebastian Pipping.
diff --git a/Misc/NEWS.d/next/Security/2025-10-07-19-31-34.gh-issue-139700.vNHU1O.rst b/Misc/NEWS.d/next/Security/2025-10-07-19-31-34.gh-issue-139700.vNHU1O.rst
new file mode 100644
index 000000000000000..a8e7a1f1878c6bb
--- /dev/null
+++ b/Misc/NEWS.d/next/Security/2025-10-07-19-31-34.gh-issue-139700.vNHU1O.rst
@@ -0,0 +1,3 @@
+Check consistency of the zip64 end of central directory record. Support
+records with "zip64 extensible data" if there are no bytes prepended to the
+ZIP file.
diff --git a/Misc/NEWS.d/next/Security/2025-12-01-09-36-45.gh-issue-142145.tcAUhg.rst b/Misc/NEWS.d/next/Security/2025-12-01-09-36-45.gh-issue-142145.tcAUhg.rst
new file mode 100644
index 000000000000000..05c7df35d14bef1
--- /dev/null
+++ b/Misc/NEWS.d/next/Security/2025-12-01-09-36-45.gh-issue-142145.tcAUhg.rst
@@ -0,0 +1,6 @@
+Remove quadratic behavior in ``xml.minidom`` node ID cache clearing. In order
+to do this without breaking existing users, we also add the *ownerDocument*
+attribute to :mod:`xml.dom.minidom` elements and attributes created by directly
+instantiating the ``Element`` or ``Attr`` class. Note that this way of creating
+nodes is not supported; creator functions like
+:py:meth:`xml.dom.Document.documentElement` should be used instead.
diff --git a/Misc/NEWS.d/next/Security/2026-01-16-11-07-36.gh-issue-143916.dpWeOD.rst b/Misc/NEWS.d/next/Security/2026-01-16-11-07-36.gh-issue-143916.dpWeOD.rst
new file mode 100644
index 000000000000000..44bd0b27059f944
--- /dev/null
+++ b/Misc/NEWS.d/next/Security/2026-01-16-11-07-36.gh-issue-143916.dpWeOD.rst
@@ -0,0 +1,2 @@
+Reject C0 control characters within wsgiref.headers.Headers fields, values,
+and parameters.
diff --git a/Misc/NEWS.d/next/Security/2026-01-16-11-13-15.gh-issue-143919.kchwZV.rst b/Misc/NEWS.d/next/Security/2026-01-16-11-13-15.gh-issue-143919.kchwZV.rst
new file mode 100644
index 000000000000000..788c3e4ac2ebf76
--- /dev/null
+++ b/Misc/NEWS.d/next/Security/2026-01-16-11-13-15.gh-issue-143919.kchwZV.rst
@@ -0,0 +1 @@
+Reject control characters in :class:`http.cookies.Morsel` fields and values.
diff --git a/Misc/NEWS.d/next/Security/2026-01-16-11-41-06.gh-issue-143921.AeCOor.rst b/Misc/NEWS.d/next/Security/2026-01-16-11-41-06.gh-issue-143921.AeCOor.rst
new file mode 100644
index 000000000000000..de62e020a95e6b2
--- /dev/null
+++ b/Misc/NEWS.d/next/Security/2026-01-16-11-41-06.gh-issue-143921.AeCOor.rst
@@ -0,0 +1,2 @@
+Reject NUL, CR and LF characters in IMAP commands. Other control
+characters are allowed and sent quoted.
diff --git a/Misc/NEWS.d/next/Security/2026-01-16-11-43-47.gh-issue-143923.DuytMe.rst b/Misc/NEWS.d/next/Security/2026-01-16-11-43-47.gh-issue-143923.DuytMe.rst
new file mode 100644
index 000000000000000..3cde4df3e0069f7
--- /dev/null
+++ b/Misc/NEWS.d/next/Security/2026-01-16-11-43-47.gh-issue-143923.DuytMe.rst
@@ -0,0 +1 @@
+Reject control characters in POP3 commands.
diff --git a/Misc/NEWS.d/next/Security/2026-01-16-11-51-19.gh-issue-143925.mrtcHW.rst b/Misc/NEWS.d/next/Security/2026-01-16-11-51-19.gh-issue-143925.mrtcHW.rst
new file mode 100644
index 000000000000000..46109dfbef3ee77
--- /dev/null
+++ b/Misc/NEWS.d/next/Security/2026-01-16-11-51-19.gh-issue-143925.mrtcHW.rst
@@ -0,0 +1 @@
+Reject control characters in ``data:`` URL media types.
diff --git a/Misc/NEWS.d/next/Security/2026-01-16-11-58-19.gh-issue-143927.aviFeG.rst b/Misc/NEWS.d/next/Security/2026-01-16-11-58-19.gh-issue-143927.aviFeG.rst
new file mode 100644
index 000000000000000..ca554997e5c3963
--- /dev/null
+++ b/Misc/NEWS.d/next/Security/2026-01-16-11-58-19.gh-issue-143927.aviFeG.rst
@@ -0,0 +1,2 @@
+Normalize all line endings (CR, CRLF, and LF) to LF+TAB when writing
+multi-line configparser values.
diff --git a/Misc/NEWS.d/next/Security/2026-01-16-12-04-49.gh-issue-143930.zYC5x3.rst b/Misc/NEWS.d/next/Security/2026-01-16-12-04-49.gh-issue-143930.zYC5x3.rst
new file mode 100644
index 000000000000000..c561023c3c2d7a7
--- /dev/null
+++ b/Misc/NEWS.d/next/Security/2026-01-16-12-04-49.gh-issue-143930.zYC5x3.rst
@@ -0,0 +1 @@
+Reject leading dashes in URLs passed to :func:`webbrowser.open`.
diff --git a/Misc/NEWS.d/next/Security/2026-01-16-14-40-31.gh-issue-143935.U2YtKl.rst b/Misc/NEWS.d/next/Security/2026-01-16-14-40-31.gh-issue-143935.U2YtKl.rst
new file mode 100644
index 000000000000000..c3d864936884ac0
--- /dev/null
+++ b/Misc/NEWS.d/next/Security/2026-01-16-14-40-31.gh-issue-143935.U2YtKl.rst
@@ -0,0 +1,6 @@
+Fixed a bug in the folding of comments when flattening an email message
+using a modern email policy. Comments consisting of a very long sequence of
+non-foldable characters could trigger a forced line wrap that omitted the
+required leading space on the continuation line, causing the remainder of
+the comment to be interpreted as a new header field. This enabled header
+injection with carefully crafted inputs.
diff --git a/Misc/NEWS.d/next/Security/2026-01-21-12-34-05.gh-issue-144125.TAz5uo.rst b/Misc/NEWS.d/next/Security/2026-01-21-12-34-05.gh-issue-144125.TAz5uo.rst
new file mode 100644
index 000000000000000..e6333e724972c59
--- /dev/null
+++ b/Misc/NEWS.d/next/Security/2026-01-21-12-34-05.gh-issue-144125.TAz5uo.rst
@@ -0,0 +1,4 @@
+:mod:`~email.generator.BytesGenerator` will now refuse to serialize (write) headers
+that are unsafely folded or delimited; see
+:attr:`~email.policy.Policy.verify_generated_headers`. (Contributed by Bas
+Bloemsaat and Petr Viktorin in :gh:`121650`).
diff --git a/Misc/NEWS.d/next/Security/2026-01-31-21-56-54.gh-issue-144370.fp9m8t.rst b/Misc/NEWS.d/next/Security/2026-01-31-21-56-54.gh-issue-144370.fp9m8t.rst
new file mode 100644
index 000000000000000..2d13a0611322c52
--- /dev/null
+++ b/Misc/NEWS.d/next/Security/2026-01-31-21-56-54.gh-issue-144370.fp9m8t.rst
@@ -0,0 +1,2 @@
+Disallow usage of control characters in status in :mod:`wsgiref.handlers` to prevent HTTP header injections.
+Patch by Benedikt Johannes.
diff --git a/Misc/NEWS.d/next/Security/2026-03-06-17-03-38.gh-issue-145599.kchwZV.rst b/Misc/NEWS.d/next/Security/2026-03-06-17-03-38.gh-issue-145599.kchwZV.rst
new file mode 100644
index 000000000000000..fc2e503779a2546
--- /dev/null
+++ b/Misc/NEWS.d/next/Security/2026-03-06-17-03-38.gh-issue-145599.kchwZV.rst
@@ -0,0 +1,4 @@
+Reject control characters in :class:`http.cookies.Morsel`
+:meth:`~http.cookies.Morsel.update` and
+:meth:`~http.cookies.BaseCookie.js_output`.
+This addresses `CVE-2026-3644 <https://www.cve.org/CVERecord?id=CVE-2026-3644>`_.
diff --git a/Misc/NEWS.d/next/Security/2026-03-14-17-31-39.gh-issue-145986.ifSSr8.rst b/Misc/NEWS.d/next/Security/2026-03-14-17-31-39.gh-issue-145986.ifSSr8.rst
new file mode 100644
index 000000000000000..cb9dbadb72d976d
--- /dev/null
+++ b/Misc/NEWS.d/next/Security/2026-03-14-17-31-39.gh-issue-145986.ifSSr8.rst
@@ -0,0 +1,4 @@
+:mod:`xml.parsers.expat`: Fixed a crash caused by unbounded C recursion when
+converting deeply nested XML content models with
+:meth:`~xml.parsers.expat.xmlparser.ElementDeclHandler`.
+This addresses `CVE-2026-4224 <https://www.cve.org/CVERecord?id=CVE-2026-4224>`_.
diff --git a/Misc/NEWS.d/next/Security/2026-03-20-09-29-42.gh-issue-146211.PQVbs7.rst b/Misc/NEWS.d/next/Security/2026-03-20-09-29-42.gh-issue-146211.PQVbs7.rst
new file mode 100644
index 000000000000000..4993633b8ebebb2
--- /dev/null
+++ b/Misc/NEWS.d/next/Security/2026-03-20-09-29-42.gh-issue-146211.PQVbs7.rst
@@ -0,0 +1,2 @@
+Reject CR/LF characters in tunnel request headers for the
+HTTPConnection.set_tunnel() method.
diff --git a/Misc/NEWS.d/next/Security/2026-03-25-00-51-03.gh-issue-146333.LqdL__bn.rst b/Misc/NEWS.d/next/Security/2026-03-25-00-51-03.gh-issue-146333.LqdL__bn.rst
new file mode 100644
index 000000000000000..96d86ecc0a0fb3e
--- /dev/null
+++ b/Misc/NEWS.d/next/Security/2026-03-25-00-51-03.gh-issue-146333.LqdL__bn.rst
@@ -0,0 +1,3 @@
+Fix quadratic backtracking in :class:`configparser.RawConfigParser` option
+parsing regexes (``OPTCRE`` and ``OPTCRE_NV``). A crafted configuration line
+with many whitespace characters could cause excessive CPU usage.
diff --git a/Misc/NEWS.d/next/Security/2026-03-29-12-51-33.gh-issue-146581.4vZfB0.rst b/Misc/NEWS.d/next/Security/2026-03-29-12-51-33.gh-issue-146581.4vZfB0.rst
new file mode 100644
index 000000000000000..98e65549d79016f
--- /dev/null
+++ b/Misc/NEWS.d/next/Security/2026-03-29-12-51-33.gh-issue-146581.4vZfB0.rst
@@ -0,0 +1,5 @@
+Fix vulnerability in :func:`shutil.unpack_archive` for ZIP files on Windows
+which allowed to write files outside of the destination tree if the patch in
+the archive contains a Windows drive prefix. Now such invalid paths will be
+skipped. Files containing ".." in the name (like "foo..bar") are no longer
+skipped.
diff --git a/Misc/NEWS.d/next/Security/2026-03-31-09-15-51.gh-issue-148169.EZJzz2.rst b/Misc/NEWS.d/next/Security/2026-03-31-09-15-51.gh-issue-148169.EZJzz2.rst
new file mode 100644
index 000000000000000..45cdeebe1b6d64a
--- /dev/null
+++ b/Misc/NEWS.d/next/Security/2026-03-31-09-15-51.gh-issue-148169.EZJzz2.rst
@@ -0,0 +1,2 @@
+A bypass in :mod:`webbrowser` allowed URLs prefixed with ``%action`` to pass
+the dash-prefix safety check.
diff --git a/Misc/NEWS.d/next/Security/2026-04-10-16-28-21.gh-issue-148395.kfzm0G.rst b/Misc/NEWS.d/next/Security/2026-04-10-16-28-21.gh-issue-148395.kfzm0G.rst
new file mode 100644
index 000000000000000..b095329bd913b03
--- /dev/null
+++ b/Misc/NEWS.d/next/Security/2026-04-10-16-28-21.gh-issue-148395.kfzm0G.rst
@@ -0,0 +1,5 @@
+Fix a dangling input pointer in :class:`lzma.LZMADecompressor`,
+:class:`bz2.BZ2Decompressor`
+when memory allocation fails with :exc:`MemoryError`, which could let a
+subsequent :meth:`!decompress` call read or write through a stale pointer to
+the already-released caller buffer.
diff --git a/Misc/NEWS.d/next/Security/2026-04-26-17-49-58.gh-issue-149017.EiVFPo.rst b/Misc/NEWS.d/next/Security/2026-04-26-17-49-58.gh-issue-149017.EiVFPo.rst
new file mode 100644
index 000000000000000..6aa7efb68a1981e
--- /dev/null
+++ b/Misc/NEWS.d/next/Security/2026-04-26-17-49-58.gh-issue-149017.EiVFPo.rst
@@ -0,0 +1 @@
+Update bundled `libexpat <https://libexpat.github.io/>`_ to version 2.8.0.
diff --git a/Misc/NEWS.d/next/Security/2026-04-26-19-30-45.gh-issue-149018.a9SqWb.rst b/Misc/NEWS.d/next/Security/2026-04-26-19-30-45.gh-issue-149018.a9SqWb.rst
new file mode 100644
index 000000000000000..d1b5b368684e6a5
--- /dev/null
+++ b/Misc/NEWS.d/next/Security/2026-04-26-19-30-45.gh-issue-149018.a9SqWb.rst
@@ -0,0 +1,3 @@
+Improved protection against XML hash-flooding attacks in
+:mod:`xml.parsers.expat` and :mod:`xml.etree.ElementTree` when Python is
+compiled with libExpat 2.8.0 or later.
diff --git a/Misc/NEWS.d/next/Security/2026-04-27-16-36-11.gh-issue-149079.vKl-LM.rst b/Misc/NEWS.d/next/Security/2026-04-27-16-36-11.gh-issue-149079.vKl-LM.rst
new file mode 100644
index 000000000000000..4ed22b58f7405f5
--- /dev/null
+++ b/Misc/NEWS.d/next/Security/2026-04-27-16-36-11.gh-issue-149079.vKl-LM.rst
@@ -0,0 +1,5 @@
+Fix a potential denial of service in :func:`unicodedata.normalize`. The
+canonical ordering step of Unicode normalization used a quadratic-time insertion
+sort for reordering combining characters, which could be exploited with
+crafted input containing many combining characters in non-canonical order.
+Replaced with a linear-time counting sort for long runs.
diff --git a/Misc/NEWS.d/next/Security/2026-05-03-21-00-00.gh-issue-149486.tarflt.rst b/Misc/NEWS.d/next/Security/2026-05-03-21-00-00.gh-issue-149486.tarflt.rst
new file mode 100644
index 000000000000000..7c69edb683cf80a
--- /dev/null
+++ b/Misc/NEWS.d/next/Security/2026-05-03-21-00-00.gh-issue-149486.tarflt.rst
@@ -0,0 +1,5 @@
+:func:`tarfile.data_filter` now validates link targets using the same
+normalised value that is written to disk, strips trailing separators from
+the member name when resolving a symlink's directory, and rejects link
+members that would replace the destination directory itself. This closes
+several path-traversal bypasses of the ``data`` extraction filter.
diff --git a/Misc/NEWS.d/next/Security/2026-05-10-18-05-32.gh-issue-87451.XkKB6M.rst b/Misc/NEWS.d/next/Security/2026-05-10-18-05-32.gh-issue-87451.XkKB6M.rst
new file mode 100644
index 000000000000000..21a79c3e0e7db74
--- /dev/null
+++ b/Misc/NEWS.d/next/Security/2026-05-10-18-05-32.gh-issue-87451.XkKB6M.rst
@@ -0,0 +1,6 @@
+The :mod:`ftplib` module's undocumented ``ftpcp`` function no longer trusts
+the IPv4 address value returned from the source server in response to the
+``PASV`` command by default, completing the fix for CVE-2021-4189. As with
+:class:`ftplib.FTP`, the former behavior can be re-enabled by setting the
+``trust_server_pasv_ipv4_address`` attribute on the source :class:`ftplib.FTP`
+instance to ``True``. Thanks to Qi Deng at Aurascape AI for the report.
diff --git a/Misc/NEWS.d/next/Security/2026-05-11-21-15-07.gh-issue-149698.OudOcW.rst b/Misc/NEWS.d/next/Security/2026-05-11-21-15-07.gh-issue-149698.OudOcW.rst
new file mode 100644
index 000000000000000..31e106997378092
--- /dev/null
+++ b/Misc/NEWS.d/next/Security/2026-05-11-21-15-07.gh-issue-149698.OudOcW.rst
@@ -0,0 +1,2 @@
+Update bundled `libexpat <https://libexpat.github.io/>`_ to version 2.8.1
+for the fix for CVE-2026-45186.
diff --git a/Misc/NEWS.d/next/Security/2026-05-30-00-00-00.gh-issue-150743.httpdos.rst b/Misc/NEWS.d/next/Security/2026-05-30-00-00-00.gh-issue-150743.httpdos.rst
new file mode 100644
index 000000000000000..4cc4e4b092da71c
--- /dev/null
+++ b/Misc/NEWS.d/next/Security/2026-05-30-00-00-00.gh-issue-150743.httpdos.rst
@@ -0,0 +1,6 @@
+:mod:`http.client` now limits the number of chunked-response trailer lines
+it will read to 100, and the number of interim (1xx) responses it will
+skip to 100. A malicious or broken server could previously stream trailer
+lines or ``100 Continue`` responses forever, hanging the client even when
+a socket timeout was in use.
+Reported by ``@YLChen-007`` via GHSA-w4q2-g22w-6fr4.
diff --git a/Misc/NEWS.d/next/Security/2026-05-30-09-36-20.gh-issue-150599.nlHqU-.rst b/Misc/NEWS.d/next/Security/2026-05-30-09-36-20.gh-issue-150599.nlHqU-.rst
new file mode 100644
index 000000000000000..a37d86cf423f820
--- /dev/null
+++ b/Misc/NEWS.d/next/Security/2026-05-30-09-36-20.gh-issue-150599.nlHqU-.rst
@@ -0,0 +1,3 @@
+Fix a possible stack buffer overflow in :mod:`bz2` when a
+:class:`bz2.BZ2Decompressor` is reused after a decompression error.
+The decompressor now becomes unusable after libbz2 reports an error.
diff --git a/Misc/NEWS.d/next/Security/2026-06-10-13-08-19.gh-issue-151558.mL74i2.rst b/Misc/NEWS.d/next/Security/2026-06-10-13-08-19.gh-issue-151558.mL74i2.rst
new file mode 100644
index 000000000000000..575081600f8bdd9
--- /dev/null
+++ b/Misc/NEWS.d/next/Security/2026-06-10-13-08-19.gh-issue-151558.mL74i2.rst
@@ -0,0 +1,3 @@
+Fixed an vulnerability in the :mod:`tarfile` ``data`` and ``tar`` extraction
+filters where crafted archives could create a symlink pointing outside the
+destination directory. This was a bypass of CVE-2025-4330.
diff --git a/Misc/NEWS.d/next/Security/2026-06-23-13-28-16.gh-issue-151981.xBHEcU.rst b/Misc/NEWS.d/next/Security/2026-06-23-13-28-16.gh-issue-151981.xBHEcU.rst
new file mode 100644
index 000000000000000..2123ab8e081b1d9
--- /dev/null
+++ b/Misc/NEWS.d/next/Security/2026-06-23-13-28-16.gh-issue-151981.xBHEcU.rst
@@ -0,0 +1,2 @@
+In :mod:`tarfile`, seeking a stream now stops when end of the stream is
+reached.
diff --git a/Misc/NEWS.d/next/Security/2026-06-23-14-19-30.gh-issue-151987.8mNIMf.rst b/Misc/NEWS.d/next/Security/2026-06-23-14-19-30.gh-issue-151987.8mNIMf.rst
new file mode 100644
index 000000000000000..9eea7b32c4d2b4f
--- /dev/null
+++ b/Misc/NEWS.d/next/Security/2026-06-23-14-19-30.gh-issue-151987.8mNIMf.rst
@@ -0,0 +1,2 @@
+The :meth:`tarfile.TarFile.extract` method now applies the given filter when
+it extracts a link target from the archive as a fallback.
diff --git a/Misc/NEWS.d/next/Security/2026-06-25-20-44-20.gh-issue-152216.VxiK5y.rst b/Misc/NEWS.d/next/Security/2026-06-25-20-44-20.gh-issue-152216.VxiK5y.rst
new file mode 100644
index 000000000000000..2769048e001d694
--- /dev/null
+++ b/Misc/NEWS.d/next/Security/2026-06-25-20-44-20.gh-issue-152216.VxiK5y.rst
@@ -0,0 +1 @@
+Update bundled `libexpat <https://libexpat.github.io/>`_ to version 2.8.2.
diff --git a/Misc/NEWS.d/next/Security/2026-06-30-13-24-13.gh-issue-152674.-2QVoL.rst b/Misc/NEWS.d/next/Security/2026-06-30-13-24-13.gh-issue-152674.-2QVoL.rst
new file mode 100644
index 000000000000000..69e73008af79f14
--- /dev/null
+++ b/Misc/NEWS.d/next/Security/2026-06-30-13-24-13.gh-issue-152674.-2QVoL.rst
@@ -0,0 +1,6 @@
+The :class:`xml.etree.ElementTree.Element` methods
+:meth:`~xml.etree.ElementTree.Element.findall`,
+:meth:`~xml.etree.ElementTree.Element.iterfind` and
+:meth:`~xml.etree.ElementTree.Element.find` avoid quadratic behavior when
+using XPath index predicates (``[1]``, ``[last()]``, ``[last()-N]``) on XML
+documents with many same-tag siblings.
diff --git a/Misc/NEWS.d/next/Security/2026-07-04-17-00-00.gh-issue-153030.RovkP6.rst b/Misc/NEWS.d/next/Security/2026-07-04-17-00-00.gh-issue-153030.RovkP6.rst
new file mode 100644
index 000000000000000..d1d60593f4ba7d2
--- /dev/null
+++ b/Misc/NEWS.d/next/Security/2026-07-04-17-00-00.gh-issue-153030.RovkP6.rst
@@ -0,0 +1,3 @@
+Fixed quadratic complexity in incremental parsing of long unterminated
+constructs (such as tags or comments) in :class:`html.parser.HTMLParser`,
+which could be exploited for a denial of service.
diff --git a/Misc/NEWS.d/next/Security/2026-07-31-16-20-17.gh-issue-155694.SsxlKG.rst b/Misc/NEWS.d/next/Security/2026-07-31-16-20-17.gh-issue-155694.SsxlKG.rst
new file mode 100644
index 000000000000000..e85d482715a83c4
--- /dev/null
+++ b/Misc/NEWS.d/next/Security/2026-07-31-16-20-17.gh-issue-155694.SsxlKG.rst
@@ -0,0 +1,4 @@
+Fix `CVE-2026-15806 <https://www.cve.org/CVERecord?id=CVE-2026-15806>`_ by scoping :class:`~urllib.request.HTTPPasswordMgr`
+credentials to the URL scheme, preventing credentials stored for an HTTPS
+URL from being used for a matching HTTP URL, while URIs without a scheme
+continue to match any scheme.
diff --git a/Misc/NEWS.d/next/Security/2026-08-06-11-43-20.gh-issue-155292.j4pHBO.rst b/Misc/NEWS.d/next/Security/2026-08-06-11-43-20.gh-issue-155292.j4pHBO.rst
new file mode 100644
index 000000000000000..7a81a8ba1eef1cd
--- /dev/null
+++ b/Misc/NEWS.d/next/Security/2026-08-06-11-43-20.gh-issue-155292.j4pHBO.rst
@@ -0,0 +1,2 @@
+Change the :mod:`stringprep` module and :mod:`encodings.idna` codec to not
+consider Unicode codepoint attributes beyond those defined in :rfc:`3454`.
diff --git a/Misc/NEWS.d/next/Security/2026-08-10-12-00-00.gh-issue-156293.sNIcbk.rst b/Misc/NEWS.d/next/Security/2026-08-10-12-00-00.gh-issue-156293.sNIcbk.rst
new file mode 100644
index 000000000000000..75b4ad2e3bc81fa
--- /dev/null
+++ b/Misc/NEWS.d/next/Security/2026-08-10-12-00-00.gh-issue-156293.sNIcbk.rst
@@ -0,0 +1,7 @@
+Fix a crash in :mod:`ssl` when an :attr:`~ssl.SSLContext.sni_callback`
+switches a connection to another :class:`~ssl.SSLContext` and the context
+that carries the callback is no longer referenced by the application.
+Servers that keep their ``sni_callback`` context alive (the usual case when
+it wraps the listening socket or is stored on the server object) were not
+affected.
+This addresses `CVE-2026-19445 <https://www.cve.org/CVERecord?id=CVE-2026-19445>`_.
diff --git a/Misc/NEWS.d/next/Security/2026-08-13-13-08-11.gh-issue-155999.Xt4rWq.rst b/Misc/NEWS.d/next/Security/2026-08-13-13-08-11.gh-issue-155999.Xt4rWq.rst
new file mode 100644
index 000000000000000..59b725e55bbffda
--- /dev/null
+++ b/Misc/NEWS.d/next/Security/2026-08-13-13-08-11.gh-issue-155999.Xt4rWq.rst
@@ -0,0 +1,5 @@
+Fix the :mod:`tarfile` ``tar`` and ``data`` extraction filters creating
+directories outside the destination for members whose name leaves the
+destination and returns to it, such as ``../evil/../dest/sub/file``. The
+containment check used the resolved path, but intermediate directories were
+created from the name as given.
diff --git a/Misc/NEWS.d/next/Security/2026-08-14-11-55-00.gh-issue-157579.Kq3Vt2.rst b/Misc/NEWS.d/next/Security/2026-08-14-11-55-00.gh-issue-157579.Kq3Vt2.rst
new file mode 100644
index 000000000000000..2dbe33be81d2f9e
--- /dev/null
+++ b/Misc/NEWS.d/next/Security/2026-08-14-11-55-00.gh-issue-157579.Kq3Vt2.rst
@@ -0,0 +1,6 @@
+Fix a race condition in the cleanup of :class:`tempfile.TemporaryDirectory`.
+When working around file system permission errors, files are now removed
+relative to open directory file descriptors instead of resolving their path
+again, so that replacing a directory of the tree with a symbolic link can no
+longer make the cleanup delete files outside of the temporary directory.
+This addresses CVE-2026-12345.
diff --git a/Misc/NEWS.d/next/Security/2026-08-18-13-54-05.gh-issue-156002.CcWXPP.rst b/Misc/NEWS.d/next/Security/2026-08-18-13-54-05.gh-issue-156002.CcWXPP.rst
new file mode 100644
index 000000000000000..4e49ad5ce8fa00a
--- /dev/null
+++ b/Misc/NEWS.d/next/Security/2026-08-18-13-54-05.gh-issue-156002.CcWXPP.rst
@@ -0,0 +1,4 @@
+Bound the amount of data :mod:`zipfile` decompresses per read for members
+compressed with bzip2, LZMA, or Zstandard, matching the existing limit for
+deflate. A small archive member could previously expand into an unbounded
+allocation even when read in small chunks.
diff --git a/Misc/NEWS.d/next/Security/2026-08-31-16-47-33.gh-issue-157953.KxCF5N.rst b/Misc/NEWS.d/next/Security/2026-08-31-16-47-33.gh-issue-157953.KxCF5N.rst
new file mode 100644
index 000000000000000..5b3ea77cb368632
--- /dev/null
+++ b/Misc/NEWS.d/next/Security/2026-08-31-16-47-33.gh-issue-157953.KxCF5N.rst
@@ -0,0 +1 @@
+Update bundled `libexpat <https://libexpat.github.io/>`_ to version 2.8.5.
diff --git a/Misc/NEWS.d/next/Security/2026-08-31-16-47-33.gh-issue-157953.KxCFSN.rst b/Misc/NEWS.d/next/Security/2026-08-31-16-47-33.gh-issue-157953.KxCFSN.rst
new file mode 100644
index 000000000000000..3220f4613a91af6
--- /dev/null
+++ b/Misc/NEWS.d/next/Security/2026-08-31-16-47-33.gh-issue-157953.KxCFSN.rst
@@ -0,0 +1 @@
+Update bundled `libexpat <https://libexpat.github.io/>`_ to version 2.9.0.
diff --git a/Misc/NEWS.d/next/Security/2026-09-06-11-02-46.gh-issue-157190.tarhln.rst b/Misc/NEWS.d/next/Security/2026-09-06-11-02-46.gh-issue-157190.tarhln.rst
new file mode 100644
index 000000000000000..c3aac4084c1859c
--- /dev/null
+++ b/Misc/NEWS.d/next/Security/2026-09-06-11-02-46.gh-issue-157190.tarhln.rst
@@ -0,0 +1,5 @@
+Fixed a vulnerability in the :mod:`tarfile` ``data`` and ``tar`` extraction
+filters where a crafted archive using a hard link to a symbolic link could
+change the permissions and modification time of a file outside the
+destination directory, and expose its contents inside the extracted tree.
+This addresses CVE 2026-82049.
diff --git a/Misc/NEWS.d/next/Security/2026-09-10-13-38-11.gh-issue-157265.-vYuMp.rst b/Misc/NEWS.d/next/Security/2026-09-10-13-38-11.gh-issue-157265.-vYuMp.rst
new file mode 100644
index 000000000000000..ba27e47f734bfb1
--- /dev/null
+++ b/Misc/NEWS.d/next/Security/2026-09-10-13-38-11.gh-issue-157265.-vYuMp.rst
@@ -0,0 +1,3 @@
+In :mod:`tarfile`, when extracting a link falls back to extracting a member
+of the archive, skip the member when the filter function returns None when
+called with the extracted member's name replaced with the link's.
diff --git a/Misc/NEWS.d/next/Security/2026-09-16-14-05-19.gh-issue-156793.Qa1T5Z.rst b/Misc/NEWS.d/next/Security/2026-09-16-14-05-19.gh-issue-156793.Qa1T5Z.rst
new file mode 100644
index 000000000000000..ce43a563f3a1df0
--- /dev/null
+++ b/Misc/NEWS.d/next/Security/2026-09-16-14-05-19.gh-issue-156793.Qa1T5Z.rst
@@ -0,0 +1,9 @@
+:meth:`ssl.SSLContext.wrap_bio` now validates its *server_side*,
+*server_hostname* and *session* arguments similar to
+:meth:`ssl.SSLContext.wrap_socket`, but for backward compatiblity reasons
+emits :exc:`DeprecationWarning` instead of :exc:`ValueError`.
+
+In particular, a context with :attr:`~ssl.SSLContext.check_hostname` enabled
+and no *server_hostname* passed to :meth:`!wrap_bio` now emits
+:exc:`DeprecationWarning` to indicate the hostname wasn't checked.
+(In Python 3.13 and later, this raises :exc:`ValueError`.)
diff --git a/Misc/NEWS.d/next/Security/2026-09-23-11-34-30.gh-issue-156793.zC_AjF.rst b/Misc/NEWS.d/next/Security/2026-09-23-11-34-30.gh-issue-156793.zC_AjF.rst
new file mode 100644
index 000000000000000..9afd6e9c454aaff
--- /dev/null
+++ b/Misc/NEWS.d/next/Security/2026-09-23-11-34-30.gh-issue-156793.zC_AjF.rst
@@ -0,0 +1,6 @@
+:mod:`asyncio`: :meth:`loop.start_tls() <asyncio.loop.start_tls>` and
+:meth:`loop.create_connection() <asyncio.loop.create_connection>` now
+validate the *server_hostname* argument if an :class:`ssl.SSLContext` is
+passed with *check_hostname* set to ``True``, emitting
+:exc:`DeprecationWarning` if *server_hostname* is missing. (This will raise
+:exc:`ValueError` in Python 3.13 and later.)
diff --git a/Misc/NEWS.d/next/Security/2026-09-29-16-32-46.gh-issue-158446.dToaPr.rst b/Misc/NEWS.d/next/Security/2026-09-29-16-32-46.gh-issue-158446.dToaPr.rst
new file mode 100644
index 000000000000000..f17a3f68b93e855
--- /dev/null
+++ b/Misc/NEWS.d/next/Security/2026-09-29-16-32-46.gh-issue-158446.dToaPr.rst
@@ -0,0 +1,5 @@
+Fix a crash or incorrect output that could occur when formatting a
+:class:`float` or :class:`complex` with a precision close to the platform's
+``INT_MAX``. :c:func:`PyOS_double_to_string` now raises :exc:`ValueError` for
+any precision of that magnitude, regardless of presentation type or value, as
+the format string parsers already did for precisions above ``INT_MAX``.
diff --git a/Misc/NEWS.d/next/Security/2026-10-15-03-47-08.gh-issue-159428.XHRKwY.rst b/Misc/NEWS.d/next/Security/2026-10-15-03-47-08.gh-issue-159428.XHRKwY.rst
new file mode 100644
index 000000000000000..d481ed2b74a20ea
--- /dev/null
+++ b/Misc/NEWS.d/next/Security/2026-10-15-03-47-08.gh-issue-159428.XHRKwY.rst
@@ -0,0 +1,2 @@
+:mod:`mimetypes` no longer reads the files listed in :data:`mimetypes.knownfiles`
+by default on Windows, and uses only the registry.
diff --git a/Misc/NEWS.d/next/Security/2026-10-21-16-55-07.gh-issue-160040.9PjFjC.rst b/Misc/NEWS.d/next/Security/2026-10-21-16-55-07.gh-issue-160040.9PjFjC.rst
new file mode 100644
index 000000000000000..7bcec7f5808cac6
--- /dev/null
+++ b/Misc/NEWS.d/next/Security/2026-10-21-16-55-07.gh-issue-160040.9PjFjC.rst
@@ -0,0 +1,12 @@
+.. cve: CVE-2026-115557
+
+:mod:`ssl` now rejects TLS certificate chains that contain any of a set of
+compromised eMudhra intermediate CA keys. This is an extraordinary measure and
+a last line of defense: removing trust is primarily the job of the platform root
+store, and the built-in denylist only blocks connections while that update has
+not yet reached the OS, a container image, or a bundled trust store in various
+Python packages. Entries are added to the denylist only under a very high
+threshold.
+The denylist matches on the public key (the SHA-256 digest of the :rfc:`5280`
+``SubjectPublicKeyInfo``), so it also covers the cross-signed certificates that
+reuse one of the distrusted keys.
diff --git a/Misc/NEWS.d/next/Security/2026-10-21-19-58-53.gh-issue-160097.-opfAL.rst b/Misc/NEWS.d/next/Security/2026-10-21-19-58-53.gh-issue-160097.-opfAL.rst
new file mode 100644
index 000000000000000..b48c8f7213ee826
--- /dev/null
+++ b/Misc/NEWS.d/next/Security/2026-10-21-19-58-53.gh-issue-160097.-opfAL.rst
@@ -0,0 +1,5 @@
+.. cve: CVE-2026-114583
+
+:func:`tarfile.tar_filter` now rejects link members whose path resolves to
+the destination directory itself, as :func:`tarfile.data_filter` already
+did.
diff --git a/Misc/NEWS.d/next/Security/2026-10-22-08-04-33.gh-issue-160300.q7Zk2B.rst b/Misc/NEWS.d/next/Security/2026-10-22-08-04-33.gh-issue-160300.q7Zk2B.rst
new file mode 100644
index 000000000000000..3cadc6c2c4fd1fb
--- /dev/null
+++ b/Misc/NEWS.d/next/Security/2026-10-22-08-04-33.gh-issue-160300.q7Zk2B.rst
@@ -0,0 +1,3 @@
+Fix :class:`http.cookiejar.DefaultCookiePolicy` accepting and returning
+cookies with a negative ``Version`` attribute without checking their domain
+against the request host. Such cookies are now ignored.
diff --git a/Misc/NEWS.d/next/Security/2026-10-29-10-30-00.gh-issue-135374.As37St.rst b/Misc/NEWS.d/next/Security/2026-10-29-10-30-00.gh-issue-135374.As37St.rst
new file mode 100644
index 000000000000000..0b71fc54b2728a4
--- /dev/null
+++ b/Misc/NEWS.d/next/Security/2026-10-29-10-30-00.gh-issue-135374.As37St.rst
@@ -0,0 +1,4 @@
+Upgrade the bundled setuptools in :mod:`ensurepip` from 47.1.0 to
+68.0.0.post3, the ActiveState security release of setuptools 68.0.0 (the
+last setuptools that supports Python 3.7). It fixes CVE-2022-40897,
+CVE-2024-6345, CVE-2025-47273 and CVE-2026-59890, and GHSA-xmj3-9gf7-h52w.
diff --git a/Misc/NEWS.d/next/Security/2026-10-29-11-30-00.gh-issue-160040.As37Pp.rst b/Misc/NEWS.d/next/Security/2026-10-29-11-30-00.gh-issue-160040.As37Pp.rst
new file mode 100644
index 000000000000000..8061e656452845b
--- /dev/null
+++ b/Misc/NEWS.d/next/Security/2026-10-29-11-30-00.gh-issue-160040.As37Pp.rst
@@ -0,0 +1,9 @@
+Upgrade the bundled pip in :mod:`ensurepip` from 23.0.1 to 24.0.post4, the
+ActiveState security release of pip 24.0 (the last pip that supports Python
+3.7). Its vendored certifi CA bundle (2026.10.23) drops the emSign roots of
+the compromised eMudhra/emSign CA except emSign Root CA - G1; it fixes
+CVE-2023-5752, CVE-2025-8869, CVE-2026-1703, CVE-2026-3219, CVE-2026-6357,
+CVE-2026-8643 and CVE-2026-13346, and its vendored urllib3 and requests fix
+CVE-2023-45803, CVE-2024-37891, CVE-2025-50181, CVE-2025-66418,
+CVE-2025-66471, CVE-2026-21441, CVE-2026-44431, CVE-2026-97687,
+CVE-2026-97689, CVE-2024-47081 and CVE-2026-25645.
diff --git a/Misc/NEWS.d/next/Tests/2024-03-24-23-49-25.gh-issue-117187.eMLT5n.rst b/Misc/NEWS.d/next/Tests/2024-03-24-23-49-25.gh-issue-117187.eMLT5n.rst
new file mode 100644
index 000000000000000..0c0b0e0f4433969
--- /dev/null
+++ b/Misc/NEWS.d/next/Tests/2024-03-24-23-49-25.gh-issue-117187.eMLT5n.rst
@@ -0,0 +1 @@
+Fix XML tests for vanilla Expat <2.6.0.
diff --git a/Misc/NEWS.d/next/macOS/2025-10-17-01-07-03.gh-issue-137586.kVzxvp.rst b/Misc/NEWS.d/next/macOS/2025-10-17-01-07-03.gh-issue-137586.kVzxvp.rst
new file mode 100644
index 000000000000000..8e42065392a2de6
--- /dev/null
+++ b/Misc/NEWS.d/next/macOS/2025-10-17-01-07-03.gh-issue-137586.kVzxvp.rst
@@ -0,0 +1 @@
+Invoke :program:`osascript` with absolute path in :mod:`webbrowser` and :mod:`!turtledemo`.
diff --git a/Modules/Setup.dist b/Modules/Setup.dist
index 344e03ada990620..9af7b666823a748 100644
--- a/Modules/Setup.dist
+++ b/Modules/Setup.dist
@@ -340,7 +340,7 @@ _symtable symtablemodule.c
# Interface to the Expat XML parser
# More information on Expat can be found at www.libexpat.org.
#
-#pyexpat expat/xmlparse.c expat/xmlrole.c expat/xmltok.c pyexpat.c -I$(srcdir)/Modules/expat -DHAVE_EXPAT_CONFIG_H -DXML_POOR_ENTROPY -DUSE_PYEXPAT_CAPI
+#pyexpat expat/xcs.c expat/xmlparse.c expat/xmlrole.c expat/xmltok.c pyexpat.c -I$(srcdir)/Modules/expat -DHAVE_EXPAT_CONFIG_H -DXML_POOR_ENTROPY -DUSE_PYEXPAT_CAPI
# Hye-Shik Chang's CJKCodecs
diff --git a/Modules/_bz2module.c b/Modules/_bz2module.c
index f0d9588fe55d6f9..ca3ccb753c38d22 100644
--- a/Modules/_bz2module.c
+++ b/Modules/_bz2module.c
@@ -40,6 +40,7 @@ typedef struct {
typedef struct {
PyObject_HEAD
bz_stream bzs;
+ int bzerror;
char eof; /* T_BOOL expects a char */
PyObject *unused_data;
char needs_input;
@@ -436,8 +437,11 @@ decompress_buf(BZ2Decompressor *d, Py_ssize_t max_length)
data_size = bzs->next_out - PyBytes_AS_STRING(result);
d->bzs_avail_in_real += bzs->avail_in;
Py_END_ALLOW_THREADS
- if (catch_bz2_error(bzret))
+ if (catch_bz2_error(bzret)) {
+ d->bzerror = bzret;
+ d->needs_input = 0;
goto error;
+ }
if (bzret == BZ_STREAM_END) {
d->eof = 1;
break;
@@ -570,6 +574,7 @@ decompress(BZ2Decompressor *d, char *data, size_t len, Py_ssize_t max_length)
return result;
error:
+ bzs->next_in = NULL;
Py_XDECREF(result);
return NULL;
}
@@ -604,10 +609,17 @@ _bz2_BZ2Decompressor_decompress_impl(BZ2Decompressor *self, Py_buffer *data,
PyObject *result = NULL;
ACQUIRE_LOCK(self);
- if (self->eof)
+ if (self->eof) {
PyErr_SetString(PyExc_EOFError, "End of stream already reached");
- else
+ }
+ else if (self->bzerror) {
+ // Re-entering BZ2_bzDecompress() after an error can write out of bounds.
+ PyErr_SetString(PyExc_ValueError,
+ "Decompressor is unusable after a previous error");
+ }
+ else {
result = decompress(self, data->buf, data->len, max_length);
+ }
RELEASE_LOCK(self);
return result;
}
@@ -644,6 +656,7 @@ _bz2_BZ2Decompressor___init___impl(BZ2Decompressor *self)
}
self->lock = lock;
+ self->bzerror = 0;
self->needs_input = 1;
self->bzs_avail_in_real = 0;
self->input_buffer = NULL;
diff --git a/Modules/_elementtree.c b/Modules/_elementtree.c
index 9d5e635b9d75f42..019e139c090b67b 100644
--- a/Modules/_elementtree.c
+++ b/Modules/_elementtree.c
@@ -3341,8 +3341,12 @@ _elementtree_XMLParser___init___impl(XMLParserObject *self, PyObject *html,
PyErr_NoMemory();
return -1;
}
- /* expat < 2.1.0 has no XML_SetHashSalt() */
- if (EXPAT(SetHashSalt) != NULL) {
+ // Prefer 16-byte entropy, only expat >= 2.8.0. See gh-149018
+ if (EXPAT(SetHashSalt16Bytes) != NULL) {
+ EXPAT(SetHashSalt16Bytes)(self->parser,
+ _Py_HashSecret.expat.hashsalt16);
+ }
+ else if (EXPAT(SetHashSalt) != NULL) {
EXPAT(SetHashSalt)(self->parser,
(unsigned long)_Py_HashSecret.expat.hashsalt);
}
@@ -3542,6 +3546,38 @@ _elementtree_XMLParser_close_impl(XMLParserObject *self)
}
}
+/*[clinic input]
+_elementtree.XMLParser.flush
+
+[clinic start generated code]*/
+
+static PyObject *
+_elementtree_XMLParser_flush_impl(XMLParserObject *self)
+/*[clinic end generated code: output=42fdb8795ca24509 input=effbecdb28715949]*/
+{
+ if (!_check_xmlparser(self)) {
+ return NULL;
+ }
+
+ if (EXPAT(SetReparseDeferralEnabled) == NULL) {
+ Py_RETURN_NONE;
+ }
+
+ // NOTE: The Expat parser in the C implementation of ElementTree is not
+ // exposed to the outside; as a result we known that reparse deferral
+ // is currently enabled, or we would not even have access to function
+ // XML_SetReparseDeferralEnabled in the first place (which we checked
+ // for, a few lines up).
+
+ EXPAT(SetReparseDeferralEnabled)(self->parser, XML_FALSE);
+
+ PyObject *res = expat_parse(self, "", 0, XML_FALSE);
+
+ EXPAT(SetReparseDeferralEnabled)(self->parser, XML_TRUE);
+
+ return res;
+}
+
/*[clinic input]
_elementtree.XMLParser.feed
@@ -3988,6 +4024,7 @@ static PyTypeObject TreeBuilder_Type = {
static PyMethodDef xmlparser_methods[] = {
_ELEMENTTREE_XMLPARSER_FEED_METHODDEF
_ELEMENTTREE_XMLPARSER_CLOSE_METHODDEF
+ _ELEMENTTREE_XMLPARSER_FLUSH_METHODDEF
_ELEMENTTREE_XMLPARSER__PARSE_WHOLE_METHODDEF
_ELEMENTTREE_XMLPARSER__SETEVENTS_METHODDEF
_ELEMENTTREE_XMLPARSER_DOCTYPE_METHODDEF
diff --git a/Modules/_lzmamodule.c b/Modules/_lzmamodule.c
index ec370897495083c..ed4ff203947aa2b 100644
--- a/Modules/_lzmamodule.c
+++ b/Modules/_lzmamodule.c
@@ -1052,6 +1052,7 @@ decompress(Decompressor *d, uint8_t *data, size_t len, Py_ssize_t max_length)
return result;
error:
+ lzs->next_in = NULL;
Py_XDECREF(result);
return NULL;
}
diff --git a/Modules/_ssl.c b/Modules/_ssl.c
index 1458888d20ce6d6..cebefa8576ed50e 100644
--- a/Modules/_ssl.c
+++ b/Modules/_ssl.c
@@ -56,6 +56,7 @@ static PySocketModule_APIObject PySocketModule;
/* Include OpenSSL header files */
#include "openssl/rsa.h"
#include "openssl/crypto.h"
+#include "openssl/evp.h"
#include "openssl/x509.h"
#include "openssl/x509v3.h"
#include "openssl/pem.h"
@@ -942,8 +943,8 @@ newPySSLSocket(PySSLContext *sslctx, PySocketSockObject *sock,
self->ssl = SSL_new(ctx);
PySSL_END_ALLOW_THREADS
if (self->ssl == NULL) {
- Py_DECREF(self);
_setSSLError(NULL, 0, __FILE__, __LINE__);
+ Py_DECREF(self);
return NULL;
}
SSL_set_app_data(self->ssl, self);
@@ -971,6 +972,7 @@ newPySSLSocket(PySSLContext *sslctx, PySocketSockObject *sock,
int (*verify_cb)(int, X509_STORE_CTX *) = NULL;
verify_cb = SSL_get_verify_callback(self->ssl);
mode |= SSL_VERIFY_POST_HANDSHAKE;
+ /* Keep ctx verify callback (distrusted SPKI check) */
SSL_set_verify(self->ssl, mode, verify_cb);
}
} else {
@@ -3003,11 +3005,70 @@ static PyTypeObject PySSLSocket_Type = {
* _SSLContext objects
*/
+#include "_ssl_distrusted_spki.h"
+
+/* Returns 1 if the SHA-256 of the cert's SPKI is on the distrusted list,
+ 0 if not (okay), -1 on an internal error. */
+static int
+_cert_has_distrusted_spki(X509 *cert)
+{
+ unsigned char *spki = NULL;
+ int spki_len = i2d_X509_PUBKEY(X509_get_X509_PUBKEY(cert), &spki);
+ if (spki_len <= 0 || spki == NULL) {
+ return -1;
+ }
+
+ unsigned char digest[EVP_MAX_MD_SIZE];
+ unsigned int digest_len = 0;
+ int ok = EVP_Digest(spki, (size_t)spki_len, digest, &digest_len,
+ EVP_sha256(), NULL);
+ OPENSSL_free(spki);
+ assert(sizeof(_distrusted_spki[0]) == SHA256_DIGEST_LENGTH);
+ if (!ok
+ || digest_len != SHA256_DIGEST_LENGTH
+ || sizeof(_distrusted_spki[0]) != SHA256_DIGEST_LENGTH) {
+ return -1;
+ }
+
+ for (size_t i = 0; i < Py_ARRAY_LENGTH(_distrusted_spki); i++) {
+ if (memcmp(digest, _distrusted_spki[i], SHA256_DIGEST_LENGTH) == 0) {
+ return 1;
+ }
+ }
+ return 0;
+}
+
+/* Verify callback, called once per cert. Reject distrusted SPKIs. */
+static int
+_ssl_verify_callback(int preverify_ok, X509_STORE_CTX *x509_ctx)
+{
+ /* Don't override an error OpenSSL already reported for this certificate. */
+ if (!preverify_ok) {
+ return preverify_ok;
+ }
+ X509 *cert = X509_STORE_CTX_get_current_cert(x509_ctx);
+ if (cert == NULL) {
+ return preverify_ok;
+ }
+ switch (_cert_has_distrusted_spki(cert)) {
+ case 0:
+ /* okay: not on the distrusted list */
+ return preverify_ok;
+ case 1:
+ /* rejected: public key is on the distrusted list */
+ X509_STORE_CTX_set_error(x509_ctx, X509_V_ERR_CERT_REJECTED);
+ return 0;
+ default:
+ /* internal error */
+ X509_STORE_CTX_set_error(x509_ctx, X509_V_ERR_UNSPECIFIED);
+ return 0;
+ }
+}
+
static int
_set_verify_mode(PySSLContext *self, enum py_ssl_cert_requirements n)
{
int mode;
- int (*verify_cb)(int, X509_STORE_CTX *) = NULL;
switch(n) {
case PY_SSL_CERT_NONE:
@@ -3028,9 +3089,7 @@ _set_verify_mode(PySSLContext *self, enum py_ssl_cert_requirements n)
/* bpo-37428: newPySSLSocket() sets SSL_VERIFY_POST_HANDSHAKE flag for
* server sockets and SSL_set_post_handshake_auth() for client. */
- /* keep current verify cb */
- verify_cb = SSL_CTX_get_verify_callback(self->ctx);
- SSL_CTX_set_verify(self->ctx, mode, verify_cb);
+ SSL_CTX_set_verify(self->ctx, mode, _ssl_verify_callback);
return 0;
}
@@ -3259,6 +3318,9 @@ context_dealloc(PySSLContext *self)
/* bpo-31095: UnTrack is needed before calling any callbacks */
PyObject_GC_UnTrack(self);
context_clear(self);
+ /* The SSL_CTX may outlive this object as the session_ctx of sockets that
+ were switched to another context; leave no Python callback behind. */
+ SSL_CTX_set_tlsext_servername_callback(self->ctx, NULL);
SSL_CTX_free(self->ctx);
#if HAVE_NPN
PyMem_FREE(self->npn_protocols);
@@ -3976,7 +4038,7 @@ _add_ca_certs(PySSLContext *self, void *data, Py_ssize_t len,
{
BIO *biobuf = NULL;
X509_STORE *store;
- int retval = 0, err, loaded = 0;
+ int retval = 0, err, loaded = 0, was_bio_eof = 0;
assert(filetype == SSL_FILETYPE_ASN1 || filetype == SSL_FILETYPE_PEM);
@@ -4004,6 +4066,10 @@ _add_ca_certs(PySSLContext *self, void *data, Py_ssize_t len,
int r;
if (filetype == SSL_FILETYPE_ASN1) {
+ if (BIO_eof(biobuf)) {
+ was_bio_eof = 1;
+ break;
+ }
cert = d2i_X509_bio(biobuf, NULL);
} else {
cert = PEM_read_bio_X509(biobuf, NULL,
@@ -4032,8 +4098,7 @@ _add_ca_certs(PySSLContext *self, void *data, Py_ssize_t len,
err = ERR_peek_last_error();
if ((filetype == SSL_FILETYPE_ASN1) &&
(loaded > 0) &&
- (ERR_GET_LIB(err) == ERR_LIB_ASN1) &&
- (ERR_GET_REASON(err) == ASN1_R_HEADER_TOO_LONG)) {
+ was_bio_eof) {
/* EOF ASN1 file, not an error */
ERR_clear_error();
retval = 0;
@@ -4382,10 +4447,11 @@ _ssl__SSLContext_set_ecdh_curve(PySSLContext *self, PyObject *name)
#if HAVE_SNI && !defined(OPENSSL_NO_TLSEXT)
static int
-_servername_callback(SSL *s, int *al, void *args)
+_servername_callback(SSL *s, int *al, void *Py_UNUSED(args))
{
int ret;
- PySSLContext *ssl_ctx = (PySSLContext *) args;
+ PySSLContext *ssl_ctx;
+ PyObject *sni_cb;
PySSLSocket *ssl;
PyObject *result;
/* The high-level ssl.SSLSocket object */
@@ -4393,15 +4459,26 @@ _servername_callback(SSL *s, int *al, void *args)
const char *servername = SSL_get_servername(s, TLSEXT_NAMETYPE_host_name);
PyGILState_STATE gstate = PyGILState_Ensure();
- if (ssl_ctx->set_sni_cb == NULL) {
- /* remove race condition in this the call back while if removing the
- * callback is in progress */
+ /* Do not use the SSL_CTX's servername arg to find the context: it is a
+ borrowed pointer to whichever _SSLContext installed the callback, and
+ that object may already be gone while OpenSSL still reaches this
+ callback through the connection's session_ctx (e.g. on the second
+ ClientHello after a HelloRetryRequest, once sni_callback has switched
+ the socket to another context). The socket's current context is
+ always alive; hold strong references to it and to the callback while
+ they are used here. */
+ ssl = SSL_get_app_data(s);
+ assert(ssl != NULL);
+ assert(PySSLSocket_Check(ssl));
+ ssl_ctx = ssl->ctx;
+ Py_INCREF(ssl_ctx);
+ sni_cb = ssl_ctx->set_sni_cb;
+ if (sni_cb == NULL) {
+ Py_DECREF(ssl_ctx);
PyGILState_Release(gstate);
return SSL_TLSEXT_ERR_OK;
}
-
- ssl = SSL_get_app_data(s);
- assert(PySSLSocket_Check(ssl));
+ Py_INCREF(sni_cb);
/* The servername callback expects an argument that represents the current
* SSL connection and that has a .context attribute that can be changed to
@@ -4423,7 +4500,7 @@ _servername_callback(SSL *s, int *al, void *args)
goto error;
if (servername == NULL) {
- result = PyObject_CallFunctionObjArgs(ssl_ctx->set_sni_cb, ssl_socket,
+ result = PyObject_CallFunctionObjArgs(sni_cb, ssl_socket,
Py_None, ssl_ctx, NULL);
}
else {
@@ -4445,14 +4522,14 @@ _servername_callback(SSL *s, int *al, void *args)
goto error;
}
result = PyObject_CallFunctionObjArgs(
- ssl_ctx->set_sni_cb, ssl_socket, servername_str,
+ sni_cb, ssl_socket, servername_str,
ssl_ctx, NULL);
Py_DECREF(servername_str);
}
Py_DECREF(ssl_socket);
if (result == NULL) {
- PyErr_WriteUnraisable(ssl_ctx->set_sni_cb);
+ PyErr_WriteUnraisable(sni_cb);
*al = SSL_AD_HANDSHAKE_FAILURE;
ret = SSL_TLSEXT_ERR_ALERT_FATAL;
}
@@ -4473,11 +4550,15 @@ _servername_callback(SSL *s, int *al, void *args)
Py_DECREF(result);
}
+ Py_DECREF(sni_cb);
+ Py_DECREF(ssl_ctx);
PyGILState_Release(gstate);
return ret;
error:
Py_DECREF(ssl_socket);
+ Py_DECREF(sni_cb);
+ Py_DECREF(ssl_ctx);
*al = SSL_AD_INTERNAL_ERROR;
ret = SSL_TLSEXT_ERR_ALERT_FATAL;
PyGILState_Release(gstate);
@@ -4519,7 +4600,6 @@ set_sni_callback(PySSLContext *self, PyObject *arg, void *c)
Py_INCREF(arg);
self->set_sni_cb = arg;
SSL_CTX_set_tlsext_servername_callback(self->ctx, _servername_callback);
- SSL_CTX_set_tlsext_servername_arg(self->ctx, self);
}
return 0;
#else
diff --git a/Modules/_ssl_distrusted_spki.h b/Modules/_ssl_distrusted_spki.h
new file mode 100644
index 000000000000000..bbdb56dcfe671ed
--- /dev/null
+++ b/Modules/_ssl_distrusted_spki.h
@@ -0,0 +1,75 @@
+/* Denylist of distrusted SubjectPublicKeyInfo (SPKI).
+ *
+ * Verification rejects any chain with a cert whose SPKI SHA-256 digest is in
+ * the _distrusted_spki table. We match the key, not a cert fingerprint, so
+ * cross-signs that reuse the key are blocked too.
+ *
+ * This is an extraordinary measure and last line of defense. Removing trust is
+ * primarily the job of the platform root store. This list only blocks keys while
+ * that update is still propagating to the OS, container images, or root CAs
+ * bundled in Python packages. The threshold for adding an entry is very high.
+ *
+ * See https://github.com/python/cpython/issues/160040 for the eMudhra discussion.
+ *
+ * To block another key, add the SHA-256 digest bytes of its SPKI:
+ *
+ * $ openssl x509 -in cert.pem -noout -pubkey | openssl pkey -pubin -outform DER | openssl dgst -sha256 -binary | xxd -i
+ */
+#ifndef Py_SSL_DISTRUSTED_SPKI_H
+#define Py_SSL_DISTRUSTED_SPKI_H
+
+#include "openssl/sha.h"
+
+static const unsigned char _distrusted_spki[][SHA256_DIGEST_LENGTH] = {
+ /* Compromised eMudhra intermediates, https://emudhra.we-have-a-problem.com/ */
+ /* CN=EM OV TLS CA - G3B,O=eMudhra BV,C=NL */
+ {
+ 0x43, 0xbf, 0xfa, 0x19, 0xd6, 0xef, 0x65, 0x14, 0x51, 0xf6, 0x84, 0x1e,
+ 0x01, 0x99, 0x31, 0xe9, 0x6f, 0x6f, 0x2e, 0xae, 0xff, 0x28, 0xab, 0x4b,
+ 0x6f, 0x1c, 0xe4, 0x57, 0x6d, 0xe2, 0x71, 0x7b,
+ },
+ /* CN=EM DV TLS CA - G3C,O=eMudhra Inc,C=US */
+ {
+ 0x49, 0xdd, 0x35, 0x3d, 0xc2, 0xb9, 0x19, 0xa3, 0x77, 0x06, 0x45, 0x11,
+ 0x7d, 0x0e, 0x60, 0xc1, 0x61, 0x15, 0xe7, 0xd1, 0xbc, 0x25, 0x9e, 0xef,
+ 0xe7, 0x34, 0xb0, 0x0e, 0xa4, 0x34, 0xfe, 0x5f,
+ },
+ /* CN=EM EV TLS CA - G3C,O=eMudhra Inc,C=US */
+ {
+ 0x5c, 0x06, 0x60, 0xa2, 0xb1, 0x64, 0x91, 0x0d, 0x8f, 0xf5, 0x89, 0xc1,
+ 0x8e, 0x14, 0x55, 0x2e, 0x9e, 0x0e, 0xc2, 0x37, 0xdc, 0xef, 0xe2, 0xcb,
+ 0x73, 0x6d, 0x41, 0xa4, 0x81, 0x1f, 0x81, 0xa0,
+ },
+ /* CN=EM OV TLS CA - G3C,O=eMudhra Inc,C=US */
+ {
+ 0x63, 0xec, 0xef, 0xee, 0x43, 0x1d, 0x49, 0x52, 0x3b, 0xc1, 0x13, 0x6c,
+ 0x4e, 0x72, 0xd0, 0x20, 0x6a, 0x47, 0x70, 0x86, 0xca, 0xf6, 0x06, 0xcd,
+ 0x5c, 0xf0, 0x25, 0x68, 0x6c, 0x10, 0x9c, 0xf8,
+ },
+ /* CN=EM OV CS CA - G3C,O=eMudhra Inc,C=US */
+ {
+ 0x92, 0xc0, 0x63, 0xd6, 0x7e, 0x8f, 0x14, 0xdb, 0x71, 0xd2, 0x88, 0xe1,
+ 0x12, 0x66, 0xff, 0xda, 0xf6, 0x9c, 0xbf, 0x11, 0x5f, 0x7b, 0xab, 0x14,
+ 0xf3, 0xbc, 0x2f, 0x68, 0xd2, 0x5e, 0x0e, 0x30,
+ },
+ /* CN=EM EV TLS CA - G3B,O=eMudhra BV,C=NL */
+ {
+ 0xe0, 0x25, 0x4c, 0x7f, 0xcd, 0xc2, 0x65, 0xe2, 0xf0, 0xf8, 0xa4, 0x28,
+ 0xda, 0xbb, 0x75, 0xab, 0x4a, 0xe0, 0x39, 0xb4, 0x87, 0x2d, 0x62, 0xf5,
+ 0xee, 0x57, 0xf0, 0x87, 0x7d, 0x5a, 0x8a, 0x41,
+ },
+ /* CN=EM DV TLS CA - G3B,O=eMudhra BV,C=NL */
+ {
+ 0xeb, 0xed, 0x64, 0x75, 0xc2, 0xc2, 0xd8, 0x0f, 0xcb, 0x19, 0x9f, 0x3f,
+ 0xe8, 0xdd, 0x6e, 0x6b, 0x3c, 0xe6, 0x0b, 0x77, 0xeb, 0x15, 0xd4, 0xe4,
+ 0x2c, 0x3d, 0xec, 0x0c, 0x6f, 0x5b, 0x51, 0x05,
+ },
+ /* CN=EM OV CS CA - G3B,O=eMudhra BV,C=NL */
+ {
+ 0xec, 0x4f, 0xfa, 0x1a, 0xd0, 0xac, 0x88, 0x08, 0x13, 0x14, 0xb7, 0xaa,
+ 0xff, 0xef, 0x6a, 0x91, 0xf9, 0x7c, 0xd5, 0x72, 0x91, 0xbb, 0x5b, 0xa6,
+ 0x86, 0xc5, 0x41, 0x5f, 0x5e, 0xb4, 0xf6, 0xb6,
+ },
+};
+
+#endif /* !Py_SSL_DISTRUSTED_SPKI_H */
diff --git a/Modules/cjkcodecs/_codecs_iso2022.c b/Modules/cjkcodecs/_codecs_iso2022.c
index 7394cf67e0e7dd5..6d906ecdd396c29 100644
--- a/Modules/cjkcodecs/_codecs_iso2022.c
+++ b/Modules/cjkcodecs/_codecs_iso2022.c
@@ -181,8 +181,9 @@ ENCODER(iso2022)
encoded = MAP_UNMAPPABLE;
for (dsg = CONFIG_DESIGNATIONS; dsg->mark; dsg++) {
+ Py_UCS4 buf[2] = {c, 0};
Py_ssize_t length = 1;
- encoded = dsg->encoder(&c, &length);
+ encoded = dsg->encoder(buf, &length);
if (encoded == MAP_MULTIPLE_AVAIL) {
/* this implementation won't work for pair
* of non-bmp characters. */
@@ -191,9 +192,11 @@ ENCODER(iso2022)
return MBERR_TOOFEW;
length = -1;
}
- else
+ else {
+ buf[1] = INCHAR2;
length = 2;
- encoded = dsg->encoder(&c, &length);
+ }
+ encoded = dsg->encoder(buf, &length);
if (encoded != MAP_UNMAPPABLE) {
insize = length;
break;
diff --git a/Modules/clinic/_elementtree.c.h b/Modules/clinic/_elementtree.c.h
index 6b558876dd0bd89..c49f684334c59ad 100644
--- a/Modules/clinic/_elementtree.c.h
+++ b/Modules/clinic/_elementtree.c.h
@@ -672,6 +672,23 @@ _elementtree_XMLParser_close(XMLParserObject *self, PyObject *Py_UNUSED(ignored)
return _elementtree_XMLParser_close_impl(self);
}
+PyDoc_STRVAR(_elementtree_XMLParser_flush__doc__,
+"flush($self, /)\n"
+"--\n"
+"\n");
+
+#define _ELEMENTTREE_XMLPARSER_FLUSH_METHODDEF \
+ {"flush", (PyCFunction)_elementtree_XMLParser_flush, METH_NOARGS, _elementtree_XMLParser_flush__doc__},
+
+static PyObject *
+_elementtree_XMLParser_flush_impl(XMLParserObject *self);
+
+static PyObject *
+_elementtree_XMLParser_flush(XMLParserObject *self, PyObject *Py_UNUSED(ignored))
+{
+ return _elementtree_XMLParser_flush_impl(self);
+}
+
PyDoc_STRVAR(_elementtree_XMLParser_feed__doc__,
"feed($self, data, /)\n"
"--\n"
@@ -749,4 +766,4 @@ _elementtree_XMLParser__setevents(XMLParserObject *self, PyObject *const *args,
exit:
return return_value;
}
-/*[clinic end generated code: output=c5a85a88bbb5cc06 input=a9049054013a1b77]*/
+/*[clinic end generated code: output=437833d15831087d input=a9049054013a1b77]*/
diff --git a/Modules/clinic/posixmodule.c.h b/Modules/clinic/posixmodule.c.h
index 22eef6893b82afa..ab36f3e949298a3 100644
--- a/Modules/clinic/posixmodule.c.h
+++ b/Modules/clinic/posixmodule.c.h
@@ -1076,7 +1076,8 @@ PyDoc_STRVAR(os_mkdir__doc__,
"dir_fd may not be implemented on your platform.\n"
" If it is unavailable, using it will raise a NotImplementedError.\n"
"\n"
-"The mode argument is ignored on Windows.");
+"On Windows, a mode of 0o700 restricts access to the new directory to the\n"
+"current user and administrators; other values of mode are ignored.");
#define OS_MKDIR_METHODDEF \
{"mkdir", (PyCFunction)os_mkdir, METH_FASTCALL|METH_KEYWORDS, os_mkdir__doc__},
@@ -6538,4 +6539,4 @@ os_getrandom(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject
#ifndef OS_GETRANDOM_METHODDEF
#define OS_GETRANDOM_METHODDEF
#endif /* !defined(OS_GETRANDOM_METHODDEF) */
-/*[clinic end generated code: output=f6eff86ac86bfce4 input=a9049054013a1b77]*/
+/*[clinic end generated code: output=af01a92e9a887611 input=a9049054013a1b77]*/
diff --git a/Modules/clinic/pyexpat.c.h b/Modules/clinic/pyexpat.c.h
index 777b6f311458120..44196bde427f746 100644
--- a/Modules/clinic/pyexpat.c.h
+++ b/Modules/clinic/pyexpat.c.h
@@ -2,6 +2,52 @@
preserve
[clinic start generated code]*/
+PyDoc_STRVAR(pyexpat_xmlparser_SetReparseDeferralEnabled__doc__,
+"SetReparseDeferralEnabled($self, enabled, /)\n"
+"--\n"
+"\n"
+"Enable/Disable reparse deferral; enabled by default with Expat >=2.6.0.");
+
+#define PYEXPAT_XMLPARSER_SETREPARSEDEFERRALENABLED_METHODDEF \
+ {"SetReparseDeferralEnabled", (PyCFunction)pyexpat_xmlparser_SetReparseDeferralEnabled, METH_O, pyexpat_xmlparser_SetReparseDeferralEnabled__doc__},
+
+static PyObject *
+pyexpat_xmlparser_SetReparseDeferralEnabled_impl(xmlparseobject *self,
+ int enabled);
+
+static PyObject *
+pyexpat_xmlparser_SetReparseDeferralEnabled(xmlparseobject *self, PyObject *arg)
+{
+ PyObject *return_value = NULL;
+ int enabled;
+
+ if (!PyArg_Parse(arg, "p:SetReparseDeferralEnabled", &enabled)) {
+ goto exit;
+ }
+ return_value = pyexpat_xmlparser_SetReparseDeferralEnabled_impl(self, enabled);
+
+exit:
+ return return_value;
+}
+
+PyDoc_STRVAR(pyexpat_xmlparser_GetReparseDeferralEnabled__doc__,
+"GetReparseDeferralEnabled($self, /)\n"
+"--\n"
+"\n"
+"Retrieve reparse deferral enabled status; always returns false with Expat <2.6.0.");
+
+#define PYEXPAT_XMLPARSER_GETREPARSEDEFERRALENABLED_METHODDEF \
+ {"GetReparseDeferralEnabled", (PyCFunction)pyexpat_xmlparser_GetReparseDeferralEnabled, METH_NOARGS, pyexpat_xmlparser_GetReparseDeferralEnabled__doc__},
+
+static PyObject *
+pyexpat_xmlparser_GetReparseDeferralEnabled_impl(xmlparseobject *self);
+
+static PyObject *
+pyexpat_xmlparser_GetReparseDeferralEnabled(xmlparseobject *self, PyObject *Py_UNUSED(ignored))
+{
+ return pyexpat_xmlparser_GetReparseDeferralEnabled_impl(self);
+}
+
PyDoc_STRVAR(pyexpat_xmlparser_Parse__doc__,
"Parse($self, data, isfinal=False, /)\n"
"--\n"
@@ -289,4 +335,4 @@ pyexpat_ErrorString(PyObject *module, PyObject *arg)
#ifndef PYEXPAT_XMLPARSER_USEFOREIGNDTD_METHODDEF
#define PYEXPAT_XMLPARSER_USEFOREIGNDTD_METHODDEF
#endif /* !defined(PYEXPAT_XMLPARSER_USEFOREIGNDTD_METHODDEF) */
-/*[clinic end generated code: output=34d02345deee104c input=a9049054013a1b77]*/
+/*[clinic end generated code: output=68f2282f9ca14a2f input=a9049054013a1b77]*/
diff --git a/Modules/expat/COPYING b/Modules/expat/COPYING
index ce9e5939291e45f..c6d184a8aae845c 100644
--- a/Modules/expat/COPYING
+++ b/Modules/expat/COPYING
@@ -1,5 +1,5 @@
Copyright (c) 1998-2000 Thai Open Source Software Center Ltd and Clark Cooper
-Copyright (c) 2001-2022 Expat maintainers
+Copyright (c) 2001-2025 Expat maintainers
Permission is hereby granted, free of charge, to any person obtaining
a copy of this software and associated documentation files (the
diff --git a/Modules/expat/ascii.h b/Modules/expat/ascii.h
index 1f594d2e54b4d29..1d9cf70bf695da1 100644
--- a/Modules/expat/ascii.h
+++ b/Modules/expat/ascii.h
@@ -31,6 +31,8 @@
DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+ SPDX-License-Identifier: MIT
*/
#define ASCII_A 0x41
diff --git a/Modules/expat/asciitab.h b/Modules/expat/asciitab.h
index af766fb24785ea3..43af0876ccd3f8a 100644
--- a/Modules/expat/asciitab.h
+++ b/Modules/expat/asciitab.h
@@ -30,6 +30,8 @@
DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+ SPDX-License-Identifier: MIT
*/
/* 0x00 */ BT_NONXML, BT_NONXML, BT_NONXML, BT_NONXML,
diff --git a/Modules/expat/expat.h b/Modules/expat/expat.h
index 523b37d8d5787d8..1b2852dd76cb487 100644
--- a/Modules/expat/expat.h
+++ b/Modules/expat/expat.h
@@ -11,7 +11,7 @@
Copyright (c) 2000-2005 Fred L. Drake, Jr. <fdrake@users.sourceforge.net>
Copyright (c) 2001-2002 Greg Stein <gstein@users.sourceforge.net>
Copyright (c) 2002-2016 Karl Waclawek <karl@waclawek.net>
- Copyright (c) 2016-2024 Sebastian Pipping <sebastian@pipping.org>
+ Copyright (c) 2016-2026 Sebastian Pipping <sebastian@pipping.org>
Copyright (c) 2016 Cristian Rodríguez <crrodriguez@opensuse.org>
Copyright (c) 2016 Thomas Beutlich <tc@tbeu.de>
Copyright (c) 2017 Rhodri James <rhodri@wildebeest.org.uk>
@@ -19,6 +19,8 @@
Copyright (c) 2023 Hanno Böck <hanno@gentoo.org>
Copyright (c) 2023 Sony Corporation / Snild Dolkow <snild@sony.com>
Copyright (c) 2024 Taichi Haradaguchi <20001722@ymail.ne.jp>
+ Copyright (c) 2025 Matthew Fernandez <matthew.fernandez@gmail.com>
+ Copyright (c) 2026 Braian Plaku <braianplaku@gmail.com>
Licensed under the MIT license:
Permission is hereby granted, free of charge, to any person obtaining
@@ -39,24 +41,27 @@
DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+ SPDX-License-Identifier: MIT
*/
#ifndef Expat_INCLUDED
-#define Expat_INCLUDED 1
+# define Expat_INCLUDED 1
-#include <stdlib.h>
-#include "expat_external.h"
+# include <stdint.h> // for int64_t, uint8_t, uint64_t
+# include <stdlib.h>
+# include "expat_external.h"
-#ifdef __cplusplus
+# ifdef __cplusplus
extern "C" {
-#endif
+# endif
struct XML_ParserStruct;
typedef struct XML_ParserStruct *XML_Parser;
typedef unsigned char XML_Bool;
-#define XML_TRUE ((XML_Bool)1)
-#define XML_FALSE ((XML_Bool)0)
+# define XML_TRUE ((XML_Bool)1)
+# define XML_FALSE ((XML_Bool)0)
/* The XML_Status enum gives the possible return values for several
API functions. The preprocessor #defines are included so this
@@ -73,11 +78,11 @@ typedef unsigned char XML_Bool;
*/
enum XML_Status {
XML_STATUS_ERROR = 0,
-#define XML_STATUS_ERROR XML_STATUS_ERROR
+# define XML_STATUS_ERROR XML_STATUS_ERROR
XML_STATUS_OK = 1,
-#define XML_STATUS_OK XML_STATUS_OK
+# define XML_STATUS_OK XML_STATUS_OK
XML_STATUS_SUSPENDED = 2
-#define XML_STATUS_SUSPENDED XML_STATUS_SUSPENDED
+# define XML_STATUS_SUSPENDED XML_STATUS_SUSPENDED
};
enum XML_Error {
@@ -135,6 +140,26 @@ enum XML_Error {
XML_ERROR_NOT_STARTED,
};
+/* Added in 2.9.0. */
+enum XML_Prop_Error {
+ XML_PROP_ERROR_NONE = 0, // i.e. success
+
+ XML_PROP_ERROR_INVALID_KEY = 1, // i.e. the property is not known at all
+ XML_PROP_ERROR_INVALID_TYPE = 2, // i.e. the property is known but expects a
+ // different type (so a different
+ // getter/setter would need to be called)
+ XML_PROP_ERROR_INVALID_VALUE = 3, // i.e. the value is invalid (with a setter
+ // function) or a NULL-pointer (with a
+ // getter function)
+
+ XML_PROP_ERROR_PARSER_NULL = 4, // i.e. the parser argument is NULL
+ XML_PROP_ERROR_PARSER_NOT_ROOT = 5, // i.e. the parser is not a root parser
+ // but a subparser (and a root parser is
+ // needed)
+
+ /* potentially more error codes upcoming here */
+};
+
enum XML_Content_Type {
XML_CTYPE_EMPTY = 1,
XML_CTYPE_ANY,
@@ -276,7 +301,7 @@ XML_ParserCreate_MM(const XML_Char *encoding,
/* Prepare a parser object to be reused. This is particularly
valuable when memory allocation overhead is disproportionately high,
- such as when a large number of small documnents need to be parsed.
+ such as when a large number of small documents need to be parsed.
All handlers are cleared from the parser, except for the
unknownEncodingHandler. The parser's external state is re-initialized
except for the values of ns and ns_triplets.
@@ -680,7 +705,7 @@ XMLPARSEAPI(void)
XML_SetUserData(XML_Parser parser, void *userData);
/* Returns the last value set by XML_SetUserData or NULL. */
-#define XML_GetUserData(parser) (*(void **)(parser))
+# define XML_GetUserData(parser) (*(void **)(parser))
/* This is equivalent to supplying an encoding argument to
XML_ParserCreate. On success XML_SetEncoding returns non-zero,
@@ -752,29 +777,6 @@ XML_GetSpecifiedAttributeCount(XML_Parser parser);
XMLPARSEAPI(int)
XML_GetIdAttributeIndex(XML_Parser parser);
-#ifdef XML_ATTR_INFO
-/* Source file byte offsets for the start and end of attribute names and values.
- The value indices are exclusive of surrounding quotes; thus in a UTF-8 source
- file an attribute value of "blah" will yield:
- info->valueEnd - info->valueStart = 4 bytes.
-*/
-typedef struct {
- XML_Index nameStart; /* Offset to beginning of the attribute name. */
- XML_Index nameEnd; /* Offset after the attribute name's last byte. */
- XML_Index valueStart; /* Offset to beginning of the attribute value. */
- XML_Index valueEnd; /* Offset after the attribute value's last byte. */
-} XML_AttrInfo;
-
-/* Returns an array of XML_AttrInfo structures for the attribute/value pairs
- passed in last call to the XML_StartElementHandler that were specified
- in the start-tag rather than defaulted. Each attribute/value pair counts
- as 1; thus the number of entries in the array is
- XML_GetSpecifiedAttributeCount(parser) / 2.
-*/
-XMLPARSEAPI(const XML_AttrInfo *)
-XML_GetAttributeInfo(XML_Parser parser);
-#endif
-
/* Parses some input. Returns XML_STATUS_ERROR if a fatal error is
detected. The last call to XML_Parse must have isFinal true; len
may be zero for this call (or any other).
@@ -916,10 +918,23 @@ XML_SetParamEntityParsing(XML_Parser parser,
function behavior. This must be called before parsing is started.
Returns 1 if successful, 0 when called after parsing has started.
Note: If parser == NULL, the function will do nothing and return 0.
+ DEPRECATED since Expat 2.8.0.
+ Please use XML_SetHashSalt16Bytes instead.
*/
+XML_ATTR_DEPRECATED("please use XML_SetHashSalt16Bytes instead")
XMLPARSEAPI(int)
XML_SetHashSalt(XML_Parser parser, unsigned long hash_salt);
+/* Sets the hash salt to use for internal hash calculations.
+ Helps in preventing DoS attacks based on predicting hash function behavior.
+ This must be called before parsing is started.
+ Returns XML_TRUE if successful, XML_FALSE when called after parsing has
+ started or when parser is NULL.
+ Added in Expat 2.8.0.
+*/
+XMLPARSEAPI(XML_Bool)
+XML_SetHashSalt16Bytes(XML_Parser parser, const uint8_t entropy[16]);
+
/* If XML_Parse or XML_ParseBuffer have returned XML_STATUS_ERROR, then
XML_GetErrorCode returns information about the error.
*/
@@ -946,15 +961,24 @@ XML_GetErrorCode(XML_Parser parser);
return 0 to indicate an error.
Note: XML_GetCurrentByteIndex returns -1 to indicate an error.
*/
+XML_ATTR_DEPRECATED("please use XML_GetCurrentLineNumber64 instead")
XMLPARSEAPI(XML_Size) XML_GetCurrentLineNumber(XML_Parser parser);
+XMLPARSEAPI(uint64_t) XML_GetCurrentLineNumber64(XML_Parser parser);
+XML_ATTR_DEPRECATED("please use XML_GetCurrentColumnNumber64 instead")
XMLPARSEAPI(XML_Size) XML_GetCurrentColumnNumber(XML_Parser parser);
+XMLPARSEAPI(uint64_t) XML_GetCurrentColumnNumber64(XML_Parser parser);
+XML_ATTR_DEPRECATED("please use XML_GetCurrentByteIndex64 instead")
XMLPARSEAPI(XML_Index) XML_GetCurrentByteIndex(XML_Parser parser);
+XMLPARSEAPI(int64_t) XML_GetCurrentByteIndex64(XML_Parser parser);
/* Return the number of bytes in the current event.
Returns 0 if the event is in an internal entity.
*/
+XML_ATTR_DEPRECATED("please use XML_GetCurrentByteCount64 instead")
XMLPARSEAPI(int)
XML_GetCurrentByteCount(XML_Parser parser);
+XMLPARSEAPI(uint64_t)
+XML_GetCurrentByteCount64(XML_Parser parser);
/* If XML_CONTEXT_BYTES is >=1, returns the input buffer, sets
the integer pointed to by offset to the offset within this buffer
@@ -966,13 +990,16 @@ XML_GetCurrentByteCount(XML_Parser parser);
NOTE: The character pointer returned should not be used outside
the handler that makes the call.
*/
+XML_ATTR_DEPRECATED("please use XML_GetInputContext64 instead")
XMLPARSEAPI(const char *)
XML_GetInputContext(XML_Parser parser, int *offset, int *size);
+XMLPARSEAPI(const char *)
+XML_GetInputContext64(XML_Parser parser, int64_t *offset, uint64_t *size);
/* For backwards compatibility with previous versions. */
-#define XML_GetErrorLineNumber XML_GetCurrentLineNumber
-#define XML_GetErrorColumnNumber XML_GetCurrentColumnNumber
-#define XML_GetErrorByteIndex XML_GetCurrentByteIndex
+# define XML_GetErrorLineNumber XML_GetCurrentLineNumber
+# define XML_GetErrorColumnNumber XML_GetCurrentColumnNumber
+# define XML_GetErrorByteIndex XML_GetCurrentByteIndex
/* Frees the content model passed to the element declaration handler */
XMLPARSEAPI(void)
@@ -1022,17 +1049,24 @@ enum XML_FeatureEnum {
XML_FEATURE_UNICODE_WCHAR_T,
XML_FEATURE_DTD,
XML_FEATURE_CONTEXT_BYTES,
+ /* Added in Expat 1.95.5, removed in Expat 2.9.0. */
XML_FEATURE_MIN_SIZE,
XML_FEATURE_SIZEOF_XML_CHAR,
XML_FEATURE_SIZEOF_XML_LCHAR,
+ /* Added in Expat 2.0.0. */
XML_FEATURE_NS,
+ /* Added in Expat 2.0.1. */
XML_FEATURE_LARGE_SIZE,
+ /* Added in Expat 2.1.0, removed in Expat 2.9.0. */
XML_FEATURE_ATTR_INFO,
/* Added in Expat 2.4.0. */
XML_FEATURE_BILLION_LAUGHS_ATTACK_PROTECTION_MAXIMUM_AMPLIFICATION_DEFAULT,
XML_FEATURE_BILLION_LAUGHS_ATTACK_PROTECTION_ACTIVATION_THRESHOLD_DEFAULT,
/* Added in Expat 2.6.0. */
- XML_FEATURE_GE
+ XML_FEATURE_GE,
+ /* Added in Expat 2.7.2. */
+ XML_FEATURE_ALLOC_TRACKER_MAXIMUM_AMPLIFICATION_DEFAULT,
+ XML_FEATURE_ALLOC_TRACKER_ACTIVATION_THRESHOLD_DEFAULT,
/* Additional features must be added to the end of this enum. */
};
@@ -1045,7 +1079,42 @@ typedef struct {
XMLPARSEAPI(const XML_Feature *)
XML_GetFeatureList(void);
-#if defined(XML_DTD) || (defined(XML_GE) && XML_GE == 1)
+/* Added in Expat 2.9.0. */
+enum XML_Parser_Property {
+# if defined(XML_TESTING)
+ XML_PROP_INVALID = 0,
+# endif
+# if XML_GE == 1
+ XML_PROP_ALLOC_TRACKER_ACTIVATION_THRESHOLD = 1, // of type `uint64_t`
+ XML_PROP_ALLOC_TRACKER_MAXIMUM_AMPLIFICATION = 2, // of type `double`
+ XML_PROP_BILLION_LAUGHS_ACTIVATION_THRESHOLD = 3, // of type `uint64_t`
+ XML_PROP_BILLION_LAUGHS_MAXIMUM_AMPLIFICATION = 4, // of type `double`
+# endif
+ XML_PROP_REPARSE_DEFERRAL_ENABLED = 5, // of type `XML_Bool`
+ /* more properties upcoming here */
+};
+
+/* Added in Expat 2.9.0. */
+XMLPARSEAPI(enum XML_Prop_Error)
+XML_GetPropertyBool(XML_Parser parser, enum XML_Parser_Property property,
+ XML_Bool *value);
+XMLPARSEAPI(enum XML_Prop_Error)
+XML_GetPropertyDouble(XML_Parser parser, enum XML_Parser_Property property,
+ double *value);
+XMLPARSEAPI(enum XML_Prop_Error)
+XML_GetPropertyUInt64(XML_Parser parser, enum XML_Parser_Property property,
+ uint64_t *value);
+XMLPARSEAPI(enum XML_Prop_Error)
+XML_SetPropertyBool(XML_Parser parser, enum XML_Parser_Property property,
+ XML_Bool value);
+XMLPARSEAPI(enum XML_Prop_Error)
+XML_SetPropertyDouble(XML_Parser parser, enum XML_Parser_Property property,
+ double value);
+XMLPARSEAPI(enum XML_Prop_Error)
+XML_SetPropertyUInt64(XML_Parser parser, enum XML_Parser_Property property,
+ uint64_t value);
+
+# if defined(XML_DTD) || (defined(XML_GE) && XML_GE == 1)
/* Added in Expat 2.4.0 for XML_DTD defined and
* added in Expat 2.6.0 for XML_GE == 1. */
XMLPARSEAPI(XML_Bool)
@@ -1057,7 +1126,17 @@ XML_SetBillionLaughsAttackProtectionMaximumAmplification(
XMLPARSEAPI(XML_Bool)
XML_SetBillionLaughsAttackProtectionActivationThreshold(
XML_Parser parser, unsigned long long activationThresholdBytes);
-#endif
+
+/* Added in Expat 2.7.2. */
+XMLPARSEAPI(XML_Bool)
+XML_SetAllocTrackerMaximumAmplification(XML_Parser parser,
+ float maximumAmplificationFactor);
+
+/* Added in Expat 2.7.2. */
+XMLPARSEAPI(XML_Bool)
+XML_SetAllocTrackerActivationThreshold(
+ XML_Parser parser, unsigned long long activationThresholdBytes);
+# endif
/* Added in Expat 2.6.0. */
XMLPARSEAPI(XML_Bool)
@@ -1066,12 +1145,12 @@ XML_SetReparseDeferralEnabled(XML_Parser parser, XML_Bool enabled);
/* Expat follows the semantic versioning convention.
See https://semver.org
*/
-#define XML_MAJOR_VERSION 2
-#define XML_MINOR_VERSION 6
-#define XML_MICRO_VERSION 4
+# define XML_MAJOR_VERSION 2
+# define XML_MINOR_VERSION 9
+# define XML_MICRO_VERSION 0
-#ifdef __cplusplus
+# ifdef __cplusplus
}
-#endif
+# endif
#endif /* not Expat_INCLUDED */
diff --git a/Modules/expat/expat_config.h b/Modules/expat/expat_config.h
index 8f1831de36faec4..f8448f5bb43f57f 100644
--- a/Modules/expat/expat_config.h
+++ b/Modules/expat/expat_config.h
@@ -3,7 +3,7 @@
* distribution.
*/
#ifndef EXPAT_CONFIG_H
-#define EXPAT_CONFIG_H
+#define EXPAT_CONFIG_H 1
#include <pyconfig.h>
#ifdef WORDS_BIGENDIAN
@@ -19,4 +19,10 @@
#define XML_GE 1
#define XML_CONTEXT_BYTES 1024
+#undef HAVE_ARC4RANDOM
+#undef HAVE_ARC4RANDOM_BUF
+#undef HAVE_GETENTROPY
+#undef HAVE_GETRANDOM
+#undef HAVE_SYSCALL_GETRANDOM
+
#endif /* EXPAT_CONFIG_H */
diff --git a/Modules/expat/expat_external.h b/Modules/expat/expat_external.h
index 8829f77091047a4..120a30e7ef3e498 100644
--- a/Modules/expat/expat_external.h
+++ b/Modules/expat/expat_external.h
@@ -12,9 +12,11 @@
Copyright (c) 2001-2002 Greg Stein <gstein@users.sourceforge.net>
Copyright (c) 2002-2006 Karl Waclawek <karl@waclawek.net>
Copyright (c) 2016 Cristian Rodríguez <crrodriguez@opensuse.org>
- Copyright (c) 2016-2019 Sebastian Pipping <sebastian@pipping.org>
+ Copyright (c) 2016-2025 Sebastian Pipping <sebastian@pipping.org>
Copyright (c) 2017 Rhodri James <rhodri@wildebeest.org.uk>
Copyright (c) 2018 Yury Gribov <tetra2005@gmail.com>
+ Copyright (c) 2026 Matthew Fernandez <matthew.fernandez@gmail.com>
+ Copyright (c) 2026 Braian Plaku <braianplaku@gmail.com>
Licensed under the MIT license:
Permission is hereby granted, free of charge, to any person obtaining
@@ -35,17 +37,22 @@
DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+ SPDX-License-Identifier: MIT
*/
#ifndef Expat_External_INCLUDED
-#define Expat_External_INCLUDED 1
+# define Expat_External_INCLUDED 1
+/* Namespace external symbols to allow multiple libexpat version to
+ co-exist. */
+#include "pyexpatns.h"
/* External API definitions */
/* Expat tries very hard to make the API boundary very specifically
defined. There are two macros defined to control this boundary;
each of these can be defined before including this header to
- achieve some different behavior, but doing so it not recommended or
+ achieve some different behavior, but doing so is not recommended or
tested frequently.
XMLCALL - The calling convention to use for all calls across the
@@ -64,12 +71,12 @@
compiled with the cdecl calling convention as the default since
system headers may assume the cdecl convention.
*/
-#ifndef XMLCALL
-# if defined(_MSC_VER)
-# define XMLCALL __cdecl
-# elif defined(__GNUC__) && defined(__i386) && ! defined(__INTEL_COMPILER)
-# define XMLCALL __attribute__((cdecl))
-# else
+# ifndef XMLCALL
+# if defined(_MSC_VER)
+# define XMLCALL __cdecl
+# elif defined(__GNUC__) && defined(__i386) && ! defined(__INTEL_COMPILER)
+# define XMLCALL __attribute__((cdecl))
+# else
/* For any platform which uses this definition and supports more than
one calling convention, we need to extend this definition to
declare the convention used on that platform, if it's possible to
@@ -80,86 +87,100 @@
pre-processor and how to specify the same calling convention as the
platform's malloc() implementation.
*/
-# define XMLCALL
-# endif
-#endif /* not defined XMLCALL */
+# define XMLCALL
+# endif
+# endif /* not defined XMLCALL */
-#if ! defined(XML_STATIC) && ! defined(XMLIMPORT)
-# ifndef XML_BUILDING_EXPAT
+# if ! defined(XML_STATIC) && ! defined(XMLIMPORT)
+# ifndef XML_BUILDING_EXPAT
/* using Expat from an application */
-# if defined(_MSC_EXTENSIONS) && ! defined(__BEOS__) && ! defined(__CYGWIN__)
-# define XMLIMPORT __declspec(dllimport)
+# if defined(_MSC_VER) && ! defined(__BEOS__) && ! defined(__CYGWIN__)
+# define XMLIMPORT __declspec(dllimport)
+# endif
+
# endif
+# endif /* not defined XML_STATIC */
+# ifndef XML_ENABLE_VISIBILITY
+# define XML_ENABLE_VISIBILITY 0
# endif
-#endif /* not defined XML_STATIC */
-
-#ifndef XML_ENABLE_VISIBILITY
-# define XML_ENABLE_VISIBILITY 0
-#endif
-#if ! defined(XMLIMPORT) && XML_ENABLE_VISIBILITY
-# define XMLIMPORT __attribute__((visibility("default")))
-#endif
+# if ! defined(XMLIMPORT) && XML_ENABLE_VISIBILITY
+# define XMLIMPORT __attribute__((visibility("default")))
+# endif
/* If we didn't define it above, define it away: */
-#ifndef XMLIMPORT
-# define XMLIMPORT
-#endif
-
-#if defined(__GNUC__) \
- && (__GNUC__ > 2 || (__GNUC__ == 2 && __GNUC_MINOR__ >= 96))
-# define XML_ATTR_MALLOC __attribute__((__malloc__))
-#else
-# define XML_ATTR_MALLOC
-#endif
-
-#if defined(__GNUC__) \
- && ((__GNUC__ > 4) || (__GNUC__ == 4 && __GNUC_MINOR__ >= 3))
-# define XML_ATTR_ALLOC_SIZE(x) __attribute__((__alloc_size__(x)))
-#else
-# define XML_ATTR_ALLOC_SIZE(x)
-#endif
-
-#define XMLPARSEAPI(type) XMLIMPORT type XMLCALL
-
-#ifdef __cplusplus
-extern "C" {
-#endif
+# ifndef XMLIMPORT
+# define XMLIMPORT
+# endif
-#ifdef XML_UNICODE_WCHAR_T
-# ifndef XML_UNICODE
-# define XML_UNICODE
+# if defined(__GNUC__) \
+ && (__GNUC__ > 2 || (__GNUC__ == 2 && __GNUC_MINOR__ >= 96))
+# define XML_ATTR_MALLOC __attribute__((__malloc__))
+# else
+# define XML_ATTR_MALLOC
# endif
-# if defined(__SIZEOF_WCHAR_T__) && (__SIZEOF_WCHAR_T__ != 2)
-# error "sizeof(wchar_t) != 2; Need -fshort-wchar for both Expat and libc"
+
+# if defined(__GNUC__) \
+ && ((__GNUC__ > 4) || (__GNUC__ == 4 && __GNUC_MINOR__ >= 3))
+# define XML_ATTR_ALLOC_SIZE(x) __attribute__((__alloc_size__(x)))
+# else
+# define XML_ATTR_ALLOC_SIZE(x)
+# endif
+
+/* Marks a function that Expat still provides but that callers should move off
+ of. */
+# if defined(__clang__) || defined(__GNUC__)
+# define XML_ATTR_DEPRECATED(message) \
+ __attribute__((__deprecated__(message)))
+# elif defined(_MSC_VER)
+# define XML_ATTR_DEPRECATED(message) __declspec(deprecated(message))
+# else
+# define XML_ATTR_DEPRECATED(message) // empty i.e. no deprecation
+# endif
+
+# define XMLPARSEAPI(type) XMLIMPORT type XMLCALL
+
+# ifdef __cplusplus
+extern "C" {
# endif
-#endif
-#ifdef XML_UNICODE /* Information is UTF-16 encoded. */
# ifdef XML_UNICODE_WCHAR_T
+# ifndef XML_UNICODE
+# define XML_UNICODE
+# endif
+# if defined(__SIZEOF_WCHAR_T__) && (__SIZEOF_WCHAR_T__ != 2)
+# error "sizeof(wchar_t) != 2; Need -fshort-wchar for both Expat and libc"
+# endif
+# endif
+
+# ifdef XML_UNICODE /* Information is UTF-16 encoded. */
+# ifdef XML_UNICODE_WCHAR_T
typedef wchar_t XML_Char;
typedef wchar_t XML_LChar;
-# else
+# else
typedef unsigned short XML_Char;
typedef char XML_LChar;
-# endif /* XML_UNICODE_WCHAR_T */
-#else /* Information is UTF-8 encoded. */
+# endif /* XML_UNICODE_WCHAR_T */
+# else /* Information is UTF-8 encoded. */
typedef char XML_Char;
typedef char XML_LChar;
-#endif /* XML_UNICODE */
+# endif /* XML_UNICODE */
-#ifdef XML_LARGE_SIZE /* Use large integers for file/stream positions. */
+# ifdef XML_LARGE_SIZE /* Use large integers for file/stream positions. */
+# if defined(__clang__) || defined(__GNUC__)
+# warning Macro XML_LARGE_SIZE is deprecated, please use the 64bit location API functions instead.
+# endif
typedef long long XML_Index;
typedef unsigned long long XML_Size;
-#else
+# else
typedef long XML_Index;
typedef unsigned long XML_Size;
-#endif /* XML_LARGE_SIZE */
+# endif /* XML_LARGE_SIZE */
-#ifdef __cplusplus
+# ifdef __cplusplus
}
-#endif
+# endif
#endif /* not Expat_External_INCLUDED */
diff --git a/Modules/expat/fallthrough.h b/Modules/expat/fallthrough.h
new file mode 100644
index 000000000000000..0152d1bdc5de8ff
--- /dev/null
+++ b/Modules/expat/fallthrough.h
@@ -0,0 +1,51 @@
+/*
+ __ __ _
+ ___\ \/ /_ __ __ _| |_
+ / _ \\ /| '_ \ / _` | __|
+ | __// \| |_) | (_| | |_
+ \___/_/\_\ .__/ \__,_|\__|
+ |_| XML parser
+
+ Copyright (c) 2026 Nick Begg <nick@stunttruck.net>
+ Licensed under the MIT license:
+
+ Permission is hereby granted, free of charge, to any person obtaining
+ a copy of this software and associated documentation files (the
+ "Software"), to deal in the Software without restriction, including
+ without limitation the rights to use, copy, modify, merge, publish,
+ distribute, sublicense, and/or sell copies of the Software, and to permit
+ persons to whom the Software is furnished to do so, subject to the
+ following conditions:
+
+ The above copyright notice and this permission notice shall be included
+ in all copies or substantial portions of the Software.
+
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN
+ NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
+ DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
+ OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
+ USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+ SPDX-License-Identifier: MIT
+*/
+
+#ifndef FALLTHROUGH_H
+# define FALLTHROUGH_H 1
+
+// Explicit fallthrough in switch case to avoid warnings
+// with compiler flag -Wimplicit-fallthrough.
+
+# define EXPAT_FALLTHROUGH \
+ do { \
+ } while (0)
+
+# if defined(__has_attribute)
+# if __has_attribute(fallthrough)
+# undef EXPAT_FALLTHROUGH
+# define EXPAT_FALLTHROUGH __attribute__((fallthrough))
+# endif
+# endif
+
+#endif // FALLTHROUGH_H
diff --git a/Modules/expat/hash_table.h b/Modules/expat/hash_table.h
new file mode 100644
index 000000000000000..f290aad18693feb
--- /dev/null
+++ b/Modules/expat/hash_table.h
@@ -0,0 +1,78 @@
+/* Hash table related internal API
+ __ __ _
+ ___\ \/ /_ __ __ _| |_
+ / _ \\ /| '_ \ / _` | __|
+ | __// \| |_) | (_| | |_
+ \___/_/\_\ .__/ \__,_|\__|
+ |_| XML parser
+
+ Copyright (c) 2026 Sebastian Pipping <sebastian@pipping.org>
+ Licensed under the MIT license:
+
+ Permission is hereby granted, free of charge, to any person obtaining
+ a copy of this software and associated documentation files (the
+ "Software"), to deal in the Software without restriction, including
+ without limitation the rights to use, copy, modify, merge, publish,
+ distribute, sublicense, and/or sell copies of the Software, and to permit
+ persons to whom the Software is furnished to do so, subject to the
+ following conditions:
+
+ The above copyright notice and this permission notice shall be included
+ in all copies or substantial portions of the Software.
+
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN
+ NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
+ DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
+ OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
+ USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+ SPDX-License-Identifier: MIT
+*/
+
+#if ! defined(HASH_TABLE_H)
+# define HASH_TABLE_H 1
+
+# include "expat.h" // for XML_Bool, XML_Parser
+# include "internal.h" // for XML_NONTESTING_STATIC
+
+# include <stddef.h> // for size_t
+
+typedef const XML_Char *KEY;
+
+typedef struct {
+ KEY name;
+} NAMED;
+
+typedef struct {
+ NAMED **v;
+ unsigned char power;
+ size_t size;
+ size_t used;
+ XML_Parser parser;
+} HASH_TABLE;
+
+typedef struct {
+ NAMED **p;
+ NAMED **end;
+} HASH_TABLE_ITER;
+
+XML_NONTESTING_STATIC NAMED *lookupWithLength(XML_Parser parser,
+ HASH_TABLE *table, KEY name,
+ size_t nameLen,
+ size_t createSize);
+XML_NONTESTING_STATIC NAMED *lookup(XML_Parser parser, HASH_TABLE *table,
+ KEY name, size_t createSize);
+
+XML_NONTESTING_STATIC void hashTableInit(HASH_TABLE *table, XML_Parser parser);
+XML_NONTESTING_STATIC void hashTableClear(HASH_TABLE *table);
+XML_NONTESTING_STATIC void hashTableDestroy(HASH_TABLE *table);
+XML_NONTESTING_STATIC void hashTableIterInit(HASH_TABLE_ITER *iter,
+ const HASH_TABLE *table);
+XML_NONTESTING_STATIC NAMED *hashTableIterNext(HASH_TABLE_ITER *iter);
+
+XML_NONTESTING_STATIC XML_Bool keyeq(KEY s1, size_t s1len, KEY s2);
+XML_NONTESTING_STATIC size_t keylen(KEY s);
+
+#endif // ! defined(HASH_TABLE_H)
diff --git a/Modules/expat/iasciitab.h b/Modules/expat/iasciitab.h
index 5d8646f2a318b8a..1de8d519c8b7bad 100644
--- a/Modules/expat/iasciitab.h
+++ b/Modules/expat/iasciitab.h
@@ -30,6 +30,8 @@
DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+ SPDX-License-Identifier: MIT
*/
/* Like asciitab.h, except that 0xD has code BT_S rather than BT_CR */
diff --git a/Modules/expat/internal.h b/Modules/expat/internal.h
index 167ec36804a43b0..de34cd7dfc55c58 100644
--- a/Modules/expat/internal.h
+++ b/Modules/expat/internal.h
@@ -6,13 +6,6 @@
The following calling convention macros are defined for frequently
called functions:
- FASTCALL - Used for those internal functions that have a simple
- body and a low number of arguments and local variables.
-
- PTRCALL - Used for functions called though function pointers.
-
- PTRFASTCALL - Like PTRCALL, but for low number of arguments.
-
inline - Used for selected internal functions for which inlining
may improve performance on some platforms.
@@ -28,11 +21,14 @@
Copyright (c) 2002-2003 Fred L. Drake, Jr. <fdrake@users.sourceforge.net>
Copyright (c) 2002-2006 Karl Waclawek <karl@waclawek.net>
Copyright (c) 2003 Greg Stein <gstein@users.sourceforge.net>
- Copyright (c) 2016-2024 Sebastian Pipping <sebastian@pipping.org>
+ Copyright (c) 2016-2026 Sebastian Pipping <sebastian@pipping.org>
Copyright (c) 2018 Yury Gribov <tetra2005@gmail.com>
Copyright (c) 2019 David Loffredo <loffredo@steptools.com>
Copyright (c) 2023-2024 Sony Corporation / Snild Dolkow <snild@sony.com>
Copyright (c) 2024 Taichi Haradaguchi <20001722@ymail.ne.jp>
+ Copyright (c) 2026 Matthew Wozniczka <mattheww@simba.com>
+ Copyright (c) 2026 Braian Plaku <braianplaku@gmail.com>
+ Copyright (c) 2026 Florian Schmaus <florian.schmaus@codasip.com>
Licensed under the MIT license:
Permission is hereby granted, free of charge, to any person obtaining
@@ -53,124 +49,106 @@
DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
USE OR OTHER DEALINGS IN THE SOFTWARE.
-*/
-
-#if defined(__GNUC__) && defined(__i386__) && ! defined(__MINGW32__)
-/* We'll use this version by default only where we know it helps.
-
- regparm() generates warnings on Solaris boxes. See SF bug #692878.
- Instability reported with egcs on a RedHat Linux 7.3.
- Let's comment out:
- #define FASTCALL __attribute__((stdcall, regparm(3)))
- and let's try this:
+ SPDX-License-Identifier: MIT
*/
-# define FASTCALL __attribute__((regparm(3)))
-# define PTRFASTCALL __attribute__((regparm(3)))
-#endif
-
-/* Using __fastcall seems to have an unexpected negative effect under
- MS VC++, especially for function pointers, so we won't use it for
- now on that platform. It may be reconsidered for a future release
- if it can be made more effective.
- Likely reason: __fastcall on Windows is like stdcall, therefore
- the compiler cannot perform stack optimizations for call clusters.
-*/
-
-/* Make sure all of these are defined if they aren't already. */
-
-#ifndef FASTCALL
-# define FASTCALL
-#endif
-#ifndef PTRCALL
-# define PTRCALL
-#endif
+#if ! defined(INTERNAL_H)
+# define INTERNAL_H 1
-#ifndef PTRFASTCALL
-# define PTRFASTCALL
-#endif
-
-#ifndef XML_MIN_SIZE
-# if ! defined(__cplusplus) && ! defined(inline)
+# if ! defined(inline)
# ifdef __GNUC__
# define inline __inline
# endif /* __GNUC__ */
# endif
-#endif /* XML_MIN_SIZE */
-#ifdef __cplusplus
-# define inline inline
-#else
# ifndef inline
# define inline
# endif
-#endif
-
-#include <limits.h> // ULONG_MAX
-#if defined(_WIN32) \
- && (! defined(__USE_MINGW_ANSI_STDIO) \
- || (1 - __USE_MINGW_ANSI_STDIO - 1 == 0))
-# define EXPAT_FMT_ULL(midpart) "%" midpart "I64u"
-# if defined(_WIN64) // Note: modifiers "td" and "zu" do not work for MinGW
-# define EXPAT_FMT_PTRDIFF_T(midpart) "%" midpart "I64d"
-# define EXPAT_FMT_SIZE_T(midpart) "%" midpart "I64u"
-# else
-# define EXPAT_FMT_PTRDIFF_T(midpart) "%" midpart "d"
-# define EXPAT_FMT_SIZE_T(midpart) "%" midpart "u"
+# if ! defined(XML_NONTESTING_STATIC)
+# if defined(XML_TESTING)
+# define XML_NONTESTING_STATIC // empty i.e. not static
+# else
+# define XML_NONTESTING_STATIC static
+# endif
# endif
-#else
-# define EXPAT_FMT_ULL(midpart) "%" midpart "llu"
-# if ! defined(ULONG_MAX)
-# error Compiler did not define ULONG_MAX for us
-# elif ULONG_MAX == 18446744073709551615u // 2^64-1
-# define EXPAT_FMT_PTRDIFF_T(midpart) "%" midpart "ld"
-# define EXPAT_FMT_SIZE_T(midpart) "%" midpart "lu"
+
+# include <limits.h> // ULONG_MAX
+# include <stddef.h> // size_t
+
+# if defined(_WIN32) \
+ && (! defined(__USE_MINGW_ANSI_STDIO) \
+ || (1 - __USE_MINGW_ANSI_STDIO - 1 == 0))
+# define EXPAT_FMT_LLX(midpart) "%" midpart "I64x"
+# define EXPAT_FMT_ULL(midpart) "%" midpart "I64u"
+# if defined(_WIN64) // Note: modifiers "td" and "zu" do not work for MinGW
+# define EXPAT_FMT_PTRDIFF_T(midpart) "%" midpart "I64d"
+# define EXPAT_FMT_SIZE_T(midpart) "%" midpart "I64u"
+# else
+# define EXPAT_FMT_PTRDIFF_T(midpart) "%" midpart "d"
+# define EXPAT_FMT_SIZE_T(midpart) "%" midpart "u"
+# endif
# else
-# define EXPAT_FMT_PTRDIFF_T(midpart) "%" midpart "d"
-# define EXPAT_FMT_SIZE_T(midpart) "%" midpart "u"
+# include <inttypes.h> // PRIdPTR, PRIuPTR
+# define EXPAT_FMT_LLX(midpart) "%" midpart "llx"
+# define EXPAT_FMT_ULL(midpart) "%" midpart "llu"
+# define EXPAT_FMT_PTRDIFF_T(midpart) "%" midpart PRIdPTR
+# define EXPAT_FMT_SIZE_T(midpart) "%" midpart PRIuPTR
# endif
-#endif
-#ifndef UNUSED_P
-# define UNUSED_P(p) (void)p
-#endif
+# ifndef UNUSED_P
+# define UNUSED_P(p) (void)p
+# endif
/* NOTE BEGIN If you ever patch these defaults to greater values
for non-attack XML payload in your environment,
please file a bug report with libexpat. Thank you!
*/
-#define EXPAT_BILLION_LAUGHS_ATTACK_PROTECTION_MAXIMUM_AMPLIFICATION_DEFAULT \
- 100.0f
-#define EXPAT_BILLION_LAUGHS_ATTACK_PROTECTION_ACTIVATION_THRESHOLD_DEFAULT \
- 8388608 // 8 MiB, 2^23
-/* NOTE END */
+# define EXPAT_BILLION_LAUGHS_ATTACK_PROTECTION_MAXIMUM_AMPLIFICATION_DEFAULT \
+ 100.0f
+# define EXPAT_BILLION_LAUGHS_ATTACK_PROTECTION_ACTIVATION_THRESHOLD_DEFAULT \
+ 8388608 // 8 MiB, 2^23
+
+# define EXPAT_ALLOC_TRACKER_MAXIMUM_AMPLIFICATION_DEFAULT 100.0f
+# define EXPAT_ALLOC_TRACKER_ACTIVATION_THRESHOLD_DEFAULT \
+ 67108864 // 64 MiB, 2^26
+
+// NOTE: If function expat_alloc was user facing, EXPAT_MALLOC_ALIGNMENT would
+// have to take sizeof(long double) into account
+union expat_align {
+ long long l;
+ void *p;
+};
+# define EXPAT_MALLOC_ALIGNMENT sizeof(union expat_align)
+# define EXPAT_MALLOC_PADDING ((EXPAT_MALLOC_ALIGNMENT) - sizeof(size_t))
-#include "expat.h" // so we can use type XML_Parser below
+/* NOTE END */
-#ifdef __cplusplus
-extern "C" {
-#endif
+# include "expat.h" // so we can use type XML_Parser below
void _INTERNAL_trim_to_complete_utf8_characters(const char *from,
const char **fromLimRef);
-#if defined(XML_GE) && XML_GE == 1
+# if defined(XML_GE) && XML_GE == 1
unsigned long long testingAccountingGetCountBytesDirect(XML_Parser parser);
unsigned long long testingAccountingGetCountBytesIndirect(XML_Parser parser);
const char *unsignedCharToPrintable(unsigned char c);
-#endif
+# endif
extern
-#if ! defined(XML_TESTING)
+# if ! defined(XML_TESTING)
const
-#endif
+# endif
XML_Bool g_reparseDeferralEnabledDefault; // written ONLY in runtests.c
-#if defined(XML_TESTING)
+# if defined(XML_TESTING)
+
+int xmlSetHashSalt(XML_Parser parser, unsigned long hash_salt);
+
+void *expat_malloc(XML_Parser parser, size_t size, int sourceLine);
+void expat_free(XML_Parser parser, void *ptr, int sourceLine);
+void *expat_realloc(XML_Parser parser, void *ptr, size_t size, int sourceLine);
extern unsigned int g_bytesScanned; // used for testing only
-#endif
+# endif
-#ifdef __cplusplus
-}
-#endif
+#endif // not defined INTERNAL_H
diff --git a/Modules/expat/latin1tab.h b/Modules/expat/latin1tab.h
index b681d278af6569b..3793f4f3cc7841d 100644
--- a/Modules/expat/latin1tab.h
+++ b/Modules/expat/latin1tab.h
@@ -30,6 +30,8 @@
DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+ SPDX-License-Identifier: MIT
*/
/* 0x80 */ BT_OTHER, BT_OTHER, BT_OTHER, BT_OTHER,
diff --git a/Modules/expat/memory_sanitizer.h b/Modules/expat/memory_sanitizer.h
new file mode 100644
index 000000000000000..f739b88f304871b
--- /dev/null
+++ b/Modules/expat/memory_sanitizer.h
@@ -0,0 +1,53 @@
+/*
+ __ __ _
+ ___\ \/ /_ __ __ _| |_
+ / _ \\ /| '_ \ / _` | __|
+ | __// \| |_) | (_| | |_
+ \___/_/\_\ .__/ \__,_|\__|
+ |_| XML parser
+
+ Copyright (c) 2026 Matthew Fernandez <matthew.fernandez@gmail.com>
+ Licensed under the MIT license:
+
+ Permission is hereby granted, free of charge, to any person obtaining
+ a copy of this software and associated documentation files (the
+ "Software"), to deal in the Software without restriction, including
+ without limitation the rights to use, copy, modify, merge, publish,
+ distribute, sublicense, and/or sell copies of the Software, and to permit
+ persons to whom the Software is furnished to do so, subject to the
+ following conditions:
+
+ The above copyright notice and this permission notice shall be included
+ in all copies or substantial portions of the Software.
+
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN
+ NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
+ DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
+ OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
+ USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+ SPDX-License-Identifier: MIT
+*/
+
+#if ! defined(MEMORY_SANITIZER_H)
+# define MEMORY_SANITIZER_H 1
+
+# if defined(__has_feature)
+# if __has_feature(memory_sanitizer)
+# include <sanitizer/msan_interface.h>
+
+// inform Memory Sanitizer that [base, base + extent) is now initialized
+# define MSAN_UNPOISON(base, extent) __msan_unpoison((base), (extent))
+
+# endif
+# endif
+
+# if ! defined(MSAN_UNPOISON)
+# define MSAN_UNPOISON(base, extent) \
+ do { \
+ } while (0)
+# endif
+
+#endif // ! defined(MEMORY_SANITIZER_H)
diff --git a/Modules/expat/nametab.h b/Modules/expat/nametab.h
index 63485446b967272..2385851562e7dcd 100644
--- a/Modules/expat/nametab.h
+++ b/Modules/expat/nametab.h
@@ -28,6 +28,8 @@
DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+ SPDX-License-Identifier: MIT
*/
static const unsigned namingBitmap[] = {
diff --git a/Modules/expat/pyexpatns.h b/Modules/expat/pyexpatns.h
index d45d9b6c4571595..8b149b9046e2ecb 100644
--- a/Modules/expat/pyexpatns.h
+++ b/Modules/expat/pyexpatns.h
@@ -50,14 +50,22 @@
#define XML_GetBase PyExpat_XML_GetBase
#define XML_GetBuffer PyExpat_XML_GetBuffer
#define XML_GetCurrentByteCount PyExpat_XML_GetCurrentByteCount
+#define XML_GetCurrentByteCount64 PyExpat_XML_GetCurrentByteCount64
#define XML_GetCurrentByteIndex PyExpat_XML_GetCurrentByteIndex
+#define XML_GetCurrentByteIndex64 PyExpat_XML_GetCurrentByteIndex64
#define XML_GetCurrentColumnNumber PyExpat_XML_GetCurrentColumnNumber
+#define XML_GetCurrentColumnNumber64 PyExpat_XML_GetCurrentColumnNumber64
#define XML_GetCurrentLineNumber PyExpat_XML_GetCurrentLineNumber
+#define XML_GetCurrentLineNumber64 PyExpat_XML_GetCurrentLineNumber64
#define XML_GetErrorCode PyExpat_XML_GetErrorCode
#define XML_GetFeatureList PyExpat_XML_GetFeatureList
#define XML_GetIdAttributeIndex PyExpat_XML_GetIdAttributeIndex
#define XML_GetInputContext PyExpat_XML_GetInputContext
+#define XML_GetInputContext64 PyExpat_XML_GetInputContext64
#define XML_GetParsingStatus PyExpat_XML_GetParsingStatus
+#define XML_GetPropertyBool PyExpat_XML_GetPropertyBool
+#define XML_GetPropertyDouble PyExpat_XML_GetPropertyDouble
+#define XML_GetPropertyUInt64 PyExpat_XML_GetPropertyUInt64
#define XML_GetSpecifiedAttributeCount PyExpat_XML_GetSpecifiedAttributeCount
#define XmlGetUtf16InternalEncoding PyExpat_XmlGetUtf16InternalEncoding
#define XmlGetUtf16InternalEncodingNS PyExpat_XmlGetUtf16InternalEncodingNS
@@ -82,6 +90,8 @@
#define XmlPrologStateInit PyExpat_XmlPrologStateInit
#define XmlPrologStateInitExternalEntity PyExpat_XmlPrologStateInitExternalEntity
#define XML_ResumeParser PyExpat_XML_ResumeParser
+#define XML_SetAllocTrackerActivationThreshold PyExpat_XML_SetAllocTrackerActivationThreshold
+#define XML_SetAllocTrackerMaximumAmplification PyExpat_XML_SetAllocTrackerMaximumAmplification
#define XML_SetAttlistDeclHandler PyExpat_XML_SetAttlistDeclHandler
#define XML_SetBase PyExpat_XML_SetBase
#define XML_SetBillionLaughsAttackProtectionActivationThreshold PyExpat_XML_SetBillionLaughsAttackProtectionActivationThreshold
@@ -103,11 +113,16 @@
#define XML_SetExternalEntityRefHandler PyExpat_XML_SetExternalEntityRefHandler
#define XML_SetExternalEntityRefHandlerArg PyExpat_XML_SetExternalEntityRefHandlerArg
#define XML_SetHashSalt PyExpat_XML_SetHashSalt
+#define XML_SetHashSalt16Bytes PyExpat_XML_SetHashSalt16Bytes
#define XML_SetNamespaceDeclHandler PyExpat_XML_SetNamespaceDeclHandler
#define XML_SetNotationDeclHandler PyExpat_XML_SetNotationDeclHandler
#define XML_SetNotStandaloneHandler PyExpat_XML_SetNotStandaloneHandler
#define XML_SetParamEntityParsing PyExpat_XML_SetParamEntityParsing
#define XML_SetProcessingInstructionHandler PyExpat_XML_SetProcessingInstructionHandler
+#define XML_SetPropertyBool PyExpat_XML_SetPropertyBool
+#define XML_SetPropertyDouble PyExpat_XML_SetPropertyDouble
+#define XML_SetPropertyUInt64 PyExpat_XML_SetPropertyUInt64
+#define XML_SetReparseDeferralEnabled PyExpat_XML_SetReparseDeferralEnabled
#define XML_SetReturnNSTriplet PyExpat_XML_SetReturnNSTriplet
#define XML_SetSkippedEntityHandler PyExpat_XML_SetSkippedEntityHandler
#define XML_SetStartCdataSectionHandler PyExpat_XML_SetStartCdataSectionHandler
diff --git a/Modules/expat/refresh.sh b/Modules/expat/refresh.sh
new file mode 100755
index 000000000000000..740065935b860df
--- /dev/null
+++ b/Modules/expat/refresh.sh
@@ -0,0 +1,77 @@
+#!/usr/bin/env bash
+#
+# Use this script to update libexpat
+
+set -e
+set -o pipefail
+
+if [[ "${BASH_VERSINFO[0]}" -lt 4 ]]; then
+ echo "A bash version >= 4 required. Got: $BASH_VERSION" >&2
+ exit 1
+fi
+
+# Update this when updating to a new version after verifying that the changes
+# the update brings in are good. These values are used for verifying the SBOM, too.
+expected_libexpat_tag="R_2_9_0"
+expected_libexpat_version="2.9.0"
+expected_libexpat_sha256="16afbb9cefead2aa278105cf27d9f597bde7fbf3dbb85015857ca7ca6a4e89ba"
+
+expat_dir="$(realpath "$(dirname -- "${BASH_SOURCE[0]}")")"
+cd ${expat_dir}
+
+# Step 1: download and copy files
+curl --location "https://github.com/libexpat/libexpat/releases/download/${expected_libexpat_tag}/expat-${expected_libexpat_version}.tar.gz" > libexpat.tar.gz
+echo "${expected_libexpat_sha256} libexpat.tar.gz" | sha256sum --check
+
+# Step 2: Pull files from the libexpat distribution
+
+tar xzvf libexpat.tar.gz "expat-${expected_libexpat_version}/COPYING" --strip-components 2
+
+declare -a lib_files
+lib_files=(
+ ascii.h
+ asciitab.h
+ expat.h
+ expat_external.h
+ fallthrough.h
+ hash_table.h
+ iasciitab.h
+ internal.h
+ latin1tab.h
+ memory_sanitizer.h
+ nametab.h
+ siphash.h
+ utf8tab.h
+ winconfig.h
+ xcs.c
+ xcs.h
+ xmlparse.c
+ xmlrole.c
+ xmlrole.h
+ xmltok.c
+ xmltok.h
+ xmltok_impl.c
+ xmltok_impl.h
+ xmltok_ns.c
+)
+for f in "${lib_files[@]}"; do
+ tar xzvf libexpat.tar.gz "expat-${expected_libexpat_version}/lib/${f}" --strip-components 2
+done
+rm libexpat.tar.gz
+
+# Step 3: Add the namespacing include to expat_external.h
+sed -i 's/# define Expat_External_INCLUDED 1/&\n\/* Namespace external symbols to allow multiple libexpat version to\n co-exist. \*\/\n#include "pyexpatns.h"/' expat_external.h
+
+# Step 4: Skip the Windows rand_s entropy path in xmlparse.c when
+# XML_POOR_ENTROPY is set.
+sed -z -i 's|#if defined(_WIN32)\n# include "random_rand_s\.h"\n#endif /\* defined(_WIN32) \*/|#if defined(_WIN32) \&\& ! defined(XML_POOR_ENTROPY)\n# include "random_rand_s.h"\n#endif /* defined(_WIN32) \&\& ! defined(XML_POOR_ENTROPY) */|' xmlparse.c
+sed -z -i 's|# ifdef _WIN32\n if (writeRandomBytes_rand_s|# if defined(_WIN32) \&\& ! defined(XML_POOR_ENTROPY)\n if (writeRandomBytes_rand_s|' xmlparse.c
+
+if ! grep -q '#if defined(_WIN32) && ! defined(XML_POOR_ENTROPY)' xmlparse.c; then
+ echo "
+Error: rand_s gate not patched in xmlparse.c;
+This may be due to source changes and will require updating this script" >&2
+ exit 1
+fi
+
+echo "Updated; verify all is okay using git diff and git status."
diff --git a/Modules/expat/siphash.h b/Modules/expat/siphash.h
index 04f6f74585b5a2f..ac1fbfaf0bde81d 100644
--- a/Modules/expat/siphash.h
+++ b/Modules/expat/siphash.h
@@ -8,6 +8,8 @@
*
* 1. https://www.131002.net/siphash/siphash24.c
* 2. https://www.131002.net/siphash/
+ *
+ * SPDX-License-Identifier: CC0-1.0
* --------------------------------------------------------------------------
* HISTORY:
*
@@ -100,6 +102,7 @@
#include <stddef.h> /* size_t */
#include <stdint.h> /* uint64_t uint32_t uint8_t */
+#include "fallthrough.h"
/*
* Workaround to not require a C++11 compiler for using ULL suffix
@@ -234,25 +237,25 @@ sip24_final(struct siphash *H) {
switch (left) {
case 7:
b |= (uint64_t)H->buf[6] << 48;
- /* fall through */
+ EXPAT_FALLTHROUGH;
case 6:
b |= (uint64_t)H->buf[5] << 40;
- /* fall through */
+ EXPAT_FALLTHROUGH;
case 5:
b |= (uint64_t)H->buf[4] << 32;
- /* fall through */
+ EXPAT_FALLTHROUGH;
case 4:
b |= (uint64_t)H->buf[3] << 24;
- /* fall through */
+ EXPAT_FALLTHROUGH;
case 3:
b |= (uint64_t)H->buf[2] << 16;
- /* fall through */
+ EXPAT_FALLTHROUGH;
case 2:
b |= (uint64_t)H->buf[1] << 8;
- /* fall through */
+ EXPAT_FALLTHROUGH;
case 1:
b |= (uint64_t)H->buf[0] << 0;
- /* fall through */
+ EXPAT_FALLTHROUGH;
case 0:
break;
}
diff --git a/Modules/expat/utf8tab.h b/Modules/expat/utf8tab.h
index 88efcf91cc16a6c..73732d1527f1420 100644
--- a/Modules/expat/utf8tab.h
+++ b/Modules/expat/utf8tab.h
@@ -30,6 +30,8 @@
DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+ SPDX-License-Identifier: MIT
*/
/* 0x80 */ BT_TRAIL, BT_TRAIL, BT_TRAIL, BT_TRAIL,
diff --git a/Modules/expat/winconfig.h b/Modules/expat/winconfig.h
index 05805514ec7fa21..9e7e8bdc22ca36c 100644
--- a/Modules/expat/winconfig.h
+++ b/Modules/expat/winconfig.h
@@ -31,6 +31,8 @@
DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+ SPDX-License-Identifier: MIT
*/
#ifndef WINCONFIG_H
diff --git a/Modules/expat/xcs.c b/Modules/expat/xcs.c
new file mode 100644
index 000000000000000..ae5e042d778fffa
--- /dev/null
+++ b/Modules/expat/xcs.c
@@ -0,0 +1,90 @@
+/*
+ __ __ _
+ ___\ \/ /_ __ __ _| |_
+ / _ \\ /| '_ \ / _` | __|
+ | __// \| |_) | (_| | |_
+ \___/_/\_\ .__/ \__,_|\__|
+ |_| XML parser
+
+ Copyright (c) 2022-2026 Sebastian Pipping <sebastian@pipping.org>
+ Licensed under the MIT license:
+
+ Permission is hereby granted, free of charge, to any person obtaining
+ a copy of this software and associated documentation files (the
+ "Software"), to deal in the Software without restriction, including
+ without limitation the rights to use, copy, modify, merge, publish,
+ distribute, sublicense, and/or sell copies of the Software, and to permit
+ persons to whom the Software is furnished to do so, subject to the
+ following conditions:
+
+ The above copyright notice and this permission notice shall be included
+ in all copies or substantial portions of the Software.
+
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN
+ NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
+ DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
+ OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
+ USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+ SPDX-License-Identifier: MIT
+*/
+
+#include "xcs.h"
+
+#if defined(XML_UNICODE)
+# if defined(XML_UNICODE_WCHAR_T)
+# include <wchar.h> // for wcscmp, wcslen, wcsncmp
+# endif
+#else
+# include <string.h> // for strcmp, strlen, strncmp
+#endif
+
+size_t
+xcslen(const XML_Char *s) {
+#ifdef XML_UNICODE
+# ifdef XML_UNICODE_WCHAR_T
+ return wcslen(s);
+# else
+ // XML_Char is unsigned short
+ size_t len = 0;
+ while (s[len]) {
+ len++;
+ }
+ return len;
+# endif
+#else
+ return strlen(s);
+#endif
+}
+
+int
+xcscmp(const XML_Char *a, const XML_Char *b) {
+#if defined(XML_UNICODE)
+# if defined(XML_UNICODE_WCHAR_T)
+ return wcscmp(a, b);
+# else
+ for (; a[0] && b[0] && a[0] == b[0]; a++, b++)
+ ;
+ return a[0] - b[0];
+# endif
+#else
+ return strcmp(a, b);
+#endif
+}
+
+int
+xcsncmp(const XML_Char *a, const XML_Char *b, size_t len) {
+#if defined(XML_UNICODE)
+# if defined(XML_UNICODE_WCHAR_T)
+ return wcsncmp(a, b, len);
+# else
+ for (; len > 0 && a[0] && b[0] && a[0] == b[0]; len--, a++, b++) {
+ }
+ return (len == 0) ? 0 : (a[0] - b[0]);
+# endif
+#else
+ return strncmp(a, b, len);
+#endif
+}
diff --git a/Modules/expat/xcs.h b/Modules/expat/xcs.h
new file mode 100644
index 000000000000000..1b942647d333093
--- /dev/null
+++ b/Modules/expat/xcs.h
@@ -0,0 +1,44 @@
+/*
+ __ __ _
+ ___\ \/ /_ __ __ _| |_
+ / _ \\ /| '_ \ / _` | __|
+ | __// \| |_) | (_| | |_
+ \___/_/\_\ .__/ \__,_|\__|
+ |_| XML parser
+
+ Copyright (c) 2026 Sebastian Pipping <sebastian@pipping.org>
+ Licensed under the MIT license:
+
+ Permission is hereby granted, free of charge, to any person obtaining
+ a copy of this software and associated documentation files (the
+ "Software"), to deal in the Software without restriction, including
+ without limitation the rights to use, copy, modify, merge, publish,
+ distribute, sublicense, and/or sell copies of the Software, and to permit
+ persons to whom the Software is furnished to do so, subject to the
+ following conditions:
+
+ The above copyright notice and this permission notice shall be included
+ in all copies or substantial portions of the Software.
+
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN
+ NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
+ DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
+ OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
+ USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+ SPDX-License-Identifier: MIT
+*/
+
+#if ! defined(EXPAT_XCS_H)
+
+# include <stddef.h> // size_t
+# include "expat_external.h" // for XML_Char
+
+extern size_t xcslen(const XML_Char *s);
+
+extern int xcscmp(const XML_Char *a, const XML_Char *b);
+extern int xcsncmp(const XML_Char *a, const XML_Char *b, size_t len);
+
+#endif // ! defined(EXPAT_XCS_H)
diff --git a/Modules/expat/xmlparse.c b/Modules/expat/xmlparse.c
index a4e091e7c33c0ae..eefc74599cbe839 100644
--- a/Modules/expat/xmlparse.c
+++ b/Modules/expat/xmlparse.c
@@ -1,4 +1,4 @@
-/* c5625880f4bf417c1463deee4eb92d86ff413f802048621c57e25fe483eb59e4 (2.6.4+)
+/* e3ca845466942140c9af531ac57808c4650116df5d6dbeda3b6546d2548a586e (2.9.0+)
__ __ _
___\ \/ /_ __ __ _| |_
/ _ \\ /| '_ \ / _` | __|
@@ -13,7 +13,7 @@
Copyright (c) 2002-2016 Karl Waclawek <karl@waclawek.net>
Copyright (c) 2005-2009 Steven Solie <steven@solie.ca>
Copyright (c) 2016 Eric Rahm <erahm@mozilla.com>
- Copyright (c) 2016-2024 Sebastian Pipping <sebastian@pipping.org>
+ Copyright (c) 2016-2026 Sebastian Pipping <sebastian@pipping.org>
Copyright (c) 2016 Gaurav <g.gupta@samsung.com>
Copyright (c) 2016 Thomas Beutlich <tc@tbeu.de>
Copyright (c) 2016 Gustavo Grieco <gustavo.grieco@imag.fr>
@@ -39,8 +39,25 @@
Copyright (c) 2022 Sean McBride <sean@rogue-research.com>
Copyright (c) 2023 Owain Davies <owaind@bath.edu>
Copyright (c) 2023-2024 Sony Corporation / Snild Dolkow <snild@sony.com>
- Copyright (c) 2024 Berkay Eren Ürün <berkay.ueruen@siemens.com>
+ Copyright (c) 2024-2025 Berkay Eren Ürün <berkay.ueruen@siemens.com>
Copyright (c) 2024 Hanno Böck <hanno@gentoo.org>
+ Copyright (c) 2025-2026 Matthew Fernandez <matthew.fernandez@gmail.com>
+ Copyright (c) 2025 Atrem Borovik <polzovatellllk@gmail.com>
+ Copyright (c) 2025 Alfonso Gregory <gfunni234@gmail.com>
+ Copyright (c) 2026 Rosen Penev <rosenp@gmail.com>
+ Copyright (c) 2026 Francesco Bertolaccini
+ Copyright (c) 2026 Christian Ng <christianrng@berkeley.edu>
+ Copyright (c) 2026 Nick Begg <nick@stunttruck.net>
+ Copyright (c) 2026 Kartik Kenchi <netliomax25@gmail.com>
+ Copyright (c) 2026 Haris Hussain <hextheshadow0x@gmail.com>
+ Copyright (c) 2026 Evgeny Kotkov <kotkov@apache.org>
+ Copyright (c) 2026 Darren Carreras <carrerasdarren@gmail.com>
+ Copyright (c) 2026 Alberto Maschietto <albertomaschietto9@gmail.com>
+ Copyright (c) 2026 Zeyou Liu <zeyouliu@tencent.com>
+ Copyright (c) 2026 Stan Ulbrych <stan@python.org>
+ Copyright (c) 2026 Braian Plaku <braianplaku@gmail.com>
+ Copyright (c) 2026 Filippo Tedeschi <filippotedeschi98@gmail.com>
+ Copyright (c) 2026 Junki Lee <junkilee80@gmail.com>
Licensed under the MIT license:
Permission is hereby granted, free of charge, to any person obtaining
@@ -61,6 +78,8 @@
DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+ SPDX-License-Identifier: MIT
*/
#define XML_BUILDING_EXPAT 1
@@ -80,28 +99,17 @@
# error XML_CONTEXT_BYTES must be defined, non-empty and >=0 (0 to disable, >=1 to enable; 1024 is a common default)
#endif
-#if defined(HAVE_SYSCALL_GETRANDOM)
-# if ! defined(_GNU_SOURCE)
-# define _GNU_SOURCE 1 /* syscall prototype */
-# endif
-#endif
-
-#ifdef _WIN32
-/* force stdlib to define rand_s() */
-# if ! defined(_CRT_RAND_S)
-# define _CRT_RAND_S
-# endif
-#endif
-
#include <stdbool.h>
#include <stddef.h>
#include <string.h> /* memset(), memcpy() */
#include <assert.h>
-#include <limits.h> /* UINT_MAX */
+#include <limits.h> /* INT_MAX, UINT_MAX */
#include <stdio.h> /* fprintf */
-#include <stdlib.h> /* getenv, rand_s */
-#include <stdint.h> /* uintptr_t */
+#include <stdlib.h> /* getenv */
+#include <stdint.h> /* SIZE_MAX, UINT64_MAX, uint64_t, uintptr_t */
#include <math.h> /* isnan */
+#include <errno.h>
+#include <wchar.h> /* wcsncmp */
#ifdef _WIN32
# define getpid GetCurrentProcessId
@@ -120,32 +128,36 @@
#include "ascii.h"
#include "expat.h"
#include "siphash.h"
+#include "xcs.h"
+
+#if defined(HAVE_ARC4RANDOM)
+# include "random_arc4random.h"
+#endif /* defined(HAVE_ARC4RANDOM) */
+
+#if defined(HAVE_ARC4RANDOM_BUF)
+# include "random_arc4random_buf.h"
+#endif // defined(HAVE_ARC4RANDOM_BUF)
+
+#if defined(XML_DEV_URANDOM)
+# include "random_dev_urandom.h"
+#endif /* defined(XML_DEV_URANDOM) */
+
+#if defined(HAVE_GETENTROPY)
+# include "random_getentropy.h"
+#endif // defined(HAVE_GETENTROPY)
#if defined(HAVE_GETRANDOM) || defined(HAVE_SYSCALL_GETRANDOM)
-# if defined(HAVE_GETRANDOM)
-# include <sys/random.h> /* getrandom */
-# else
-# include <unistd.h> /* syscall */
-# include <sys/syscall.h> /* SYS_getrandom */
-# endif
-# if ! defined(GRND_NONBLOCK)
-# define GRND_NONBLOCK 0x0001
-# endif /* defined(GRND_NONBLOCK) */
-#endif /* defined(HAVE_GETRANDOM) || defined(HAVE_SYSCALL_GETRANDOM) */
-
-#if defined(HAVE_LIBBSD) \
- && (defined(HAVE_ARC4RANDOM_BUF) || defined(HAVE_ARC4RANDOM))
-# include <bsd/stdlib.h>
-#endif
+# include "random_getrandom.h"
+#endif /* defined(HAVE_GETRANDOM) || defined(HAVE_SYSCALL_GETRANDOM) */
-#if defined(_WIN32) && ! defined(LOAD_LIBRARY_SEARCH_SYSTEM32)
-# define LOAD_LIBRARY_SEARCH_SYSTEM32 0x00000800
-#endif
+#if defined(_WIN32) && ! defined(XML_POOR_ENTROPY)
+# include "random_rand_s.h"
+#endif /* defined(_WIN32) && ! defined(XML_POOR_ENTROPY) */
#if ! defined(HAVE_GETRANDOM) && ! defined(HAVE_SYSCALL_GETRANDOM) \
&& ! defined(HAVE_ARC4RANDOM_BUF) && ! defined(HAVE_ARC4RANDOM) \
- && ! defined(XML_DEV_URANDOM) && ! defined(_WIN32) \
- && ! defined(XML_POOR_ENTROPY)
+ && ! defined(HAVE_GETENTROPY) && ! defined(XML_DEV_URANDOM) \
+ && ! defined(_WIN32) && ! defined(XML_POOR_ENTROPY)
# error You do not have support for any sources of high quality entropy \
enabled. For end user security, that is probably not what you want. \
\
@@ -154,13 +166,12 @@
* Linux >=3.17 + glibc (including <2.25) (syscall SYS_getrandom): HAVE_SYSCALL_GETRANDOM, \
* BSD / macOS >=10.7 / glibc >=2.36 (arc4random_buf): HAVE_ARC4RANDOM_BUF, \
* BSD / macOS (including <10.7) / glibc >=2.36 (arc4random): HAVE_ARC4RANDOM, \
- * libbsd (arc4random_buf): HAVE_ARC4RANDOM_BUF + HAVE_LIBBSD, \
- * libbsd (arc4random): HAVE_ARC4RANDOM + HAVE_LIBBSD, \
+ * BSD / macOS >=10.12 / glibc >=2.25 (getentropy): HAVE_GETENTROPY, \
* Linux (including <3.17) / BSD / macOS (including <10.7) / Solaris >=8 (/dev/urandom): XML_DEV_URANDOM, \
* Windows >=Vista (rand_s): _WIN32. \
\
- If insist on not using any of these, bypass this error by defining \
- XML_POOR_ENTROPY; you have been warned. \
+ If you insist on not using any of these, bypass this error by defining \
+ XML_POOR_ENTROPY and be vulnerable to hash flooding; you have been warned. \
\
If you have reasons to patch this detection code away or need changes \
to the build system, please open a bug. Thank you!
@@ -220,25 +231,10 @@ typedef char ICHAR;
#define EXPAT_MIN(a, b) (((a) < (b)) ? (a) : (b))
#include "internal.h"
+#include "hash_table.h"
#include "xmltok.h"
#include "xmlrole.h"
-typedef const XML_Char *KEY;
-
-typedef struct {
- KEY name;
-} NAMED;
-
-typedef struct {
- NAMED **v;
- unsigned char power;
- size_t size;
- size_t used;
- const XML_Memory_Handling_Suite *mem;
-} HASH_TABLE;
-
-static size_t keylen(KEY s);
-
static void copy_salt_to_sipkey(XML_Parser parser, struct sipkey *key);
/* For probing (after a collision) we need a step size relative prime
@@ -254,11 +250,6 @@ static void copy_salt_to_sipkey(XML_Parser parser, struct sipkey *key);
#define PROBE_STEP(hash, mask, power) \
((unsigned char)((SECOND_HASH(hash, mask, power)) | 1))
-typedef struct {
- NAMED **p;
- NAMED **end;
-} HASH_TABLE_ITER;
-
#define INIT_TAG_BUF_SIZE 32 /* must be a multiple of sizeof(XML_Char) */
#define INIT_DATA_BUF_SIZE 1024
#define INIT_ATTS_SIZE 16
@@ -274,8 +265,8 @@ typedef struct binding {
struct binding *prevPrefixBinding;
const struct attribute_id *attId;
XML_Char *uri;
- int uriLen;
- int uriAlloc;
+ size_t uriLen;
+ size_t uriAlloc;
} BINDING;
typedef struct prefix {
@@ -287,9 +278,9 @@ typedef struct {
const XML_Char *str;
const XML_Char *localPart;
const XML_Char *prefix;
- int strLen;
- int uriLen;
- int prefixLen;
+ size_t strLen;
+ size_t uriLen;
+ size_t prefixLen;
} TAG_NAME;
/* TAG represents an open element.
@@ -310,8 +301,11 @@ typedef struct tag {
const char *rawName; /* tagName in the original encoding */
int rawNameLength;
TAG_NAME name; /* tagName in the API encoding */
- char *buf; /* buffer for name components */
- char *bufEnd; /* end of the buffer */
+ union {
+ char *raw; /* for byte-level access (rawName storage) */
+ XML_Char *str; /* for character-level access (converted name) */
+ } buf; /* buffer for name components */
+ char *bufEnd; /* end of the buffer */
BINDING *bindings;
} TAG;
@@ -324,9 +318,13 @@ typedef struct {
const XML_Char *base;
const XML_Char *publicId;
const XML_Char *notation;
- XML_Bool open;
- XML_Bool is_param;
- XML_Bool is_internal; /* true if declared in internal subset outside PE */
+ bool open;
+ bool hasMore; /* true if entity has not been completely processed */
+ /* An entity can be open while being already completely processed (!hasMore).
+ The reason is the delayed closing of entities until their inner
+ entities are processed and closed */
+ bool is_param;
+ bool is_internal; /* true if declared in internal subset outside PE */
} ENTITY;
typedef struct {
@@ -344,7 +342,7 @@ typedef struct {
typedef struct block {
struct block *next;
int size;
- XML_Char s[1];
+ XML_Char s[];
} BLOCK;
typedef struct {
@@ -353,7 +351,7 @@ typedef struct {
const XML_Char *end;
XML_Char *ptr;
XML_Char *start;
- const XML_Memory_Handling_Suite *mem;
+ XML_Parser parser;
} STRING_POOL;
/* The XML_Char before the name is used to determine whether
@@ -361,8 +359,8 @@ typedef struct {
typedef struct attribute_id {
XML_Char *name;
PREFIX *prefix;
- XML_Bool maybeTokenized;
- XML_Bool xmlns;
+ bool maybeTokenized;
+ bool xmlns;
} ATTRIBUTE_ID;
typedef struct {
@@ -371,6 +369,22 @@ typedef struct {
const XML_Char *value;
} DEFAULT_ATTRIBUTE;
+// This structure allows mapping attribute names to instances of
+// `DEFAULT_ATTRIBUTE`.
+typedef struct {
+ // Member `name` goes first to make this structure compatible with structure
+ // `NAMED` (further up), which is needed to support use of structure
+ // `NAME_AND_DEFAULT_ATTRIBUTE` in a hash table as implemented by function
+ // `lookup` (further down).
+ const XML_Char *name;
+ // We would store a `DEFAULT_ATTRIBUTE *` here but the backing array
+ // can be reallocated which would invalidate the pointer. Using an index
+ // into the array instead, avoids that problem.
+ size_t attIndex;
+ // This is set to `false` by function `lookup`.
+ bool initialized;
+} NAME_AND_DEFAULT_ATTRIBUTE;
+
typedef struct {
unsigned long version;
unsigned long hash;
@@ -381,9 +395,10 @@ typedef struct {
const XML_Char *name;
PREFIX *prefix;
const ATTRIBUTE_ID *idAtt;
- int nDefaultAtts;
- int allocDefaultAtts;
+ size_t nDefaultAtts;
+ size_t allocDefaultAtts;
DEFAULT_ATTRIBUTE *defaultAtts;
+ HASH_TABLE defaultAttForName;
} ELEMENT_TYPE;
typedef struct {
@@ -401,27 +416,35 @@ typedef struct {
XML_Bool standalone;
#ifdef XML_DTD
/* indicates if external PE has been read */
- XML_Bool paramEntityRead;
+ bool paramEntityRead;
HASH_TABLE paramEntities;
#endif /* XML_DTD */
PREFIX defaultPrefix;
/* === scaffolding for building content model === */
- XML_Bool in_eldecl;
+ bool in_eldecl;
CONTENT_SCAFFOLD *scaffold;
unsigned contentStringLen;
unsigned scaffSize;
unsigned scaffCount;
int scaffLevel;
int *scaffIndex;
+ size_t scaffIndexSize;
} DTD;
+enum EntityType {
+ ENTITY_INTERNAL,
+ ENTITY_ATTRIBUTE,
+ ENTITY_VALUE,
+};
+
typedef struct open_internal_entity {
const char *internalEventPtr;
const char *internalEventEndPtr;
struct open_internal_entity *next;
ENTITY *entity;
int startTagLevel;
- XML_Bool betweenDecl; /* WFC: PE Between Declarations */
+ bool betweenDecl; /* WFC: PE Between Declarations */
+ enum EntityType type;
} OPEN_INTERNAL_ENTITY;
enum XML_Account {
@@ -441,6 +464,14 @@ typedef struct accounting {
unsigned long long activationThresholdBytes;
} ACCOUNTING;
+typedef struct MALLOC_TRACKER {
+ size_t bytesAllocated;
+ size_t peakBytesAllocated; // updated live only for debug level >=2
+ unsigned long debugLevel;
+ float maximumAmplificationFactor; // >=1.0
+ XmlBigCount activationThresholdBytes;
+} MALLOC_TRACKER;
+
typedef struct entity_stats {
unsigned int countEverOpened;
unsigned int currentDepth;
@@ -449,8 +480,8 @@ typedef struct entity_stats {
} ENTITY_STATS;
#endif /* XML_GE == 1 */
-typedef enum XML_Error PTRCALL Processor(XML_Parser parser, const char *start,
- const char *end, const char **endPtr);
+typedef enum XML_Error Processor(XML_Parser parser, const char *start,
+ const char *end, const char **endPtr);
static Processor prologProcessor;
static Processor prologInitProcessor;
@@ -481,8 +512,8 @@ static enum XML_Error doProlog(XML_Parser parser, const ENCODING *enc,
const char *next, const char **nextPtr,
XML_Bool haveMore, XML_Bool allowClosingDoctype,
enum XML_Account account);
-static enum XML_Error processInternalEntity(XML_Parser parser, ENTITY *entity,
- XML_Bool betweenDecl);
+static enum XML_Error processEntity(XML_Parser parser, ENTITY *entity,
+ bool betweenDecl, enum EntityType type);
static enum XML_Error doContent(XML_Parser parser, int startTagLevel,
const ENCODING *enc, const char *start,
const char *end, const char **endPtr,
@@ -513,18 +544,22 @@ static enum XML_Error storeAttributeValue(XML_Parser parser,
const char *ptr, const char *end,
STRING_POOL *pool,
enum XML_Account account);
-static enum XML_Error appendAttributeValue(XML_Parser parser,
- const ENCODING *enc,
- XML_Bool isCdata, const char *ptr,
- const char *end, STRING_POOL *pool,
- enum XML_Account account);
+static enum XML_Error
+appendAttributeValue(XML_Parser parser, const ENCODING *enc, XML_Bool isCdata,
+ const char *ptr, const char *end, STRING_POOL *pool,
+ enum XML_Account account, const char **nextPtr);
static ATTRIBUTE_ID *getAttributeId(XML_Parser parser, const ENCODING *enc,
const char *start, const char *end);
static int setElementTypePrefix(XML_Parser parser, ELEMENT_TYPE *elementType);
#if XML_GE == 1
static enum XML_Error storeEntityValue(XML_Parser parser, const ENCODING *enc,
const char *start, const char *end,
- enum XML_Account account);
+ enum XML_Account account,
+ const char **nextPtr);
+static enum XML_Error callStoreEntityValue(XML_Parser parser,
+ const ENCODING *enc,
+ const char *start, const char *end,
+ enum XML_Account account);
#else
static enum XML_Error storeSelfEntityValue(XML_Parser parser, ENTITY *entity);
#endif
@@ -538,57 +573,47 @@ static void reportDefault(XML_Parser parser, const ENCODING *enc,
static const XML_Char *getContext(XML_Parser parser);
static XML_Bool setContext(XML_Parser parser, const XML_Char *context);
-static void FASTCALL normalizePublicId(XML_Char *s);
+static void normalizePublicId(XML_Char *s);
-static DTD *dtdCreate(const XML_Memory_Handling_Suite *ms);
+static DTD *dtdCreate(XML_Parser parser);
/* do not call if m_parentParser != NULL */
-static void dtdReset(DTD *p, const XML_Memory_Handling_Suite *ms);
-static void dtdDestroy(DTD *p, XML_Bool isDocEntity,
- const XML_Memory_Handling_Suite *ms);
+static void dtdReset(DTD *p, XML_Parser parser);
+static void dtdDestroy(DTD *p, XML_Bool isDocEntity, XML_Parser parser);
static int dtdCopy(XML_Parser oldParser, DTD *newDtd, const DTD *oldDtd,
- const XML_Memory_Handling_Suite *ms);
+ XML_Parser parser);
static int copyEntityTable(XML_Parser oldParser, HASH_TABLE *newTable,
STRING_POOL *newPool, const HASH_TABLE *oldTable);
-static NAMED *lookup(XML_Parser parser, HASH_TABLE *table, KEY name,
- size_t createSize);
-static void FASTCALL hashTableInit(HASH_TABLE *table,
- const XML_Memory_Handling_Suite *ms);
-static void FASTCALL hashTableClear(HASH_TABLE *table);
-static void FASTCALL hashTableDestroy(HASH_TABLE *table);
-static void FASTCALL hashTableIterInit(HASH_TABLE_ITER *iter,
- const HASH_TABLE *table);
-static NAMED *FASTCALL hashTableIterNext(HASH_TABLE_ITER *iter);
-
-static void FASTCALL poolInit(STRING_POOL *pool,
- const XML_Memory_Handling_Suite *ms);
-static void FASTCALL poolClear(STRING_POOL *pool);
-static void FASTCALL poolDestroy(STRING_POOL *pool);
+
+static void poolInit(STRING_POOL *pool, XML_Parser parser);
+static void poolClear(STRING_POOL *pool);
+static void poolDestroy(STRING_POOL *pool);
static XML_Char *poolAppend(STRING_POOL *pool, const ENCODING *enc,
const char *ptr, const char *end);
static XML_Char *poolStoreString(STRING_POOL *pool, const ENCODING *enc,
const char *ptr, const char *end);
-static XML_Bool FASTCALL poolGrow(STRING_POOL *pool);
-static const XML_Char *FASTCALL poolCopyString(STRING_POOL *pool,
- const XML_Char *s);
+static XML_Bool poolGrow(STRING_POOL *pool);
+static bool poolGrowUntil(STRING_POOL *pool, size_t needed);
+static const XML_Char *poolCopyString(STRING_POOL *pool, const XML_Char *s);
+static const XML_Char *poolCopyStringNoFinish(STRING_POOL *pool,
+ const XML_Char *s);
static const XML_Char *poolCopyStringN(STRING_POOL *pool, const XML_Char *s,
int n);
-static const XML_Char *FASTCALL poolAppendString(STRING_POOL *pool,
- const XML_Char *s);
+static const XML_Char *poolAppendString(STRING_POOL *pool, const XML_Char *s);
-static int FASTCALL nextScaffoldPart(XML_Parser parser);
+static int nextScaffoldPart(XML_Parser parser);
static XML_Content *build_model(XML_Parser parser);
static ELEMENT_TYPE *getElementType(XML_Parser parser, const ENCODING *enc,
const char *ptr, const char *end);
-static XML_Char *copyString(const XML_Char *s,
- const XML_Memory_Handling_Suite *memsuite);
+static XML_Char *copyString(const XML_Char *s, XML_Parser parser);
-static unsigned long generate_hash_secret_salt(XML_Parser parser);
+static struct sipkey generate_hash_secret_salt(void);
static XML_Bool startParsing(XML_Parser parser);
static XML_Parser parserCreate(const XML_Char *encodingName,
const XML_Memory_Handling_Suite *memsuite,
- const XML_Char *nameSep, DTD *dtd);
+ const XML_Char *nameSep, DTD *dtd,
+ XML_Parser parentParser);
static void parserInit(XML_Parser parser, const XML_Char *encodingName);
@@ -612,24 +637,67 @@ static void entityTrackingOnOpen(XML_Parser parser, ENTITY *entity,
int sourceLine);
static void entityTrackingOnClose(XML_Parser parser, ENTITY *entity,
int sourceLine);
+#endif /* XML_GE == 1 */
static XML_Parser getRootParserOf(XML_Parser parser,
unsigned int *outLevelDiff);
-#endif /* XML_GE == 1 */
static unsigned long getDebugLevel(const char *variableName,
unsigned long defaultDebugLevel);
+static bool poolAppendChar(STRING_POOL *pool, XML_Char c);
+
+static bool poolAppendChars(STRING_POOL *pool, const XML_Char *s, size_t len);
+
+#if XML_GE == 1
+static enum XML_Prop_Error setBillionLaughsAttackProtectionMaximumAmplification(
+ XML_Parser parser, float maximumAmplificationFactor);
+
+static enum XML_Prop_Error setBillionLaughsAttackProtectionActivationThreshold(
+ XML_Parser parser, unsigned long long activationThresholdBytes);
+
+static enum XML_Prop_Error
+setAllocTrackerMaximumAmplification(XML_Parser parser,
+ float maximumAmplificationFactor);
+
+static enum XML_Prop_Error
+setAllocTrackerActivationThreshold(XML_Parser parser,
+ unsigned long long activationThresholdBytes);
+#endif /* XML_GE == 1 */
+
+static enum XML_Prop_Error setReparseDeferralEnabled(XML_Parser parser,
+ XML_Bool enabled);
+
#define poolStart(pool) ((pool)->start)
#define poolLength(pool) ((pool)->ptr - (pool)->start)
#define poolChop(pool) ((void)--(pool->ptr))
#define poolLastChar(pool) (((pool)->ptr)[-1])
#define poolDiscard(pool) ((pool)->ptr = (pool)->start)
#define poolFinish(pool) ((pool)->start = (pool)->ptr)
-#define poolAppendChar(pool, c) \
- (((pool)->ptr == (pool)->end && ! poolGrow(pool)) \
- ? 0 \
- : ((*((pool)->ptr)++ = c), 1))
+
+bool
+poolAppendChar(STRING_POOL *pool, XML_Char c) {
+ if (pool->ptr == pool->end && ! poolGrow(pool))
+ return false;
+
+ *(pool->ptr)++ = c;
+ return true;
+}
+
+bool
+poolAppendChars(STRING_POOL *pool, const XML_Char *s, size_t len) {
+ // Detect and prevent integer overflow
+ if (len > SIZE_MAX / sizeof(XML_Char))
+ return false;
+
+ if (! poolGrowUntil(pool, len))
+ return false;
+
+ memcpy(pool->ptr, s, len * sizeof(XML_Char));
+ pool->ptr += len;
+
+ return true;
+}
#if ! defined(XML_TESTING)
const
@@ -661,7 +729,7 @@ struct XML_ParserStruct {
char *m_bufferEnd; // past last character to be parsed
const char *m_bufferLim; // allocated end of m_buffer
- XML_Index m_parseEndByteIndex;
+ uint64_t m_parseEndByteIndex;
const char *m_parseEndPtr;
size_t m_partialTokenBytesBefore; /* used in heuristic to avoid O(n^2) */
XML_Bool m_reparseDeferralEnabled;
@@ -700,6 +768,8 @@ struct XML_ParserStruct {
void *m_unknownEncodingMem;
void *m_unknownEncodingData;
void *m_unknownEncodingHandlerData;
+ // Application callback invoked by callUnknownEncodingConvert.
+ int(XMLCALL *m_unknownEncodingConvert)(void *, const char *);
void(XMLCALL *m_unknownEncodingRelease)(void *);
PROLOG_STATE m_prologState;
Processor *m_processor;
@@ -708,7 +778,9 @@ struct XML_ParserStruct {
const char *m_eventEndPtr;
const char *m_positionPtr;
OPEN_INTERNAL_ENTITY *m_openInternalEntities;
- OPEN_INTERNAL_ENTITY *m_freeInternalEntities;
+ OPEN_INTERNAL_ENTITY *m_openAttributeEntities;
+ OPEN_INTERNAL_ENTITY *m_openValueEntities;
+ OPEN_INTERNAL_ENTITY *m_freeEntities;
XML_Bool m_defaultExpandInternalEntities;
int m_tagLevel;
ENTITY *m_declEntity;
@@ -728,21 +800,18 @@ struct XML_ParserStruct {
TAG *m_freeTagList;
BINDING *m_inheritedBindings;
BINDING *m_freeBindingList;
- int m_attsSize;
+ size_t m_attsSize;
int m_nSpecifiedAtts;
int m_idAttIndex;
ATTRIBUTE *m_atts;
NS_ATT *m_nsAtts;
unsigned long m_nsAttsVersion;
unsigned char m_nsAttsPower;
-#ifdef XML_ATTR_INFO
- XML_AttrInfo *m_attInfo;
-#endif
POSITION m_position;
STRING_POOL m_tempPool;
STRING_POOL m_temp2Pool;
char *m_groupConnector;
- unsigned int m_groupSize;
+ size_t m_groupSize;
XML_Char m_namespaceSeparator;
XML_Parser m_parentParser;
XML_ParsingStatus m_parsingStatus;
@@ -751,165 +820,265 @@ struct XML_ParserStruct {
XML_Bool m_useForeignDTD;
enum XML_ParamEntityParsing m_paramEntityParsing;
#endif
- unsigned long m_hash_secret_salt;
+ struct sipkey m_hash_secret_salt_128;
+ XML_Bool m_hash_secret_salt_set;
#if XML_GE == 1
ACCOUNTING m_accounting;
+ MALLOC_TRACKER m_alloc_tracker;
ENTITY_STATS m_entity_stats;
#endif
+ XML_Bool m_reenter;
+ unsigned m_handlerCallDepth;
};
-#define MALLOC(parser, s) (parser->m_mem.malloc_fcn((s)))
-#define REALLOC(parser, p, s) (parser->m_mem.realloc_fcn((p), (s)))
-#define FREE(parser, p) (parser->m_mem.free_fcn((p)))
-
-XML_Parser XMLCALL
-XML_ParserCreate(const XML_Char *encodingName) {
- return XML_ParserCreate_MM(encodingName, NULL, NULL);
-}
+#if XML_GE == 1
+# define MALLOC(parser, s) (expat_malloc((parser), (s), __LINE__))
+# define REALLOC(parser, p, s) (expat_realloc((parser), (p), (s), __LINE__))
+# define FREE(parser, p) (expat_free((parser), (p), __LINE__))
+#else
+# define MALLOC(parser, s) (parser->m_mem.malloc_fcn((s)))
+# define REALLOC(parser, p, s) (parser->m_mem.realloc_fcn((p), (s)))
+# define FREE(parser, p) (parser->m_mem.free_fcn((p)))
+#endif
-XML_Parser XMLCALL
-XML_ParserCreateNS(const XML_Char *encodingName, XML_Char nsSep) {
- XML_Char tmp[2] = {nsSep, 0};
- return XML_ParserCreate_MM(encodingName, NULL, tmp);
+#if XML_GE == 1
+static void
+expat_heap_stat(XML_Parser rootParser, char operator, size_t absDiff,
+ size_t newTotal, size_t peakTotal, int sourceLine) {
+ // NOTE: This can be +infinity or -nan
+ const float amplification
+ = (float)newTotal / (float)rootParser->m_accounting.countBytesDirect;
+ fprintf(
+ stderr,
+ "expat: Allocations(%p): Direct " EXPAT_FMT_ULL("10") ", allocated %c" EXPAT_FMT_SIZE_T(
+ "10") " to " EXPAT_FMT_SIZE_T("10") " (" EXPAT_FMT_SIZE_T("10") " peak), amplification %8.2f (xmlparse.c:%d)\n",
+ (void *)rootParser, rootParser->m_accounting.countBytesDirect, operator,
+ absDiff, newTotal, peakTotal, (double)amplification, sourceLine);
}
-// "xml=http://www.w3.org/XML/1998/namespace"
-static const XML_Char implicitContext[]
- = {ASCII_x, ASCII_m, ASCII_l, ASCII_EQUALS, ASCII_h,
- ASCII_t, ASCII_t, ASCII_p, ASCII_COLON, ASCII_SLASH,
- ASCII_SLASH, ASCII_w, ASCII_w, ASCII_w, ASCII_PERIOD,
- ASCII_w, ASCII_3, ASCII_PERIOD, ASCII_o, ASCII_r,
- ASCII_g, ASCII_SLASH, ASCII_X, ASCII_M, ASCII_L,
- ASCII_SLASH, ASCII_1, ASCII_9, ASCII_9, ASCII_8,
- ASCII_SLASH, ASCII_n, ASCII_a, ASCII_m, ASCII_e,
- ASCII_s, ASCII_p, ASCII_a, ASCII_c, ASCII_e,
- '\0'};
+static bool
+expat_heap_increase_tolerable(XML_Parser rootParser, size_t increase,
+ int sourceLine) {
+ assert(rootParser != NULL);
+ assert(increase > 0);
-/* To avoid warnings about unused functions: */
-#if ! defined(HAVE_ARC4RANDOM_BUF) && ! defined(HAVE_ARC4RANDOM)
+ size_t newTotal = 0;
+ bool tolerable = true;
-# if defined(HAVE_GETRANDOM) || defined(HAVE_SYSCALL_GETRANDOM)
+ // Detect integer overflow
+ if (SIZE_MAX - rootParser->m_alloc_tracker.bytesAllocated < increase) {
+ tolerable = false;
+ } else {
+ newTotal = rootParser->m_alloc_tracker.bytesAllocated + increase;
+
+ if (newTotal >= rootParser->m_alloc_tracker.activationThresholdBytes) {
+ assert(newTotal > 0);
+ // NOTE: This can be +infinity when dividing by zero but not -nan
+ const float amplification
+ = (float)newTotal / (float)rootParser->m_accounting.countBytesDirect;
+ if (amplification
+ > rootParser->m_alloc_tracker.maximumAmplificationFactor) {
+ tolerable = false;
+ }
+ }
+ }
-/* Obtain entropy on Linux 3.17+ */
-static int
-writeRandomBytes_getrandom_nonblock(void *target, size_t count) {
- int success = 0; /* full count bytes written? */
- size_t bytesWrittenTotal = 0;
- const unsigned int getrandomFlags = GRND_NONBLOCK;
+ if (! tolerable && (rootParser->m_alloc_tracker.debugLevel >= 1)) {
+ expat_heap_stat(rootParser, '+', increase, newTotal, newTotal, sourceLine);
+ }
- do {
- void *const currentTarget = (void *)((char *)target + bytesWrittenTotal);
- const size_t bytesToWrite = count - bytesWrittenTotal;
+ return tolerable;
+}
- const int bytesWrittenMore =
-# if defined(HAVE_GETRANDOM)
- getrandom(currentTarget, bytesToWrite, getrandomFlags);
-# else
- syscall(SYS_getrandom, currentTarget, bytesToWrite, getrandomFlags);
-# endif
+# if defined(XML_TESTING)
+void *
+# else
+static void *
+# endif
+expat_malloc(XML_Parser parser, size_t size, int sourceLine) {
+ // Detect integer overflow
+ if (SIZE_MAX - size < sizeof(size_t) + EXPAT_MALLOC_PADDING) {
+ return NULL;
+ }
- if (bytesWrittenMore > 0) {
- bytesWrittenTotal += bytesWrittenMore;
- if (bytesWrittenTotal >= count)
- success = 1;
- }
- } while (! success && (errno == EINTR));
+ const XML_Parser rootParser = getRootParserOf(parser, NULL);
+ assert(rootParser->m_parentParser == NULL);
- return success;
-}
+ const size_t bytesToAllocate = sizeof(size_t) + EXPAT_MALLOC_PADDING + size;
-# endif /* defined(HAVE_GETRANDOM) || defined(HAVE_SYSCALL_GETRANDOM) */
+ if (SIZE_MAX - rootParser->m_alloc_tracker.bytesAllocated < bytesToAllocate) {
+ return NULL; // i.e. signal integer overflow as out-of-memory
+ }
-# if ! defined(_WIN32) && defined(XML_DEV_URANDOM)
+ if (! expat_heap_increase_tolerable(rootParser, bytesToAllocate,
+ sourceLine)) {
+ return NULL; // i.e. signal violation as out-of-memory
+ }
-/* Extract entropy from /dev/urandom */
-static int
-writeRandomBytes_dev_urandom(void *target, size_t count) {
- int success = 0; /* full count bytes written? */
- size_t bytesWrittenTotal = 0;
+ // Actually allocate
+ void *const mallocedPtr = parser->m_mem.malloc_fcn(bytesToAllocate);
- const int fd = open("/dev/urandom", O_RDONLY);
- if (fd < 0) {
- return 0;
+ if (mallocedPtr == NULL) {
+ return NULL;
}
- do {
- void *const currentTarget = (void *)((char *)target + bytesWrittenTotal);
- const size_t bytesToWrite = count - bytesWrittenTotal;
+ // Update in-block recorded size
+ *(size_t *)mallocedPtr = size;
- const ssize_t bytesWrittenMore = read(fd, currentTarget, bytesToWrite);
+ // Update accounting
+ rootParser->m_alloc_tracker.bytesAllocated += bytesToAllocate;
- if (bytesWrittenMore > 0) {
- bytesWrittenTotal += bytesWrittenMore;
- if (bytesWrittenTotal >= count)
- success = 1;
+ // Report as needed
+ if (rootParser->m_alloc_tracker.debugLevel >= 2) {
+ if (rootParser->m_alloc_tracker.bytesAllocated
+ > rootParser->m_alloc_tracker.peakBytesAllocated) {
+ rootParser->m_alloc_tracker.peakBytesAllocated
+ = rootParser->m_alloc_tracker.bytesAllocated;
}
- } while (! success && (errno == EINTR));
+ expat_heap_stat(rootParser, '+', bytesToAllocate,
+ rootParser->m_alloc_tracker.bytesAllocated,
+ rootParser->m_alloc_tracker.peakBytesAllocated, sourceLine);
+ }
- close(fd);
- return success;
+ return (char *)mallocedPtr + sizeof(size_t) + EXPAT_MALLOC_PADDING;
}
-# endif /* ! defined(_WIN32) && defined(XML_DEV_URANDOM) */
+# if defined(XML_TESTING)
+void
+# else
+static void
+# endif
+expat_free(XML_Parser parser, void *ptr, int sourceLine) {
+ assert(parser != NULL);
-#endif /* ! defined(HAVE_ARC4RANDOM_BUF) && ! defined(HAVE_ARC4RANDOM) */
+ if (ptr == NULL) {
+ return;
+ }
-#if defined(HAVE_ARC4RANDOM) && ! defined(HAVE_ARC4RANDOM_BUF)
+ const XML_Parser rootParser = getRootParserOf(parser, NULL);
+ assert(rootParser->m_parentParser == NULL);
-static void
-writeRandomBytes_arc4random(void *target, size_t count) {
- size_t bytesWrittenTotal = 0;
+ // Extract size (to the eyes of malloc_fcn/realloc_fcn) and
+ // the original pointer returned by malloc/realloc
+ void *const mallocedPtr = (char *)ptr - EXPAT_MALLOC_PADDING - sizeof(size_t);
+ const size_t bytesAllocated
+ = sizeof(size_t) + EXPAT_MALLOC_PADDING + *(size_t *)mallocedPtr;
- while (bytesWrittenTotal < count) {
- const uint32_t random32 = arc4random();
- size_t i = 0;
+ // Update accounting
+ assert(rootParser->m_alloc_tracker.bytesAllocated >= bytesAllocated);
+ rootParser->m_alloc_tracker.bytesAllocated -= bytesAllocated;
- for (; (i < sizeof(random32)) && (bytesWrittenTotal < count);
- i++, bytesWrittenTotal++) {
- const uint8_t random8 = (uint8_t)(random32 >> (i * 8));
- ((uint8_t *)target)[bytesWrittenTotal] = random8;
- }
+ // Report as needed
+ if (rootParser->m_alloc_tracker.debugLevel >= 2) {
+ expat_heap_stat(rootParser, '-', bytesAllocated,
+ rootParser->m_alloc_tracker.bytesAllocated,
+ rootParser->m_alloc_tracker.peakBytesAllocated, sourceLine);
}
+
+ // NOTE: This may be freeing rootParser, so freeing has to come last
+ parser->m_mem.free_fcn(mallocedPtr);
}
-#endif /* defined(HAVE_ARC4RANDOM) && ! defined(HAVE_ARC4RANDOM_BUF) */
+# if defined(XML_TESTING)
+void *
+# else
+static void *
+# endif
+expat_realloc(XML_Parser parser, void *ptr, size_t size, int sourceLine) {
+ assert(parser != NULL);
-#ifdef _WIN32
+ if (ptr == NULL) {
+ return expat_malloc(parser, size, sourceLine);
+ }
-/* Provide declaration of rand_s() for MinGW-32 (not 64, which has it),
- as it didn't declare it in its header prior to version 5.3.0 of its
- runtime package (mingwrt, containing stdlib.h). The upstream fix
- was introduced at https://osdn.net/projects/mingw/ticket/39658 . */
-# if defined(__MINGW32__) && defined(__MINGW32_VERSION) \
- && __MINGW32_VERSION < 5003000L && ! defined(__MINGW64_VERSION_MAJOR)
-__declspec(dllimport) int rand_s(unsigned int *);
-# endif
+ if (size == 0) {
+ expat_free(parser, ptr, sourceLine);
+ return NULL;
+ }
-/* Obtain entropy on Windows using the rand_s() function which
- * generates cryptographically secure random numbers. Internally it
- * uses RtlGenRandom API which is present in Windows XP and later.
- */
-static int
-writeRandomBytes_rand_s(void *target, size_t count) {
- size_t bytesWrittenTotal = 0;
+ const XML_Parser rootParser = getRootParserOf(parser, NULL);
+ assert(rootParser->m_parentParser == NULL);
+
+ // Extract original size (to the eyes of the caller) and the original
+ // pointer returned by malloc/realloc
+ void *mallocedPtr = (char *)ptr - EXPAT_MALLOC_PADDING - sizeof(size_t);
+ const size_t prevSize = *(size_t *)mallocedPtr;
+
+ // Classify upcoming change
+ const bool isIncrease = (size > prevSize);
+ const size_t absDiff
+ = (size > prevSize) ? (size - prevSize) : (prevSize - size);
+
+ // Ask for permission from accounting
+ if (isIncrease) {
+ if (! expat_heap_increase_tolerable(rootParser, absDiff, sourceLine)) {
+ return NULL; // i.e. signal violation as out-of-memory
+ }
+ }
+
+ // Detect and prevent integer overflow
+ if (size > SIZE_MAX - sizeof(size_t) - EXPAT_MALLOC_PADDING) {
+ return NULL;
+ }
+
+ // Actually allocate
+ mallocedPtr = parser->m_mem.realloc_fcn(
+ mallocedPtr, sizeof(size_t) + EXPAT_MALLOC_PADDING + size);
- while (bytesWrittenTotal < count) {
- unsigned int random32 = 0;
- size_t i = 0;
+ if (mallocedPtr == NULL) {
+ return NULL;
+ }
- if (rand_s(&random32))
- return 0; /* failure */
+ // Update accounting
+ if (isIncrease) {
+ assert(SIZE_MAX - rootParser->m_alloc_tracker.bytesAllocated >= absDiff);
+ rootParser->m_alloc_tracker.bytesAllocated += absDiff;
+ } else { // i.e. decrease
+ assert(rootParser->m_alloc_tracker.bytesAllocated >= absDiff);
+ rootParser->m_alloc_tracker.bytesAllocated -= absDiff;
+ }
- for (; (i < sizeof(random32)) && (bytesWrittenTotal < count);
- i++, bytesWrittenTotal++) {
- const uint8_t random8 = (uint8_t)(random32 >> (i * 8));
- ((uint8_t *)target)[bytesWrittenTotal] = random8;
+ // Report as needed
+ if (rootParser->m_alloc_tracker.debugLevel >= 2) {
+ if (rootParser->m_alloc_tracker.bytesAllocated
+ > rootParser->m_alloc_tracker.peakBytesAllocated) {
+ rootParser->m_alloc_tracker.peakBytesAllocated
+ = rootParser->m_alloc_tracker.bytesAllocated;
}
+ expat_heap_stat(rootParser, isIncrease ? '+' : '-', absDiff,
+ rootParser->m_alloc_tracker.bytesAllocated,
+ rootParser->m_alloc_tracker.peakBytesAllocated, sourceLine);
}
- return 1; /* success */
+
+ // Update in-block recorded size
+ *(size_t *)mallocedPtr = size;
+
+ return (char *)mallocedPtr + sizeof(size_t) + EXPAT_MALLOC_PADDING;
+}
+#endif // XML_GE == 1
+
+XML_Parser XMLCALL
+XML_ParserCreate(const XML_Char *encodingName) {
+ return XML_ParserCreate_MM(encodingName, NULL, NULL);
}
-#endif /* _WIN32 */
+XML_Parser XMLCALL
+XML_ParserCreateNS(const XML_Char *encodingName, XML_Char nsSep) {
+ XML_Char tmp[2] = {nsSep, 0};
+ return XML_ParserCreate_MM(encodingName, NULL, tmp);
+}
+
+// "xml=http://www.w3.org/XML/1998/namespace"
+static const XML_Char implicitContext[]
+ = {ASCII_x, ASCII_m, ASCII_l, ASCII_EQUALS, ASCII_h,
+ ASCII_t, ASCII_t, ASCII_p, ASCII_COLON, ASCII_SLASH,
+ ASCII_SLASH, ASCII_w, ASCII_w, ASCII_w, ASCII_PERIOD,
+ ASCII_w, ASCII_3, ASCII_PERIOD, ASCII_o, ASCII_r,
+ ASCII_g, ASCII_SLASH, ASCII_X, ASCII_M, ASCII_L,
+ ASCII_SLASH, ASCII_1, ASCII_9, ASCII_9, ASCII_8,
+ ASCII_SLASH, ASCII_n, ASCII_a, ASCII_m, ASCII_e,
+ ASCII_s, ASCII_p, ASCII_a, ASCII_c, ASCII_e,
+ '\0'};
#if ! defined(HAVE_ARC4RANDOM_BUF) && ! defined(HAVE_ARC4RANDOM)
@@ -938,63 +1107,104 @@ gather_time_entropy(void) {
#endif /* ! defined(HAVE_ARC4RANDOM_BUF) && ! defined(HAVE_ARC4RANDOM) */
-static unsigned long
-ENTROPY_DEBUG(const char *label, unsigned long entropy) {
+static struct sipkey
+ENTROPY_DEBUG(const char *label, struct sipkey entropy_128) {
if (getDebugLevel("EXPAT_ENTROPY_DEBUG", 0) >= 1u) {
- fprintf(stderr, "expat: Entropy: %s --> 0x%0*lx (%lu bytes)\n", label,
- (int)sizeof(entropy) * 2, entropy, (unsigned long)sizeof(entropy));
+ fprintf(stderr,
+ "expat: Entropy: %s --> [0x" EXPAT_FMT_LLX(
+ "016") ", 0x" EXPAT_FMT_LLX("016") "] (16 bytes)\n",
+ label, (unsigned long long)entropy_128.k[0],
+ (unsigned long long)entropy_128.k[1]);
}
- return entropy;
+ return entropy_128;
}
-static unsigned long
-generate_hash_secret_salt(XML_Parser parser) {
- unsigned long entropy;
- (void)parser;
+static struct sipkey
+generate_hash_secret_salt(void) {
+ struct sipkey entropy;
/* "Failproof" high quality providers: */
#if defined(HAVE_ARC4RANDOM_BUF)
- arc4random_buf(&entropy, sizeof(entropy));
+ writeRandomBytes_arc4random_buf(&entropy, sizeof(entropy));
return ENTROPY_DEBUG("arc4random_buf", entropy);
#elif defined(HAVE_ARC4RANDOM)
- writeRandomBytes_arc4random((void *)&entropy, sizeof(entropy));
+ writeRandomBytes_arc4random(&entropy, sizeof(entropy));
return ENTROPY_DEBUG("arc4random", entropy);
#else
/* Try high quality providers first .. */
-# ifdef _WIN32
- if (writeRandomBytes_rand_s((void *)&entropy, sizeof(entropy))) {
+# if defined(_WIN32) && ! defined(XML_POOR_ENTROPY)
+ if (writeRandomBytes_rand_s(&entropy, sizeof(entropy))) {
return ENTROPY_DEBUG("rand_s", entropy);
}
+# elif defined(HAVE_GETENTROPY)
+ if (writeRandomBytes_getentropy(&entropy, sizeof(entropy))) {
+ return ENTROPY_DEBUG("getentropy", entropy);
+ }
+ errno = 0;
# elif defined(HAVE_GETRANDOM) || defined(HAVE_SYSCALL_GETRANDOM)
- if (writeRandomBytes_getrandom_nonblock((void *)&entropy, sizeof(entropy))) {
+ if (writeRandomBytes_getrandom_nonblock(&entropy, sizeof(entropy))) {
return ENTROPY_DEBUG("getrandom", entropy);
}
# endif
# if ! defined(_WIN32) && defined(XML_DEV_URANDOM)
- if (writeRandomBytes_dev_urandom((void *)&entropy, sizeof(entropy))) {
+ if (writeRandomBytes_dev_urandom(&entropy, sizeof(entropy))) {
return ENTROPY_DEBUG("/dev/urandom", entropy);
}
# endif /* ! defined(_WIN32) && defined(XML_DEV_URANDOM) */
/* .. and self-made low quality for backup: */
+ entropy.k[0] = 0;
+ entropy.k[1] = gather_time_entropy();
+# if ! defined(__wasi__)
/* Process ID is 0 bits entropy if attacker has local access */
- entropy = gather_time_entropy() ^ getpid();
+ entropy.k[1] ^= getpid();
+# endif
/* Factors are 2^31-1 and 2^61-1 (Mersenne primes M31 and M61) */
if (sizeof(unsigned long) == 4) {
- return ENTROPY_DEBUG("fallback(4)", entropy * 2147483647);
+ entropy.k[1] *= 2147483647;
+ return ENTROPY_DEBUG("fallback(4)", entropy);
} else {
- return ENTROPY_DEBUG("fallback(8)",
- entropy * (unsigned long)2305843009213693951ULL);
+ entropy.k[1] *= 2305843009213693951ULL;
+ return ENTROPY_DEBUG("fallback(8)", entropy);
}
#endif
}
-static unsigned long
-get_hash_secret_salt(XML_Parser parser) {
- if (parser->m_parentParser != NULL)
- return get_hash_secret_salt(parser->m_parentParser);
- return parser->m_hash_secret_salt;
+static void
+beforeHandler(XML_Parser parser) {
+ assert(parser->m_handlerCallDepth < UINT_MAX);
+ parser->m_handlerCallDepth++;
+}
+
+static void
+afterHandler(XML_Parser parser) {
+ assert(parser->m_handlerCallDepth > 0);
+ parser->m_handlerCallDepth--;
+}
+
+static bool
+isCalledFromInsideHandler(XML_Parser parser) {
+ return parser->m_handlerCallDepth > 0;
+}
+
+static void
+callUnknownEncodingRelease(XML_Parser parser) {
+ beforeHandler(parser);
+ parser->m_unknownEncodingRelease(parser->m_unknownEncodingData);
+ afterHandler(parser);
+ parser->m_unknownEncodingRelease = NULL;
+ parser->m_unknownEncodingData = NULL;
+}
+
+static int XMLCALL
+callUnknownEncodingConvert(void *data, const char *p) {
+ XML_Parser parser = data;
+ beforeHandler(parser);
+ const int result
+ = parser->m_unknownEncodingConvert(parser->m_unknownEncodingData, p);
+ afterHandler(parser);
+ return result;
}
static enum XML_Error
@@ -1028,7 +1238,29 @@ callProcessor(XML_Parser parser, const char *start, const char *end,
#if defined(XML_TESTING)
g_bytesScanned += (unsigned)have_now;
#endif
- const enum XML_Error ret = parser->m_processor(parser, start, end, endPtr);
+ // Run in a loop to eliminate dangerous recursion depths
+ enum XML_Error ret;
+ *endPtr = start;
+ while (1) {
+ // Use endPtr as the new start in each iteration, since it will
+ // be set to the next start point by m_processor.
+ ret = parser->m_processor(parser, *endPtr, end, endPtr);
+
+ // Make parsing status (and in particular XML_SUSPENDED) take
+ // precedence over re-enter flag when they disagree
+ if (parser->m_parsingStatus.parsing != XML_PARSING) {
+ parser->m_reenter = XML_FALSE;
+ }
+
+ if (! parser->m_reenter) {
+ break;
+ }
+
+ parser->m_reenter = XML_FALSE;
+ if (ret != XML_ERROR_NONE)
+ return ret;
+ }
+
if (ret == XML_ERROR_NONE) {
// if we consumed nothing, remember what we had on this parse attempt.
if (*endPtr == start) {
@@ -1043,8 +1275,10 @@ callProcessor(XML_Parser parser, const char *start, const char *end,
static XML_Bool /* only valid for root parser */
startParsing(XML_Parser parser) {
/* hash functions must be initialized before setContext() is called */
- if (parser->m_hash_secret_salt == 0)
- parser->m_hash_secret_salt = generate_hash_secret_salt(parser);
+ if (parser->m_hash_secret_salt_set != XML_TRUE) {
+ parser->m_hash_secret_salt_128 = generate_hash_secret_salt();
+ parser->m_hash_secret_salt_set = XML_TRUE;
+ }
if (parser->m_ns) {
/* implicit context only set for root parser, since child
parsers (i.e. external entity parsers) will inherit it
@@ -1058,19 +1292,43 @@ XML_Parser XMLCALL
XML_ParserCreate_MM(const XML_Char *encodingName,
const XML_Memory_Handling_Suite *memsuite,
const XML_Char *nameSep) {
- return parserCreate(encodingName, memsuite, nameSep, NULL);
+ return parserCreate(encodingName, memsuite, nameSep, NULL, NULL);
}
static XML_Parser
parserCreate(const XML_Char *encodingName,
const XML_Memory_Handling_Suite *memsuite, const XML_Char *nameSep,
- DTD *dtd) {
- XML_Parser parser;
+ DTD *dtd, XML_Parser parentParser) {
+ XML_Parser parser = NULL;
+
+#if XML_GE == 1
+ const size_t increase
+ = sizeof(size_t) + EXPAT_MALLOC_PADDING + sizeof(struct XML_ParserStruct);
+
+ if (parentParser != NULL) {
+ const XML_Parser rootParser = getRootParserOf(parentParser, NULL);
+ if (! expat_heap_increase_tolerable(rootParser, increase, __LINE__)) {
+ return NULL;
+ }
+ }
+#else
+ UNUSED_P(parentParser);
+#endif
if (memsuite) {
XML_Memory_Handling_Suite *mtemp;
+#if XML_GE == 1
+ void *const sizeAndParser
+ = memsuite->malloc_fcn(sizeof(size_t) + EXPAT_MALLOC_PADDING
+ + sizeof(struct XML_ParserStruct));
+ if (sizeAndParser != NULL) {
+ *(size_t *)sizeAndParser = sizeof(struct XML_ParserStruct);
+ parser = (XML_Parser)((char *)sizeAndParser + sizeof(size_t)
+ + EXPAT_MALLOC_PADDING);
+#else
parser = memsuite->malloc_fcn(sizeof(struct XML_ParserStruct));
if (parser != NULL) {
+#endif
mtemp = (XML_Memory_Handling_Suite *)&(parser->m_mem);
mtemp->malloc_fcn = memsuite->malloc_fcn;
mtemp->realloc_fcn = memsuite->realloc_fcn;
@@ -1078,44 +1336,80 @@ parserCreate(const XML_Char *encodingName,
}
} else {
XML_Memory_Handling_Suite *mtemp;
- parser = (XML_Parser)malloc(sizeof(struct XML_ParserStruct));
+#if XML_GE == 1
+ void *const sizeAndParser = malloc(sizeof(size_t) + EXPAT_MALLOC_PADDING
+ + sizeof(struct XML_ParserStruct));
+ if (sizeAndParser != NULL) {
+ *(size_t *)sizeAndParser = sizeof(struct XML_ParserStruct);
+ parser = (XML_Parser)((char *)sizeAndParser + sizeof(size_t)
+ + EXPAT_MALLOC_PADDING);
+#else
+ parser = malloc(sizeof(struct XML_ParserStruct));
if (parser != NULL) {
+#endif
mtemp = (XML_Memory_Handling_Suite *)&(parser->m_mem);
mtemp->malloc_fcn = malloc;
mtemp->realloc_fcn = realloc;
mtemp->free_fcn = free;
}
- }
+ } // cppcheck-suppress[memleak symbolName=sizeAndParser] // Cppcheck >=2.18.0
if (! parser)
return parser;
+#if XML_GE == 1
+ // Initialize .m_alloc_tracker
+ memset(&parser->m_alloc_tracker, 0, sizeof(MALLOC_TRACKER));
+ if (parentParser == NULL) {
+ parser->m_alloc_tracker.debugLevel
+ = getDebugLevel("EXPAT_MALLOC_DEBUG", 0u);
+ parser->m_alloc_tracker.maximumAmplificationFactor
+ = EXPAT_ALLOC_TRACKER_MAXIMUM_AMPLIFICATION_DEFAULT;
+ parser->m_alloc_tracker.activationThresholdBytes
+ = EXPAT_ALLOC_TRACKER_ACTIVATION_THRESHOLD_DEFAULT;
+
+ // NOTE: This initialization needs to come this early because these fields
+ // are read by allocation tracking code
+ parser->m_parentParser = NULL;
+ parser->m_accounting.countBytesDirect = 0;
+ } else {
+ parser->m_parentParser = parentParser;
+ }
+
+ // Record XML_ParserStruct allocation we did a few lines up before
+ const XML_Parser rootParser = getRootParserOf(parser, NULL);
+ assert(rootParser->m_parentParser == NULL);
+ assert(SIZE_MAX - rootParser->m_alloc_tracker.bytesAllocated >= increase);
+ rootParser->m_alloc_tracker.bytesAllocated += increase;
+
+ // Report on allocation
+ if (rootParser->m_alloc_tracker.debugLevel >= 2) {
+ if (rootParser->m_alloc_tracker.bytesAllocated
+ > rootParser->m_alloc_tracker.peakBytesAllocated) {
+ rootParser->m_alloc_tracker.peakBytesAllocated
+ = rootParser->m_alloc_tracker.bytesAllocated;
+ }
+
+ expat_heap_stat(rootParser, '+', increase,
+ rootParser->m_alloc_tracker.bytesAllocated,
+ rootParser->m_alloc_tracker.peakBytesAllocated, __LINE__);
+ }
+#else
+ parser->m_parentParser = NULL;
+#endif // XML_GE == 1
+
parser->m_buffer = NULL;
parser->m_bufferLim = NULL;
parser->m_attsSize = INIT_ATTS_SIZE;
- parser->m_atts
- = (ATTRIBUTE *)MALLOC(parser, parser->m_attsSize * sizeof(ATTRIBUTE));
+ parser->m_atts = MALLOC(parser, parser->m_attsSize * sizeof(ATTRIBUTE));
if (parser->m_atts == NULL) {
FREE(parser, parser);
return NULL;
}
-#ifdef XML_ATTR_INFO
- parser->m_attInfo = (XML_AttrInfo *)MALLOC(
- parser, parser->m_attsSize * sizeof(XML_AttrInfo));
- if (parser->m_attInfo == NULL) {
- FREE(parser, parser->m_atts);
- FREE(parser, parser);
- return NULL;
- }
-#endif
- parser->m_dataBuf
- = (XML_Char *)MALLOC(parser, INIT_DATA_BUF_SIZE * sizeof(XML_Char));
+ parser->m_dataBuf = MALLOC(parser, INIT_DATA_BUF_SIZE * sizeof(XML_Char));
if (parser->m_dataBuf == NULL) {
FREE(parser, parser->m_atts);
-#ifdef XML_ATTR_INFO
- FREE(parser, parser->m_attInfo);
-#endif
FREE(parser, parser);
return NULL;
}
@@ -1124,13 +1418,10 @@ parserCreate(const XML_Char *encodingName,
if (dtd)
parser->m_dtd = dtd;
else {
- parser->m_dtd = dtdCreate(&parser->m_mem);
+ parser->m_dtd = dtdCreate(parser);
if (parser->m_dtd == NULL) {
FREE(parser, parser->m_dataBuf);
FREE(parser, parser->m_atts);
-#ifdef XML_ATTR_INFO
- FREE(parser, parser->m_attInfo);
-#endif
FREE(parser, parser);
return NULL;
}
@@ -1138,7 +1429,7 @@ parserCreate(const XML_Char *encodingName,
parser->m_freeBindingList = NULL;
parser->m_freeTagList = NULL;
- parser->m_freeInternalEntities = NULL;
+ parser->m_freeEntities = NULL;
parser->m_groupSize = 0;
parser->m_groupConnector = NULL;
@@ -1156,8 +1447,8 @@ parserCreate(const XML_Char *encodingName,
parser->m_protocolEncodingName = NULL;
- poolInit(&parser->m_tempPool, &(parser->m_mem));
- poolInit(&parser->m_temp2Pool, &(parser->m_mem));
+ poolInit(&parser->m_tempPool, parser);
+ poolInit(&parser->m_temp2Pool, parser);
parserInit(parser, encodingName);
if (encodingName && ! parser->m_protocolEncodingName) {
@@ -1189,7 +1480,7 @@ parserInit(XML_Parser parser, const XML_Char *encodingName) {
parser->m_processor = prologInitProcessor;
XmlPrologStateInit(&parser->m_prologState);
if (encodingName != NULL) {
- parser->m_protocolEncodingName = copyString(encodingName, &(parser->m_mem));
+ parser->m_protocolEncodingName = copyString(encodingName, parser);
}
parser->m_curBase = NULL;
XmlInitEncoding(&parser->m_initEncoding, &parser->m_encoding, 0);
@@ -1241,22 +1532,29 @@ parserInit(XML_Parser parser, const XML_Char *encodingName) {
parser->m_eventEndPtr = NULL;
parser->m_positionPtr = NULL;
parser->m_openInternalEntities = NULL;
+ parser->m_openAttributeEntities = NULL;
+ parser->m_openValueEntities = NULL;
parser->m_defaultExpandInternalEntities = XML_TRUE;
parser->m_tagLevel = 0;
parser->m_tagStack = NULL;
parser->m_inheritedBindings = NULL;
parser->m_nSpecifiedAtts = 0;
parser->m_unknownEncodingMem = NULL;
+ parser->m_unknownEncodingConvert = NULL;
parser->m_unknownEncodingRelease = NULL;
parser->m_unknownEncodingData = NULL;
- parser->m_parentParser = NULL;
parser->m_parsingStatus.parsing = XML_INITIALIZED;
+ // Reentry can only be triggered inside m_processor calls
+ parser->m_reenter = XML_FALSE;
+ parser->m_handlerCallDepth = 0;
#ifdef XML_DTD
parser->m_isParamEntity = XML_FALSE;
parser->m_useForeignDTD = XML_FALSE;
parser->m_paramEntityParsing = XML_PARAM_ENTITY_PARSING_NEVER;
#endif
- parser->m_hash_secret_salt = 0;
+ parser->m_hash_secret_salt_128.k[0] = 0;
+ parser->m_hash_secret_salt_128.k[1] = 0;
+ parser->m_hash_secret_salt_set = XML_FALSE;
#if XML_GE == 1
memset(&parser->m_accounting, 0, sizeof(ACCOUNTING));
@@ -1272,7 +1570,7 @@ parserInit(XML_Parser parser, const XML_Char *encodingName) {
}
/* moves list of bindings to m_freeBindingList */
-static void FASTCALL
+static void
moveToFreeBindingList(XML_Parser parser, BINDING *bindings) {
while (bindings) {
BINDING *b = bindings;
@@ -1282,16 +1580,30 @@ moveToFreeBindingList(XML_Parser parser, BINDING *bindings) {
}
}
+/* Moves a list of entities onto the start of another list. */
+static void
+moveEntityList(OPEN_INTERNAL_ENTITY **dst, OPEN_INTERNAL_ENTITY **src) {
+ for (OPEN_INTERNAL_ENTITY *head = *src; head != NULL;) {
+ OPEN_INTERNAL_ENTITY *const openEntity = head;
+ head = head->next;
+ openEntity->next = *dst;
+ *dst = openEntity;
+ }
+}
+
XML_Bool XMLCALL
XML_ParserReset(XML_Parser parser, const XML_Char *encodingName) {
TAG *tStk;
- OPEN_INTERNAL_ENTITY *openEntityList;
- if (parser == NULL)
+ if ((parser == NULL) || isCalledFromInsideHandler(parser))
return XML_FALSE;
if (parser->m_parentParser)
return XML_FALSE;
+ // The application-defined release callback may access the parser, so it
+ // must run before any parser state is freed.
+ if (parser->m_unknownEncodingRelease)
+ callUnknownEncodingRelease(parser);
/* move m_tagStack to m_freeTagList */
tStk = parser->m_tagStack;
while (tStk) {
@@ -1302,27 +1614,42 @@ XML_ParserReset(XML_Parser parser, const XML_Char *encodingName) {
tag->bindings = NULL;
parser->m_freeTagList = tag;
}
- /* move m_openInternalEntities to m_freeInternalEntities */
- openEntityList = parser->m_openInternalEntities;
- while (openEntityList) {
- OPEN_INTERNAL_ENTITY *openEntity = openEntityList;
- openEntityList = openEntity->next;
- openEntity->next = parser->m_freeInternalEntities;
- parser->m_freeInternalEntities = openEntity;
- }
+ /* move m_openInternalEntities to m_freeEntities */
+ moveEntityList(&parser->m_freeEntities, &parser->m_openInternalEntities);
+ /* move m_openAttributeEntities to m_freeEntities (i.e. same task but for
+ * attributes) */
+ moveEntityList(&parser->m_freeEntities, &parser->m_openAttributeEntities);
+ /* move m_openValueEntities to m_freeEntities (i.e. same task but for value
+ * entities) */
+ moveEntityList(&parser->m_freeEntities, &parser->m_openValueEntities);
moveToFreeBindingList(parser, parser->m_inheritedBindings);
FREE(parser, parser->m_unknownEncodingMem);
- if (parser->m_unknownEncodingRelease)
- parser->m_unknownEncodingRelease(parser->m_unknownEncodingData);
poolClear(&parser->m_tempPool);
poolClear(&parser->m_temp2Pool);
FREE(parser, (void *)parser->m_protocolEncodingName);
parser->m_protocolEncodingName = NULL;
parserInit(parser, encodingName);
- dtdReset(parser->m_dtd, &parser->m_mem);
+ dtdReset(parser->m_dtd, parser);
+ if (encodingName && ! parser->m_protocolEncodingName) {
+ parser->m_errorCode = XML_ERROR_NO_MEMORY;
+ return XML_FALSE;
+ }
return XML_TRUE;
}
+static XML_Bool
+parserBusy(XML_Parser parser) {
+ switch (parser->m_parsingStatus.parsing) {
+ case XML_PARSING:
+ case XML_SUSPENDED:
+ return XML_TRUE;
+ case XML_INITIALIZED:
+ case XML_FINISHED:
+ default:
+ return XML_FALSE;
+ }
+}
+
enum XML_Status XMLCALL
XML_SetEncoding(XML_Parser parser, const XML_Char *encodingName) {
if (parser == NULL)
@@ -1331,8 +1658,7 @@ XML_SetEncoding(XML_Parser parser, const XML_Char *encodingName) {
XXX There's no way for the caller to determine which of the
XXX possible error cases caused the XML_STATUS_ERROR return.
*/
- if (parser->m_parsingStatus.parsing == XML_PARSING
- || parser->m_parsingStatus.parsing == XML_SUSPENDED)
+ if (parserBusy(parser))
return XML_STATUS_ERROR;
/* Get rid of any previous encoding name */
@@ -1343,7 +1669,7 @@ XML_SetEncoding(XML_Parser parser, const XML_Char *encodingName) {
parser->m_protocolEncodingName = NULL;
else {
/* Copy the new encoding name into allocated memory */
- parser->m_protocolEncodingName = copyString(encodingName, &(parser->m_mem));
+ parser->m_protocolEncodingName = copyString(encodingName, parser);
if (! parser->m_protocolEncodingName)
return XML_STATUS_ERROR;
}
@@ -1372,6 +1698,7 @@ XML_ExternalEntityParserCreate(XML_Parser oldParser, const XML_Char *context,
XML_ExternalEntityRefHandler oldExternalEntityRefHandler;
XML_SkippedEntityHandler oldSkippedEntityHandler;
XML_UnknownEncodingHandler oldUnknownEncodingHandler;
+ void *oldUnknownEncodingHandlerData;
XML_ElementDeclHandler oldElementDeclHandler;
XML_AttlistDeclHandler oldAttlistDeclHandler;
XML_EntityDeclHandler oldEntityDeclHandler;
@@ -1392,7 +1719,8 @@ XML_ExternalEntityParserCreate(XML_Parser oldParser, const XML_Char *context,
from hash tables associated with either parser without us having
to worry which hash secrets each table has.
*/
- unsigned long oldhash_secret_salt;
+ struct sipkey oldhash_secret_salt_128;
+ XML_Bool oldhash_secret_salt_set;
XML_Bool oldReparseDeferralEnabled;
/* Validate the oldParser parameter before we pull everything out of it */
@@ -1417,6 +1745,7 @@ XML_ExternalEntityParserCreate(XML_Parser oldParser, const XML_Char *context,
oldExternalEntityRefHandler = parser->m_externalEntityRefHandler;
oldSkippedEntityHandler = parser->m_skippedEntityHandler;
oldUnknownEncodingHandler = parser->m_unknownEncodingHandler;
+ oldUnknownEncodingHandlerData = parser->m_unknownEncodingHandlerData;
oldElementDeclHandler = parser->m_elementDeclHandler;
oldAttlistDeclHandler = parser->m_attlistDeclHandler;
oldEntityDeclHandler = parser->m_entityDeclHandler;
@@ -1437,7 +1766,8 @@ XML_ExternalEntityParserCreate(XML_Parser oldParser, const XML_Char *context,
from hash tables associated with either parser without us having
to worry which hash secrets each table has.
*/
- oldhash_secret_salt = parser->m_hash_secret_salt;
+ oldhash_secret_salt_128 = parser->m_hash_secret_salt_128;
+ oldhash_secret_salt_set = parser->m_hash_secret_salt_set;
oldReparseDeferralEnabled = parser->m_reparseDeferralEnabled;
#ifdef XML_DTD
@@ -1445,16 +1775,12 @@ XML_ExternalEntityParserCreate(XML_Parser oldParser, const XML_Char *context,
newDtd = oldDtd;
#endif /* XML_DTD */
- /* Note that the magical uses of the pre-processor to make field
- access look more like C++ require that `parser' be overwritten
- here. This makes this function more painful to follow than it
- would be otherwise.
- */
if (parser->m_ns) {
XML_Char tmp[2] = {parser->m_namespaceSeparator, 0};
- parser = parserCreate(encodingName, &parser->m_mem, tmp, newDtd);
+ parser = parserCreate(encodingName, &parser->m_mem, tmp, newDtd, oldParser);
} else {
- parser = parserCreate(encodingName, &parser->m_mem, NULL, newDtd);
+ parser
+ = parserCreate(encodingName, &parser->m_mem, NULL, newDtd, oldParser);
}
if (! parser)
@@ -1476,6 +1802,7 @@ XML_ExternalEntityParserCreate(XML_Parser oldParser, const XML_Char *context,
parser->m_externalEntityRefHandler = oldExternalEntityRefHandler;
parser->m_skippedEntityHandler = oldSkippedEntityHandler;
parser->m_unknownEncodingHandler = oldUnknownEncodingHandler;
+ parser->m_unknownEncodingHandlerData = oldUnknownEncodingHandlerData;
parser->m_elementDeclHandler = oldElementDeclHandler;
parser->m_attlistDeclHandler = oldAttlistDeclHandler;
parser->m_entityDeclHandler = oldEntityDeclHandler;
@@ -1490,7 +1817,8 @@ XML_ExternalEntityParserCreate(XML_Parser oldParser, const XML_Char *context,
parser->m_externalEntityRefHandlerArg = oldExternalEntityRefHandlerArg;
parser->m_defaultExpandInternalEntities = oldDefaultExpandInternalEntities;
parser->m_ns_triplets = oldns_triplets;
- parser->m_hash_secret_salt = oldhash_secret_salt;
+ parser->m_hash_secret_salt_128 = oldhash_secret_salt_128;
+ parser->m_hash_secret_salt_set = oldhash_secret_salt_set;
parser->m_reparseDeferralEnabled = oldReparseDeferralEnabled;
parser->m_parentParser = oldParser;
#ifdef XML_DTD
@@ -1498,7 +1826,7 @@ XML_ExternalEntityParserCreate(XML_Parser oldParser, const XML_Char *context,
parser->m_prologState.inEntityValue = oldInEntityValue;
if (context) {
#endif /* XML_DTD */
- if (! dtdCopy(oldParser, parser->m_dtd, oldDtd, &parser->m_mem)
+ if (! dtdCopy(oldParser, parser->m_dtd, oldDtd, parser)
|| ! setContext(parser, context)) {
XML_ParserFree(parser);
return NULL;
@@ -1521,7 +1849,7 @@ XML_ExternalEntityParserCreate(XML_Parser oldParser, const XML_Char *context,
return parser;
}
-static void FASTCALL
+static void
destroyBindings(BINDING *bindings, XML_Parser parser) {
for (;;) {
BINDING *b = bindings;
@@ -1536,9 +1864,12 @@ destroyBindings(BINDING *bindings, XML_Parser parser) {
void XMLCALL
XML_ParserFree(XML_Parser parser) {
TAG *tagList;
- OPEN_INTERNAL_ENTITY *entityList;
- if (parser == NULL)
+ if ((parser == NULL) || isCalledFromInsideHandler(parser))
return;
+ // The application-defined release callback may access the parser, so it
+ // must run before any parser state is freed.
+ if (parser->m_unknownEncodingRelease)
+ callUnknownEncodingRelease(parser);
/* free m_tagStack and m_freeTagList */
tagList = parser->m_tagStack;
for (;;) {
@@ -1551,25 +1882,39 @@ XML_ParserFree(XML_Parser parser) {
}
p = tagList;
tagList = tagList->parent;
- FREE(parser, p->buf);
+ FREE(parser, p->buf.raw);
destroyBindings(p->bindings, parser);
FREE(parser, p);
}
- /* free m_openInternalEntities and m_freeInternalEntities */
- entityList = parser->m_openInternalEntities;
- for (;;) {
- OPEN_INTERNAL_ENTITY *openEntity;
- if (entityList == NULL) {
- if (parser->m_freeInternalEntities == NULL)
- break;
- entityList = parser->m_freeInternalEntities;
- parser->m_freeInternalEntities = NULL;
- }
- openEntity = entityList;
+ /* free m_openInternalEntities */
+ for (OPEN_INTERNAL_ENTITY *entityList = parser->m_openInternalEntities;
+ entityList != NULL;) {
+ OPEN_INTERNAL_ENTITY *const openEntity = entityList;
entityList = entityList->next;
FREE(parser, openEntity);
}
-
+ /* free m_openAttributeEntities */
+ for (OPEN_INTERNAL_ENTITY *entityList = parser->m_openAttributeEntities;
+ entityList != NULL;) {
+ OPEN_INTERNAL_ENTITY *const openEntity = entityList;
+ entityList = entityList->next;
+ FREE(parser, openEntity);
+ }
+ /* free m_openValueEntities */
+ for (OPEN_INTERNAL_ENTITY *entityList = parser->m_openValueEntities;
+ entityList != NULL;) {
+ OPEN_INTERNAL_ENTITY *const openEntity = entityList;
+ entityList = entityList->next;
+ FREE(parser, openEntity);
+ }
+ /* free m_freeEntities */
+ for (OPEN_INTERNAL_ENTITY *entityList = parser->m_freeEntities;
+ entityList != NULL;) {
+ OPEN_INTERNAL_ENTITY *const openEntity = entityList;
+ entityList = entityList->next;
+ FREE(parser, openEntity);
+ }
+ parser->m_freeEntities = NULL;
destroyBindings(parser->m_freeBindingList, parser);
destroyBindings(parser->m_inheritedBindings, parser);
poolDestroy(&parser->m_tempPool);
@@ -1583,19 +1928,16 @@ XML_ParserFree(XML_Parser parser) {
#else
if (parser->m_dtd)
#endif /* XML_DTD */
- dtdDestroy(parser->m_dtd, (XML_Bool)! parser->m_parentParser,
- &parser->m_mem);
- FREE(parser, (void *)parser->m_atts);
-#ifdef XML_ATTR_INFO
- FREE(parser, (void *)parser->m_attInfo);
-#endif
+ dtdDestroy(parser->m_dtd, (XML_Bool)! parser->m_parentParser, parser);
+ FREE(parser, parser->m_atts);
FREE(parser, parser->m_groupConnector);
- FREE(parser, parser->m_buffer);
+ // NOTE: We are avoiding FREE(..) here because parser->m_buffer
+ // is not being allocated with MALLOC(..) but with plain
+ // .malloc_fcn(..).
+ parser->m_mem.free_fcn(parser->m_buffer);
FREE(parser, parser->m_dataBuf);
FREE(parser, parser->m_nsAtts);
FREE(parser, parser->m_unknownEncodingMem);
- if (parser->m_unknownEncodingRelease)
- parser->m_unknownEncodingRelease(parser->m_unknownEncodingData);
FREE(parser, parser);
}
@@ -1611,8 +1953,7 @@ XML_UseForeignDTD(XML_Parser parser, XML_Bool useDTD) {
return XML_ERROR_INVALID_ARGUMENT;
#ifdef XML_DTD
/* block after XML_Parse()/XML_ParseBuffer() has been called */
- if (parser->m_parsingStatus.parsing == XML_PARSING
- || parser->m_parsingStatus.parsing == XML_SUSPENDED)
+ if (parserBusy(parser))
return XML_ERROR_CANT_CHANGE_FEATURE_ONCE_PARSING;
parser->m_useForeignDTD = useDTD;
return XML_ERROR_NONE;
@@ -1627,8 +1968,7 @@ XML_SetReturnNSTriplet(XML_Parser parser, int do_nst) {
if (parser == NULL)
return;
/* block after XML_Parse()/XML_ParseBuffer() has been called */
- if (parser->m_parsingStatus.parsing == XML_PARSING
- || parser->m_parsingStatus.parsing == XML_SUSPENDED)
+ if (parserBusy(parser))
return;
parser->m_ns_triplets = do_nst ? XML_TRUE : XML_FALSE;
}
@@ -1678,15 +2018,6 @@ XML_GetIdAttributeIndex(XML_Parser parser) {
return parser->m_idAttIndex;
}
-#ifdef XML_ATTR_INFO
-const XML_AttrInfo *XMLCALL
-XML_GetAttributeInfo(XML_Parser parser) {
- if (parser == NULL)
- return NULL;
- return parser->m_attInfo;
-}
-#endif
-
void XMLCALL
XML_SetElementHandler(XML_Parser parser, XML_StartElementHandler start,
XML_EndElementHandler end) {
@@ -1897,8 +2228,7 @@ XML_SetParamEntityParsing(XML_Parser parser,
if (parser == NULL)
return 0;
/* block after XML_Parse()/XML_ParseBuffer() has been called */
- if (parser->m_parsingStatus.parsing == XML_PARSING
- || parser->m_parsingStatus.parsing == XML_SUSPENDED)
+ if (parserBusy(parser))
return 0;
#ifdef XML_DTD
parser->m_paramEntityParsing = peParsing;
@@ -1908,20 +2238,65 @@ XML_SetParamEntityParsing(XML_Parser parser,
#endif
}
-int XMLCALL
-XML_SetHashSalt(XML_Parser parser, unsigned long hash_salt) {
+/* The body of XML_SetHashSalt, so that Expat's own tests can reach it
+ without tripping the deprecation of the public function. */
+XML_NONTESTING_STATIC int
+xmlSetHashSalt(XML_Parser parser, unsigned long hash_salt) {
if (parser == NULL)
return 0;
- if (parser->m_parentParser)
- return XML_SetHashSalt(parser->m_parentParser, hash_salt);
+
+ const XML_Parser rootParser = getRootParserOf(parser, NULL);
+ assert(! rootParser->m_parentParser);
+
/* block after XML_Parse()/XML_ParseBuffer() has been called */
- if (parser->m_parsingStatus.parsing == XML_PARSING
- || parser->m_parsingStatus.parsing == XML_SUSPENDED)
+ if (parserBusy(rootParser))
return 0;
- parser->m_hash_secret_salt = hash_salt;
+
+ rootParser->m_hash_secret_salt_128.k[0] = 0;
+ rootParser->m_hash_secret_salt_128.k[1] = hash_salt;
+
+ if (hash_salt != 0) { // to remain backwards compatible
+ rootParser->m_hash_secret_salt_set = XML_TRUE;
+
+ if (sizeof(unsigned long) == 4)
+ ENTROPY_DEBUG("explicit(4)", rootParser->m_hash_secret_salt_128);
+ else
+ ENTROPY_DEBUG("explicit(8)", rootParser->m_hash_secret_salt_128);
+ }
+
return 1;
}
+// DEPRECATED since Expat 2.8.0.
+int XMLCALL
+XML_SetHashSalt(XML_Parser parser, unsigned long hash_salt) {
+ return xmlSetHashSalt(parser, hash_salt);
+}
+
+XML_Bool XMLCALL
+XML_SetHashSalt16Bytes(XML_Parser parser, const uint8_t entropy[16]) {
+ if (parser == NULL)
+ return XML_FALSE;
+
+ if (entropy == NULL)
+ return XML_FALSE;
+
+ const XML_Parser rootParser = getRootParserOf(parser, NULL);
+ assert(! rootParser->m_parentParser);
+
+ /* block after XML_Parse()/XML_ParseBuffer() has been called */
+ if (parserBusy(rootParser))
+ return XML_FALSE;
+
+ sip_tokey(&(rootParser->m_hash_secret_salt_128), entropy);
+
+ rootParser->m_hash_secret_salt_set = XML_TRUE;
+
+ ENTROPY_DEBUG("explicit(16)", rootParser->m_hash_secret_salt_128);
+
+ return XML_TRUE;
+}
+
enum XML_Status XMLCALL
XML_Parse(XML_Parser parser, const char *s, int len, int isFinal) {
if ((parser == NULL) || (len < 0) || ((s == NULL) && (len != 0))) {
@@ -1929,6 +2304,8 @@ XML_Parse(XML_Parser parser, const char *s, int len, int isFinal) {
parser->m_errorCode = XML_ERROR_INVALID_ARGUMENT;
return XML_STATUS_ERROR;
}
+ if (isCalledFromInsideHandler(parser))
+ return XML_STATUS_ERROR;
switch (parser->m_parsingStatus.parsing) {
case XML_SUSPENDED:
parser->m_errorCode = XML_ERROR_SUSPENDED;
@@ -1941,7 +2318,7 @@ XML_Parse(XML_Parser parser, const char *s, int len, int isFinal) {
parser->m_errorCode = XML_ERROR_NO_MEMORY;
return XML_STATUS_ERROR;
}
- /* fall through */
+ EXPAT_FALLTHROUGH;
default:
parser->m_parsingStatus.parsing = XML_PARSING;
}
@@ -1952,7 +2329,7 @@ XML_Parse(XML_Parser parser, const char *s, int len, int isFinal) {
int nLeftOver;
enum XML_Status result;
/* Detect overflow (a+b > MAX <==> b > MAX-a) */
- if ((XML_Size)len > ((XML_Size)-1) / 2 - parser->m_parseEndByteIndex) {
+ if ((uint64_t)len > UINT64_MAX - parser->m_parseEndByteIndex) {
parser->m_errorCode = XML_ERROR_NO_MEMORY;
parser->m_eventPtr = parser->m_eventEndPtr = NULL;
parser->m_processor = errorProcessor;
@@ -1983,7 +2360,7 @@ XML_Parse(XML_Parser parser, const char *s, int len, int isFinal) {
parser->m_parsingStatus.parsing = XML_FINISHED;
return XML_STATUS_OK;
}
- /* fall through */
+ EXPAT_FALLTHROUGH;
default:
result = XML_STATUS_OK;
}
@@ -2038,7 +2415,7 @@ XML_ParseBuffer(XML_Parser parser, int len, int isFinal) {
const char *start;
enum XML_Status result = XML_STATUS_OK;
- if (parser == NULL)
+ if ((parser == NULL) || isCalledFromInsideHandler(parser))
return XML_STATUS_ERROR;
if (len < 0) {
@@ -2064,11 +2441,24 @@ XML_ParseBuffer(XML_Parser parser, int len, int isFinal) {
parser->m_errorCode = XML_ERROR_NO_MEMORY;
return XML_STATUS_ERROR;
}
- /* fall through */
+ EXPAT_FALLTHROUGH;
default:
parser->m_parsingStatus.parsing = XML_PARSING;
}
+ if (len > EXPAT_SAFE_PTR_DIFF(parser->m_bufferLim, parser->m_bufferEnd)) {
+ parser->m_errorCode = XML_ERROR_INVALID_ARGUMENT;
+ return XML_STATUS_ERROR;
+ }
+
+ // Detect and avoid integer overflow
+ if ((uint64_t)len > UINT64_MAX - parser->m_parseEndByteIndex) {
+ parser->m_errorCode = XML_ERROR_NO_MEMORY;
+ parser->m_eventPtr = parser->m_eventEndPtr = NULL;
+ parser->m_processor = errorProcessor;
+ return XML_STATUS_ERROR;
+ }
+
start = parser->m_bufferPtr;
parser->m_positionPtr = start;
parser->m_bufferEnd += len;
@@ -2094,6 +2484,7 @@ XML_ParseBuffer(XML_Parser parser, int len, int isFinal) {
parser->m_parsingStatus.parsing = XML_FINISHED;
return result;
}
+ break;
default:; /* should not happen */
}
}
@@ -2104,9 +2495,32 @@ XML_ParseBuffer(XML_Parser parser, int len, int isFinal) {
return result;
}
+/* Modifies `parser`’s buffer to be backed by `newBuf`. */
+static void
+setParserBuffer(XML_Parser parser, char *newBuf, int newBufSize, int keep) {
+ parser->m_bufferLim = newBuf + newBufSize;
+ if (parser->m_bufferPtr) {
+ const int parsing
+ = (int)EXPAT_SAFE_PTR_DIFF(parser->m_bufferEnd, parser->m_bufferPtr);
+ memcpy(newBuf, parser->m_bufferPtr - keep, parsing + keep);
+ // NOTE: We are avoiding FREE(..) here because parser->m_buffer
+ // is not being allocated with MALLOC(..) but with plain
+ // .malloc_fcn(..).
+ parser->m_mem.free_fcn(parser->m_buffer);
+ parser->m_buffer = newBuf;
+ parser->m_bufferEnd = newBuf + parsing + keep;
+ parser->m_bufferPtr = newBuf + keep;
+ } else {
+ /* This must be a brand new buffer with no data in it yet */
+ parser->m_buffer = newBuf;
+ parser->m_bufferEnd = newBuf;
+ parser->m_bufferPtr = newBuf;
+ }
+}
+
void *XMLCALL
XML_GetBuffer(XML_Parser parser, int len) {
- if (parser == NULL)
+ if ((parser == NULL) || isCalledFromInsideHandler(parser))
return NULL;
if (len < 0) {
parser->m_errorCode = XML_ERROR_NO_MEMORY;
@@ -2127,9 +2541,6 @@ XML_GetBuffer(XML_Parser parser, int len) {
parser->m_lastBufferRequestSize = len;
if (len > EXPAT_SAFE_PTR_DIFF(parser->m_bufferLim, parser->m_bufferEnd)
|| parser->m_buffer == NULL) {
-#if XML_CONTEXT_BYTES > 0
- int keep;
-#endif /* XML_CONTEXT_BYTES > 0 */
/* Do not invoke signed arithmetic overflow: */
int neededSize = (int)((unsigned)len
+ (unsigned)EXPAT_SAFE_PTR_DIFF(
@@ -2139,7 +2550,9 @@ XML_GetBuffer(XML_Parser parser, int len) {
return NULL;
}
#if XML_CONTEXT_BYTES > 0
- keep = (int)EXPAT_SAFE_PTR_DIFF(parser->m_bufferPtr, parser->m_buffer);
+ const int parsed
+ = (int)EXPAT_SAFE_PTR_DIFF(parser->m_bufferPtr, parser->m_buffer);
+ int keep = parsed;
if (keep > XML_CONTEXT_BYTES)
keep = XML_CONTEXT_BYTES;
/* Detect and prevent integer overflow */
@@ -2147,16 +2560,16 @@ XML_GetBuffer(XML_Parser parser, int len) {
parser->m_errorCode = XML_ERROR_NO_MEMORY;
return NULL;
}
- neededSize += keep;
+#else
+ int keep = 0;
#endif /* XML_CONTEXT_BYTES > 0 */
+ neededSize += keep;
if (parser->m_buffer && parser->m_bufferPtr
&& neededSize
<= EXPAT_SAFE_PTR_DIFF(parser->m_bufferLim, parser->m_buffer)) {
#if XML_CONTEXT_BYTES > 0
- if (keep < EXPAT_SAFE_PTR_DIFF(parser->m_bufferPtr, parser->m_buffer)) {
- int offset
- = (int)EXPAT_SAFE_PTR_DIFF(parser->m_bufferPtr, parser->m_buffer)
- - keep;
+ if (keep < parsed) {
+ int offset = parsed - keep;
/* The buffer pointers cannot be NULL here; we have at least some bytes
* in the buffer */
memmove(parser->m_buffer, &parser->m_buffer[offset],
@@ -2173,7 +2586,6 @@ XML_GetBuffer(XML_Parser parser, int len) {
parser->m_bufferPtr = parser->m_buffer;
#endif /* XML_CONTEXT_BYTES > 0 */
} else {
- char *newBuf;
int bufferSize
= (int)EXPAT_SAFE_PTR_DIFF(parser->m_bufferLim, parser->m_buffer);
if (bufferSize == 0)
@@ -2186,43 +2598,14 @@ XML_GetBuffer(XML_Parser parser, int len) {
parser->m_errorCode = XML_ERROR_NO_MEMORY;
return NULL;
}
- newBuf = (char *)MALLOC(parser, bufferSize);
- if (newBuf == 0) {
+ // NOTE: We are avoiding MALLOC(..) here to leave limiting
+ // the input size to the application using Expat.
+ char *const newBuf = parser->m_mem.malloc_fcn(bufferSize);
+ if (newBuf == NULL) {
parser->m_errorCode = XML_ERROR_NO_MEMORY;
return NULL;
}
- parser->m_bufferLim = newBuf + bufferSize;
-#if XML_CONTEXT_BYTES > 0
- if (parser->m_bufferPtr) {
- memcpy(newBuf, &parser->m_bufferPtr[-keep],
- EXPAT_SAFE_PTR_DIFF(parser->m_bufferEnd, parser->m_bufferPtr)
- + keep);
- FREE(parser, parser->m_buffer);
- parser->m_buffer = newBuf;
- parser->m_bufferEnd
- = parser->m_buffer
- + EXPAT_SAFE_PTR_DIFF(parser->m_bufferEnd, parser->m_bufferPtr)
- + keep;
- parser->m_bufferPtr = parser->m_buffer + keep;
- } else {
- /* This must be a brand new buffer with no data in it yet */
- parser->m_bufferEnd = newBuf;
- parser->m_bufferPtr = parser->m_buffer = newBuf;
- }
-#else
- if (parser->m_bufferPtr) {
- memcpy(newBuf, parser->m_bufferPtr,
- EXPAT_SAFE_PTR_DIFF(parser->m_bufferEnd, parser->m_bufferPtr));
- FREE(parser, parser->m_buffer);
- parser->m_bufferEnd
- = newBuf
- + EXPAT_SAFE_PTR_DIFF(parser->m_bufferEnd, parser->m_bufferPtr);
- } else {
- /* This must be a brand new buffer with no data in it yet */
- parser->m_bufferEnd = newBuf;
- }
- parser->m_bufferPtr = parser->m_buffer = newBuf;
-#endif /* XML_CONTEXT_BYTES > 0 */
+ setParserBuffer(parser, newBuf, bufferSize, keep);
}
parser->m_eventPtr = parser->m_eventEndPtr = NULL;
parser->m_positionPtr = NULL;
@@ -2230,6 +2613,11 @@ XML_GetBuffer(XML_Parser parser, int len) {
return parser->m_bufferEnd;
}
+static void
+triggerReenter(XML_Parser parser) {
+ parser->m_reenter = XML_TRUE;
+}
+
enum XML_Status XMLCALL
XML_StopParser(XML_Parser parser, XML_Bool resumable) {
if (parser == NULL)
@@ -2270,7 +2658,7 @@ enum XML_Status XMLCALL
XML_ResumeParser(XML_Parser parser) {
enum XML_Status result = XML_STATUS_OK;
- if (parser == NULL)
+ if ((parser == NULL) || isCalledFromInsideHandler(parser))
return XML_STATUS_ERROR;
if (parser->m_parsingStatus.parsing != XML_SUSPENDED) {
parser->m_errorCode = XML_ERROR_NOT_SUSPENDED;
@@ -2296,6 +2684,7 @@ XML_ResumeParser(XML_Parser parser) {
parser->m_parsingStatus.parsing = XML_FINISHED;
return result;
}
+ break;
default:;
}
}
@@ -2321,35 +2710,57 @@ XML_GetErrorCode(XML_Parser parser) {
return parser->m_errorCode;
}
-XML_Index XMLCALL
-XML_GetCurrentByteIndex(XML_Parser parser) {
+int64_t XMLCALL
+XML_GetCurrentByteIndex64(XML_Parser parser) {
if (parser == NULL)
return -1;
- if (parser->m_eventPtr)
- return (XML_Index)(parser->m_parseEndByteIndex
- - (parser->m_parseEndPtr - parser->m_eventPtr));
+ if (parser->m_eventPtr) {
+ return (int64_t)(parser->m_parseEndByteIndex
+ - (parser->m_parseEndPtr - parser->m_eventPtr));
+ }
return -1;
}
-int XMLCALL
-XML_GetCurrentByteCount(XML_Parser parser) {
+// DEPRECATED since Expat 2.9.0.
+XML_Index XMLCALL
+XML_GetCurrentByteIndex(XML_Parser parser) {
+ // NOTE: XML_Index is known to wrap around for >2 GiB content
+ // on 32bit machines and 64bit Windows, unless (non-default and
+ // uncommon) XML_LARGE_SIZE is defined.
+ // That's a bug and it only lives on because we cannot break
+ // ABI compatibility of public API.
+ return (XML_Index)XML_GetCurrentByteIndex64(parser);
+}
+
+uint64_t XMLCALL
+XML_GetCurrentByteCount64(XML_Parser parser) {
if (parser == NULL)
return 0;
- if (parser->m_eventEndPtr && parser->m_eventPtr)
- return (int)(parser->m_eventEndPtr - parser->m_eventPtr);
+ if (parser->m_eventEndPtr && parser->m_eventPtr) {
+ return parser->m_eventEndPtr - parser->m_eventPtr;
+ }
return 0;
}
+// DEPRECATED since Expat 2.9.0.
+int XMLCALL
+XML_GetCurrentByteCount(XML_Parser parser) {
+ // NOTE: int is known to wrap around for >2 GiB content.
+ // That's a bug and it only lives on because we cannot break
+ // ABI compatibility of public API.
+ return (int)XML_GetCurrentByteCount64(parser);
+}
+
const char *XMLCALL
-XML_GetInputContext(XML_Parser parser, int *offset, int *size) {
+XML_GetInputContext64(XML_Parser parser, int64_t *offset, uint64_t *size) {
#if XML_CONTEXT_BYTES > 0
if (parser == NULL)
return NULL;
if (parser->m_eventPtr && parser->m_buffer) {
if (offset != NULL)
- *offset = (int)(parser->m_eventPtr - parser->m_buffer);
+ *offset = parser->m_eventPtr - parser->m_buffer;
if (size != NULL)
- *size = (int)(parser->m_bufferEnd - parser->m_buffer);
+ *size = parser->m_bufferEnd - parser->m_buffer;
return parser->m_buffer;
}
#else
@@ -2357,11 +2768,44 @@ XML_GetInputContext(XML_Parser parser, int *offset, int *size) {
(void)offset;
(void)size;
#endif /* XML_CONTEXT_BYTES > 0 */
- return (const char *)0;
+ return NULL;
}
-XML_Size XMLCALL
-XML_GetCurrentLineNumber(XML_Parser parser) {
+// DEPRECATED since Expat 2.9.0.
+const char *XMLCALL
+XML_GetInputContext(XML_Parser parser, int *offset, int *size) {
+#if XML_CONTEXT_BYTES > 0
+ if (parser == NULL)
+ return NULL;
+
+ int64_t offset64;
+ uint64_t size64;
+
+ const char *const buffer = XML_GetInputContext64(parser, &offset64, &size64);
+
+ if (buffer == NULL)
+ return NULL;
+
+ // NOTE: int is known to wrap around for >2 GiB content.
+ // That's a bug and it only lives on because we cannot break
+ // ABI compatibility of public API.
+ if (offset != NULL)
+ *offset = (int)offset64;
+
+ if (size != NULL)
+ *size = (int)size64;
+
+ return buffer;
+#else
+ (void)parser;
+ (void)offset;
+ (void)size;
+#endif /* XML_CONTEXT_BYTES > 0 */
+ return NULL;
+}
+
+uint64_t XMLCALL
+XML_GetCurrentLineNumber64(XML_Parser parser) {
if (parser == NULL)
return 0;
if (parser->m_eventPtr && parser->m_eventPtr >= parser->m_positionPtr) {
@@ -2372,8 +2816,19 @@ XML_GetCurrentLineNumber(XML_Parser parser) {
return parser->m_position.lineNumber + 1;
}
+// DEPRECATED since Expat 2.9.0.
XML_Size XMLCALL
-XML_GetCurrentColumnNumber(XML_Parser parser) {
+XML_GetCurrentLineNumber(XML_Parser parser) {
+ // NOTE: XML_Size is known to wrap around for >4 GiB content
+ // on 32bit machines and 64bit Windows, unless (non-default and
+ // uncommon) XML_LARGE_SIZE is defined.
+ // That's a bug and it only lives on because we cannot break
+ // ABI compatibility of public API.
+ return (XML_Size)XML_GetCurrentLineNumber64(parser);
+}
+
+uint64_t XMLCALL
+XML_GetCurrentColumnNumber64(XML_Parser parser) {
if (parser == NULL)
return 0;
if (parser->m_eventPtr && parser->m_eventPtr >= parser->m_positionPtr) {
@@ -2384,35 +2839,61 @@ XML_GetCurrentColumnNumber(XML_Parser parser) {
return parser->m_position.columnNumber;
}
+// DEPRECATED since Expat 2.9.0.
+XML_Size XMLCALL
+XML_GetCurrentColumnNumber(XML_Parser parser) {
+ // NOTE: XML_Size is known to wrap around for >4 GiB content
+ // on 32bit machines and 64bit Windows, unless (non-default and
+ // uncommon) XML_LARGE_SIZE is defined.
+ // That's a bug and it only lives on because we cannot break
+ // ABI compatibility of public API.
+ return (XML_Size)XML_GetCurrentColumnNumber64(parser);
+}
+
void XMLCALL
XML_FreeContentModel(XML_Parser parser, XML_Content *model) {
- if (parser != NULL)
- FREE(parser, model);
+ if (parser == NULL)
+ return;
+
+ // NOTE: We are avoiding FREE(..) here because the content model
+ // has been created using plain .malloc_fcn(..) rather than MALLOC(..).
+ parser->m_mem.free_fcn(model);
}
void *XMLCALL
XML_MemMalloc(XML_Parser parser, size_t size) {
if (parser == NULL)
return NULL;
- return MALLOC(parser, size);
+
+ // NOTE: We are avoiding MALLOC(..) here to not include
+ // user allocations with allocation tracking and limiting.
+ return parser->m_mem.malloc_fcn(size);
}
void *XMLCALL
XML_MemRealloc(XML_Parser parser, void *ptr, size_t size) {
if (parser == NULL)
return NULL;
- return REALLOC(parser, ptr, size);
+
+ // NOTE: We are avoiding REALLOC(..) here to not include
+ // user allocations with allocation tracking and limiting.
+ return parser->m_mem.realloc_fcn(ptr, size);
}
void XMLCALL
XML_MemFree(XML_Parser parser, void *ptr) {
- if (parser != NULL)
- FREE(parser, ptr);
+ if (parser == NULL)
+ return;
+
+ // NOTE: We are avoiding FREE(..) here because XML_MemMalloc and
+ // XML_MemRealloc are not using MALLOC(..) and REALLOC(..)
+ // but plain .malloc_fcn(..) and .realloc_fcn(..), internally.
+ parser->m_mem.free_fcn(ptr);
}
void XMLCALL
XML_DefaultCurrent(XML_Parser parser) {
- if (parser == NULL)
+ if (parser == NULL || ! isCalledFromInsideHandler(parser))
return;
if (parser->m_defaultHandler) {
if (parser->m_openInternalEntities)
@@ -2583,67 +3064,326 @@ XML_GetFeatureList(void) {
{XML_FEATURE_CONTEXT_BYTES, XML_L("XML_CONTEXT_BYTES"),
XML_CONTEXT_BYTES},
#endif
-#ifdef XML_MIN_SIZE
- {XML_FEATURE_MIN_SIZE, XML_L("XML_MIN_SIZE"), 0},
-#endif
#ifdef XML_NS
{XML_FEATURE_NS, XML_L("XML_NS"), 0},
#endif
#ifdef XML_LARGE_SIZE
{XML_FEATURE_LARGE_SIZE, XML_L("XML_LARGE_SIZE"), 0},
#endif
-#ifdef XML_ATTR_INFO
- {XML_FEATURE_ATTR_INFO, XML_L("XML_ATTR_INFO"), 0},
+#if XML_GE == 1
+ /* Added in Expat 2.4.0 for XML_DTD defined and
+ * added in Expat 2.6.0 for XML_GE == 1. */
+ {XML_FEATURE_BILLION_LAUGHS_ATTACK_PROTECTION_MAXIMUM_AMPLIFICATION_DEFAULT,
+ XML_L("XML_BLAP_MAX_AMP"),
+ (long int)
+ EXPAT_BILLION_LAUGHS_ATTACK_PROTECTION_MAXIMUM_AMPLIFICATION_DEFAULT},
+ {XML_FEATURE_BILLION_LAUGHS_ATTACK_PROTECTION_ACTIVATION_THRESHOLD_DEFAULT,
+ XML_L("XML_BLAP_ACT_THRES"),
+ EXPAT_BILLION_LAUGHS_ATTACK_PROTECTION_ACTIVATION_THRESHOLD_DEFAULT},
+ /* Added in Expat 2.6.0. */
+ {XML_FEATURE_GE, XML_L("XML_GE"), 0},
+ /* Added in Expat 2.7.2. */
+ {XML_FEATURE_ALLOC_TRACKER_MAXIMUM_AMPLIFICATION_DEFAULT,
+ XML_L("XML_AT_MAX_AMP"),
+ (long int)EXPAT_ALLOC_TRACKER_MAXIMUM_AMPLIFICATION_DEFAULT},
+ {XML_FEATURE_ALLOC_TRACKER_ACTIVATION_THRESHOLD_DEFAULT,
+ XML_L("XML_AT_ACT_THRES"),
+ (long int)EXPAT_ALLOC_TRACKER_ACTIVATION_THRESHOLD_DEFAULT},
+#endif
+ {XML_FEATURE_END, NULL, 0}};
+
+ return features;
+}
+
+#if XML_GE == 1
+static enum XML_Prop_Error
+setBillionLaughsAttackProtectionMaximumAmplification(
+ XML_Parser parser, float maximumAmplificationFactor) {
+ if (parser == NULL)
+ return XML_PROP_ERROR_PARSER_NULL;
+
+ if (parser->m_parentParser != NULL)
+ return XML_PROP_ERROR_PARSER_NOT_ROOT;
+
+ if (isnan(maximumAmplificationFactor) || (maximumAmplificationFactor < 1.0f))
+ return XML_PROP_ERROR_INVALID_VALUE;
+
+ parser->m_accounting.maximumAmplificationFactor = maximumAmplificationFactor;
+
+ return XML_PROP_ERROR_NONE;
+}
+
+XML_Bool XMLCALL
+XML_SetBillionLaughsAttackProtectionMaximumAmplification(
+ XML_Parser parser, float maximumAmplificationFactor) {
+ return (setBillionLaughsAttackProtectionMaximumAmplification(
+ parser, maximumAmplificationFactor)
+ == XML_PROP_ERROR_NONE)
+ ? XML_TRUE
+ : XML_FALSE;
+}
+
+static enum XML_Prop_Error
+setBillionLaughsAttackProtectionActivationThreshold(
+ XML_Parser parser, unsigned long long activationThresholdBytes) {
+ if (parser == NULL)
+ return XML_PROP_ERROR_PARSER_NULL;
+
+ if (parser->m_parentParser != NULL)
+ return XML_PROP_ERROR_PARSER_NOT_ROOT;
+
+ parser->m_accounting.activationThresholdBytes = activationThresholdBytes;
+
+ return XML_PROP_ERROR_NONE;
+}
+
+XML_Bool XMLCALL
+XML_SetBillionLaughsAttackProtectionActivationThreshold(
+ XML_Parser parser, unsigned long long activationThresholdBytes) {
+ return (setBillionLaughsAttackProtectionActivationThreshold(
+ parser, activationThresholdBytes)
+ == XML_PROP_ERROR_NONE)
+ ? XML_TRUE
+ : XML_FALSE;
+}
+
+static enum XML_Prop_Error
+setAllocTrackerMaximumAmplification(XML_Parser parser,
+ float maximumAmplificationFactor) {
+ if (parser == NULL)
+ return XML_PROP_ERROR_PARSER_NULL;
+
+ if (parser->m_parentParser != NULL)
+ return XML_PROP_ERROR_PARSER_NOT_ROOT;
+
+ if (isnan(maximumAmplificationFactor) || (maximumAmplificationFactor < 1.0f))
+ return XML_PROP_ERROR_INVALID_VALUE;
+
+ parser->m_alloc_tracker.maximumAmplificationFactor
+ = maximumAmplificationFactor;
+
+ return XML_PROP_ERROR_NONE;
+}
+
+XML_Bool XMLCALL
+XML_SetAllocTrackerMaximumAmplification(XML_Parser parser,
+ float maximumAmplificationFactor) {
+ return (setAllocTrackerMaximumAmplification(parser,
+ maximumAmplificationFactor)
+ == XML_PROP_ERROR_NONE)
+ ? XML_TRUE
+ : XML_FALSE;
+}
+
+static enum XML_Prop_Error
+setAllocTrackerActivationThreshold(
+ XML_Parser parser, unsigned long long activationThresholdBytes) {
+ if (parser == NULL)
+ return XML_PROP_ERROR_PARSER_NULL;
+
+ if (parser->m_parentParser != NULL)
+ return XML_PROP_ERROR_PARSER_NOT_ROOT;
+
+ parser->m_alloc_tracker.activationThresholdBytes = activationThresholdBytes;
+
+ return XML_PROP_ERROR_NONE;
+}
+
+XML_Bool XMLCALL
+XML_SetAllocTrackerActivationThreshold(
+ XML_Parser parser, unsigned long long activationThresholdBytes) {
+ return (setAllocTrackerActivationThreshold(parser, activationThresholdBytes)
+ == XML_PROP_ERROR_NONE)
+ ? XML_TRUE
+ : XML_FALSE;
+}
+#endif /* XML_GE == 1 */
+
+static enum XML_Prop_Error
+setReparseDeferralEnabled(XML_Parser parser, XML_Bool enabled) {
+ if (parser == NULL)
+ return XML_PROP_ERROR_PARSER_NULL;
+
+ if (enabled != XML_TRUE && enabled != XML_FALSE)
+ return XML_PROP_ERROR_INVALID_VALUE;
+
+ parser->m_reparseDeferralEnabled = enabled;
+
+ return XML_PROP_ERROR_NONE;
+}
+
+XML_Bool XMLCALL
+XML_SetReparseDeferralEnabled(XML_Parser parser, XML_Bool enabled) {
+ return (setReparseDeferralEnabled(parser, enabled) == XML_PROP_ERROR_NONE)
+ ? XML_TRUE
+ : XML_FALSE;
+}
+
+enum XML_Prop_Error XMLCALL
+XML_SetPropertyBool(XML_Parser parser, enum XML_Parser_Property property,
+ XML_Bool value) {
+ if (parser == NULL)
+ return XML_PROP_ERROR_PARSER_NULL;
+
+ switch (property) {
+#if XML_GE == 1
+ case XML_PROP_ALLOC_TRACKER_ACTIVATION_THRESHOLD:
+ case XML_PROP_ALLOC_TRACKER_MAXIMUM_AMPLIFICATION:
+ case XML_PROP_BILLION_LAUGHS_ACTIVATION_THRESHOLD:
+ case XML_PROP_BILLION_LAUGHS_MAXIMUM_AMPLIFICATION:
+ return XML_PROP_ERROR_INVALID_TYPE;
+#endif /* XML_GE == 1 */
+ case XML_PROP_REPARSE_DEFERRAL_ENABLED:
+ return setReparseDeferralEnabled(parser, value);
+ default:
+ return XML_PROP_ERROR_INVALID_KEY;
+ }
+
+ assert(0 && "considered unreachable");
+}
+
+enum XML_Prop_Error XMLCALL
+XML_SetPropertyDouble(XML_Parser parser, enum XML_Parser_Property property,
+ double value) {
+ if (parser == NULL)
+ return XML_PROP_ERROR_PARSER_NULL;
+
+#if XML_GE == 0
+ UNUSED_P(value);
#endif
+
+ switch (property) {
#if XML_GE == 1
- /* Added in Expat 2.4.0 for XML_DTD defined and
- * added in Expat 2.6.0 for XML_GE == 1. */
- {XML_FEATURE_BILLION_LAUGHS_ATTACK_PROTECTION_MAXIMUM_AMPLIFICATION_DEFAULT,
- XML_L("XML_BLAP_MAX_AMP"),
- (long int)
- EXPAT_BILLION_LAUGHS_ATTACK_PROTECTION_MAXIMUM_AMPLIFICATION_DEFAULT},
- {XML_FEATURE_BILLION_LAUGHS_ATTACK_PROTECTION_ACTIVATION_THRESHOLD_DEFAULT,
- XML_L("XML_BLAP_ACT_THRES"),
- EXPAT_BILLION_LAUGHS_ATTACK_PROTECTION_ACTIVATION_THRESHOLD_DEFAULT},
- /* Added in Expat 2.6.0. */
- {XML_FEATURE_GE, XML_L("XML_GE"), 0},
-#endif
- {XML_FEATURE_END, NULL, 0}};
+ case XML_PROP_ALLOC_TRACKER_MAXIMUM_AMPLIFICATION:
+ return setAllocTrackerMaximumAmplification(parser, (float)value);
+ case XML_PROP_BILLION_LAUGHS_MAXIMUM_AMPLIFICATION:
+ return setBillionLaughsAttackProtectionMaximumAmplification(parser,
+ (float)value);
+ case XML_PROP_ALLOC_TRACKER_ACTIVATION_THRESHOLD:
+ case XML_PROP_BILLION_LAUGHS_ACTIVATION_THRESHOLD:
+#endif /* XML_GE == 1 */
+ case XML_PROP_REPARSE_DEFERRAL_ENABLED:
+ return XML_PROP_ERROR_INVALID_TYPE;
+ default:
+ return XML_PROP_ERROR_INVALID_KEY;
+ }
- return features;
+ assert(0 && "considered unreachable");
}
+enum XML_Prop_Error XMLCALL
+XML_SetPropertyUInt64(XML_Parser parser, enum XML_Parser_Property property,
+ uint64_t value) {
+ if (parser == NULL)
+ return XML_PROP_ERROR_PARSER_NULL;
+
+#if XML_GE == 0
+ UNUSED_P(value);
+#endif
+
+ switch (property) {
#if XML_GE == 1
-XML_Bool XMLCALL
-XML_SetBillionLaughsAttackProtectionMaximumAmplification(
- XML_Parser parser, float maximumAmplificationFactor) {
- if ((parser == NULL) || (parser->m_parentParser != NULL)
- || isnan(maximumAmplificationFactor)
- || (maximumAmplificationFactor < 1.0f)) {
- return XML_FALSE;
+ case XML_PROP_ALLOC_TRACKER_ACTIVATION_THRESHOLD:
+ return setAllocTrackerActivationThreshold(parser,
+ (unsigned long long)value);
+ case XML_PROP_BILLION_LAUGHS_ACTIVATION_THRESHOLD:
+ return setBillionLaughsAttackProtectionActivationThreshold(
+ parser, (unsigned long long)value);
+ case XML_PROP_ALLOC_TRACKER_MAXIMUM_AMPLIFICATION:
+ case XML_PROP_BILLION_LAUGHS_MAXIMUM_AMPLIFICATION:
+#endif /* XML_GE == 1 */
+ case XML_PROP_REPARSE_DEFERRAL_ENABLED:
+ return XML_PROP_ERROR_INVALID_TYPE;
+ default:
+ return XML_PROP_ERROR_INVALID_KEY;
}
- parser->m_accounting.maximumAmplificationFactor = maximumAmplificationFactor;
- return XML_TRUE;
+
+ assert(0 && "considered unreachable");
}
-XML_Bool XMLCALL
-XML_SetBillionLaughsAttackProtectionActivationThreshold(
- XML_Parser parser, unsigned long long activationThresholdBytes) {
- if ((parser == NULL) || (parser->m_parentParser != NULL)) {
- return XML_FALSE;
+enum XML_Prop_Error XMLCALL
+XML_GetPropertyBool(XML_Parser parser, enum XML_Parser_Property property,
+ XML_Bool *value) {
+ if (parser == NULL)
+ return XML_PROP_ERROR_PARSER_NULL;
+
+ if (value == NULL)
+ return XML_PROP_ERROR_INVALID_VALUE;
+
+ switch (property) {
+#if XML_GE == 1
+ case XML_PROP_ALLOC_TRACKER_ACTIVATION_THRESHOLD:
+ case XML_PROP_ALLOC_TRACKER_MAXIMUM_AMPLIFICATION:
+ case XML_PROP_BILLION_LAUGHS_ACTIVATION_THRESHOLD:
+ case XML_PROP_BILLION_LAUGHS_MAXIMUM_AMPLIFICATION:
+ return XML_PROP_ERROR_INVALID_TYPE;
+#endif /* XML_GE == 1 */
+ case XML_PROP_REPARSE_DEFERRAL_ENABLED:
+ *value = parser->m_reparseDeferralEnabled;
+ break;
+ default:
+ return XML_PROP_ERROR_INVALID_KEY;
}
- parser->m_accounting.activationThresholdBytes = activationThresholdBytes;
- return XML_TRUE;
+
+ return XML_PROP_ERROR_NONE;
}
+
+enum XML_Prop_Error XMLCALL
+XML_GetPropertyDouble(XML_Parser parser, enum XML_Parser_Property property,
+ double *value) {
+ if (parser == NULL)
+ return XML_PROP_ERROR_PARSER_NULL;
+
+ if (value == NULL)
+ return XML_PROP_ERROR_INVALID_VALUE;
+
+ switch (property) {
+#if XML_GE == 1
+ case XML_PROP_BILLION_LAUGHS_MAXIMUM_AMPLIFICATION:
+ *value = (double)parser->m_accounting.maximumAmplificationFactor;
+ break;
+ case XML_PROP_ALLOC_TRACKER_MAXIMUM_AMPLIFICATION:
+ *value = (double)parser->m_alloc_tracker.maximumAmplificationFactor;
+ break;
+ case XML_PROP_ALLOC_TRACKER_ACTIVATION_THRESHOLD:
+ case XML_PROP_BILLION_LAUGHS_ACTIVATION_THRESHOLD:
#endif /* XML_GE == 1 */
+ case XML_PROP_REPARSE_DEFERRAL_ENABLED:
+ return XML_PROP_ERROR_INVALID_TYPE;
+ default:
+ return XML_PROP_ERROR_INVALID_KEY;
+ }
-XML_Bool XMLCALL
-XML_SetReparseDeferralEnabled(XML_Parser parser, XML_Bool enabled) {
- if (parser != NULL && (enabled == XML_TRUE || enabled == XML_FALSE)) {
- parser->m_reparseDeferralEnabled = enabled;
- return XML_TRUE;
+ return XML_PROP_ERROR_NONE;
+}
+
+enum XML_Prop_Error XMLCALL
+XML_GetPropertyUInt64(XML_Parser parser, enum XML_Parser_Property property,
+ uint64_t *value) {
+ if (parser == NULL)
+ return XML_PROP_ERROR_PARSER_NULL;
+
+ if (value == NULL)
+ return XML_PROP_ERROR_INVALID_VALUE;
+
+ switch (property) {
+#if XML_GE == 1
+ case XML_PROP_BILLION_LAUGHS_ACTIVATION_THRESHOLD:
+ *value = parser->m_accounting.activationThresholdBytes;
+ break;
+ case XML_PROP_ALLOC_TRACKER_ACTIVATION_THRESHOLD:
+ *value = parser->m_alloc_tracker.activationThresholdBytes;
+ break;
+ case XML_PROP_ALLOC_TRACKER_MAXIMUM_AMPLIFICATION:
+ case XML_PROP_BILLION_LAUGHS_MAXIMUM_AMPLIFICATION:
+#endif /* XML_GE == 1 */
+ case XML_PROP_REPARSE_DEFERRAL_ENABLED:
+ return XML_PROP_ERROR_INVALID_TYPE;
+ default:
+ return XML_PROP_ERROR_INVALID_KEY;
}
- return XML_FALSE;
+
+ return XML_PROP_ERROR_NONE;
}
/* Initially tag->rawName always points into the parse buffer;
@@ -2655,10 +3395,10 @@ static XML_Bool
storeRawNames(XML_Parser parser) {
TAG *tag = parser->m_tagStack;
while (tag) {
- int bufSize;
- int nameLen = sizeof(XML_Char) * (tag->name.strLen + 1);
+ size_t bufSize;
+ size_t nameLen = sizeof(XML_Char) * (tag->name.strLen + 1);
size_t rawNameLen;
- char *rawNameBuf = tag->buf + nameLen;
+ char *rawNameBuf = tag->buf.raw + nameLen;
/* Stop if already stored. Since m_tagStack is a stack, we can stop
at the first entry that has already been copied; everything
below it in the stack is already been accounted for in a
@@ -2671,25 +3411,25 @@ storeRawNames(XML_Parser parser) {
*/
rawNameLen = ROUND_UP(tag->rawNameLength, sizeof(XML_Char));
/* Detect and prevent integer overflow. */
- if (rawNameLen > (size_t)INT_MAX - nameLen)
+ if (rawNameLen > SIZE_MAX - nameLen)
return XML_FALSE;
- bufSize = nameLen + (int)rawNameLen;
- if (bufSize > tag->bufEnd - tag->buf) {
- char *temp = (char *)REALLOC(parser, tag->buf, bufSize);
+ bufSize = nameLen + rawNameLen;
+ if (bufSize > (size_t)(tag->bufEnd - tag->buf.raw)) {
+ char *temp = REALLOC(parser, tag->buf.raw, bufSize);
if (temp == NULL)
return XML_FALSE;
- /* if tag->name.str points to tag->buf (only when namespace
+ /* if tag->name.str points to tag->buf.str (only when namespace
processing is off) then we have to update it
*/
- if (tag->name.str == (XML_Char *)tag->buf)
+ if (tag->name.str == tag->buf.str)
tag->name.str = (XML_Char *)temp;
/* if tag->name.localPart is set (when namespace processing is on)
then update it as well, since it will always point into tag->buf
*/
if (tag->name.localPart)
tag->name.localPart
- = (XML_Char *)temp + (tag->name.localPart - (XML_Char *)tag->buf);
- tag->buf = temp;
+ = (XML_Char *)temp + (tag->name.localPart - tag->buf.str);
+ tag->buf.raw = temp;
tag->bufEnd = temp + bufSize;
rawNameBuf = temp + nameLen;
}
@@ -2700,12 +3440,13 @@ storeRawNames(XML_Parser parser) {
return XML_TRUE;
}
-static enum XML_Error PTRCALL
+static enum XML_Error
contentProcessor(XML_Parser parser, const char *start, const char *end,
const char **endPtr) {
enum XML_Error result = doContent(
- parser, 0, parser->m_encoding, start, end, endPtr,
- (XML_Bool)! parser->m_parsingStatus.finalBuffer, XML_ACCOUNT_DIRECT);
+ parser, parser->m_parentParser ? 1 : 0, parser->m_encoding, start, end,
+ endPtr, (XML_Bool)! parser->m_parsingStatus.finalBuffer,
+ XML_ACCOUNT_DIRECT);
if (result == XML_ERROR_NONE) {
if (! storeRawNames(parser))
return XML_ERROR_NO_MEMORY;
@@ -2713,7 +3454,7 @@ contentProcessor(XML_Parser parser, const char *start, const char *end,
return result;
}
-static enum XML_Error PTRCALL
+static enum XML_Error
externalEntityInitProcessor(XML_Parser parser, const char *start,
const char *end, const char **endPtr) {
enum XML_Error result = initializeEncoding(parser);
@@ -2723,7 +3464,7 @@ externalEntityInitProcessor(XML_Parser parser, const char *start,
return externalEntityInitProcessor2(parser, start, end, endPtr);
}
-static enum XML_Error PTRCALL
+static enum XML_Error
externalEntityInitProcessor2(XML_Parser parser, const char *start,
const char *end, const char **endPtr) {
const char *next = start; /* XmlContentTok doesn't always set the last arg */
@@ -2768,7 +3509,7 @@ externalEntityInitProcessor2(XML_Parser parser, const char *start,
return externalEntityInitProcessor3(parser, start, end, endPtr);
}
-static enum XML_Error PTRCALL
+static enum XML_Error
externalEntityInitProcessor3(XML_Parser parser, const char *start,
const char *end, const char **endPtr) {
int tok;
@@ -2793,6 +3534,11 @@ externalEntityInitProcessor3(XML_Parser parser, const char *start,
return XML_ERROR_NONE;
case XML_FINISHED:
return XML_ERROR_ABORTED;
+ case XML_PARSING:
+ if (parser->m_reenter) {
+ return XML_ERROR_UNEXPECTED_STATE; // LCOV_EXCL_LINE
+ }
+ EXPAT_FALLTHROUGH;
default:
start = next;
}
@@ -2815,7 +3561,7 @@ externalEntityInitProcessor3(XML_Parser parser, const char *start,
return externalEntityContentProcessor(parser, start, end, endPtr);
}
-static enum XML_Error PTRCALL
+static enum XML_Error
externalEntityContentProcessor(XML_Parser parser, const char *start,
const char *end, const char **endPtr) {
enum XML_Error result
@@ -2871,7 +3617,9 @@ doContent(XML_Parser parser, int startTagLevel, const ENCODING *enc,
*eventEndPP = end;
if (parser->m_characterDataHandler) {
XML_Char c = 0xA;
+ beforeHandler(parser);
parser->m_characterDataHandler(parser->m_handlerArg, &c, 1);
+ afterHandler(parser);
} else if (parser->m_defaultHandler)
reportDefault(parser, enc, s, end);
/* We are at the end of the final buffer, should we check for
@@ -2924,9 +3672,11 @@ doContent(XML_Parser parser, int startTagLevel, const ENCODING *enc,
((char *)&ch) + sizeof(XML_Char), __LINE__,
XML_ACCOUNT_ENTITY_EXPANSION);
#endif /* XML_GE == 1 */
- if (parser->m_characterDataHandler)
+ if (parser->m_characterDataHandler) {
+ beforeHandler(parser);
parser->m_characterDataHandler(parser->m_handlerArg, &ch, 1);
- else if (parser->m_defaultHandler)
+ afterHandler(parser);
+ } else if (parser->m_defaultHandler)
reportDefault(parser, enc, s, next);
break;
}
@@ -2946,9 +3696,11 @@ doContent(XML_Parser parser, int startTagLevel, const ENCODING *enc,
else if (! entity->is_internal)
return XML_ERROR_ENTITY_DECLARED_IN_PE;
} else if (! entity) {
- if (parser->m_skippedEntityHandler)
+ if (parser->m_skippedEntityHandler) {
+ beforeHandler(parser);
parser->m_skippedEntityHandler(parser->m_handlerArg, name, 0);
- else if (parser->m_defaultHandler)
+ afterHandler(parser);
+ } else if (parser->m_defaultHandler)
reportDefault(parser, enc, s, next);
break;
}
@@ -2959,26 +3711,31 @@ doContent(XML_Parser parser, int startTagLevel, const ENCODING *enc,
if (entity->textPtr) {
enum XML_Error result;
if (! parser->m_defaultExpandInternalEntities) {
- if (parser->m_skippedEntityHandler)
+ if (parser->m_skippedEntityHandler) {
+ beforeHandler(parser);
parser->m_skippedEntityHandler(parser->m_handlerArg, entity->name,
0);
- else if (parser->m_defaultHandler)
+ afterHandler(parser);
+ } else if (parser->m_defaultHandler)
reportDefault(parser, enc, s, next);
break;
}
- result = processInternalEntity(parser, entity, XML_FALSE);
+ result = processEntity(parser, entity, XML_FALSE, ENTITY_INTERNAL);
if (result != XML_ERROR_NONE)
return result;
} else if (parser->m_externalEntityRefHandler) {
const XML_Char *context;
- entity->open = XML_TRUE;
+ entity->open = true;
context = getContext(parser);
- entity->open = XML_FALSE;
+ entity->open = false;
if (! context)
return XML_ERROR_NO_MEMORY;
- if (! parser->m_externalEntityRefHandler(
- parser->m_externalEntityRefHandlerArg, context, entity->base,
- entity->systemId, entity->publicId))
+ beforeHandler(parser);
+ const int status = parser->m_externalEntityRefHandler(
+ parser->m_externalEntityRefHandlerArg, context, entity->base,
+ entity->systemId, entity->publicId);
+ afterHandler(parser);
+ if (! status)
return XML_ERROR_EXTERNAL_ENTITY_HANDLING;
poolDiscard(&parser->m_tempPool);
} else if (parser->m_defaultHandler)
@@ -2986,7 +3743,6 @@ doContent(XML_Parser parser, int startTagLevel, const ENCODING *enc,
break;
}
case XML_TOK_START_TAG_NO_ATTS:
- /* fall through */
case XML_TOK_START_TAG_WITH_ATTS: {
TAG *tag;
enum XML_Error result;
@@ -2995,15 +3751,15 @@ doContent(XML_Parser parser, int startTagLevel, const ENCODING *enc,
tag = parser->m_freeTagList;
parser->m_freeTagList = parser->m_freeTagList->parent;
} else {
- tag = (TAG *)MALLOC(parser, sizeof(TAG));
+ tag = MALLOC(parser, sizeof(TAG));
if (! tag)
return XML_ERROR_NO_MEMORY;
- tag->buf = (char *)MALLOC(parser, INIT_TAG_BUF_SIZE);
- if (! tag->buf) {
+ tag->buf.raw = MALLOC(parser, INIT_TAG_BUF_SIZE);
+ if (! tag->buf.raw) {
FREE(parser, tag);
return XML_ERROR_NO_MEMORY;
}
- tag->bufEnd = tag->buf + INIT_TAG_BUF_SIZE;
+ tag->bufEnd = tag->buf.raw + INIT_TAG_BUF_SIZE;
}
tag->bindings = NULL;
tag->parent = parser->m_tagStack;
@@ -3016,46 +3772,47 @@ doContent(XML_Parser parser, int startTagLevel, const ENCODING *enc,
{
const char *rawNameEnd = tag->rawName + tag->rawNameLength;
const char *fromPtr = tag->rawName;
- toPtr = (XML_Char *)tag->buf;
+ toPtr = tag->buf.str;
for (;;) {
- int bufSize;
- int convLen;
const enum XML_Convert_Result convert_res
= XmlConvert(enc, &fromPtr, rawNameEnd, (ICHAR **)&toPtr,
(ICHAR *)tag->bufEnd - 1);
- convLen = (int)(toPtr - (XML_Char *)tag->buf);
+ const size_t convLen = (size_t)(toPtr - tag->buf.str);
if ((fromPtr >= rawNameEnd)
|| (convert_res == XML_CONVERT_INPUT_INCOMPLETE)) {
tag->name.strLen = convLen;
break;
}
- bufSize = (int)(tag->bufEnd - tag->buf) << 1;
+ if (SIZE_MAX / 2 < (size_t)(tag->bufEnd - tag->buf.raw))
+ return XML_ERROR_NO_MEMORY;
+ const size_t bufSize = (size_t)(tag->bufEnd - tag->buf.raw) * 2;
{
- char *temp = (char *)REALLOC(parser, tag->buf, bufSize);
+ char *temp = REALLOC(parser, tag->buf.raw, bufSize);
if (temp == NULL)
return XML_ERROR_NO_MEMORY;
- tag->buf = temp;
+ tag->buf.raw = temp;
tag->bufEnd = temp + bufSize;
toPtr = (XML_Char *)temp + convLen;
}
}
}
- tag->name.str = (XML_Char *)tag->buf;
+ tag->name.str = tag->buf.str;
*toPtr = XML_T('\0');
result
= storeAtts(parser, enc, s, &(tag->name), &(tag->bindings), account);
if (result)
return result;
- if (parser->m_startElementHandler)
+ if (parser->m_startElementHandler) {
+ beforeHandler(parser);
parser->m_startElementHandler(parser->m_handlerArg, tag->name.str,
(const XML_Char **)parser->m_atts);
- else if (parser->m_defaultHandler)
+ afterHandler(parser);
+ } else if (parser->m_defaultHandler)
reportDefault(parser, enc, s, next);
poolClear(&parser->m_tempPool);
break;
}
case XML_TOK_EMPTY_ELEMENT_NO_ATTS:
- /* fall through */
case XML_TOK_EMPTY_ELEMENT_WITH_ATTS: {
const char *rawName = s + enc->minBytesPerChar;
enum XML_Error result;
@@ -3075,14 +3832,18 @@ doContent(XML_Parser parser, int startTagLevel, const ENCODING *enc,
}
poolFinish(&parser->m_tempPool);
if (parser->m_startElementHandler) {
+ beforeHandler(parser);
parser->m_startElementHandler(parser->m_handlerArg, name.str,
(const XML_Char **)parser->m_atts);
+ afterHandler(parser);
noElmHandlers = XML_FALSE;
}
if (parser->m_endElementHandler) {
if (parser->m_startElementHandler)
*eventPP = *eventEndPP;
+ beforeHandler(parser);
parser->m_endElementHandler(parser->m_handlerArg, name.str);
+ afterHandler(parser);
noElmHandlers = XML_FALSE;
}
if (noElmHandlers && parser->m_defaultHandler)
@@ -3092,7 +3853,9 @@ doContent(XML_Parser parser, int startTagLevel, const ENCODING *enc,
}
if ((parser->m_tagLevel == 0)
&& (parser->m_parsingStatus.parsing != XML_FINISHED)) {
- if (parser->m_parsingStatus.parsing == XML_SUSPENDED)
+ if (parser->m_parsingStatus.parsing == XML_SUSPENDED
+ || (parser->m_parsingStatus.parsing == XML_PARSING
+ && parser->m_reenter))
parser->m_processor = epilogProcessor;
else
return epilogProcessor(parser, next, end, nextPtr);
@@ -3138,14 +3901,19 @@ doContent(XML_Parser parser, int startTagLevel, const ENCODING *enc,
}
*uri = XML_T('\0');
}
+ beforeHandler(parser);
parser->m_endElementHandler(parser->m_handlerArg, tag->name.str);
+ afterHandler(parser);
} else if (parser->m_defaultHandler)
reportDefault(parser, enc, s, next);
while (tag->bindings) {
BINDING *b = tag->bindings;
- if (parser->m_endNamespaceDeclHandler)
+ if (parser->m_endNamespaceDeclHandler) {
+ beforeHandler(parser);
parser->m_endNamespaceDeclHandler(parser->m_handlerArg,
b->prefix->name);
+ afterHandler(parser);
+ }
tag->bindings = tag->bindings->nextTagBinding;
b->nextTagBinding = parser->m_freeBindingList;
parser->m_freeBindingList = b;
@@ -3153,7 +3921,9 @@ doContent(XML_Parser parser, int startTagLevel, const ENCODING *enc,
}
if ((parser->m_tagLevel == 0)
&& (parser->m_parsingStatus.parsing != XML_FINISHED)) {
- if (parser->m_parsingStatus.parsing == XML_SUSPENDED)
+ if (parser->m_parsingStatus.parsing == XML_SUSPENDED
+ || (parser->m_parsingStatus.parsing == XML_PARSING
+ && parser->m_reenter))
parser->m_processor = epilogProcessor;
else
return epilogProcessor(parser, next, end, nextPtr);
@@ -3166,8 +3936,10 @@ doContent(XML_Parser parser, int startTagLevel, const ENCODING *enc,
return XML_ERROR_BAD_CHAR_REF;
if (parser->m_characterDataHandler) {
XML_Char buf[XML_ENCODE_MAX];
+ beforeHandler(parser);
parser->m_characterDataHandler(parser->m_handlerArg, buf,
XmlEncode(n, (ICHAR *)buf));
+ afterHandler(parser);
} else if (parser->m_defaultHandler)
reportDefault(parser, enc, s, next);
} break;
@@ -3176,32 +3948,38 @@ doContent(XML_Parser parser, int startTagLevel, const ENCODING *enc,
case XML_TOK_DATA_NEWLINE:
if (parser->m_characterDataHandler) {
XML_Char c = 0xA;
+ beforeHandler(parser);
parser->m_characterDataHandler(parser->m_handlerArg, &c, 1);
+ afterHandler(parser);
} else if (parser->m_defaultHandler)
reportDefault(parser, enc, s, next);
break;
case XML_TOK_CDATA_SECT_OPEN: {
enum XML_Error result;
- if (parser->m_startCdataSectionHandler)
+ if (parser->m_startCdataSectionHandler) {
+ beforeHandler(parser);
parser->m_startCdataSectionHandler(parser->m_handlerArg);
- /* BEGIN disabled code */
- /* Suppose you doing a transformation on a document that involves
- changing only the character data. You set up a defaultHandler
- and a characterDataHandler. The defaultHandler simply copies
- characters through. The characterDataHandler does the
- transformation and writes the characters out escaping them as
- necessary. This case will fail to work if we leave out the
- following two lines (because & and < inside CDATA sections will
- be incorrectly escaped).
-
- However, now we have a start/endCdataSectionHandler, so it seems
- easier to let the user deal with this.
- */
- else if ((0) && parser->m_characterDataHandler)
+ afterHandler(parser);
+ /* BEGIN disabled code */
+ /* Suppose you doing a transformation on a document that involves
+ changing only the character data. You set up a defaultHandler
+ and a characterDataHandler. The defaultHandler simply copies
+ characters through. The characterDataHandler does the
+ transformation and writes the characters out escaping them as
+ necessary. This case will fail to work if we leave out the
+ following two lines (because & and < inside CDATA sections will
+ be incorrectly escaped).
+
+ However, now we have a start/endCdataSectionHandler, so it seems
+ easier to let the user deal with this.
+ */
+ } else if ((0) && parser->m_characterDataHandler) {
+ beforeHandler(parser);
parser->m_characterDataHandler(parser->m_handlerArg, parser->m_dataBuf,
0);
- /* END disabled code */
- else if (parser->m_defaultHandler)
+ afterHandler(parser);
+ /* END disabled code */
+ } else if (parser->m_defaultHandler)
reportDefault(parser, enc, s, next);
result
= doCdataSection(parser, enc, &next, end, nextPtr, haveMore, account);
@@ -3221,13 +3999,18 @@ doContent(XML_Parser parser, int startTagLevel, const ENCODING *enc,
if (MUST_CONVERT(enc, s)) {
ICHAR *dataPtr = (ICHAR *)parser->m_dataBuf;
XmlConvert(enc, &s, end, &dataPtr, (ICHAR *)parser->m_dataBufEnd);
+ beforeHandler(parser);
parser->m_characterDataHandler(
parser->m_handlerArg, parser->m_dataBuf,
(int)(dataPtr - (ICHAR *)parser->m_dataBuf));
- } else
+ afterHandler(parser);
+ } else {
+ beforeHandler(parser);
parser->m_characterDataHandler(
parser->m_handlerArg, (const XML_Char *)s,
(int)((const XML_Char *)end - (const XML_Char *)s));
+ afterHandler(parser);
+ }
} else if (parser->m_defaultHandler)
reportDefault(parser, enc, s, end);
/* We are at the end of the final buffer, should we check for
@@ -3252,16 +4035,21 @@ doContent(XML_Parser parser, int startTagLevel, const ENCODING *enc,
const enum XML_Convert_Result convert_res = XmlConvert(
enc, &s, next, &dataPtr, (ICHAR *)parser->m_dataBufEnd);
*eventEndPP = s;
+ beforeHandler(parser);
charDataHandler(parser->m_handlerArg, parser->m_dataBuf,
(int)(dataPtr - (ICHAR *)parser->m_dataBuf));
+ afterHandler(parser);
if ((convert_res == XML_CONVERT_COMPLETED)
|| (convert_res == XML_CONVERT_INPUT_INCOMPLETE))
break;
*eventPP = s;
}
- } else
+ } else {
+ beforeHandler(parser);
charDataHandler(parser->m_handlerArg, (const XML_Char *)s,
(int)((const XML_Char *)next - (const XML_Char *)s));
+ afterHandler(parser);
+ }
} else if (parser->m_defaultHandler)
reportDefault(parser, enc, s, next);
} break;
@@ -3286,14 +4074,22 @@ doContent(XML_Parser parser, int startTagLevel, const ENCODING *enc,
break;
/* LCOV_EXCL_STOP */
}
- *eventPP = s = next;
switch (parser->m_parsingStatus.parsing) {
case XML_SUSPENDED:
+ *eventPP = next;
*nextPtr = next;
return XML_ERROR_NONE;
case XML_FINISHED:
+ *eventPP = next;
return XML_ERROR_ABORTED;
+ case XML_PARSING:
+ if (parser->m_reenter) {
+ *nextPtr = next;
+ return XML_ERROR_NONE;
+ }
+ EXPAT_FALLTHROUGH;
default:;
+ *eventPP = s = next;
}
}
/* not reached */
@@ -3311,8 +4107,11 @@ freeBindings(XML_Parser parser, BINDING *bindings) {
/* m_startNamespaceDeclHandler will have been called for this
* binding in addBindings(), so call the end handler now.
*/
- if (parser->m_endNamespaceDeclHandler)
+ if (parser->m_endNamespaceDeclHandler) {
+ beforeHandler(parser);
parser->m_endNamespaceDeclHandler(parser->m_handlerArg, b->prefix->name);
+ afterHandler(parser);
+ }
bindings = bindings->nextTagBinding;
b->nextTagBinding = parser->m_freeBindingList;
@@ -3336,20 +4135,14 @@ storeAtts(XML_Parser parser, const ENCODING *enc, const char *attStr,
TAG_NAME *tagNamePtr, BINDING **bindingsPtr,
enum XML_Account account) {
DTD *const dtd = parser->m_dtd; /* save one level of indirection */
- ELEMENT_TYPE *elementType;
- int nDefaultAtts;
- const XML_Char **appAtts; /* the attribute list for the application */
int attIndex = 0;
- int prefixLen;
- int i;
- int n;
XML_Char *uri;
int nPrefixes = 0;
BINDING *binding;
const XML_Char *localPart;
/* lookup the element type name */
- elementType
+ ELEMENT_TYPE *elementType
= (ELEMENT_TYPE *)lookup(parser, &dtd->elementTypes, tagNamePtr->str, 0);
if (! elementType) {
const XML_Char *name = poolCopyString(&dtd->pool, tagNamePtr->str);
@@ -3359,98 +4152,68 @@ storeAtts(XML_Parser parser, const ENCODING *enc, const char *attStr,
sizeof(ELEMENT_TYPE));
if (! elementType)
return XML_ERROR_NO_MEMORY;
+ if (! elementType->defaultAttForName.parser)
+ hashTableInit(&(elementType->defaultAttForName), parser);
if (parser->m_ns && ! setElementTypePrefix(parser, elementType))
return XML_ERROR_NO_MEMORY;
}
- nDefaultAtts = elementType->nDefaultAtts;
+ const size_t nDefaultAtts = elementType->nDefaultAtts;
+
+ /* Detect and prevent integer overflow. */
+ if (parser->m_attsSize > (size_t)INT_MAX)
+ return XML_ERROR_NO_MEMORY;
/* get the attributes from the tokenizer */
- n = XmlGetAttributes(enc, attStr, parser->m_attsSize, parser->m_atts);
+ size_t n = (size_t)XmlGetAttributes(enc, attStr, (int)parser->m_attsSize,
+ parser->m_atts);
/* Detect and prevent integer overflow */
- if (n > INT_MAX - nDefaultAtts) {
+ if (n > SIZE_MAX - nDefaultAtts) {
return XML_ERROR_NO_MEMORY;
}
if (n + nDefaultAtts > parser->m_attsSize) {
- int oldAttsSize = parser->m_attsSize;
- ATTRIBUTE *temp;
-#ifdef XML_ATTR_INFO
- XML_AttrInfo *temp2;
-#endif
+ size_t oldAttsSize = parser->m_attsSize;
/* Detect and prevent integer overflow */
- if ((nDefaultAtts > INT_MAX - INIT_ATTS_SIZE)
- || (n > INT_MAX - (nDefaultAtts + INIT_ATTS_SIZE))) {
+ if ((nDefaultAtts > SIZE_MAX - INIT_ATTS_SIZE)
+ || (n > SIZE_MAX - (nDefaultAtts + INIT_ATTS_SIZE))) {
return XML_ERROR_NO_MEMORY;
}
parser->m_attsSize = n + nDefaultAtts + INIT_ATTS_SIZE;
- /* Detect and prevent integer overflow.
- * The preprocessor guard addresses the "always false" warning
- * from -Wtype-limits on platforms where
- * sizeof(unsigned int) < sizeof(size_t), e.g. on x86_64. */
-#if UINT_MAX >= SIZE_MAX
- if ((unsigned)parser->m_attsSize > (size_t)(-1) / sizeof(ATTRIBUTE)) {
+ /* Detect and prevent integer overflow. */
+ if (parser->m_attsSize > SIZE_MAX / sizeof(ATTRIBUTE)) {
parser->m_attsSize = oldAttsSize;
return XML_ERROR_NO_MEMORY;
}
-#endif
- temp = (ATTRIBUTE *)REALLOC(parser, (void *)parser->m_atts,
- parser->m_attsSize * sizeof(ATTRIBUTE));
+ ATTRIBUTE *const temp = REALLOC(parser, parser->m_atts,
+ parser->m_attsSize * sizeof(ATTRIBUTE));
if (temp == NULL) {
parser->m_attsSize = oldAttsSize;
return XML_ERROR_NO_MEMORY;
}
parser->m_atts = temp;
-#ifdef XML_ATTR_INFO
- /* Detect and prevent integer overflow.
- * The preprocessor guard addresses the "always false" warning
- * from -Wtype-limits on platforms where
- * sizeof(unsigned int) < sizeof(size_t), e.g. on x86_64. */
-# if UINT_MAX >= SIZE_MAX
- if ((unsigned)parser->m_attsSize > (size_t)(-1) / sizeof(XML_AttrInfo)) {
- parser->m_attsSize = oldAttsSize;
- return XML_ERROR_NO_MEMORY;
- }
-# endif
-
- temp2 = (XML_AttrInfo *)REALLOC(parser, (void *)parser->m_attInfo,
- parser->m_attsSize * sizeof(XML_AttrInfo));
- if (temp2 == NULL) {
- parser->m_attsSize = oldAttsSize;
- return XML_ERROR_NO_MEMORY;
+ if (n > oldAttsSize) {
+ /* Detect and prevent integer overflow. */
+ if (n > (size_t)INT_MAX)
+ return XML_ERROR_NO_MEMORY;
+ XmlGetAttributes(enc, attStr, (int)n, parser->m_atts);
}
- parser->m_attInfo = temp2;
-#endif
- if (n > oldAttsSize)
- XmlGetAttributes(enc, attStr, n, parser->m_atts);
}
- appAtts = (const XML_Char **)parser->m_atts;
- for (i = 0; i < n; i++) {
+ /* the attribute list for the application */
+ const XML_Char **const appAtts = (const XML_Char **)parser->m_atts;
+ for (size_t i = 0; i < n; i++) {
ATTRIBUTE *currAtt = &parser->m_atts[i];
-#ifdef XML_ATTR_INFO
- XML_AttrInfo *currAttInfo = &parser->m_attInfo[i];
-#endif
/* add the name and value to the attribute list */
ATTRIBUTE_ID *attId
= getAttributeId(parser, enc, currAtt->name,
currAtt->name + XmlNameLength(enc, currAtt->name));
if (! attId)
return XML_ERROR_NO_MEMORY;
-#ifdef XML_ATTR_INFO
- currAttInfo->nameStart
- = parser->m_parseEndByteIndex - (parser->m_parseEndPtr - currAtt->name);
- currAttInfo->nameEnd
- = currAttInfo->nameStart + XmlNameLength(enc, currAtt->name);
- currAttInfo->valueStart = parser->m_parseEndByteIndex
- - (parser->m_parseEndPtr - currAtt->valuePtr);
- currAttInfo->valueEnd = parser->m_parseEndByteIndex
- - (parser->m_parseEndPtr - currAtt->valueEnd);
-#endif
/* Detect duplicate attributes by their QNames. This does not work when
namespace processing is turned on and different prefixes for the same
namespace are used. For this case we have a check further down.
@@ -3463,22 +4226,23 @@ storeAtts(XML_Parser parser, const ENCODING *enc, const char *attStr,
(attId->name)[-1] = 1;
appAtts[attIndex++] = attId->name;
if (! parser->m_atts[i].normalized) {
- enum XML_Error result;
XML_Bool isCdata = XML_TRUE;
/* figure out whether declared as other than CDATA */
if (attId->maybeTokenized) {
- int j;
- for (j = 0; j < nDefaultAtts; j++) {
- if (attId == elementType->defaultAtts[j].id) {
- isCdata = elementType->defaultAtts[j].isCdata;
- break;
- }
+ NAME_AND_DEFAULT_ATTRIBUTE *const nameAndDefaultAttribute
+ = (NAME_AND_DEFAULT_ATTRIBUTE *)lookup(
+ parser, &(elementType->defaultAttForName), attId->name, 0);
+ if (nameAndDefaultAttribute != NULL) {
+ assert(nameAndDefaultAttribute->attIndex < elementType->nDefaultAtts);
+ const DEFAULT_ATTRIBUTE *const att
+ = elementType->defaultAtts + nameAndDefaultAttribute->attIndex;
+ isCdata = att->isCdata;
}
}
/* normalize the attribute value */
- result = storeAttributeValue(
+ const enum XML_Error result = storeAttributeValue(
parser, enc, isCdata, parser->m_atts[i].valuePtr,
parser->m_atts[i].valueEnd, &parser->m_tempPool, account);
if (result)
@@ -3516,7 +4280,7 @@ storeAtts(XML_Parser parser, const ENCODING *enc, const char *attStr,
/* set-up for XML_GetSpecifiedAttributeCount and XML_GetIdAttributeIndex */
parser->m_nSpecifiedAtts = attIndex;
if (elementType->idAtt && (elementType->idAtt->name)[-1]) {
- for (i = 0; i < attIndex; i += 2)
+ for (int i = 0; i < attIndex; i += 2)
if (appAtts[i] == elementType->idAtt->name) {
parser->m_idAttIndex = i;
break;
@@ -3525,7 +4289,7 @@ storeAtts(XML_Parser parser, const ENCODING *enc, const char *attStr,
parser->m_idAttIndex = -1;
/* do attribute defaulting */
- for (i = 0; i < nDefaultAtts; i++) {
+ for (size_t i = 0; i < nDefaultAtts; i++) {
const DEFAULT_ATTRIBUTE *da = elementType->defaultAtts + i;
if (! (da->id->name)[-1] && da->value) {
if (da->id->prefix) {
@@ -3551,9 +4315,9 @@ storeAtts(XML_Parser parser, const ENCODING *enc, const char *attStr,
/* expand prefixed attribute names, check for duplicates,
and clear flags that say whether attributes were specified */
- i = 0;
+ int i = 0;
if (nPrefixes) {
- int j; /* hash table index */
+ unsigned int j; /* hash table index */
unsigned long version = parser->m_nsAttsVersion;
/* Detect and prevent invalid shift */
@@ -3564,9 +4328,8 @@ storeAtts(XML_Parser parser, const ENCODING *enc, const char *attStr,
unsigned int nsAttsSize = 1u << parser->m_nsAttsPower;
unsigned char oldNsAttsPower = parser->m_nsAttsPower;
/* size of hash table must be at least 2 * (# of prefixed attributes) */
- if ((nPrefixes << 1)
- >> parser->m_nsAttsPower) { /* true for m_nsAttsPower = 0 */
- NS_ATT *temp;
+ if (parser->m_nsAttsPower == 0
+ || (nPrefixes >> (parser->m_nsAttsPower - 1))) {
/* hash table size must also be a power of 2 and >= 8 */
while (nPrefixes >> parser->m_nsAttsPower++)
;
@@ -3587,15 +4350,15 @@ storeAtts(XML_Parser parser, const ENCODING *enc, const char *attStr,
* from -Wtype-limits on platforms where
* sizeof(unsigned int) < sizeof(size_t), e.g. on x86_64. */
#if UINT_MAX >= SIZE_MAX
- if (nsAttsSize > (size_t)(-1) / sizeof(NS_ATT)) {
+ if (nsAttsSize > SIZE_MAX / sizeof(NS_ATT)) {
/* Restore actual size of memory in m_nsAtts */
parser->m_nsAttsPower = oldNsAttsPower;
return XML_ERROR_NO_MEMORY;
}
#endif
- temp = (NS_ATT *)REALLOC(parser, parser->m_nsAtts,
- nsAttsSize * sizeof(NS_ATT));
+ NS_ATT *const temp
+ = REALLOC(parser, parser->m_nsAtts, nsAttsSize * sizeof(NS_ATT));
if (! temp) {
/* Restore actual size of memory in m_nsAtts */
parser->m_nsAttsPower = oldNsAttsPower;
@@ -3616,9 +4379,6 @@ storeAtts(XML_Parser parser, const ENCODING *enc, const char *attStr,
for (; i < attIndex; i += 2) {
const XML_Char *s = appAtts[i];
if (s[-1] == 2) { /* prefixed */
- ATTRIBUTE_ID *id;
- const BINDING *b;
- unsigned long uriHash;
struct siphash sip_state;
struct sipkey sip_key;
@@ -3626,7 +4386,8 @@ storeAtts(XML_Parser parser, const ENCODING *enc, const char *attStr,
sip24_init(&sip_state, &sip_key);
((XML_Char *)s)[-1] = 0; /* clear flag */
- id = (ATTRIBUTE_ID *)lookup(parser, &dtd->attributeIds, s, 0);
+ ATTRIBUTE_ID *const id
+ = (ATTRIBUTE_ID *)lookup(parser, &dtd->attributeIds, s, 0);
if (! id || ! id->prefix) {
/* This code is walking through the appAtts array, dealing
* with (in this case) a prefixed attribute name. To be in
@@ -3634,7 +4395,7 @@ storeAtts(XML_Parser parser, const ENCODING *enc, const char *attStr,
* has to have passed through the hash table lookup once
* already. That implies that an entry for it already
* exists, so the lookup above will return a pointer to
- * already allocated memory. There is no opportunaity for
+ * already allocated memory. There is no opportunity for
* the allocator to fail, so the condition above cannot be
* fulfilled.
*
@@ -3644,15 +4405,12 @@ storeAtts(XML_Parser parser, const ENCODING *enc, const char *attStr,
*/
return XML_ERROR_NO_MEMORY; /* LCOV_EXCL_LINE */
}
- b = id->prefix->binding;
+ const BINDING *const b = id->prefix->binding;
if (! b)
return XML_ERROR_UNBOUND_PREFIX;
- for (j = 0; j < b->uriLen; j++) {
- const XML_Char c = b->uri[j];
- if (! poolAppendChar(&parser->m_tempPool, c))
- return XML_ERROR_NO_MEMORY;
- }
+ if (! poolAppendChars(&parser->m_tempPool, b->uri, b->uriLen))
+ return XML_ERROR_NO_MEMORY;
sip24_update(&sip_state, b->uri, b->uriLen * sizeof(XML_Char));
@@ -3661,12 +4419,13 @@ storeAtts(XML_Parser parser, const ENCODING *enc, const char *attStr,
sip24_update(&sip_state, s, keylen(s) * sizeof(XML_Char));
- do { /* copies null terminator */
- if (! poolAppendChar(&parser->m_tempPool, *s))
+ {
+ const size_t len = xcslen(s) + /*null terminator*/ 1;
+ if (! poolAppendChars(&parser->m_tempPool, s, len))
return XML_ERROR_NO_MEMORY;
- } while (*s++);
+ }
- uriHash = (unsigned long)sip24_final(&sip_state);
+ const unsigned long uriHash = (unsigned long)sip24_final(&sip_state);
{ /* Check hash table for duplicate of expanded name (uriName).
Derived from code in lookup(parser, HASH_TABLE *table, ...).
@@ -3694,10 +4453,9 @@ storeAtts(XML_Parser parser, const ENCODING *enc, const char *attStr,
if (parser->m_ns_triplets) { /* append namespace separator and prefix */
parser->m_tempPool.ptr[-1] = parser->m_namespaceSeparator;
s = b->prefix->name;
- do {
- if (! poolAppendChar(&parser->m_tempPool, *s))
- return XML_ERROR_NO_MEMORY;
- } while (*s++);
+ const size_t len = xcslen(s) + /*null terminator*/ 1;
+ if (! poolAppendChars(&parser->m_tempPool, s, len))
+ return XML_ERROR_NO_MEMORY;
}
/* store expanded name in attribute list */
@@ -3740,48 +4498,36 @@ storeAtts(XML_Parser parser, const ENCODING *enc, const char *attStr,
localPart = tagNamePtr->str;
} else
return XML_ERROR_NONE;
- prefixLen = 0;
- if (parser->m_ns_triplets && binding->prefix->name) {
- for (; binding->prefix->name[prefixLen++];)
- ; /* prefixLen includes null terminator */
- }
+ size_t prefixLen = 0;
+ if (parser->m_ns_triplets && binding->prefix->name)
+ prefixLen = xcslen(binding->prefix->name) + /*null terminator*/ 1;
tagNamePtr->localPart = localPart;
tagNamePtr->uriLen = binding->uriLen;
tagNamePtr->prefix = binding->prefix->name;
tagNamePtr->prefixLen = prefixLen;
- for (i = 0; localPart[i++];)
- ; /* i includes null terminator */
+
+ const size_t localPartLen = xcslen(localPart) + /*null terminator*/ 1;
/* Detect and prevent integer overflow */
- if (binding->uriLen > INT_MAX - prefixLen
- || i > INT_MAX - (binding->uriLen + prefixLen)) {
+ if (binding->uriLen > SIZE_MAX - prefixLen
+ || localPartLen > SIZE_MAX - (binding->uriLen + prefixLen)) {
return XML_ERROR_NO_MEMORY;
}
- n = i + binding->uriLen + prefixLen;
- if (n > binding->uriAlloc) {
- TAG *p;
-
+ const size_t totalLen = localPartLen + binding->uriLen + prefixLen;
+ if (totalLen > binding->uriAlloc) {
/* Detect and prevent integer overflow */
- if (n > INT_MAX - EXPAND_SPARE) {
- return XML_ERROR_NO_MEMORY;
- }
- /* Detect and prevent integer overflow.
- * The preprocessor guard addresses the "always false" warning
- * from -Wtype-limits on platforms where
- * sizeof(unsigned int) < sizeof(size_t), e.g. on x86_64. */
-#if UINT_MAX >= SIZE_MAX
- if ((unsigned)(n + EXPAND_SPARE) > (size_t)(-1) / sizeof(XML_Char)) {
+ if (totalLen > SIZE_MAX - EXPAND_SPARE
+ || totalLen + EXPAND_SPARE > SIZE_MAX / sizeof(XML_Char)) {
return XML_ERROR_NO_MEMORY;
}
-#endif
- uri = (XML_Char *)MALLOC(parser, (n + EXPAND_SPARE) * sizeof(XML_Char));
+ uri = MALLOC(parser, (totalLen + EXPAND_SPARE) * sizeof(XML_Char));
if (! uri)
return XML_ERROR_NO_MEMORY;
- binding->uriAlloc = n + EXPAND_SPARE;
+ binding->uriAlloc = totalLen + EXPAND_SPARE;
memcpy(uri, binding->uri, binding->uriLen * sizeof(XML_Char));
- for (p = parser->m_tagStack; p; p = p->parent)
+ for (TAG *p = parser->m_tagStack; p; p = p->parent)
if (p->name.str == binding->uri)
p->name.str = uri;
FREE(parser, binding->uri);
@@ -3789,10 +4535,14 @@ storeAtts(XML_Parser parser, const ENCODING *enc, const char *attStr,
}
/* if m_namespaceSeparator != '\0' then uri includes it already */
uri = binding->uri + binding->uriLen;
- memcpy(uri, localPart, i * sizeof(XML_Char));
+ /* Detect and prevent integer overflow */
+ if (localPartLen > SIZE_MAX / sizeof(XML_Char)) {
+ return XML_ERROR_NO_MEMORY;
+ }
+ memcpy(uri, localPart, localPartLen * sizeof(XML_Char));
/* we always have a namespace separator between localPart and prefix */
if (prefixLen) {
- uri += i - 1;
+ uri += localPartLen - 1;
*uri = parser->m_namespaceSeparator; /* replace null terminator */
memcpy(uri + 1, binding->prefix->name, prefixLen * sizeof(XML_Char));
}
@@ -3927,7 +4677,7 @@ addBinding(XML_Parser parser, PREFIX *prefix, const ATTRIBUTE_ID *attId,
ASCII_8, ASCII_SLASH, ASCII_n, ASCII_a, ASCII_m,
ASCII_e, ASCII_s, ASCII_p, ASCII_a, ASCII_c,
ASCII_e, '\0'};
- static const int xmlLen = (int)sizeof(xmlNamespace) / sizeof(XML_Char) - 1;
+ static const size_t xmlLen = sizeof(xmlNamespace) / sizeof(XML_Char) - 1;
// "http://www.w3.org/2000/xmlns/"
static const XML_Char xmlnsNamespace[]
= {ASCII_h, ASCII_t, ASCII_t, ASCII_p, ASCII_COLON, ASCII_SLASH,
@@ -3935,15 +4685,14 @@ addBinding(XML_Parser parser, PREFIX *prefix, const ATTRIBUTE_ID *attId,
ASCII_3, ASCII_PERIOD, ASCII_o, ASCII_r, ASCII_g, ASCII_SLASH,
ASCII_2, ASCII_0, ASCII_0, ASCII_0, ASCII_SLASH, ASCII_x,
ASCII_m, ASCII_l, ASCII_n, ASCII_s, ASCII_SLASH, '\0'};
- static const int xmlnsLen
- = (int)sizeof(xmlnsNamespace) / sizeof(XML_Char) - 1;
+ static const size_t xmlnsLen = sizeof(xmlnsNamespace) / sizeof(XML_Char) - 1;
XML_Bool mustBeXML = XML_FALSE;
XML_Bool isXML = XML_TRUE;
XML_Bool isXMLNS = XML_TRUE;
BINDING *b;
- int len;
+ size_t len;
/* empty URI is only valid for default namespace per XML NS 1.0 (not 1.1) */
if (*uri == XML_T('\0') && prefix->name)
@@ -3962,6 +4711,10 @@ addBinding(XML_Parser parser, PREFIX *prefix, const ATTRIBUTE_ID *attId,
}
for (len = 0; uri[len]; len++) {
+ /* Detect and prevent integer overflow */
+ if (len == SIZE_MAX) {
+ return XML_ERROR_NO_MEMORY;
+ }
if (isXML && (len > xmlLen || uri[len] != xmlNamespace[len]))
isXML = XML_FALSE;
@@ -4002,28 +4755,24 @@ addBinding(XML_Parser parser, PREFIX *prefix, const ATTRIBUTE_ID *attId,
if (isXMLNS)
return XML_ERROR_RESERVED_NAMESPACE_URI;
- if (parser->m_namespaceSeparator)
+ if (parser->m_namespaceSeparator) {
+ /* Detect and prevent integer overflow */
+ if (len == SIZE_MAX) {
+ return XML_ERROR_NO_MEMORY;
+ }
len++;
+ }
if (parser->m_freeBindingList) {
b = parser->m_freeBindingList;
if (len > b->uriAlloc) {
/* Detect and prevent integer overflow */
- if (len > INT_MAX - EXPAND_SPARE) {
- return XML_ERROR_NO_MEMORY;
- }
-
- /* Detect and prevent integer overflow.
- * The preprocessor guard addresses the "always false" warning
- * from -Wtype-limits on platforms where
- * sizeof(unsigned int) < sizeof(size_t), e.g. on x86_64. */
-#if UINT_MAX >= SIZE_MAX
- if ((unsigned)(len + EXPAND_SPARE) > (size_t)(-1) / sizeof(XML_Char)) {
+ if (len > SIZE_MAX - EXPAND_SPARE
+ || len + EXPAND_SPARE > SIZE_MAX / sizeof(XML_Char)) {
return XML_ERROR_NO_MEMORY;
}
-#endif
- XML_Char *temp = (XML_Char *)REALLOC(
- parser, b->uri, sizeof(XML_Char) * (len + EXPAND_SPARE));
+ XML_Char *temp
+ = REALLOC(parser, b->uri, sizeof(XML_Char) * (len + EXPAND_SPARE));
if (temp == NULL)
return XML_ERROR_NO_MEMORY;
b->uri = temp;
@@ -4031,26 +4780,18 @@ addBinding(XML_Parser parser, PREFIX *prefix, const ATTRIBUTE_ID *attId,
}
parser->m_freeBindingList = b->nextTagBinding;
} else {
- b = (BINDING *)MALLOC(parser, sizeof(BINDING));
+ b = MALLOC(parser, sizeof(BINDING));
if (! b)
return XML_ERROR_NO_MEMORY;
/* Detect and prevent integer overflow */
- if (len > INT_MAX - EXPAND_SPARE) {
- return XML_ERROR_NO_MEMORY;
- }
- /* Detect and prevent integer overflow.
- * The preprocessor guard addresses the "always false" warning
- * from -Wtype-limits on platforms where
- * sizeof(unsigned int) < sizeof(size_t), e.g. on x86_64. */
-#if UINT_MAX >= SIZE_MAX
- if ((unsigned)(len + EXPAND_SPARE) > (size_t)(-1) / sizeof(XML_Char)) {
+ if (len > SIZE_MAX - EXPAND_SPARE
+ || len + EXPAND_SPARE > SIZE_MAX / sizeof(XML_Char)) {
+ FREE(parser, b);
return XML_ERROR_NO_MEMORY;
}
-#endif
- b->uri
- = (XML_Char *)MALLOC(parser, sizeof(XML_Char) * (len + EXPAND_SPARE));
+ b->uri = MALLOC(parser, sizeof(XML_Char) * (len + EXPAND_SPARE));
if (! b->uri) {
FREE(parser, b);
return XML_ERROR_NO_MEMORY;
@@ -4072,16 +4813,19 @@ addBinding(XML_Parser parser, PREFIX *prefix, const ATTRIBUTE_ID *attId,
b->nextTagBinding = *bindingsPtr;
*bindingsPtr = b;
/* if attId == NULL then we are not starting a namespace scope */
- if (attId && parser->m_startNamespaceDeclHandler)
+ if (attId && parser->m_startNamespaceDeclHandler) {
+ beforeHandler(parser);
parser->m_startNamespaceDeclHandler(parser->m_handlerArg, prefix->name,
prefix->binding ? uri : 0);
+ afterHandler(parser);
+ }
return XML_ERROR_NONE;
}
/* The idea here is to avoid using stack for each CDATA section when
the whole file is parsed with one call.
*/
-static enum XML_Error PTRCALL
+static enum XML_Error
cdataSectionProcessor(XML_Parser parser, const char *start, const char *end,
const char **endPtr) {
enum XML_Error result = doCdataSection(
@@ -4136,15 +4880,20 @@ doCdataSection(XML_Parser parser, const ENCODING *enc, const char **startPtr,
*eventEndPP = next;
switch (tok) {
case XML_TOK_CDATA_SECT_CLOSE:
- if (parser->m_endCdataSectionHandler)
+ if (parser->m_endCdataSectionHandler) {
+ beforeHandler(parser);
parser->m_endCdataSectionHandler(parser->m_handlerArg);
+ afterHandler(parser);
+ }
/* BEGIN disabled code */
/* see comment under XML_TOK_CDATA_SECT_OPEN */
- else if ((0) && parser->m_characterDataHandler)
+ else if ((0) && parser->m_characterDataHandler) {
+ beforeHandler(parser);
parser->m_characterDataHandler(parser->m_handlerArg, parser->m_dataBuf,
0);
- /* END disabled code */
- else if (parser->m_defaultHandler)
+ afterHandler(parser);
+ /* END disabled code */
+ } else if (parser->m_defaultHandler)
reportDefault(parser, enc, s, next);
*startPtr = next;
*nextPtr = next;
@@ -4155,7 +4904,9 @@ doCdataSection(XML_Parser parser, const ENCODING *enc, const char **startPtr,
case XML_TOK_DATA_NEWLINE:
if (parser->m_characterDataHandler) {
XML_Char c = 0xA;
+ beforeHandler(parser);
parser->m_characterDataHandler(parser->m_handlerArg, &c, 1);
+ afterHandler(parser);
} else if (parser->m_defaultHandler)
reportDefault(parser, enc, s, next);
break;
@@ -4168,16 +4919,21 @@ doCdataSection(XML_Parser parser, const ENCODING *enc, const char **startPtr,
const enum XML_Convert_Result convert_res = XmlConvert(
enc, &s, next, &dataPtr, (ICHAR *)parser->m_dataBufEnd);
*eventEndPP = next;
+ beforeHandler(parser);
charDataHandler(parser->m_handlerArg, parser->m_dataBuf,
(int)(dataPtr - (ICHAR *)parser->m_dataBuf));
+ afterHandler(parser);
if ((convert_res == XML_CONVERT_COMPLETED)
|| (convert_res == XML_CONVERT_INPUT_INCOMPLETE))
break;
*eventPP = s;
}
- } else
+ } else {
+ beforeHandler(parser);
charDataHandler(parser->m_handlerArg, (const XML_Char *)s,
(int)((const XML_Char *)next - (const XML_Char *)s));
+ afterHandler(parser);
+ }
} else if (parser->m_defaultHandler)
reportDefault(parser, enc, s, next);
} break;
@@ -4210,14 +4966,21 @@ doCdataSection(XML_Parser parser, const ENCODING *enc, const char **startPtr,
/* LCOV_EXCL_STOP */
}
- *eventPP = s = next;
switch (parser->m_parsingStatus.parsing) {
case XML_SUSPENDED:
+ *eventPP = next;
*nextPtr = next;
return XML_ERROR_NONE;
case XML_FINISHED:
+ *eventPP = next;
return XML_ERROR_ABORTED;
+ case XML_PARSING:
+ if (parser->m_reenter) {
+ return XML_ERROR_UNEXPECTED_STATE; // LCOV_EXCL_LINE
+ }
+ EXPAT_FALLTHROUGH;
default:;
+ *eventPP = s = next;
}
}
/* not reached */
@@ -4228,7 +4991,7 @@ doCdataSection(XML_Parser parser, const ENCODING *enc, const char **startPtr,
/* The idea here is to avoid using stack for each IGNORE section when
the whole file is parsed with one call.
*/
-static enum XML_Error PTRCALL
+static enum XML_Error
ignoreSectionProcessor(XML_Parser parser, const char *start, const char *end,
const char **endPtr) {
enum XML_Error result
@@ -4409,8 +5172,10 @@ processXmlDecl(XML_Parser parser, int isGeneralTextEntity, const char *s,
if (! storedversion)
return XML_ERROR_NO_MEMORY;
}
+ beforeHandler(parser);
parser->m_xmlDeclHandler(parser->m_handlerArg, storedversion, storedEncName,
standalone);
+ afterHandler(parser);
} else if (parser->m_defaultHandler)
reportDefault(parser, parser->m_encoding, s, next);
if (parser->m_protocolEncodingName == NULL) {
@@ -4460,31 +5225,43 @@ handleUnknownEncoding(XML_Parser parser, const XML_Char *encodingName) {
info.convert = NULL;
info.data = NULL;
info.release = NULL;
- if (parser->m_unknownEncodingHandler(parser->m_unknownEncodingHandlerData,
- encodingName, &info)) {
+ beforeHandler(parser);
+ const int status = parser->m_unknownEncodingHandler(
+ parser->m_unknownEncodingHandlerData, encodingName, &info);
+ afterHandler(parser);
+
+ parser->m_unknownEncodingRelease = info.release;
+ parser->m_unknownEncodingData = info.data;
+
+ if (status) {
ENCODING *enc;
parser->m_unknownEncodingMem = MALLOC(parser, XmlSizeOfUnknownEncoding());
if (! parser->m_unknownEncodingMem) {
- if (info.release)
- info.release(info.data);
+ if (parser->m_unknownEncodingRelease)
+ callUnknownEncodingRelease(parser);
+ else
+ parser->m_unknownEncodingData = NULL;
return XML_ERROR_NO_MEMORY;
}
+ parser->m_unknownEncodingConvert = info.convert;
enc = (parser->m_ns ? XmlInitUnknownEncodingNS : XmlInitUnknownEncoding)(
- parser->m_unknownEncodingMem, info.map, info.convert, info.data);
+ parser->m_unknownEncodingMem, info.map,
+ info.convert ? callUnknownEncodingConvert : NULL, parser);
if (enc) {
- parser->m_unknownEncodingData = info.data;
- parser->m_unknownEncodingRelease = info.release;
parser->m_encoding = enc;
return XML_ERROR_NONE;
}
+ parser->m_unknownEncodingConvert = NULL;
}
- if (info.release != NULL)
- info.release(info.data);
+ if (parser->m_unknownEncodingRelease != NULL)
+ callUnknownEncodingRelease(parser);
+ else
+ parser->m_unknownEncodingData = NULL;
}
return XML_ERROR_UNKNOWN_ENCODING;
}
-static enum XML_Error PTRCALL
+static enum XML_Error
prologInitProcessor(XML_Parser parser, const char *s, const char *end,
const char **nextPtr) {
enum XML_Error result = initializeEncoding(parser);
@@ -4496,7 +5273,7 @@ prologInitProcessor(XML_Parser parser, const char *s, const char *end,
#ifdef XML_DTD
-static enum XML_Error PTRCALL
+static enum XML_Error
externalParEntInitProcessor(XML_Parser parser, const char *s, const char *end,
const char **nextPtr) {
enum XML_Error result = initializeEncoding(parser);
@@ -4505,7 +5282,7 @@ externalParEntInitProcessor(XML_Parser parser, const char *s, const char *end,
/* we know now that XML_Parse(Buffer) has been called,
so we consider the external parameter entity read */
- parser->m_dtd->paramEntityRead = XML_TRUE;
+ parser->m_dtd->paramEntityRead = true;
if (parser->m_prologState.inEntityValue) {
parser->m_processor = entityValueInitProcessor;
@@ -4516,7 +5293,7 @@ externalParEntInitProcessor(XML_Parser parser, const char *s, const char *end,
}
}
-static enum XML_Error PTRCALL
+static enum XML_Error
entityValueInitProcessor(XML_Parser parser, const char *s, const char *end,
const char **nextPtr) {
int tok;
@@ -4549,7 +5326,7 @@ entityValueInitProcessor(XML_Parser parser, const char *s, const char *end,
}
/* found end of entity value - can store it now */
return storeEntityValue(parser, parser->m_encoding, s, end,
- XML_ACCOUNT_DIRECT);
+ XML_ACCOUNT_DIRECT, NULL);
} else if (tok == XML_TOK_XML_DECL) {
enum XML_Error result;
result = processXmlDecl(parser, 0, start, next);
@@ -4589,7 +5366,7 @@ entityValueInitProcessor(XML_Parser parser, const char *s, const char *end,
}
/* If we get this token, we have the start of what might be a
normal tag, but not a declaration (i.e. it doesn't begin with
- "<!"). In a DTD context, that isn't legal.
+ "<!" or "<?"). In a DTD context, that isn't legal.
*/
else if (tok == XML_TOK_INSTANCE_START) {
*nextPtr = next;
@@ -4600,7 +5377,7 @@ entityValueInitProcessor(XML_Parser parser, const char *s, const char *end,
}
}
-static enum XML_Error PTRCALL
+static enum XML_Error
externalParEntProcessor(XML_Parser parser, const char *s, const char *end,
const char **nextPtr) {
const char *next = s;
@@ -4646,7 +5423,7 @@ externalParEntProcessor(XML_Parser parser, const char *s, const char *end,
XML_ACCOUNT_DIRECT);
}
-static enum XML_Error PTRCALL
+static enum XML_Error
entityValueProcessor(XML_Parser parser, const char *s, const char *end,
const char **nextPtr) {
const char *start = s;
@@ -4676,15 +5453,24 @@ entityValueProcessor(XML_Parser parser, const char *s, const char *end,
break;
}
/* found end of entity value - can store it now */
- return storeEntityValue(parser, enc, s, end, XML_ACCOUNT_DIRECT);
+ return storeEntityValue(parser, enc, s, end, XML_ACCOUNT_DIRECT, NULL);
+ }
+ /* If we get this token, we have the start of what might be a
+ normal tag, but not a declaration (i.e. it doesn't begin with
+ "<!" or "<?"). In a DTD context, that isn't legal.
+ */
+ else if (tok == XML_TOK_INSTANCE_START) {
+ *nextPtr = next;
+ return XML_ERROR_SYNTAX;
}
+
start = next;
}
}
#endif /* XML_DTD */
-static enum XML_Error PTRCALL
+static enum XML_Error
prologProcessor(XML_Parser parser, const char *s, const char *end,
const char **nextPtr) {
const char *next = s;
@@ -4829,9 +5615,11 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end,
break;
case XML_ROLE_DOCTYPE_INTERNAL_SUBSET:
if (parser->m_startDoctypeDeclHandler) {
+ beforeHandler(parser);
parser->m_startDoctypeDeclHandler(
parser->m_handlerArg, parser->m_doctypeName, parser->m_doctypeSysid,
parser->m_doctypePubid, 1);
+ afterHandler(parser);
parser->m_doctypeName = NULL;
poolClear(&parser->m_tempPool);
handleDefault = XML_FALSE;
@@ -4870,7 +5658,7 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end,
handleDefault = XML_FALSE;
goto alreadyChecked;
}
- /* fall through */
+ EXPAT_FALLTHROUGH;
case XML_ROLE_ENTITY_PUBLIC_ID:
if (! XmlIsPublicId(enc, s, next, eventPP))
return XML_ERROR_PUBLICID;
@@ -4898,9 +5686,11 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end,
}
if (parser->m_doctypeName) {
+ beforeHandler(parser);
parser->m_startDoctypeDeclHandler(
parser->m_handlerArg, parser->m_doctypeName, parser->m_doctypeSysid,
parser->m_doctypePubid, 0);
+ afterHandler(parser);
poolClear(&parser->m_tempPool);
handleDefault = XML_FALSE;
}
@@ -4926,15 +5716,23 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end,
}
if (parser->m_useForeignDTD)
entity->base = parser->m_curBase;
- dtd->paramEntityRead = XML_FALSE;
- if (! parser->m_externalEntityRefHandler(
- parser->m_externalEntityRefHandlerArg, 0, entity->base,
- entity->systemId, entity->publicId))
+ dtd->paramEntityRead = false;
+ beforeHandler(parser);
+ const int status = parser->m_externalEntityRefHandler(
+ parser->m_externalEntityRefHandlerArg, 0, entity->base,
+ entity->systemId, entity->publicId);
+ afterHandler(parser);
+ if (! status)
return XML_ERROR_EXTERNAL_ENTITY_HANDLING;
if (dtd->paramEntityRead) {
- if (! dtd->standalone && parser->m_notStandaloneHandler
- && ! parser->m_notStandaloneHandler(parser->m_handlerArg))
- return XML_ERROR_NOT_STANDALONE;
+ if (! dtd->standalone && parser->m_notStandaloneHandler) {
+ beforeHandler(parser);
+ const int handlerStatus
+ = parser->m_notStandaloneHandler(parser->m_handlerArg);
+ afterHandler(parser);
+ if (! handlerStatus)
+ return XML_ERROR_NOT_STANDALONE;
+ }
}
/* if we didn't read the foreign DTD then this means that there
is no external subset and we must reset dtd->hasParamEntityRefs
@@ -4947,7 +5745,9 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end,
}
#endif /* XML_DTD */
if (parser->m_endDoctypeDeclHandler) {
+ beforeHandler(parser);
parser->m_endDoctypeDeclHandler(parser->m_handlerArg);
+ afterHandler(parser);
handleDefault = XML_FALSE;
}
break;
@@ -4966,15 +5766,23 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end,
if (! entity)
return XML_ERROR_NO_MEMORY;
entity->base = parser->m_curBase;
- dtd->paramEntityRead = XML_FALSE;
- if (! parser->m_externalEntityRefHandler(
- parser->m_externalEntityRefHandlerArg, 0, entity->base,
- entity->systemId, entity->publicId))
+ dtd->paramEntityRead = false;
+ beforeHandler(parser);
+ const int status = parser->m_externalEntityRefHandler(
+ parser->m_externalEntityRefHandlerArg, 0, entity->base,
+ entity->systemId, entity->publicId);
+ afterHandler(parser);
+ if (! status)
return XML_ERROR_EXTERNAL_ENTITY_HANDLING;
if (dtd->paramEntityRead) {
- if (! dtd->standalone && parser->m_notStandaloneHandler
- && ! parser->m_notStandaloneHandler(parser->m_handlerArg))
- return XML_ERROR_NOT_STANDALONE;
+ if (! dtd->standalone && parser->m_notStandaloneHandler) {
+ beforeHandler(parser);
+ const int handlerStatus
+ = parser->m_notStandaloneHandler(parser->m_handlerArg);
+ afterHandler(parser);
+ if (! handlerStatus)
+ return XML_ERROR_NOT_STANDALONE;
+ }
}
/* if we didn't read the foreign DTD then this means that there
is no external subset and we must reset dtd->hasParamEntityRefs
@@ -5067,10 +5875,12 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end,
poolFinish(&parser->m_tempPool);
}
*eventEndPP = s;
+ beforeHandler(parser);
parser->m_attlistDeclHandler(
parser->m_handlerArg, parser->m_declElementType->name,
parser->m_declAttributeId->name, parser->m_declAttributeType, 0,
role == XML_ROLE_REQUIRED_ATTRIBUTE_VALUE);
+ afterHandler(parser);
handleDefault = XML_FALSE;
}
}
@@ -5105,10 +5915,12 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end,
poolFinish(&parser->m_tempPool);
}
*eventEndPP = s;
+ beforeHandler(parser);
parser->m_attlistDeclHandler(
parser->m_handlerArg, parser->m_declElementType->name,
parser->m_declAttributeId->name, parser->m_declAttributeType,
attVal, role == XML_ROLE_FIXED_ATTRIBUTE_VALUE);
+ afterHandler(parser);
poolClear(&parser->m_tempPool);
handleDefault = XML_FALSE;
}
@@ -5119,20 +5931,26 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end,
#if XML_GE == 1
// This will store the given replacement text in
// parser->m_declEntity->textPtr.
- enum XML_Error result
- = storeEntityValue(parser, enc, s + enc->minBytesPerChar,
- next - enc->minBytesPerChar, XML_ACCOUNT_NONE);
+ enum XML_Error result = callStoreEntityValue(
+ parser, enc, s + enc->minBytesPerChar, next - enc->minBytesPerChar,
+ XML_ACCOUNT_NONE);
if (parser->m_declEntity) {
+ /* Detect and prevent signed integer overflow */
+ if ((size_t)poolLength(&dtd->entityValuePool) > (size_t)INT_MAX) {
+ return XML_ERROR_NO_MEMORY;
+ }
parser->m_declEntity->textPtr = poolStart(&dtd->entityValuePool);
parser->m_declEntity->textLen
= (int)(poolLength(&dtd->entityValuePool));
poolFinish(&dtd->entityValuePool);
if (parser->m_entityDeclHandler) {
*eventEndPP = s;
+ beforeHandler(parser);
parser->m_entityDeclHandler(
parser->m_handlerArg, parser->m_declEntity->name,
parser->m_declEntity->is_param, parser->m_declEntity->textPtr,
parser->m_declEntity->textLen, parser->m_curBase, 0, 0, 0);
+ afterHandler(parser);
handleDefault = XML_FALSE;
}
} else
@@ -5150,10 +5968,12 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end,
if (parser->m_entityDeclHandler) {
*eventEndPP = s;
+ beforeHandler(parser);
parser->m_entityDeclHandler(
parser->m_handlerArg, parser->m_declEntity->name,
parser->m_declEntity->is_param, parser->m_declEntity->textPtr,
parser->m_declEntity->textLen, parser->m_curBase, 0, 0, 0);
+ afterHandler(parser);
handleDefault = XML_FALSE;
}
}
@@ -5184,9 +6004,13 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end,
#ifdef XML_DTD
&& ! parser->m_paramEntityParsing
#endif /* XML_DTD */
- && parser->m_notStandaloneHandler
- && ! parser->m_notStandaloneHandler(parser->m_handlerArg))
- return XML_ERROR_NOT_STANDALONE;
+ && parser->m_notStandaloneHandler) {
+ beforeHandler(parser);
+ const int status = parser->m_notStandaloneHandler(parser->m_handlerArg);
+ afterHandler(parser);
+ if (! status)
+ return XML_ERROR_NOT_STANDALONE;
+ }
#ifndef XML_DTD
break;
#else /* XML_DTD */
@@ -5198,7 +6022,7 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end,
parser->m_declEntity->publicId = NULL;
}
#endif /* XML_DTD */
- /* fall through */
+ EXPAT_FALLTHROUGH;
case XML_ROLE_ENTITY_SYSTEM_ID:
if (dtd->keepProcessing && parser->m_declEntity) {
parser->m_declEntity->systemId
@@ -5229,10 +6053,12 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end,
if (dtd->keepProcessing && parser->m_declEntity
&& parser->m_entityDeclHandler) {
*eventEndPP = s;
+ beforeHandler(parser);
parser->m_entityDeclHandler(
parser->m_handlerArg, parser->m_declEntity->name,
parser->m_declEntity->is_param, 0, 0, parser->m_declEntity->base,
parser->m_declEntity->systemId, parser->m_declEntity->publicId, 0);
+ afterHandler(parser);
handleDefault = XML_FALSE;
}
break;
@@ -5245,17 +6071,21 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end,
poolFinish(&dtd->pool);
if (parser->m_unparsedEntityDeclHandler) {
*eventEndPP = s;
+ beforeHandler(parser);
parser->m_unparsedEntityDeclHandler(
parser->m_handlerArg, parser->m_declEntity->name,
parser->m_declEntity->base, parser->m_declEntity->systemId,
parser->m_declEntity->publicId, parser->m_declEntity->notation);
+ afterHandler(parser);
handleDefault = XML_FALSE;
} else if (parser->m_entityDeclHandler) {
*eventEndPP = s;
+ beforeHandler(parser);
parser->m_entityDeclHandler(
parser->m_handlerArg, parser->m_declEntity->name, 0, 0, 0,
parser->m_declEntity->base, parser->m_declEntity->systemId,
parser->m_declEntity->publicId, parser->m_declEntity->notation);
+ afterHandler(parser);
handleDefault = XML_FALSE;
}
}
@@ -5279,7 +6109,7 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end,
} else {
poolFinish(&dtd->pool);
parser->m_declEntity->publicId = NULL;
- parser->m_declEntity->is_param = XML_FALSE;
+ parser->m_declEntity->is_param = false;
/* if we have a parent parser or are reading an internal parameter
entity, then the entity declaration is not considered "internal"
*/
@@ -5309,7 +6139,7 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end,
} else {
poolFinish(&dtd->pool);
parser->m_declEntity->publicId = NULL;
- parser->m_declEntity->is_param = XML_TRUE;
+ parser->m_declEntity->is_param = true;
/* if we have a parent parser or are reading an internal parameter
entity, then the entity declaration is not considered "internal"
*/
@@ -5362,9 +6192,11 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end,
if (! systemId)
return XML_ERROR_NO_MEMORY;
*eventEndPP = s;
+ beforeHandler(parser);
parser->m_notationDeclHandler(
parser->m_handlerArg, parser->m_declNotationName, parser->m_curBase,
systemId, parser->m_declNotationPublicId);
+ afterHandler(parser);
handleDefault = XML_FALSE;
}
poolClear(&parser->m_tempPool);
@@ -5372,9 +6204,11 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end,
case XML_ROLE_NOTATION_NO_SYSTEM_ID:
if (parser->m_declNotationPublicId && parser->m_notationDeclHandler) {
*eventEndPP = s;
+ beforeHandler(parser);
parser->m_notationDeclHandler(
parser->m_handlerArg, parser->m_declNotationName, parser->m_curBase,
0, parser->m_declNotationPublicId);
+ afterHandler(parser);
handleDefault = XML_FALSE;
}
poolClear(&parser->m_tempPool);
@@ -5408,41 +6242,20 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end,
case XML_ROLE_GROUP_OPEN:
if (parser->m_prologState.level >= parser->m_groupSize) {
if (parser->m_groupSize) {
- {
- /* Detect and prevent integer overflow */
- if (parser->m_groupSize > (unsigned int)(-1) / 2u) {
- return XML_ERROR_NO_MEMORY;
- }
-
- char *const new_connector = (char *)REALLOC(
- parser, parser->m_groupConnector, parser->m_groupSize *= 2);
- if (new_connector == NULL) {
- parser->m_groupSize /= 2;
- return XML_ERROR_NO_MEMORY;
- }
- parser->m_groupConnector = new_connector;
+ /* Detect and prevent integer overflow */
+ if (parser->m_groupSize > SIZE_MAX / 2) {
+ return XML_ERROR_NO_MEMORY;
}
- if (dtd->scaffIndex) {
- /* Detect and prevent integer overflow.
- * The preprocessor guard addresses the "always false" warning
- * from -Wtype-limits on platforms where
- * sizeof(unsigned int) < sizeof(size_t), e.g. on x86_64. */
-#if UINT_MAX >= SIZE_MAX
- if (parser->m_groupSize > (size_t)(-1) / sizeof(int)) {
- return XML_ERROR_NO_MEMORY;
- }
-#endif
-
- int *const new_scaff_index = (int *)REALLOC(
- parser, dtd->scaffIndex, parser->m_groupSize * sizeof(int));
- if (new_scaff_index == NULL)
- return XML_ERROR_NO_MEMORY;
- dtd->scaffIndex = new_scaff_index;
+ char *const new_connector = REALLOC(parser, parser->m_groupConnector,
+ parser->m_groupSize *= 2);
+ if (new_connector == NULL) {
+ parser->m_groupSize /= 2;
+ return XML_ERROR_NO_MEMORY;
}
+ parser->m_groupConnector = new_connector;
} else {
- parser->m_groupConnector
- = (char *)MALLOC(parser, parser->m_groupSize = 32);
+ parser->m_groupConnector = MALLOC(parser, parser->m_groupSize = 32);
if (! parser->m_groupConnector) {
parser->m_groupSize = 0;
return XML_ERROR_NO_MEMORY;
@@ -5455,6 +6268,21 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end,
if (myindex < 0)
return XML_ERROR_NO_MEMORY;
assert(dtd->scaffIndex != NULL);
+ if ((size_t)dtd->scaffLevel >= dtd->scaffIndexSize) {
+ /* Detect and prevent integer overflow */
+ if (dtd->scaffIndexSize > SIZE_MAX / 2 / sizeof(int)) {
+ return XML_ERROR_NO_MEMORY;
+ }
+ assert(dtd->scaffIndexSize > 0);
+ const size_t new_size = dtd->scaffIndexSize * 2;
+ int *const new_scaff_index
+ = REALLOC(parser, dtd->scaffIndex, new_size * sizeof(int));
+ if (new_scaff_index == NULL) {
+ return XML_ERROR_NO_MEMORY;
+ }
+ dtd->scaffIndex = new_scaff_index;
+ dtd->scaffIndexSize = new_size;
+ }
dtd->scaffIndex[dtd->scaffLevel] = myindex;
dtd->scaffLevel++;
dtd->scaffold[myindex].type = XML_CTYPE_SEQ;
@@ -5535,7 +6363,9 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end,
/* cannot report skipped entities in declarations */
if ((role == XML_ROLE_PARAM_ENTITY_REF)
&& parser->m_skippedEntityHandler) {
+ beforeHandler(parser);
parser->m_skippedEntityHandler(parser->m_handlerArg, name, 1);
+ afterHandler(parser);
handleDefault = XML_FALSE;
}
break;
@@ -5544,27 +6374,29 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end,
return XML_ERROR_RECURSIVE_ENTITY_REF;
if (entity->textPtr) {
enum XML_Error result;
- XML_Bool betweenDecl
- = (role == XML_ROLE_PARAM_ENTITY_REF ? XML_TRUE : XML_FALSE);
- result = processInternalEntity(parser, entity, betweenDecl);
+ bool betweenDecl = (role == XML_ROLE_PARAM_ENTITY_REF);
+ result = processEntity(parser, entity, betweenDecl, ENTITY_INTERNAL);
if (result != XML_ERROR_NONE)
return result;
handleDefault = XML_FALSE;
break;
}
if (parser->m_externalEntityRefHandler) {
- dtd->paramEntityRead = XML_FALSE;
- entity->open = XML_TRUE;
+ dtd->paramEntityRead = false;
+ entity->open = true;
entityTrackingOnOpen(parser, entity, __LINE__);
- if (! parser->m_externalEntityRefHandler(
- parser->m_externalEntityRefHandlerArg, 0, entity->base,
- entity->systemId, entity->publicId)) {
+ beforeHandler(parser);
+ const int status = parser->m_externalEntityRefHandler(
+ parser->m_externalEntityRefHandlerArg, 0, entity->base,
+ entity->systemId, entity->publicId);
+ afterHandler(parser);
+ if (! status) {
entityTrackingOnClose(parser, entity, __LINE__);
- entity->open = XML_FALSE;
+ entity->open = false;
return XML_ERROR_EXTERNAL_ENTITY_HANDLING;
}
entityTrackingOnClose(parser, entity, __LINE__);
- entity->open = XML_FALSE;
+ entity->open = false;
handleDefault = XML_FALSE;
if (! dtd->paramEntityRead) {
dtd->keepProcessing = dtd->standalone;
@@ -5576,9 +6408,13 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end,
}
}
#endif /* XML_DTD */
- if (! dtd->standalone && parser->m_notStandaloneHandler
- && ! parser->m_notStandaloneHandler(parser->m_handlerArg))
- return XML_ERROR_NOT_STANDALONE;
+ if (! dtd->standalone && parser->m_notStandaloneHandler) {
+ beforeHandler(parser);
+ const int status = parser->m_notStandaloneHandler(parser->m_handlerArg);
+ afterHandler(parser);
+ if (! status)
+ return XML_ERROR_NOT_STANDALONE;
+ }
break;
/* Element declaration stuff */
@@ -5590,7 +6426,7 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end,
return XML_ERROR_NO_MEMORY;
dtd->scaffLevel = 0;
dtd->scaffCount = 0;
- dtd->in_eldecl = XML_TRUE;
+ dtd->in_eldecl = true;
handleDefault = XML_FALSE;
}
break;
@@ -5599,8 +6435,11 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end,
case XML_ROLE_CONTENT_EMPTY:
if (dtd->in_eldecl) {
if (parser->m_elementDeclHandler) {
- XML_Content *content
- = (XML_Content *)MALLOC(parser, sizeof(XML_Content));
+ // NOTE: We are avoiding MALLOC(..) here to so that
+ // applications that are not using XML_FreeContentModel but
+ // plain free(..) or .free_fcn() to free the content model's
+ // memory are safe.
+ XML_Content *content = parser->m_mem.malloc_fcn(sizeof(XML_Content));
if (! content)
return XML_ERROR_NO_MEMORY;
content->quant = XML_CQUANT_NONE;
@@ -5610,11 +6449,13 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end,
content->type = ((role == XML_ROLE_CONTENT_ANY) ? XML_CTYPE_ANY
: XML_CTYPE_EMPTY);
*eventEndPP = s;
+ beforeHandler(parser);
parser->m_elementDeclHandler(
parser->m_handlerArg, parser->m_declElementType->name, content);
+ afterHandler(parser);
handleDefault = XML_FALSE;
}
- dtd->in_eldecl = XML_FALSE;
+ dtd->in_eldecl = false;
}
break;
@@ -5655,9 +6496,7 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end,
return XML_ERROR_NO_MEMORY;
name = el->name;
dtd->scaffold[myindex].name = name;
- nameLen = 0;
- for (; name[nameLen++];)
- ;
+ nameLen = xcslen(name) + /*null terminator*/ 1;
/* Detect and prevent integer overflow */
if (nameLen > UINT_MAX - dtd->contentStringLen) {
@@ -5693,10 +6532,12 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end,
if (! model)
return XML_ERROR_NO_MEMORY;
*eventEndPP = s;
+ beforeHandler(parser);
parser->m_elementDeclHandler(
parser->m_handlerArg, parser->m_declElementType->name, model);
+ afterHandler(parser);
}
- dtd->in_eldecl = XML_FALSE;
+ dtd->in_eldecl = false;
dtd->contentStringLen = 0;
}
}
@@ -5751,6 +6592,12 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end,
return XML_ERROR_NONE;
case XML_FINISHED:
return XML_ERROR_ABORTED;
+ case XML_PARSING:
+ if (parser->m_reenter) {
+ *nextPtr = next;
+ return XML_ERROR_NONE;
+ }
+ EXPAT_FALLTHROUGH;
default:
s = next;
tok = XmlPrologTok(enc, s, end, &next);
@@ -5759,7 +6606,7 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end,
/* not reached */
}
-static enum XML_Error PTRCALL
+static enum XML_Error
epilogProcessor(XML_Parser parser, const char *s, const char *end,
const char **nextPtr) {
parser->m_processor = epilogProcessor;
@@ -5818,84 +6665,90 @@ epilogProcessor(XML_Parser parser, const char *s, const char *end,
default:
return XML_ERROR_JUNK_AFTER_DOC_ELEMENT;
}
- parser->m_eventPtr = s = next;
switch (parser->m_parsingStatus.parsing) {
case XML_SUSPENDED:
+ parser->m_eventPtr = next;
*nextPtr = next;
return XML_ERROR_NONE;
case XML_FINISHED:
+ parser->m_eventPtr = next;
return XML_ERROR_ABORTED;
+ case XML_PARSING:
+ if (parser->m_reenter) {
+ return XML_ERROR_UNEXPECTED_STATE; // LCOV_EXCL_LINE
+ }
+ EXPAT_FALLTHROUGH;
default:;
+ parser->m_eventPtr = s = next;
}
}
}
static enum XML_Error
-processInternalEntity(XML_Parser parser, ENTITY *entity, XML_Bool betweenDecl) {
- const char *textStart, *textEnd;
- const char *next;
- enum XML_Error result;
- OPEN_INTERNAL_ENTITY *openEntity;
+processEntity(XML_Parser parser, ENTITY *entity, bool betweenDecl,
+ enum EntityType type) {
+ OPEN_INTERNAL_ENTITY *openEntity, **openEntityList;
+ OPEN_INTERNAL_ENTITY **const freeEntityList = &parser->m_freeEntities;
+ switch (type) {
+ case ENTITY_INTERNAL:
+ parser->m_processor = internalEntityProcessor;
+ openEntityList = &parser->m_openInternalEntities;
+ break;
+ case ENTITY_ATTRIBUTE:
+ openEntityList = &parser->m_openAttributeEntities;
+ break;
+ case ENTITY_VALUE:
+ openEntityList = &parser->m_openValueEntities;
+ break;
+ /* default case serves merely as a safety net in case of a
+ * wrong entityType. Therefore we exclude the following lines
+ * from the test coverage.
+ *
+ * LCOV_EXCL_START
+ */
+ default:
+ // Should not reach here
+ assert(0);
+ /* LCOV_EXCL_STOP */
+ }
- if (parser->m_freeInternalEntities) {
- openEntity = parser->m_freeInternalEntities;
- parser->m_freeInternalEntities = openEntity->next;
+ if (*freeEntityList) {
+ openEntity = *freeEntityList;
+ *freeEntityList = openEntity->next;
} else {
- openEntity
- = (OPEN_INTERNAL_ENTITY *)MALLOC(parser, sizeof(OPEN_INTERNAL_ENTITY));
+ openEntity = MALLOC(parser, sizeof(OPEN_INTERNAL_ENTITY));
if (! openEntity)
return XML_ERROR_NO_MEMORY;
}
- entity->open = XML_TRUE;
+ entity->open = true;
+ entity->hasMore = true;
#if XML_GE == 1
entityTrackingOnOpen(parser, entity, __LINE__);
#endif
entity->processed = 0;
- openEntity->next = parser->m_openInternalEntities;
- parser->m_openInternalEntities = openEntity;
+ openEntity->next = *openEntityList;
+ *openEntityList = openEntity;
openEntity->entity = entity;
+ openEntity->type = type;
openEntity->startTagLevel = parser->m_tagLevel;
openEntity->betweenDecl = betweenDecl;
openEntity->internalEventPtr = NULL;
openEntity->internalEventEndPtr = NULL;
- textStart = (const char *)entity->textPtr;
- textEnd = (const char *)(entity->textPtr + entity->textLen);
- /* Set a safe default value in case 'next' does not get set */
- next = textStart;
-
- if (entity->is_param) {
- int tok
- = XmlPrologTok(parser->m_internalEncoding, textStart, textEnd, &next);
- result = doProlog(parser, parser->m_internalEncoding, textStart, textEnd,
- tok, next, &next, XML_FALSE, XML_FALSE,
- XML_ACCOUNT_ENTITY_EXPANSION);
- } else {
- result = doContent(parser, parser->m_tagLevel, parser->m_internalEncoding,
- textStart, textEnd, &next, XML_FALSE,
- XML_ACCOUNT_ENTITY_EXPANSION);
- }
- if (result == XML_ERROR_NONE) {
- if (textEnd != next && parser->m_parsingStatus.parsing == XML_SUSPENDED) {
- entity->processed = (int)(next - textStart);
- parser->m_processor = internalEntityProcessor;
- } else if (parser->m_openInternalEntities->entity == entity) {
-#if XML_GE == 1
- entityTrackingOnClose(parser, entity, __LINE__);
-#endif /* XML_GE == 1 */
- entity->open = XML_FALSE;
- parser->m_openInternalEntities = openEntity->next;
- /* put openEntity back in list of free instances */
- openEntity->next = parser->m_freeInternalEntities;
- parser->m_freeInternalEntities = openEntity;
- }
+ // Only internal entities make use of the reenter flag
+ // therefore no need to set it for other entity types
+ if (type == ENTITY_INTERNAL) {
+ triggerReenter(parser);
}
- return result;
+ return XML_ERROR_NONE;
}
-static enum XML_Error PTRCALL
+static enum XML_Error
internalEntityProcessor(XML_Parser parser, const char *s, const char *end,
const char **nextPtr) {
+ UNUSED_P(s);
+ UNUSED_P(end);
+ UNUSED_P(nextPtr);
ENTITY *entity;
const char *textStart, *textEnd;
const char *next;
@@ -5905,71 +6758,74 @@ internalEntityProcessor(XML_Parser parser, const char *s, const char *end,
return XML_ERROR_UNEXPECTED_STATE;
entity = openEntity->entity;
- textStart = ((const char *)entity->textPtr) + entity->processed;
- textEnd = (const char *)(entity->textPtr + entity->textLen);
- /* Set a safe default value in case 'next' does not get set */
- next = textStart;
-
- if (entity->is_param) {
- int tok
- = XmlPrologTok(parser->m_internalEncoding, textStart, textEnd, &next);
- result = doProlog(parser, parser->m_internalEncoding, textStart, textEnd,
- tok, next, &next, XML_FALSE, XML_TRUE,
- XML_ACCOUNT_ENTITY_EXPANSION);
- } else {
- result = doContent(parser, openEntity->startTagLevel,
- parser->m_internalEncoding, textStart, textEnd, &next,
- XML_FALSE, XML_ACCOUNT_ENTITY_EXPANSION);
- }
- if (result != XML_ERROR_NONE)
- return result;
+ // This will return early
+ if (entity->hasMore) {
+ textStart = ((const char *)entity->textPtr) + entity->processed;
+ textEnd = (const char *)(entity->textPtr + entity->textLen);
+ /* Set a safe default value in case 'next' does not get set */
+ next = textStart;
+
+ if (entity->is_param) {
+ int tok
+ = XmlPrologTok(parser->m_internalEncoding, textStart, textEnd, &next);
+ result = doProlog(parser, parser->m_internalEncoding, textStart, textEnd,
+ tok, next, &next, XML_FALSE, XML_FALSE,
+ XML_ACCOUNT_ENTITY_EXPANSION);
+ } else {
+ result = doContent(parser, openEntity->startTagLevel,
+ parser->m_internalEncoding, textStart, textEnd, &next,
+ XML_FALSE, XML_ACCOUNT_ENTITY_EXPANSION);
+ }
+
+ if (result != XML_ERROR_NONE)
+ return result;
+ // Check if entity is complete, if not, mark down how much of it is
+ // processed
+ if (textEnd != next
+ && (parser->m_parsingStatus.parsing == XML_SUSPENDED
+ || (parser->m_parsingStatus.parsing == XML_PARSING
+ && parser->m_reenter))) {
+ entity->processed = (int)(next - (const char *)entity->textPtr);
+ return result;
+ }
- if (textEnd != next && parser->m_parsingStatus.parsing == XML_SUSPENDED) {
- entity->processed = (int)(next - (const char *)entity->textPtr);
+ // Entity is complete. We cannot close it here since we need to first
+ // process its possible inner entities (which are added to the
+ // m_openInternalEntities during doProlog or doContent calls above)
+ entity->hasMore = false;
+ if (! entity->is_param
+ && (openEntity->startTagLevel != parser->m_tagLevel)) {
+ return XML_ERROR_ASYNC_ENTITY;
+ }
+ triggerReenter(parser);
return result;
- }
+ } // End of entity processing, "if" block will return here
+ // Remove fully processed openEntity from open entity list.
#if XML_GE == 1
entityTrackingOnClose(parser, entity, __LINE__);
#endif
- entity->open = XML_FALSE;
- parser->m_openInternalEntities = openEntity->next;
+ // openEntity is m_openInternalEntities' head, as we set it at the start of
+ // this function and we skipped doProlog and doContent calls with hasMore set
+ // to false. This means we can directly remove the head of
+ // m_openInternalEntities
+ assert(parser->m_openInternalEntities == openEntity);
+ entity->open = false;
+ parser->m_openInternalEntities = parser->m_openInternalEntities->next;
+
/* put openEntity back in list of free instances */
- openEntity->next = parser->m_freeInternalEntities;
- parser->m_freeInternalEntities = openEntity;
-
- // If there are more open entities we want to stop right here and have the
- // upcoming call to XML_ResumeParser continue with entity content, or it would
- // be ignored altogether.
- if (parser->m_openInternalEntities != NULL
- && parser->m_parsingStatus.parsing == XML_SUSPENDED) {
- return XML_ERROR_NONE;
- }
+ openEntity->next = parser->m_freeEntities;
+ parser->m_freeEntities = openEntity;
- if (entity->is_param) {
- int tok;
- parser->m_processor = prologProcessor;
- tok = XmlPrologTok(parser->m_encoding, s, end, &next);
- return doProlog(parser, parser->m_encoding, s, end, tok, next, nextPtr,
- (XML_Bool)! parser->m_parsingStatus.finalBuffer, XML_TRUE,
- XML_ACCOUNT_DIRECT);
- } else {
- parser->m_processor = contentProcessor;
- /* see externalEntityContentProcessor vs contentProcessor */
- result = doContent(parser, parser->m_parentParser ? 1 : 0,
- parser->m_encoding, s, end, nextPtr,
- (XML_Bool)! parser->m_parsingStatus.finalBuffer,
- XML_ACCOUNT_DIRECT);
- if (result == XML_ERROR_NONE) {
- if (! storeRawNames(parser))
- return XML_ERROR_NO_MEMORY;
- }
- return result;
+ if (parser->m_openInternalEntities == NULL) {
+ parser->m_processor = entity->is_param ? prologProcessor : contentProcessor;
}
+ triggerReenter(parser);
+ return XML_ERROR_NONE;
}
-static enum XML_Error PTRCALL
+static enum XML_Error
errorProcessor(XML_Parser parser, const char *s, const char *end,
const char **nextPtr) {
UNUSED_P(s);
@@ -5982,8 +6838,71 @@ static enum XML_Error
storeAttributeValue(XML_Parser parser, const ENCODING *enc, XML_Bool isCdata,
const char *ptr, const char *end, STRING_POOL *pool,
enum XML_Account account) {
- enum XML_Error result
- = appendAttributeValue(parser, enc, isCdata, ptr, end, pool, account);
+ const char *next = ptr;
+ enum XML_Error result = XML_ERROR_NONE;
+
+ while (1) {
+ if (! parser->m_openAttributeEntities) {
+ result = appendAttributeValue(parser, enc, isCdata, next, end, pool,
+ account, &next);
+ } else {
+ OPEN_INTERNAL_ENTITY *const openEntity = parser->m_openAttributeEntities;
+ if (! openEntity)
+ return XML_ERROR_UNEXPECTED_STATE;
+
+ ENTITY *const entity = openEntity->entity;
+ const char *const textStart
+ = ((const char *)entity->textPtr) + entity->processed;
+ const char *const textEnd
+ = (const char *)(entity->textPtr + entity->textLen);
+ /* Set a safe default value in case 'next' does not get set */
+ const char *nextInEntity = textStart;
+ if (entity->hasMore) {
+ result = appendAttributeValue(
+ parser, parser->m_internalEncoding, isCdata, textStart, textEnd,
+ pool, XML_ACCOUNT_ENTITY_EXPANSION, &nextInEntity);
+ if (result != XML_ERROR_NONE)
+ break;
+ // Check if entity is complete, if not, mark down how much of it is
+ // processed. A XML_SUSPENDED check here is not required as
+ // appendAttributeValue will never suspend the parser.
+ if (nextInEntity < textEnd) {
+ entity->processed
+ = (int)(nextInEntity - (const char *)entity->textPtr);
+ continue;
+ }
+ assert(nextInEntity == textEnd);
+
+ // Entity is complete. We cannot close it here since we need to first
+ // process its possible inner entities (which are added to the
+ // m_openAttributeEntities during appendAttributeValue)
+ entity->hasMore = false;
+ continue;
+ } // End of entity processing, "if" block skips the rest
+
+ // Remove fully processed openEntity from open entity list.
+#if XML_GE == 1
+ entityTrackingOnClose(parser, entity, __LINE__);
+#endif
+ // openEntity is m_openAttributeEntities' head, since we set it at the
+ // start of this function and because we skipped appendAttributeValue call
+ // with hasMore set to false. This means we can directly remove the head
+ // of m_openAttributeEntities
+ assert(parser->m_openAttributeEntities == openEntity);
+ entity->open = false;
+ parser->m_openAttributeEntities = parser->m_openAttributeEntities->next;
+
+ /* put openEntity back in list of free instances */
+ openEntity->next = parser->m_freeEntities;
+ parser->m_freeEntities = openEntity;
+ }
+
+ // Break if an error occurred or there is nothing left to process
+ if (result || (parser->m_openAttributeEntities == NULL && end == next)) {
+ break;
+ }
+ }
+
if (result)
return result;
if (! isCdata && poolLength(pool) && poolLastChar(pool) == 0x20)
@@ -5996,7 +6915,7 @@ storeAttributeValue(XML_Parser parser, const ENCODING *enc, XML_Bool isCdata,
static enum XML_Error
appendAttributeValue(XML_Parser parser, const ENCODING *enc, XML_Bool isCdata,
const char *ptr, const char *end, STRING_POOL *pool,
- enum XML_Account account) {
+ enum XML_Account account, const char **nextPtr) {
DTD *const dtd = parser->m_dtd; /* save one level of indirection */
#ifndef XML_DTD
UNUSED_P(account);
@@ -6014,6 +6933,9 @@ appendAttributeValue(XML_Parser parser, const ENCODING *enc, XML_Bool isCdata,
#endif
switch (tok) {
case XML_TOK_NONE:
+ if (nextPtr) {
+ *nextPtr = next;
+ }
return XML_ERROR_NONE;
case XML_TOK_INVALID:
if (enc == parser->m_encoding)
@@ -6025,7 +6947,6 @@ appendAttributeValue(XML_Parser parser, const ENCODING *enc, XML_Bool isCdata,
return XML_ERROR_INVALID_TOKEN;
case XML_TOK_CHAR_REF: {
XML_Char buf[XML_ENCODE_MAX];
- int i;
int n = XmlCharRefNumber(enc, ptr);
if (n < 0) {
if (enc == parser->m_encoding)
@@ -6045,10 +6966,9 @@ appendAttributeValue(XML_Parser parser, const ENCODING *enc, XML_Bool isCdata,
* XmlEncode() is never passed a value it might return an
* error for.
*/
- for (i = 0; i < n; i++) {
- if (! poolAppendChar(pool, buf[i]))
- return XML_ERROR_NO_MEMORY;
- }
+
+ if (! poolAppendChars(pool, buf, n))
+ return XML_ERROR_NO_MEMORY;
} break;
case XML_TOK_DATA_CHARS:
if (! poolAppend(pool, enc, ptr, next))
@@ -6056,7 +6976,7 @@ appendAttributeValue(XML_Parser parser, const ENCODING *enc, XML_Bool isCdata,
break;
case XML_TOK_TRAILING_CR:
next = ptr + enc->minBytesPerChar;
- /* fall through */
+ EXPAT_FALLTHROUGH;
case XML_TOK_ATTRIBUTE_VALUE_S:
case XML_TOK_DATA_NEWLINE:
if (! isCdata && (poolLength(pool) == 0 || poolLastChar(pool) == 0x20))
@@ -6067,7 +6987,7 @@ appendAttributeValue(XML_Parser parser, const ENCODING *enc, XML_Bool isCdata,
case XML_TOK_ENTITY_REF: {
const XML_Char *name;
ENTITY *entity;
- char checkEntityDecl;
+ bool checkEntityDecl;
XML_Char ch = (XML_Char)XmlPredefinedEntityName(
enc, ptr + enc->minBytesPerChar, next - enc->minBytesPerChar);
if (ch) {
@@ -6110,8 +7030,11 @@ appendAttributeValue(XML_Parser parser, const ENCODING *enc, XML_Bool isCdata,
} else if (! entity) {
/* Cannot report skipped entity here - see comments on
parser->m_skippedEntityHandler.
- if (parser->m_skippedEntityHandler)
+ if (parser->m_skippedEntityHandler) {
+ beforeHandler(parser);
parser->m_skippedEntityHandler(parser->m_handlerArg, name, 0);
+ afterHandler(parser);
+ }
*/
/* Cannot call the default handler because this would be
out of sync with the call to the startElementHandler.
@@ -6154,21 +7077,11 @@ appendAttributeValue(XML_Parser parser, const ENCODING *enc, XML_Bool isCdata,
return XML_ERROR_ATTRIBUTE_EXTERNAL_ENTITY_REF;
} else {
enum XML_Error result;
- const XML_Char *textEnd = entity->textPtr + entity->textLen;
- entity->open = XML_TRUE;
-#if XML_GE == 1
- entityTrackingOnOpen(parser, entity, __LINE__);
-#endif
- result = appendAttributeValue(parser, parser->m_internalEncoding,
- isCdata, (const char *)entity->textPtr,
- (const char *)textEnd, pool,
- XML_ACCOUNT_ENTITY_EXPANSION);
-#if XML_GE == 1
- entityTrackingOnClose(parser, entity, __LINE__);
-#endif
- entity->open = XML_FALSE;
- if (result)
- return result;
+ result = processEntity(parser, entity, XML_FALSE, ENTITY_ATTRIBUTE);
+ if ((result == XML_ERROR_NONE) && (nextPtr != NULL)) {
+ *nextPtr = next;
+ }
+ return result;
}
} break;
default:
@@ -6197,7 +7110,7 @@ appendAttributeValue(XML_Parser parser, const ENCODING *enc, XML_Bool isCdata,
static enum XML_Error
storeEntityValue(XML_Parser parser, const ENCODING *enc,
const char *entityTextPtr, const char *entityTextEnd,
- enum XML_Account account) {
+ enum XML_Account account, const char **nextPtr) {
DTD *const dtd = parser->m_dtd; /* save one level of indirection */
STRING_POOL *pool = &(dtd->entityValuePool);
enum XML_Error result = XML_ERROR_NONE;
@@ -6215,8 +7128,16 @@ storeEntityValue(XML_Parser parser, const ENCODING *enc,
return XML_ERROR_NO_MEMORY;
}
+ const char *next = entityTextPtr;
+
+ /* Nothing to tokenize. */
+ if (entityTextPtr >= entityTextEnd) {
+ result = XML_ERROR_NONE;
+ goto endEntityValue;
+ }
+
for (;;) {
- const char *next
+ next
= entityTextPtr; /* XmlEntityValueTok doesn't always set the last arg */
int tok = XmlEntityValueTok(enc, entityTextPtr, entityTextEnd, &next);
@@ -6246,8 +7167,11 @@ storeEntityValue(XML_Parser parser, const ENCODING *enc,
/* not a well-formedness error - see XML 1.0: WFC Entity Declared */
/* cannot report skipped entity here - see comments on
parser->m_skippedEntityHandler
- if (parser->m_skippedEntityHandler)
+ if (parser->m_skippedEntityHandler) {
+ beforeHandler(parser);
parser->m_skippedEntityHandler(parser->m_handlerArg, name, 0);
+ afterHandler(parser);
+ }
*/
dtd->keepProcessing = dtd->standalone;
goto endEntityValue;
@@ -6260,34 +7184,29 @@ storeEntityValue(XML_Parser parser, const ENCODING *enc,
}
if (entity->systemId) {
if (parser->m_externalEntityRefHandler) {
- dtd->paramEntityRead = XML_FALSE;
- entity->open = XML_TRUE;
+ dtd->paramEntityRead = false;
+ entity->open = true;
entityTrackingOnOpen(parser, entity, __LINE__);
- if (! parser->m_externalEntityRefHandler(
- parser->m_externalEntityRefHandlerArg, 0, entity->base,
- entity->systemId, entity->publicId)) {
+ beforeHandler(parser);
+ const int status = parser->m_externalEntityRefHandler(
+ parser->m_externalEntityRefHandlerArg, 0, entity->base,
+ entity->systemId, entity->publicId);
+ afterHandler(parser);
+ if (! status) {
entityTrackingOnClose(parser, entity, __LINE__);
- entity->open = XML_FALSE;
+ entity->open = false;
result = XML_ERROR_EXTERNAL_ENTITY_HANDLING;
goto endEntityValue;
}
entityTrackingOnClose(parser, entity, __LINE__);
- entity->open = XML_FALSE;
+ entity->open = false;
if (! dtd->paramEntityRead)
dtd->keepProcessing = dtd->standalone;
} else
dtd->keepProcessing = dtd->standalone;
} else {
- entity->open = XML_TRUE;
- entityTrackingOnOpen(parser, entity, __LINE__);
- result = storeEntityValue(
- parser, parser->m_internalEncoding, (const char *)entity->textPtr,
- (const char *)(entity->textPtr + entity->textLen),
- XML_ACCOUNT_ENTITY_EXPANSION);
- entityTrackingOnClose(parser, entity, __LINE__);
- entity->open = XML_FALSE;
- if (result)
- goto endEntityValue;
+ result = processEntity(parser, entity, XML_FALSE, ENTITY_VALUE);
+ goto endEntityValue;
}
break;
}
@@ -6309,17 +7228,15 @@ storeEntityValue(XML_Parser parser, const ENCODING *enc,
break;
case XML_TOK_TRAILING_CR:
next = entityTextPtr + enc->minBytesPerChar;
- /* fall through */
+ EXPAT_FALLTHROUGH;
case XML_TOK_DATA_NEWLINE:
- if (pool->end == pool->ptr && ! poolGrow(pool)) {
+ if (! poolAppendChar(pool, 0xA)) {
result = XML_ERROR_NO_MEMORY;
goto endEntityValue;
}
- *(pool->ptr)++ = 0xA;
break;
case XML_TOK_CHAR_REF: {
XML_Char buf[XML_ENCODE_MAX];
- int i;
int n = XmlCharRefNumber(enc, entityTextPtr);
if (n < 0) {
if (enc == parser->m_encoding)
@@ -6337,12 +7254,9 @@ storeEntityValue(XML_Parser parser, const ENCODING *enc,
* XmlEncode() is never passed a value it might return an
* error for.
*/
- for (i = 0; i < n; i++) {
- if (pool->end == pool->ptr && ! poolGrow(pool)) {
- result = XML_ERROR_NO_MEMORY;
- goto endEntityValue;
- }
- *(pool->ptr)++ = buf[i];
+ if (! poolAppendChars(pool, buf, n)) {
+ result = XML_ERROR_NO_MEMORY;
+ goto endEntityValue;
}
} break;
case XML_TOK_PARTIAL:
@@ -6375,6 +7289,81 @@ storeEntityValue(XML_Parser parser, const ENCODING *enc,
# ifdef XML_DTD
parser->m_prologState.inEntityValue = oldInEntityValue;
# endif /* XML_DTD */
+ // If 'nextPtr' is given, it should be updated during the processing
+ if (nextPtr != NULL) {
+ *nextPtr = next;
+ }
+ return result;
+}
+
+static enum XML_Error
+callStoreEntityValue(XML_Parser parser, const ENCODING *enc,
+ const char *entityTextPtr, const char *entityTextEnd,
+ enum XML_Account account) {
+ const char *next = entityTextPtr;
+ enum XML_Error result = XML_ERROR_NONE;
+ while (1) {
+ if (! parser->m_openValueEntities) {
+ result
+ = storeEntityValue(parser, enc, next, entityTextEnd, account, &next);
+ } else {
+ OPEN_INTERNAL_ENTITY *const openEntity = parser->m_openValueEntities;
+ if (! openEntity)
+ return XML_ERROR_UNEXPECTED_STATE;
+
+ ENTITY *const entity = openEntity->entity;
+ const char *const textStart
+ = ((const char *)entity->textPtr) + entity->processed;
+ const char *const textEnd
+ = (const char *)(entity->textPtr + entity->textLen);
+ /* Set a safe default value in case 'next' does not get set */
+ const char *nextInEntity = textStart;
+ if (entity->hasMore) {
+ result = storeEntityValue(parser, parser->m_internalEncoding, textStart,
+ textEnd, XML_ACCOUNT_ENTITY_EXPANSION,
+ &nextInEntity);
+ if (result != XML_ERROR_NONE)
+ break;
+ // Check if entity is complete, if not, mark down how much of it is
+ // processed. A XML_SUSPENDED check here is not required as
+ // appendAttributeValue will never suspend the parser.
+ if (textEnd != nextInEntity) {
+ entity->processed
+ = (int)(nextInEntity - (const char *)entity->textPtr);
+ continue;
+ }
+
+ // Entity is complete. We cannot close it here since we need to first
+ // process its possible inner entities (which are added to the
+ // m_openValueEntities during storeEntityValue)
+ entity->hasMore = false;
+ continue;
+ } // End of entity processing, "if" block skips the rest
+
+ // Remove fully processed openEntity from open entity list.
+# if XML_GE == 1
+ entityTrackingOnClose(parser, entity, __LINE__);
+# endif
+ // openEntity is m_openValueEntities' head, since we set it at the
+ // start of this function and because we skipped storeEntityValue call
+ // with hasMore set to false. This means we can directly remove the head
+ // of m_openValueEntities
+ assert(parser->m_openValueEntities == openEntity);
+ entity->open = false;
+ parser->m_openValueEntities = parser->m_openValueEntities->next;
+
+ /* put openEntity back in list of free instances */
+ openEntity->next = parser->m_freeEntities;
+ parser->m_freeEntities = openEntity;
+ }
+
+ // Break if an error occurred or there is nothing left to process
+ if (result
+ || (parser->m_openValueEntities == NULL && entityTextEnd == next)) {
+ break;
+ }
+ }
+
return result;
}
@@ -6395,6 +7384,11 @@ storeSelfEntityValue(XML_Parser parser, ENTITY *entity) {
return XML_ERROR_NO_MEMORY;
}
+ /* Detect and prevent signed integer overflow */
+ if ((size_t)poolLength(pool) > (size_t)INT_MAX) {
+ poolDiscard(pool);
+ return XML_ERROR_NO_MEMORY;
+ }
entity->textPtr = poolStart(pool);
entity->textLen = (int)(poolLength(pool));
poolFinish(pool);
@@ -6404,7 +7398,7 @@ storeSelfEntityValue(XML_Parser parser, ENTITY *entity) {
#endif /* XML_GE == 0 */
-static void FASTCALL
+static void
normalizeLines(XML_Char *s) {
XML_Char *p;
for (;; s++) {
@@ -6447,7 +7441,9 @@ reportProcessingInstruction(XML_Parser parser, const ENCODING *enc,
if (! data)
return 0;
normalizeLines(data);
+ beforeHandler(parser);
parser->m_processingInstructionHandler(parser->m_handlerArg, target, data);
+ afterHandler(parser);
poolClear(&parser->m_tempPool);
return 1;
}
@@ -6467,7 +7463,9 @@ reportComment(XML_Parser parser, const ENCODING *enc, const char *start,
if (! data)
return 0;
normalizeLines(data);
+ beforeHandler(parser);
parser->m_commentHandler(parser->m_handlerArg, data);
+ afterHandler(parser);
poolClear(&parser->m_tempPool);
return 1;
}
@@ -6508,15 +7506,20 @@ reportDefault(XML_Parser parser, const ENCODING *enc, const char *s,
convert_res
= XmlConvert(enc, &s, end, &dataPtr, (ICHAR *)parser->m_dataBufEnd);
*eventEndPP = s;
+ beforeHandler(parser);
parser->m_defaultHandler(parser->m_handlerArg, parser->m_dataBuf,
(int)(dataPtr - (ICHAR *)parser->m_dataBuf));
+ afterHandler(parser);
*eventPP = s;
} while ((convert_res != XML_CONVERT_COMPLETED)
&& (convert_res != XML_CONVERT_INPUT_INCOMPLETE));
- } else
+ } else {
+ beforeHandler(parser);
parser->m_defaultHandler(
parser->m_handlerArg, (const XML_Char *)s,
(int)((const XML_Char *)end - (const XML_Char *)s));
+ afterHandler(parser);
+ }
}
static int
@@ -6526,56 +7529,61 @@ defineAttribute(ELEMENT_TYPE *type, ATTRIBUTE_ID *attId, XML_Bool isCdata,
if (value || isId) {
/* The handling of default attributes gets messed up if we have
a default which duplicates a non-default. */
- int i;
- for (i = 0; i < type->nDefaultAtts; i++)
- if (attId == type->defaultAtts[i].id)
- return 1;
+ NAMED *const nameFound
+ = lookup(parser, &(type->defaultAttForName), attId->name, 0);
+ if (nameFound)
+ return 1;
if (isId && ! type->idAtt && ! attId->xmlns)
type->idAtt = attId;
}
if (type->nDefaultAtts == type->allocDefaultAtts) {
- if (type->allocDefaultAtts == 0) {
- type->allocDefaultAtts = 8;
- type->defaultAtts = (DEFAULT_ATTRIBUTE *)MALLOC(
- parser, type->allocDefaultAtts * sizeof(DEFAULT_ATTRIBUTE));
- if (! type->defaultAtts) {
- type->allocDefaultAtts = 0;
- return 0;
- }
- } else {
- DEFAULT_ATTRIBUTE *temp;
-
- /* Detect and prevent integer overflow */
- if (type->allocDefaultAtts > INT_MAX / 2) {
- return 0;
- }
-
- int count = type->allocDefaultAtts * 2;
+ /* Detect and prevent integer overflow */
+ if (type->allocDefaultAtts > SIZE_MAX / 2) {
+ return 0;
+ }
- /* Detect and prevent integer overflow.
- * The preprocessor guard addresses the "always false" warning
- * from -Wtype-limits on platforms where
- * sizeof(unsigned int) < sizeof(size_t), e.g. on x86_64. */
-#if UINT_MAX >= SIZE_MAX
- if ((unsigned)count > (size_t)(-1) / sizeof(DEFAULT_ATTRIBUTE)) {
- return 0;
- }
-#endif
+ size_t count = type->allocDefaultAtts * 2;
+ if (count == 0) {
+ count = 8;
+ }
- temp = (DEFAULT_ATTRIBUTE *)REALLOC(parser, type->defaultAtts,
- (count * sizeof(DEFAULT_ATTRIBUTE)));
- if (temp == NULL)
- return 0;
- type->allocDefaultAtts = count;
- type->defaultAtts = temp;
+ /* Detect and prevent integer overflow. */
+ if (count > SIZE_MAX / sizeof(DEFAULT_ATTRIBUTE)) {
+ return 0;
}
+
+ DEFAULT_ATTRIBUTE *const temp = REALLOC(
+ parser, type->defaultAtts, (count * sizeof(DEFAULT_ATTRIBUTE)));
+ if (temp == NULL)
+ return 0;
+ type->allocDefaultAtts = count;
+ type->defaultAtts = temp;
}
att = type->defaultAtts + type->nDefaultAtts;
att->id = attId;
att->value = value;
att->isCdata = isCdata;
if (! isCdata)
- attId->maybeTokenized = XML_TRUE;
+ attId->maybeTokenized = true;
+
+ NAME_AND_DEFAULT_ATTRIBUTE *const nameAndDefaultAttribute
+ = (NAME_AND_DEFAULT_ATTRIBUTE *)lookup(
+ parser, &(type->defaultAttForName), attId->name,
+ sizeof(NAME_AND_DEFAULT_ATTRIBUTE));
+ if (! nameAndDefaultAttribute)
+ return 0;
+
+ assert(nameAndDefaultAttribute->name == attId->name);
+
+ // NOTE: The XML 1.0r4 spec says:
+ // "When more than one definition is provided for the same attribute of a
+ // given element type, the first declaration is binding and later
+ // declarations are ignored."
+ if (! nameAndDefaultAttribute->initialized) {
+ nameAndDefaultAttribute->attIndex = type->nDefaultAtts;
+ nameAndDefaultAttribute->initialized = true;
+ }
+
type->nDefaultAtts += 1;
return 1;
}
@@ -6641,17 +7649,18 @@ getAttributeId(XML_Parser parser, const ENCODING *enc, const char *start,
else
id->prefix = (PREFIX *)lookup(parser, &dtd->prefixes, name + 6,
sizeof(PREFIX));
- id->xmlns = XML_TRUE;
+ id->xmlns = true;
} else {
int i;
for (i = 0; name[i]; i++) {
+ /* Detect and prevent signed integer overflow */
+ if (i == INT_MAX) {
+ return NULL;
+ }
/* attributes without prefix are *not* in the default namespace */
if (name[i] == XML_T(ASCII_COLON)) {
- int j;
- for (j = 0; j < i; j++) {
- if (! poolAppendChar(&dtd->pool, name[j]))
- return NULL;
- }
+ if (! poolAppendChars(&dtd->pool, name, i))
+ return NULL;
if (! poolAppendChar(&dtd->pool, XML_T('\0')))
return NULL;
id->prefix = (PREFIX *)lookup(parser, &dtd->prefixes,
@@ -6679,46 +7688,39 @@ getContext(XML_Parser parser) {
XML_Bool needSep = XML_FALSE;
if (dtd->defaultPrefix.binding) {
- int i;
- int len;
if (! poolAppendChar(&parser->m_tempPool, XML_T(ASCII_EQUALS)))
return NULL;
- len = dtd->defaultPrefix.binding->uriLen;
+ size_t len = dtd->defaultPrefix.binding->uriLen;
if (parser->m_namespaceSeparator)
len--;
- for (i = 0; i < len; i++) {
- if (! poolAppendChar(&parser->m_tempPool,
- dtd->defaultPrefix.binding->uri[i])) {
- /* Because of memory caching, I don't believe this line can be
- * executed.
- *
- * This is part of a loop copying the default prefix binding
- * URI into the parser's temporary string pool. Previously,
- * that URI was copied into the same string pool, with a
- * terminating NUL character, as part of setContext(). When
- * the pool was cleared, that leaves a block definitely big
- * enough to hold the URI on the free block list of the pool.
- * The URI copy in getContext() therefore cannot run out of
- * memory.
- *
- * If the pool is used between the setContext() and
- * getContext() calls, the worst it can do is leave a bigger
- * block on the front of the free list. Given that this is
- * all somewhat inobvious and program logic can be changed, we
- * don't delete the line but we do exclude it from the test
- * coverage statistics.
- */
- return NULL; /* LCOV_EXCL_LINE */
- }
+ if (! poolAppendChars(&parser->m_tempPool, dtd->defaultPrefix.binding->uri,
+ len)) {
+ /* Because of memory caching, I don't believe this line can be
+ * executed.
+ *
+ * This is part of a loop copying the default prefix binding
+ * URI into the parser's temporary string pool. Previously,
+ * that URI was copied into the same string pool, with a
+ * terminating NUL character, as part of setContext(). When
+ * the pool was cleared, that leaves a block definitely big
+ * enough to hold the URI on the free block list of the pool.
+ * The URI copy in getContext() therefore cannot run out of
+ * memory.
+ *
+ * If the pool is used between the setContext() and
+ * getContext() calls, the worst it can do is leave a bigger
+ * block on the front of the free list. Given that this is
+ * all somewhat inobvious and program logic can be changed, we
+ * don't delete the line but we do exclude it from the test
+ * coverage statistics.
+ */
+ return NULL; /* LCOV_EXCL_LINE */
}
needSep = XML_TRUE;
}
hashTableIterInit(&iter, &(dtd->prefixes));
for (;;) {
- int i;
- int len;
- const XML_Char *s;
PREFIX *prefix = (PREFIX *)hashTableIterNext(&iter);
if (! prefix)
break;
@@ -6733,23 +7735,21 @@ getContext(XML_Parser parser) {
}
if (needSep && ! poolAppendChar(&parser->m_tempPool, CONTEXT_SEP))
return NULL;
- for (s = prefix->name; *s; s++)
- if (! poolAppendChar(&parser->m_tempPool, *s))
- return NULL;
+ if (! poolAppendChars(&parser->m_tempPool, prefix->name,
+ xcslen(prefix->name)))
+ return NULL;
if (! poolAppendChar(&parser->m_tempPool, XML_T(ASCII_EQUALS)))
return NULL;
- len = prefix->binding->uriLen;
+ size_t len = prefix->binding->uriLen;
if (parser->m_namespaceSeparator)
len--;
- for (i = 0; i < len; i++)
- if (! poolAppendChar(&parser->m_tempPool, prefix->binding->uri[i]))
- return NULL;
+ if (! poolAppendChars(&parser->m_tempPool, prefix->binding->uri, len))
+ return NULL;
needSep = XML_TRUE;
}
hashTableIterInit(&iter, &(dtd->generalEntities));
for (;;) {
- const XML_Char *s;
ENTITY *e = (ENTITY *)hashTableIterNext(&iter);
if (! e)
break;
@@ -6757,9 +7757,8 @@ getContext(XML_Parser parser) {
continue;
if (needSep && ! poolAppendChar(&parser->m_tempPool, CONTEXT_SEP))
return NULL;
- for (s = e->name; *s; s++)
- if (! poolAppendChar(&parser->m_tempPool, *s))
- return 0;
+ if (! poolAppendChars(&parser->m_tempPool, e->name, xcslen(e->name)))
+ return NULL;
needSep = XML_TRUE;
}
@@ -6785,7 +7784,7 @@ setContext(XML_Parser parser, const XML_Char *context) {
e = (ENTITY *)lookup(parser, &dtd->generalEntities,
poolStart(&parser->m_tempPool), 0);
if (e)
- e->open = XML_TRUE;
+ e->open = true;
if (*s != XML_T('\0'))
s++;
context = s;
@@ -6797,16 +7796,24 @@ setContext(XML_Parser parser, const XML_Char *context) {
else {
if (! poolAppendChar(&parser->m_tempPool, XML_T('\0')))
return XML_FALSE;
- prefix
- = (PREFIX *)lookup(parser, &dtd->prefixes,
- poolStart(&parser->m_tempPool), sizeof(PREFIX));
- if (! prefix)
+ const XML_Char *const prefixName = poolCopyStringNoFinish(
+ &dtd->pool, poolStart(&parser->m_tempPool));
+ if (! prefixName) {
return XML_FALSE;
- if (prefix->name == poolStart(&parser->m_tempPool)) {
- prefix->name = poolCopyString(&dtd->pool, prefix->name);
- if (! prefix->name)
- return XML_FALSE;
}
+
+ prefix = (PREFIX *)lookup(parser, &dtd->prefixes, prefixName,
+ sizeof(PREFIX));
+
+ const bool prefixNameUsed = prefix && prefix->name == prefixName;
+ if (prefixNameUsed)
+ poolFinish(&dtd->pool);
+ else
+ poolDiscard(&dtd->pool);
+
+ if (! prefix)
+ return XML_FALSE;
+
poolDiscard(&parser->m_tempPool);
}
for (context = s + 1; *context != CONTEXT_SEP && *context != XML_T('\0');
@@ -6832,7 +7839,7 @@ setContext(XML_Parser parser, const XML_Char *context) {
return XML_TRUE;
}
-static void FASTCALL
+static void
normalizePublicId(XML_Char *publicId) {
XML_Char *p = publicId;
XML_Char *s;
@@ -6854,25 +7861,26 @@ normalizePublicId(XML_Char *publicId) {
}
static DTD *
-dtdCreate(const XML_Memory_Handling_Suite *ms) {
- DTD *p = ms->malloc_fcn(sizeof(DTD));
+dtdCreate(XML_Parser parser) {
+ DTD *p = MALLOC(parser, sizeof(DTD));
if (p == NULL)
return p;
- poolInit(&(p->pool), ms);
- poolInit(&(p->entityValuePool), ms);
- hashTableInit(&(p->generalEntities), ms);
- hashTableInit(&(p->elementTypes), ms);
- hashTableInit(&(p->attributeIds), ms);
- hashTableInit(&(p->prefixes), ms);
+ poolInit(&(p->pool), parser);
+ poolInit(&(p->entityValuePool), parser);
+ hashTableInit(&(p->generalEntities), parser);
+ hashTableInit(&(p->elementTypes), parser);
+ hashTableInit(&(p->attributeIds), parser);
+ hashTableInit(&(p->prefixes), parser);
#ifdef XML_DTD
- p->paramEntityRead = XML_FALSE;
- hashTableInit(&(p->paramEntities), ms);
+ p->paramEntityRead = false;
+ hashTableInit(&(p->paramEntities), parser);
#endif /* XML_DTD */
p->defaultPrefix.name = NULL;
p->defaultPrefix.binding = NULL;
- p->in_eldecl = XML_FALSE;
+ p->in_eldecl = false;
p->scaffIndex = NULL;
+ p->scaffIndexSize = 0;
p->scaffold = NULL;
p->scaffLevel = 0;
p->scaffSize = 0;
@@ -6886,19 +7894,19 @@ dtdCreate(const XML_Memory_Handling_Suite *ms) {
}
static void
-dtdReset(DTD *p, const XML_Memory_Handling_Suite *ms) {
+dtdReset(DTD *p, XML_Parser parser) {
HASH_TABLE_ITER iter;
hashTableIterInit(&iter, &(p->elementTypes));
for (;;) {
ELEMENT_TYPE *e = (ELEMENT_TYPE *)hashTableIterNext(&iter);
if (! e)
break;
- if (e->allocDefaultAtts != 0)
- ms->free_fcn(e->defaultAtts);
+ hashTableDestroy(&(e->defaultAttForName));
+ FREE(parser, e->defaultAtts);
}
hashTableClear(&(p->generalEntities));
#ifdef XML_DTD
- p->paramEntityRead = XML_FALSE;
+ p->paramEntityRead = false;
hashTableClear(&(p->paramEntities));
#endif /* XML_DTD */
hashTableClear(&(p->elementTypes));
@@ -6909,11 +7917,12 @@ dtdReset(DTD *p, const XML_Memory_Handling_Suite *ms) {
p->defaultPrefix.name = NULL;
p->defaultPrefix.binding = NULL;
- p->in_eldecl = XML_FALSE;
+ p->in_eldecl = false;
- ms->free_fcn(p->scaffIndex);
+ FREE(parser, p->scaffIndex);
p->scaffIndex = NULL;
- ms->free_fcn(p->scaffold);
+ p->scaffIndexSize = 0;
+ FREE(parser, p->scaffold);
p->scaffold = NULL;
p->scaffLevel = 0;
@@ -6927,15 +7936,15 @@ dtdReset(DTD *p, const XML_Memory_Handling_Suite *ms) {
}
static void
-dtdDestroy(DTD *p, XML_Bool isDocEntity, const XML_Memory_Handling_Suite *ms) {
+dtdDestroy(DTD *p, XML_Bool isDocEntity, XML_Parser parser) {
HASH_TABLE_ITER iter;
hashTableIterInit(&iter, &(p->elementTypes));
for (;;) {
ELEMENT_TYPE *e = (ELEMENT_TYPE *)hashTableIterNext(&iter);
if (! e)
break;
- if (e->allocDefaultAtts != 0)
- ms->free_fcn(e->defaultAtts);
+ hashTableDestroy(&(e->defaultAttForName));
+ FREE(parser, e->defaultAtts);
}
hashTableDestroy(&(p->generalEntities));
#ifdef XML_DTD
@@ -6947,10 +7956,10 @@ dtdDestroy(DTD *p, XML_Bool isDocEntity, const XML_Memory_Handling_Suite *ms) {
poolDestroy(&(p->pool));
poolDestroy(&(p->entityValuePool));
if (isDocEntity) {
- ms->free_fcn(p->scaffIndex);
- ms->free_fcn(p->scaffold);
+ FREE(parser, p->scaffIndex);
+ FREE(parser, p->scaffold);
}
- ms->free_fcn(p);
+ FREE(parser, p);
}
/* Do a deep copy of the DTD. Return 0 for out of memory, non-zero otherwise.
@@ -6958,7 +7967,7 @@ dtdDestroy(DTD *p, XML_Bool isDocEntity, const XML_Memory_Handling_Suite *ms) {
*/
static int
dtdCopy(XML_Parser oldParser, DTD *newDtd, const DTD *oldDtd,
- const XML_Memory_Handling_Suite *ms) {
+ XML_Parser parser) {
HASH_TABLE_ITER iter;
/* Copy the prefix table. */
@@ -7014,7 +8023,6 @@ dtdCopy(XML_Parser oldParser, DTD *newDtd, const DTD *oldDtd,
hashTableIterInit(&iter, &(oldDtd->elementTypes));
for (;;) {
- int i;
ELEMENT_TYPE *newE;
const XML_Char *name;
const ELEMENT_TYPE *oldE = (ELEMENT_TYPE *)hashTableIterNext(&iter);
@@ -7027,19 +8035,17 @@ dtdCopy(XML_Parser oldParser, DTD *newDtd, const DTD *oldDtd,
sizeof(ELEMENT_TYPE));
if (! newE)
return 0;
+
+ if (! newE->defaultAttForName.parser)
+ hashTableInit(&(newE->defaultAttForName), parser);
+
if (oldE->nDefaultAtts) {
- /* Detect and prevent integer overflow.
- * The preprocessor guard addresses the "always false" warning
- * from -Wtype-limits on platforms where
- * sizeof(int) < sizeof(size_t), e.g. on x86_64. */
-#if UINT_MAX >= SIZE_MAX
- if ((size_t)oldE->nDefaultAtts
- > ((size_t)(-1) / sizeof(DEFAULT_ATTRIBUTE))) {
+ /* Detect and prevent integer overflow. */
+ if (oldE->nDefaultAtts > SIZE_MAX / sizeof(DEFAULT_ATTRIBUTE)) {
return 0;
}
-#endif
newE->defaultAtts
- = ms->malloc_fcn(oldE->nDefaultAtts * sizeof(DEFAULT_ATTRIBUTE));
+ = MALLOC(parser, oldE->nDefaultAtts * sizeof(DEFAULT_ATTRIBUTE));
if (! newE->defaultAtts) {
return 0;
}
@@ -7051,9 +8057,10 @@ dtdCopy(XML_Parser oldParser, DTD *newDtd, const DTD *oldDtd,
if (oldE->prefix)
newE->prefix = (PREFIX *)lookup(oldParser, &(newDtd->prefixes),
oldE->prefix->name, 0);
- for (i = 0; i < newE->nDefaultAtts; i++) {
+ for (size_t i = 0; i < newE->nDefaultAtts; i++) {
+ const XML_Char *const attributeName = oldE->defaultAtts[i].id->name;
newE->defaultAtts[i].id = (ATTRIBUTE_ID *)lookup(
- oldParser, &(newDtd->attributeIds), oldE->defaultAtts[i].id->name, 0);
+ oldParser, &(newDtd->attributeIds), attributeName, 0);
newE->defaultAtts[i].isCdata = oldE->defaultAtts[i].isCdata;
if (oldE->defaultAtts[i].value) {
newE->defaultAtts[i].value
@@ -7062,6 +8069,23 @@ dtdCopy(XML_Parser oldParser, DTD *newDtd, const DTD *oldDtd,
return 0;
} else
newE->defaultAtts[i].value = NULL;
+
+ NAME_AND_DEFAULT_ATTRIBUTE *const nameAndDefaultAttribute
+ = (NAME_AND_DEFAULT_ATTRIBUTE *)lookup(
+ parser, &(newE->defaultAttForName), attributeName,
+ sizeof(NAME_AND_DEFAULT_ATTRIBUTE));
+ if (! nameAndDefaultAttribute) {
+ return 0;
+ }
+
+ // NOTE: The XML 1.0r4 spec says:
+ // "When more than one definition is provided for the same attribute of a
+ // given element type, the first declaration is binding and later
+ // declarations are ignored."
+ if (! nameAndDefaultAttribute->initialized) {
+ nameAndDefaultAttribute->attIndex = i;
+ nameAndDefaultAttribute->initialized = true;
+ }
}
}
@@ -7088,6 +8112,7 @@ dtdCopy(XML_Parser oldParser, DTD *newDtd, const DTD *oldDtd,
newDtd->scaffSize = oldDtd->scaffSize;
newDtd->scaffLevel = oldDtd->scaffLevel;
newDtd->scaffIndex = oldDtd->scaffIndex;
+ newDtd->scaffIndexSize = oldDtd->scaffIndexSize;
return 1;
} /* End dtdCopy */
@@ -7157,41 +8182,72 @@ copyEntityTable(XML_Parser oldParser, HASH_TABLE *newTable,
#define INIT_POWER 6
-static XML_Bool FASTCALL
-keyeq(KEY s1, KEY s2) {
- for (; *s1 == *s2; s1++, s2++)
- if (*s1 == 0)
- return XML_TRUE;
- return XML_FALSE;
+// Compares two strings `s1` and `s2` whereas:
+// - `s2` is zero-terminated but
+// - `s1` is made up of exactly (not just up to) `s1len` non-zero characters.
+XML_NONTESTING_STATIC XML_Bool
+keyeq(KEY s1, size_t s1len, KEY s2) {
+#ifdef XML_UNICODE
+# ifdef XML_UNICODE_WCHAR_T
+ return (wcsncmp(s1, s2, s1len) == 0 && s2[s1len] == L'\0') ? XML_TRUE
+ : XML_FALSE;
+# else
+ for (; s1len > 0 && *s1 == *s2; s1len--, s1++, s2++)
+ ; /* no loop body! */
+ return ((s1len == 0) && (*s2 == 0)) ? XML_TRUE : XML_FALSE;
+# endif
+#else
+ return (strncmp(s1, s2, s1len) == 0 && s2[s1len] == '\0') ? XML_TRUE
+ : XML_FALSE;
+#endif
}
-static size_t
+XML_NONTESTING_STATIC size_t
keylen(KEY s) {
- size_t len = 0;
- for (; *s; s++, len++)
- ;
- return len;
+ return xcslen(s);
}
static void
copy_salt_to_sipkey(XML_Parser parser, struct sipkey *key) {
- key->k[0] = 0;
- key->k[1] = get_hash_secret_salt(parser);
+ const XML_Parser rootParser = getRootParserOf(parser, NULL);
+ assert(! rootParser->m_parentParser);
+
+ *key = rootParser->m_hash_secret_salt_128;
}
-static unsigned long FASTCALL
-hash(XML_Parser parser, KEY s) {
+static unsigned long
+hash(XML_Parser parser, KEY s, size_t keyLen) {
struct siphash state;
struct sipkey key;
(void)sip24_valid;
copy_salt_to_sipkey(parser, &key);
sip24_init(&state, &key);
- sip24_update(&state, s, keylen(s) * sizeof(XML_Char));
+ sip24_update(&state, s, keyLen * sizeof(XML_Char));
return (unsigned long)sip24_final(&state);
}
-static NAMED *
-lookup(XML_Parser parser, HASH_TABLE *table, KEY name, size_t createSize) {
+// Function `lookupWithLength` can be used to either…
+//
+// a) check whether an element with key `name` exists in the given hash table
+// (read-only mode where `createSize == 0`) or
+//
+// b) check whether an element with key `name` exists in the given hash table
+// *and* insert it if missing (i.e. read-write mode where `createSize != 0`.
+//
+// When inserting, a block of `createSize` number of bytes will be allocated
+// and set to zero, and the resulting block of memory will be considered
+// to start with a `NAMED` structure, and `->name = name;` is performed.
+// The fact that all other bytes in the structure are initially zero can
+// be used to tell cases "existed and found" and "newly inserted" apart
+// with the structure returned.
+//
+// NOTE: Read-only lookup does not need zero-terminated keys but
+// read-write mode does, because keys can be re-hashed later and the
+// hash table does not store key length information.
+//
+XML_NONTESTING_STATIC NAMED *
+lookupWithLength(XML_Parser parser, HASH_TABLE *table, KEY name, size_t nameLen,
+ size_t createSize) {
size_t i;
if (table->size == 0) {
size_t tsize;
@@ -7201,20 +8257,20 @@ lookup(XML_Parser parser, HASH_TABLE *table, KEY name, size_t createSize) {
/* table->size is a power of 2 */
table->size = (size_t)1 << INIT_POWER;
tsize = table->size * sizeof(NAMED *);
- table->v = table->mem->malloc_fcn(tsize);
+ table->v = MALLOC(table->parser, tsize);
if (! table->v) {
table->size = 0;
return NULL;
}
memset(table->v, 0, tsize);
- i = hash(parser, name) & ((unsigned long)table->size - 1);
+ i = hash(parser, name, nameLen) & ((unsigned long)table->size - 1);
} else {
- unsigned long h = hash(parser, name);
+ unsigned long h = hash(parser, name, nameLen);
unsigned long mask = (unsigned long)table->size - 1;
unsigned char step = 0;
i = h & mask;
while (table->v[i]) {
- if (keyeq(name, table->v[i]->name))
+ if (keyeq(name, nameLen, table->v[i]->name))
return table->v[i];
if (! step)
step = PROBE_STEP(h, mask, table->power);
@@ -7236,18 +8292,19 @@ lookup(XML_Parser parser, HASH_TABLE *table, KEY name, size_t createSize) {
unsigned long newMask = (unsigned long)newSize - 1;
/* Detect and prevent integer overflow */
- if (newSize > (size_t)(-1) / sizeof(NAMED *)) {
+ if (newSize > SIZE_MAX / sizeof(NAMED *)) {
return NULL;
}
size_t tsize = newSize * sizeof(NAMED *);
- NAMED **newV = table->mem->malloc_fcn(tsize);
+ NAMED **newV = MALLOC(table->parser, tsize);
if (! newV)
return NULL;
memset(newV, 0, tsize);
for (i = 0; i < table->size; i++)
if (table->v[i]) {
- unsigned long newHash = hash(parser, table->v[i]->name);
+ KEY const key = table->v[i]->name;
+ unsigned long newHash = hash(parser, key, keylen(key));
size_t j = newHash & newMask;
step = 0;
while (newV[j]) {
@@ -7257,7 +8314,7 @@ lookup(XML_Parser parser, HASH_TABLE *table, KEY name, size_t createSize) {
}
newV[j] = table->v[i];
}
- table->mem->free_fcn(table->v);
+ FREE(table->parser, table->v);
table->v = newV;
table->power = newPower;
table->size = newSize;
@@ -7270,49 +8327,70 @@ lookup(XML_Parser parser, HASH_TABLE *table, KEY name, size_t createSize) {
}
}
}
- table->v[i] = table->mem->malloc_fcn(createSize);
+ assert(createSize >= sizeof(NAMED));
+ table->v[i] = MALLOC(table->parser, createSize);
if (! table->v[i])
return NULL;
memset(table->v[i], 0, createSize);
- table->v[i]->name = name;
+ table->v[i]->name = name; // NOTE: This requires and assumes zero termination!
(table->used)++;
return table->v[i];
}
-static void FASTCALL
+// Function `lookup` can be used to either…
+//
+// a) check whether an element with key `name` exists in the given hash table
+// (read-only mode where `createSize == 0`) or
+//
+// b) check whether an element with key `name` exists in the given hash table
+// *and* insert it if missing (i.e. read-write mode where `createSize != 0`.
+//
+// When inserting, a block of `createSize` number of bytes will be allocated
+// and set to zero, and the resulting block of memory will be considered
+// to start with a `NAMED` structure, and `->name = name;` is performed.
+// The fact that all other bytes in the structure are initially zero can
+// be used to tell cases "existed and found" and "newly inserted" apart
+// with the structure returned.
+//
+XML_NONTESTING_STATIC NAMED *
+lookup(XML_Parser parser, HASH_TABLE *table, KEY name, size_t createSize) {
+ return lookupWithLength(parser, table, name, keylen(name), createSize);
+}
+
+XML_NONTESTING_STATIC void
hashTableClear(HASH_TABLE *table) {
size_t i;
for (i = 0; i < table->size; i++) {
- table->mem->free_fcn(table->v[i]);
+ FREE(table->parser, table->v[i]);
table->v[i] = NULL;
}
table->used = 0;
}
-static void FASTCALL
+XML_NONTESTING_STATIC void
hashTableDestroy(HASH_TABLE *table) {
size_t i;
for (i = 0; i < table->size; i++)
- table->mem->free_fcn(table->v[i]);
- table->mem->free_fcn(table->v);
+ FREE(table->parser, table->v[i]);
+ FREE(table->parser, table->v);
}
-static void FASTCALL
-hashTableInit(HASH_TABLE *p, const XML_Memory_Handling_Suite *ms) {
+XML_NONTESTING_STATIC void
+hashTableInit(HASH_TABLE *p, XML_Parser parser) {
p->power = 0;
p->size = 0;
p->used = 0;
p->v = NULL;
- p->mem = ms;
+ p->parser = parser;
}
-static void FASTCALL
+XML_NONTESTING_STATIC void
hashTableIterInit(HASH_TABLE_ITER *iter, const HASH_TABLE *table) {
iter->p = table->v;
iter->end = iter->p ? iter->p + table->size : NULL;
}
-static NAMED *FASTCALL
+XML_NONTESTING_STATIC NAMED *
hashTableIterNext(HASH_TABLE_ITER *iter) {
while (iter->p != iter->end) {
NAMED *tem = *(iter->p)++;
@@ -7322,17 +8400,17 @@ hashTableIterNext(HASH_TABLE_ITER *iter) {
return NULL;
}
-static void FASTCALL
-poolInit(STRING_POOL *pool, const XML_Memory_Handling_Suite *ms) {
+static void
+poolInit(STRING_POOL *pool, XML_Parser parser) {
pool->blocks = NULL;
pool->freeBlocks = NULL;
pool->start = NULL;
pool->ptr = NULL;
pool->end = NULL;
- pool->mem = ms;
+ pool->parser = parser;
}
-static void FASTCALL
+static void
poolClear(STRING_POOL *pool) {
if (! pool->freeBlocks)
pool->freeBlocks = pool->blocks;
@@ -7351,18 +8429,18 @@ poolClear(STRING_POOL *pool) {
pool->end = NULL;
}
-static void FASTCALL
+static void
poolDestroy(STRING_POOL *pool) {
BLOCK *p = pool->blocks;
while (p) {
BLOCK *tem = p->next;
- pool->mem->free_fcn(p);
+ FREE(pool->parser, p);
p = tem;
}
p = pool->freeBlocks;
while (p) {
BLOCK *tem = p->next;
- pool->mem->free_fcn(p);
+ FREE(pool->parser, p);
p = tem;
}
}
@@ -7384,17 +8462,32 @@ poolAppend(STRING_POOL *pool, const ENCODING *enc, const char *ptr,
return pool->start;
}
-static const XML_Char *FASTCALL
+static const XML_Char *
poolCopyString(STRING_POOL *pool, const XML_Char *s) {
- do {
- if (! poolAppendChar(pool, *s))
- return NULL;
- } while (*s++);
+ if (! poolAppendChars(pool, s, xcslen(s) + /*null terminator*/ 1))
+ return NULL;
s = pool->start;
poolFinish(pool);
return s;
}
+// A version of `poolCopyString` that does not call `poolFinish`
+// and reverts any partial advancement upon failure.
+static const XML_Char *
+poolCopyStringNoFinish(STRING_POOL *pool, const XML_Char *s) {
+ const XML_Char *const original = s;
+ do {
+ if (! poolAppendChar(pool, *s)) {
+ // Revert any previously successful advancement
+ const ptrdiff_t advancedBy = s - original;
+ if (advancedBy > 0)
+ pool->ptr -= advancedBy;
+ return NULL;
+ }
+ } while (*s++);
+ return pool->start;
+}
+
static const XML_Char *
poolCopyStringN(STRING_POOL *pool, const XML_Char *s, int n) {
if (! pool->ptr && ! poolGrow(pool)) {
@@ -7411,22 +8504,17 @@ poolCopyStringN(STRING_POOL *pool, const XML_Char *s, int n) {
*/
return NULL; /* LCOV_EXCL_LINE */
}
- for (; n > 0; --n, s++) {
- if (! poolAppendChar(pool, *s))
- return NULL;
- }
+ if (n > 0 && ! poolAppendChars(pool, s, n))
+ return NULL;
s = pool->start;
poolFinish(pool);
return s;
}
-static const XML_Char *FASTCALL
+static const XML_Char *
poolAppendString(STRING_POOL *pool, const XML_Char *s) {
- while (*s) {
- if (! poolAppendChar(pool, *s))
- return NULL;
- s++;
- }
+ if (! poolAppendChars(pool, s, xcslen(s)))
+ return NULL;
return pool->start;
}
@@ -7435,9 +8523,8 @@ poolStoreString(STRING_POOL *pool, const ENCODING *enc, const char *ptr,
const char *end) {
if (! poolAppend(pool, enc, ptr, end))
return NULL;
- if (pool->ptr == pool->end && ! poolGrow(pool))
+ if (! poolAppendChar(pool, 0))
return NULL;
- *(pool->ptr)++ = 0;
return pool->start;
}
@@ -7469,10 +8556,10 @@ poolBytesToAllocateFor(int blockSize) {
}
}
-static XML_Bool FASTCALL
+static XML_Bool
poolGrow(STRING_POOL *pool) {
if (pool->freeBlocks) {
- if (pool->start == 0) {
+ if (pool->start == NULL) {
pool->blocks = pool->freeBlocks;
pool->freeBlocks = pool->freeBlocks->next;
pool->blocks->next = NULL;
@@ -7488,7 +8575,7 @@ poolGrow(STRING_POOL *pool) {
pool->freeBlocks = tem;
memcpy(pool->blocks->s, pool->start,
(pool->end - pool->start) * sizeof(XML_Char));
- pool->ptr = pool->blocks->s + (pool->ptr - pool->start);
+ pool->ptr = pool->blocks->s + EXPAT_SAFE_PTR_DIFF(pool->ptr, pool->start);
pool->start = pool->blocks->s;
pool->end = pool->start + pool->blocks->size;
return XML_TRUE;
@@ -7501,7 +8588,8 @@ poolGrow(STRING_POOL *pool) {
/* NOTE: Needs to be calculated prior to calling `realloc`
to avoid dangling pointers: */
- const ptrdiff_t offsetInsideBlock = pool->ptr - pool->start;
+ const ptrdiff_t offsetInsideBlock
+ = EXPAT_SAFE_PTR_DIFF(pool->ptr, pool->start);
if (blockSize < 0) {
/* This condition traps a situation where either more than
@@ -7517,8 +8605,7 @@ poolGrow(STRING_POOL *pool) {
if (bytesToAllocate == 0)
return XML_FALSE;
- temp = (BLOCK *)pool->mem->realloc_fcn(pool->blocks,
- (unsigned)bytesToAllocate);
+ temp = REALLOC(pool->parser, pool->blocks, bytesToAllocate);
if (temp == NULL)
return XML_FALSE;
pool->blocks = temp;
@@ -7558,43 +8645,58 @@ poolGrow(STRING_POOL *pool) {
if (bytesToAllocate == 0)
return XML_FALSE;
- tem = pool->mem->malloc_fcn(bytesToAllocate);
+ tem = MALLOC(pool->parser, bytesToAllocate);
if (! tem)
return XML_FALSE;
tem->size = blockSize;
tem->next = pool->blocks;
pool->blocks = tem;
if (pool->ptr != pool->start)
- memcpy(tem->s, pool->start, (pool->ptr - pool->start) * sizeof(XML_Char));
- pool->ptr = tem->s + (pool->ptr - pool->start);
+ memcpy(tem->s, pool->start,
+ EXPAT_SAFE_PTR_DIFF(pool->ptr, pool->start) * sizeof(XML_Char));
+ pool->ptr = tem->s + EXPAT_SAFE_PTR_DIFF(pool->ptr, pool->start);
pool->start = tem->s;
pool->end = tem->s + blockSize;
}
return XML_TRUE;
}
-static int FASTCALL
+static bool
+poolGrowUntil(STRING_POOL *pool, size_t needed) {
+ for (;;) {
+ const size_t available = pool->end - pool->ptr;
+ if (available >= needed) {
+ return true;
+ }
+ if (! poolGrow(pool)) {
+ return false;
+ }
+ }
+}
+
+static int
nextScaffoldPart(XML_Parser parser) {
DTD *const dtd = parser->m_dtd; /* save one level of indirection */
CONTENT_SCAFFOLD *me;
int next;
if (! dtd->scaffIndex) {
- /* Detect and prevent integer overflow.
- * The preprocessor guard addresses the "always false" warning
- * from -Wtype-limits on platforms where
- * sizeof(unsigned int) < sizeof(size_t), e.g. on x86_64. */
-#if UINT_MAX >= SIZE_MAX
- if (parser->m_groupSize > ((size_t)(-1) / sizeof(int))) {
+ /* Detect and prevent integer overflow. */
+ if (parser->m_groupSize > SIZE_MAX / sizeof(int)) {
return -1;
}
-#endif
- dtd->scaffIndex = (int *)MALLOC(parser, parser->m_groupSize * sizeof(int));
+ dtd->scaffIndex = MALLOC(parser, parser->m_groupSize * sizeof(int));
if (! dtd->scaffIndex)
return -1;
+ dtd->scaffIndexSize = parser->m_groupSize;
dtd->scaffIndex[0] = 0;
}
+ // Will casting to int be safe further down?
+ if (dtd->scaffCount > INT_MAX) {
+ return -1;
+ }
+
if (dtd->scaffCount >= dtd->scaffSize) {
CONTENT_SCAFFOLD *temp;
if (dtd->scaffold) {
@@ -7607,26 +8709,25 @@ nextScaffoldPart(XML_Parser parser) {
* from -Wtype-limits on platforms where
* sizeof(unsigned int) < sizeof(size_t), e.g. on x86_64. */
#if UINT_MAX >= SIZE_MAX
- if (dtd->scaffSize > (size_t)(-1) / 2u / sizeof(CONTENT_SCAFFOLD)) {
+ if (dtd->scaffSize > SIZE_MAX / 2u / sizeof(CONTENT_SCAFFOLD)) {
return -1;
}
#endif
- temp = (CONTENT_SCAFFOLD *)REALLOC(
- parser, dtd->scaffold, dtd->scaffSize * 2 * sizeof(CONTENT_SCAFFOLD));
+ temp = REALLOC(parser, dtd->scaffold,
+ dtd->scaffSize * 2 * sizeof(CONTENT_SCAFFOLD));
if (temp == NULL)
return -1;
dtd->scaffSize *= 2;
} else {
- temp = (CONTENT_SCAFFOLD *)MALLOC(parser, INIT_SCAFFOLD_ELEMENTS
- * sizeof(CONTENT_SCAFFOLD));
+ temp = MALLOC(parser, INIT_SCAFFOLD_ELEMENTS * sizeof(CONTENT_SCAFFOLD));
if (temp == NULL)
return -1;
dtd->scaffSize = INIT_SCAFFOLD_ELEMENTS;
}
dtd->scaffold = temp;
}
- next = dtd->scaffCount++;
+ next = (int)dtd->scaffCount++;
me = &dtd->scaffold[next];
if (dtd->scaffLevel) {
CONTENT_SCAFFOLD *parent
@@ -7658,22 +8759,25 @@ build_model(XML_Parser parser) {
* from -Wtype-limits on platforms where
* sizeof(unsigned int) < sizeof(size_t), e.g. on x86_64. */
#if UINT_MAX >= SIZE_MAX
- if (dtd->scaffCount > (size_t)(-1) / sizeof(XML_Content)) {
+ if (dtd->scaffCount > SIZE_MAX / sizeof(XML_Content)) {
return NULL;
}
- if (dtd->contentStringLen > (size_t)(-1) / sizeof(XML_Char)) {
+ if (dtd->contentStringLen > SIZE_MAX / sizeof(XML_Char)) {
return NULL;
}
#endif
if (dtd->scaffCount * sizeof(XML_Content)
- > (size_t)(-1) - dtd->contentStringLen * sizeof(XML_Char)) {
+ > SIZE_MAX - dtd->contentStringLen * sizeof(XML_Char)) {
return NULL;
}
const size_t allocsize = (dtd->scaffCount * sizeof(XML_Content)
+ (dtd->contentStringLen * sizeof(XML_Char)));
- ret = (XML_Content *)MALLOC(parser, allocsize);
+ // NOTE: We are avoiding MALLOC(..) here to so that
+ // applications that are not using XML_FreeContentModel but plain
+ // free(..) or .free_fcn() to free the content model's memory are safe.
+ ret = parser->m_mem.malloc_fcn(allocsize);
if (! ret)
return NULL;
@@ -7745,12 +8849,21 @@ build_model(XML_Parser parser) {
const XML_Char *src;
dest->name = str;
src = dtd->scaffold[src_node].name;
- for (;;) {
- *str++ = *src;
- if (! *src)
- break;
- src++;
+
+ const size_t nameLen = xcslen(src) + /* null terminator*/ 1;
+
+ // Detect and prevent integer overflow
+ if (nameLen > SIZE_MAX / sizeof(XML_Char)) {
+ // NOTE: We are avoiding FREE(..) here because the model
+ // is not being allocated with MALLOC(..) but with plain
+ // .malloc_fcn(..).
+ parser->m_mem.free_fcn(ret);
+ return NULL;
}
+
+ memcpy(str, src, nameLen * sizeof(XML_Char));
+ str += nameLen;
+
dest->numchildren = 0;
dest->children = NULL;
} else {
@@ -7783,6 +8896,8 @@ getElementType(XML_Parser parser, const ENCODING *enc, const char *ptr,
sizeof(ELEMENT_TYPE));
if (! ret)
return NULL;
+ if (! ret->defaultAttForName.parser)
+ hashTableInit(&(ret->defaultAttForName), getRootParserOf(parser, NULL));
if (ret->name != name)
poolDiscard(&dtd->pool);
else {
@@ -7794,23 +8909,25 @@ getElementType(XML_Parser parser, const ENCODING *enc, const char *ptr,
}
static XML_Char *
-copyString(const XML_Char *s, const XML_Memory_Handling_Suite *memsuite) {
- size_t charsRequired = 0;
- XML_Char *result;
-
+copyString(const XML_Char *s, XML_Parser parser) {
/* First determine how long the string is */
- while (s[charsRequired] != 0) {
- charsRequired++;
- }
- /* Include the terminator */
- charsRequired++;
+ const size_t charsRequired = xcslen(s) + /*null terminator*/ 1;
+
+ /* Detect and prevent integer overflow */
+ if (charsRequired > SIZE_MAX / sizeof(XML_Char))
+ return NULL;
+
+ const size_t bytesRequired = charsRequired * sizeof(XML_Char);
/* Now allocate space for the copy */
- result = memsuite->malloc_fcn(charsRequired * sizeof(XML_Char));
+ XML_Char *const result = MALLOC(parser, bytesRequired);
+
if (result == NULL)
return NULL;
+
/* Copy the original into place */
- memcpy(result, s, charsRequired * sizeof(XML_Char));
+ memcpy(result, s, bytesRequired);
+
return result;
}
@@ -7825,10 +8942,10 @@ accountingGetCurrentAmplification(XML_Parser rootParser) {
+ rootParser->m_accounting.countBytesIndirect;
const float amplificationFactor
= rootParser->m_accounting.countBytesDirect
- ? (countBytesOutput
+ ? ((float)countBytesOutput
/ (float)(rootParser->m_accounting.countBytesDirect))
- : ((lenOfShortestInclude
- + rootParser->m_accounting.countBytesIndirect)
+ : ((float)(lenOfShortestInclude
+ + rootParser->m_accounting.countBytesIndirect)
/ (float)lenOfShortestInclude);
assert(! rootParser->m_parentParser);
return amplificationFactor;
@@ -7977,15 +9094,25 @@ entityTrackingReportStats(XML_Parser rootParser, ENTITY *entity,
const char *const entityName = entity->name;
# endif
+ const bool limitingWanted = rootParser->m_entity_stats.debugLevel < 2;
+ const int maxLimitedDepth = 10; // somewhat arbitrary
+ const int candidateIndentDepth
+ = (int)rootParser->m_entity_stats.currentDepth - 1;
+ const bool limitingNeeded
+ = limitingWanted && (candidateIndentDepth > maxLimitedDepth);
+ const char *const ellipisOrEmpty = limitingNeeded ? " [..] " : "";
+ const int indentDepth
+ = limitingNeeded ? (maxLimitedDepth - /* make space for ellipis */ 2)
+ : candidateIndentDepth;
+
fprintf(
stderr,
- "expat: Entities(%p): Count %9u, depth %2u/%2u %*s%s%s; %s length %d (xmlparse.c:%d)\n",
+ "expat: Entities(%p): Count %9u, depth %2u/%2u %*s%s%s%s; %s length %d (xmlparse.c:%d)\n",
(void *)rootParser, rootParser->m_entity_stats.countEverOpened,
rootParser->m_entity_stats.currentDepth,
- rootParser->m_entity_stats.maximumDepthSeen,
- (rootParser->m_entity_stats.currentDepth - 1) * 2, "",
- entity->is_param ? "%" : "&", entityName, action, entity->textLen,
- sourceLine);
+ rootParser->m_entity_stats.maximumDepthSeen, indentDepth * 2, "",
+ ellipisOrEmpty, entity->is_param ? "%" : "&", entityName, action,
+ entity->textLen, sourceLine);
}
static void
@@ -8012,6 +9139,8 @@ entityTrackingOnClose(XML_Parser originParser, ENTITY *entity, int sourceLine) {
rootParser->m_entity_stats.currentDepth--;
}
+#endif /* XML_GE == 1 */
+
static XML_Parser
getRootParserOf(XML_Parser parser, unsigned int *outLevelDiff) {
XML_Parser rootParser = parser;
@@ -8027,6 +9156,8 @@ getRootParserOf(XML_Parser parser, unsigned int *outLevelDiff) {
return rootParser;
}
+#if XML_GE == 1
+
const char *
unsignedCharToPrintable(unsigned char c) {
switch (c) {
@@ -8542,11 +9673,13 @@ unsignedCharToPrintable(unsigned char c) {
return "\\xFE";
case 255:
return "\\xFF";
+ // LCOV_EXCL_START
default:
assert(0); /* never gets here */
return "dead code";
}
assert(0); /* never gets here */
+ // LCOV_EXCL_STOP
}
#endif /* XML_GE == 1 */
diff --git a/Modules/expat/xmlrole.c b/Modules/expat/xmlrole.c
index 2c48bf408679538..dbf4a2d1b8f7080 100644
--- a/Modules/expat/xmlrole.c
+++ b/Modules/expat/xmlrole.c
@@ -12,10 +12,11 @@
Copyright (c) 2002-2006 Karl Waclawek <karl@waclawek.net>
Copyright (c) 2002-2003 Fred L. Drake, Jr. <fdrake@users.sourceforge.net>
Copyright (c) 2005-2009 Steven Solie <steven@solie.ca>
- Copyright (c) 2016-2023 Sebastian Pipping <sebastian@pipping.org>
+ Copyright (c) 2016-2026 Sebastian Pipping <sebastian@pipping.org>
Copyright (c) 2017 Rhodri James <rhodri@wildebeest.org.uk>
Copyright (c) 2019 David Loffredo <loffredo@steptools.com>
Copyright (c) 2021 Donghee Na <donghee.na@python.org>
+ Copyright (c) 2025 Alfonso Gregory <gfunni234@gmail.com>
Licensed under the MIT license:
Permission is hereby granted, free of charge, to any person obtaining
@@ -36,6 +37,8 @@
DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+ SPDX-License-Identifier: MIT
*/
#include "expat_config.h"
@@ -46,7 +49,6 @@
# include "winconfig.h"
#endif
-#include "expat_external.h"
#include "internal.h"
#include "xmlrole.h"
#include "ascii.h"
@@ -119,9 +121,8 @@ static const char KW_SYSTEM[]
# define setTopLevel(state) ((state)->handler = internalSubset)
#endif /* not XML_DTD */
-typedef int PTRCALL PROLOG_HANDLER(PROLOG_STATE *state, int tok,
- const char *ptr, const char *end,
- const ENCODING *enc);
+typedef int PROLOG_HANDLER(PROLOG_STATE *state, int tok, const char *ptr,
+ const char *end, const ENCODING *enc);
static PROLOG_HANDLER prolog0, prolog1, prolog2, doctype0, doctype1, doctype2,
doctype3, doctype4, doctype5, internalSubset, entity0, entity1, entity2,
@@ -135,9 +136,9 @@ static PROLOG_HANDLER prolog0, prolog1, prolog2, doctype0, doctype1, doctype2,
#endif /* XML_DTD */
declClose, error;
-static int FASTCALL common(PROLOG_STATE *state, int tok);
+static int common(PROLOG_STATE *state, int tok);
-static int PTRCALL
+static int
prolog0(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
const ENCODING *enc) {
switch (tok) {
@@ -168,7 +169,7 @@ prolog0(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
return common(state, tok);
}
-static int PTRCALL
+static int
prolog1(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
const ENCODING *enc) {
switch (tok) {
@@ -200,7 +201,7 @@ prolog1(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
return common(state, tok);
}
-static int PTRCALL
+static int
prolog2(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
const ENCODING *enc) {
UNUSED_P(ptr);
@@ -220,7 +221,7 @@ prolog2(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
return common(state, tok);
}
-static int PTRCALL
+static int
doctype0(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
const ENCODING *enc) {
UNUSED_P(ptr);
@@ -237,7 +238,7 @@ doctype0(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
return common(state, tok);
}
-static int PTRCALL
+static int
doctype1(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
const ENCODING *enc) {
switch (tok) {
@@ -263,7 +264,7 @@ doctype1(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
return common(state, tok);
}
-static int PTRCALL
+static int
doctype2(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
const ENCODING *enc) {
UNUSED_P(ptr);
@@ -279,7 +280,7 @@ doctype2(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
return common(state, tok);
}
-static int PTRCALL
+static int
doctype3(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
const ENCODING *enc) {
UNUSED_P(ptr);
@@ -295,7 +296,7 @@ doctype3(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
return common(state, tok);
}
-static int PTRCALL
+static int
doctype4(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
const ENCODING *enc) {
UNUSED_P(ptr);
@@ -314,7 +315,7 @@ doctype4(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
return common(state, tok);
}
-static int PTRCALL
+static int
doctype5(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
const ENCODING *enc) {
UNUSED_P(ptr);
@@ -330,7 +331,7 @@ doctype5(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
return common(state, tok);
}
-static int PTRCALL
+static int
internalSubset(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
const ENCODING *enc) {
switch (tok) {
@@ -375,7 +376,7 @@ internalSubset(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
#ifdef XML_DTD
-static int PTRCALL
+static int
externalSubset0(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
const ENCODING *enc) {
state->handler = externalSubset1;
@@ -384,7 +385,7 @@ externalSubset0(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
return externalSubset1(state, tok, ptr, end, enc);
}
-static int PTRCALL
+static int
externalSubset1(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
const ENCODING *enc) {
switch (tok) {
@@ -412,7 +413,7 @@ externalSubset1(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
#endif /* XML_DTD */
-static int PTRCALL
+static int
entity0(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
const ENCODING *enc) {
UNUSED_P(ptr);
@@ -431,7 +432,7 @@ entity0(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
return common(state, tok);
}
-static int PTRCALL
+static int
entity1(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
const ENCODING *enc) {
UNUSED_P(ptr);
@@ -447,7 +448,7 @@ entity1(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
return common(state, tok);
}
-static int PTRCALL
+static int
entity2(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
const ENCODING *enc) {
switch (tok) {
@@ -471,7 +472,7 @@ entity2(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
return common(state, tok);
}
-static int PTRCALL
+static int
entity3(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
const ENCODING *enc) {
UNUSED_P(ptr);
@@ -487,7 +488,7 @@ entity3(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
return common(state, tok);
}
-static int PTRCALL
+static int
entity4(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
const ENCODING *enc) {
UNUSED_P(ptr);
@@ -503,7 +504,7 @@ entity4(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
return common(state, tok);
}
-static int PTRCALL
+static int
entity5(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
const ENCODING *enc) {
switch (tok) {
@@ -522,7 +523,7 @@ entity5(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
return common(state, tok);
}
-static int PTRCALL
+static int
entity6(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
const ENCODING *enc) {
UNUSED_P(ptr);
@@ -539,7 +540,7 @@ entity6(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
return common(state, tok);
}
-static int PTRCALL
+static int
entity7(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
const ENCODING *enc) {
switch (tok) {
@@ -563,7 +564,7 @@ entity7(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
return common(state, tok);
}
-static int PTRCALL
+static int
entity8(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
const ENCODING *enc) {
UNUSED_P(ptr);
@@ -579,7 +580,7 @@ entity8(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
return common(state, tok);
}
-static int PTRCALL
+static int
entity9(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
const ENCODING *enc) {
UNUSED_P(ptr);
@@ -595,7 +596,7 @@ entity9(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
return common(state, tok);
}
-static int PTRCALL
+static int
entity10(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
const ENCODING *enc) {
UNUSED_P(ptr);
@@ -611,7 +612,7 @@ entity10(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
return common(state, tok);
}
-static int PTRCALL
+static int
notation0(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
const ENCODING *enc) {
UNUSED_P(ptr);
@@ -627,7 +628,7 @@ notation0(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
return common(state, tok);
}
-static int PTRCALL
+static int
notation1(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
const ENCODING *enc) {
switch (tok) {
@@ -647,7 +648,7 @@ notation1(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
return common(state, tok);
}
-static int PTRCALL
+static int
notation2(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
const ENCODING *enc) {
UNUSED_P(ptr);
@@ -663,7 +664,7 @@ notation2(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
return common(state, tok);
}
-static int PTRCALL
+static int
notation3(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
const ENCODING *enc) {
UNUSED_P(ptr);
@@ -680,7 +681,7 @@ notation3(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
return common(state, tok);
}
-static int PTRCALL
+static int
notation4(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
const ENCODING *enc) {
UNUSED_P(ptr);
@@ -700,7 +701,7 @@ notation4(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
return common(state, tok);
}
-static int PTRCALL
+static int
attlist0(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
const ENCODING *enc) {
UNUSED_P(ptr);
@@ -717,7 +718,7 @@ attlist0(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
return common(state, tok);
}
-static int PTRCALL
+static int
attlist1(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
const ENCODING *enc) {
UNUSED_P(ptr);
@@ -737,7 +738,7 @@ attlist1(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
return common(state, tok);
}
-static int PTRCALL
+static int
attlist2(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
const ENCODING *enc) {
switch (tok) {
@@ -767,7 +768,7 @@ attlist2(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
return common(state, tok);
}
-static int PTRCALL
+static int
attlist3(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
const ENCODING *enc) {
UNUSED_P(ptr);
@@ -785,7 +786,7 @@ attlist3(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
return common(state, tok);
}
-static int PTRCALL
+static int
attlist4(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
const ENCODING *enc) {
UNUSED_P(ptr);
@@ -804,7 +805,7 @@ attlist4(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
return common(state, tok);
}
-static int PTRCALL
+static int
attlist5(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
const ENCODING *enc) {
UNUSED_P(ptr);
@@ -820,7 +821,7 @@ attlist5(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
return common(state, tok);
}
-static int PTRCALL
+static int
attlist6(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
const ENCODING *enc) {
UNUSED_P(ptr);
@@ -836,7 +837,7 @@ attlist6(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
return common(state, tok);
}
-static int PTRCALL
+static int
attlist7(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
const ENCODING *enc) {
UNUSED_P(ptr);
@@ -856,7 +857,7 @@ attlist7(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
}
/* default value */
-static int PTRCALL
+static int
attlist8(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
const ENCODING *enc) {
switch (tok) {
@@ -886,7 +887,7 @@ attlist8(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
return common(state, tok);
}
-static int PTRCALL
+static int
attlist9(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
const ENCODING *enc) {
UNUSED_P(ptr);
@@ -902,7 +903,7 @@ attlist9(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
return common(state, tok);
}
-static int PTRCALL
+static int
element0(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
const ENCODING *enc) {
UNUSED_P(ptr);
@@ -919,7 +920,7 @@ element0(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
return common(state, tok);
}
-static int PTRCALL
+static int
element1(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
const ENCODING *enc) {
switch (tok) {
@@ -945,7 +946,7 @@ element1(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
return common(state, tok);
}
-static int PTRCALL
+static int
element2(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
const ENCODING *enc) {
switch (tok) {
@@ -979,7 +980,7 @@ element2(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
return common(state, tok);
}
-static int PTRCALL
+static int
element3(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
const ENCODING *enc) {
UNUSED_P(ptr);
@@ -1003,7 +1004,7 @@ element3(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
return common(state, tok);
}
-static int PTRCALL
+static int
element4(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
const ENCODING *enc) {
UNUSED_P(ptr);
@@ -1020,7 +1021,7 @@ element4(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
return common(state, tok);
}
-static int PTRCALL
+static int
element5(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
const ENCODING *enc) {
UNUSED_P(ptr);
@@ -1040,7 +1041,7 @@ element5(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
return common(state, tok);
}
-static int PTRCALL
+static int
element6(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
const ENCODING *enc) {
UNUSED_P(ptr);
@@ -1069,7 +1070,7 @@ element6(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
return common(state, tok);
}
-static int PTRCALL
+static int
element7(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
const ENCODING *enc) {
UNUSED_P(ptr);
@@ -1118,7 +1119,7 @@ element7(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
#ifdef XML_DTD
-static int PTRCALL
+static int
condSect0(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
const ENCODING *enc) {
switch (tok) {
@@ -1138,7 +1139,7 @@ condSect0(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
return common(state, tok);
}
-static int PTRCALL
+static int
condSect1(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
const ENCODING *enc) {
UNUSED_P(ptr);
@@ -1155,7 +1156,7 @@ condSect1(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
return common(state, tok);
}
-static int PTRCALL
+static int
condSect2(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
const ENCODING *enc) {
UNUSED_P(ptr);
@@ -1173,7 +1174,7 @@ condSect2(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
#endif /* XML_DTD */
-static int PTRCALL
+static int
declClose(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
const ENCODING *enc) {
UNUSED_P(ptr);
@@ -1209,7 +1210,7 @@ declClose(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
*
* LCOV_EXCL_START
*/
-static int PTRCALL
+static int
error(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
const ENCODING *enc) {
UNUSED_P(state);
@@ -1221,7 +1222,7 @@ error(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
}
/* LCOV_EXCL_STOP */
-static int FASTCALL
+static int
common(PROLOG_STATE *state, int tok) {
#ifdef XML_DTD
if (! state->documentEntity && tok == XML_TOK_PARAM_ENTITY_REF)
diff --git a/Modules/expat/xmlrole.h b/Modules/expat/xmlrole.h
index a7904274c91d4ec..f4aba4079f5cdfa 100644
--- a/Modules/expat/xmlrole.h
+++ b/Modules/expat/xmlrole.h
@@ -10,7 +10,7 @@
Copyright (c) 2000 Clark Cooper <coopercc@users.sourceforge.net>
Copyright (c) 2002 Karl Waclawek <karl@waclawek.net>
Copyright (c) 2002 Fred L. Drake, Jr. <fdrake@users.sourceforge.net>
- Copyright (c) 2017-2024 Sebastian Pipping <sebastian@pipping.org>
+ Copyright (c) 2017-2026 Sebastian Pipping <sebastian@pipping.org>
Licensed under the MIT license:
Permission is hereby granted, free of charge, to any person obtaining
@@ -31,22 +31,14 @@
DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+ SPDX-License-Identifier: MIT
*/
#ifndef XmlRole_INCLUDED
-#define XmlRole_INCLUDED 1
-
-#ifdef __VMS
-/* 0 1 2 3 0 1 2 3
- 1234567890123456789012345678901 1234567890123456789012345678901 */
-# define XmlPrologStateInitExternalEntity XmlPrologStateInitExternalEnt
-#endif
+# define XmlRole_INCLUDED 1
-#include "xmltok.h"
-
-#ifdef __cplusplus
-extern "C" {
-#endif
+# include "xmltok.h"
enum {
XML_ROLE_ERROR = -1,
@@ -107,36 +99,32 @@ enum {
XML_ROLE_CONTENT_ELEMENT_PLUS,
XML_ROLE_PI,
XML_ROLE_COMMENT,
-#ifdef XML_DTD
+# ifdef XML_DTD
XML_ROLE_TEXT_DECL,
XML_ROLE_IGNORE_SECT,
XML_ROLE_INNER_PARAM_ENTITY_REF,
-#endif /* XML_DTD */
+# endif /* XML_DTD */
XML_ROLE_PARAM_ENTITY_REF
};
typedef struct prolog_state {
- int(PTRCALL *handler)(struct prolog_state *state, int tok, const char *ptr,
- const char *end, const ENCODING *enc);
+ int (*handler)(struct prolog_state *state, int tok, const char *ptr,
+ const char *end, const ENCODING *enc);
unsigned level;
int role_none;
-#ifdef XML_DTD
+# ifdef XML_DTD
unsigned includeLevel;
int documentEntity;
int inEntityValue;
-#endif /* XML_DTD */
+# endif /* XML_DTD */
} PROLOG_STATE;
void XmlPrologStateInit(PROLOG_STATE *state);
-#ifdef XML_DTD
+# ifdef XML_DTD
void XmlPrologStateInitExternalEntity(PROLOG_STATE *state);
-#endif /* XML_DTD */
-
-#define XmlTokenRole(state, tok, ptr, end, enc) \
- (((state)->handler)(state, tok, ptr, end, enc))
+# endif /* XML_DTD */
-#ifdef __cplusplus
-}
-#endif
+# define XmlTokenRole(state, tok, ptr, end, enc) \
+ (((state)->handler)(state, tok, ptr, end, enc))
#endif /* not XmlRole_INCLUDED */
diff --git a/Modules/expat/xmltok.c b/Modules/expat/xmltok.c
index 29a66d72ceea5e5..2f837acc4cbf72a 100644
--- a/Modules/expat/xmltok.c
+++ b/Modules/expat/xmltok.c
@@ -12,7 +12,7 @@
Copyright (c) 2002 Greg Stein <gstein@users.sourceforge.net>
Copyright (c) 2002-2016 Karl Waclawek <karl@waclawek.net>
Copyright (c) 2005-2009 Steven Solie <steven@solie.ca>
- Copyright (c) 2016-2024 Sebastian Pipping <sebastian@pipping.org>
+ Copyright (c) 2016-2026 Sebastian Pipping <sebastian@pipping.org>
Copyright (c) 2016 Pascal Cuoq <cuoq@trust-in-soft.com>
Copyright (c) 2016 Don Lewis <truckman@apache.org>
Copyright (c) 2017 Rhodri James <rhodri@wildebeest.org.uk>
@@ -24,6 +24,10 @@
Copyright (c) 2022 Martin Ettl <ettl.martin78@googlemail.com>
Copyright (c) 2022 Sean McBride <sean@rogue-research.com>
Copyright (c) 2023 Hanno Böck <hanno@gentoo.org>
+ Copyright (c) 2025 Alfonso Gregory <gfunni234@gmail.com>
+ Copyright (c) 2026 Nick Begg <nick@stunttruck.net>
+ Copyright (c) 2026 Kartik Kenchi <netliomax25@gmail.com>
+ Copyright (c) 2026 Afonso Januário <afonso-januario@hotmail.com>
Licensed under the MIT license:
Permission is hereby granted, free of charge, to any person obtaining
@@ -44,6 +48,8 @@
DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+ SPDX-License-Identifier: MIT
*/
#include "expat_config.h"
@@ -56,8 +62,8 @@
# include "winconfig.h"
#endif
-#include "expat_external.h"
#include "internal.h"
+#include "fallthrough.h"
#include "xmltok.h"
#include "nametab.h"
@@ -131,20 +137,20 @@
: ((p)[1] & 0x80) == 0 \
|| ((*p) == 0xF4 ? (p)[1] > 0x8F : ((p)[1] & 0xC0) == 0xC0)))
-static int PTRFASTCALL
+static int
isNever(const ENCODING *enc, const char *p) {
UNUSED_P(enc);
UNUSED_P(p);
return 0;
}
-static int PTRFASTCALL
+static int
utf8_isName2(const ENCODING *enc, const char *p) {
UNUSED_P(enc);
return UTF8_GET_NAMING2(namePages, (const unsigned char *)p);
}
-static int PTRFASTCALL
+static int
utf8_isName3(const ENCODING *enc, const char *p) {
UNUSED_P(enc);
return UTF8_GET_NAMING3(namePages, (const unsigned char *)p);
@@ -152,13 +158,13 @@ utf8_isName3(const ENCODING *enc, const char *p) {
#define utf8_isName4 isNever
-static int PTRFASTCALL
+static int
utf8_isNmstrt2(const ENCODING *enc, const char *p) {
UNUSED_P(enc);
return UTF8_GET_NAMING2(nmstrtPages, (const unsigned char *)p);
}
-static int PTRFASTCALL
+static int
utf8_isNmstrt3(const ENCODING *enc, const char *p) {
UNUSED_P(enc);
return UTF8_GET_NAMING3(nmstrtPages, (const unsigned char *)p);
@@ -166,19 +172,19 @@ utf8_isNmstrt3(const ENCODING *enc, const char *p) {
#define utf8_isNmstrt4 isNever
-static int PTRFASTCALL
+static int
utf8_isInvalid2(const ENCODING *enc, const char *p) {
UNUSED_P(enc);
return UTF8_INVALID2((const unsigned char *)p);
}
-static int PTRFASTCALL
+static int
utf8_isInvalid3(const ENCODING *enc, const char *p) {
UNUSED_P(enc);
return UTF8_INVALID3((const unsigned char *)p);
}
-static int PTRFASTCALL
+static int
utf8_isInvalid4(const ENCODING *enc, const char *p) {
UNUSED_P(enc);
return UTF8_INVALID4((const unsigned char *)p);
@@ -187,36 +193,20 @@ utf8_isInvalid4(const ENCODING *enc, const char *p) {
struct normal_encoding {
ENCODING enc;
unsigned char type[256];
-#ifdef XML_MIN_SIZE
- int(PTRFASTCALL *byteType)(const ENCODING *, const char *);
- int(PTRFASTCALL *isNameMin)(const ENCODING *, const char *);
- int(PTRFASTCALL *isNmstrtMin)(const ENCODING *, const char *);
- int(PTRFASTCALL *byteToAscii)(const ENCODING *, const char *);
- int(PTRCALL *charMatches)(const ENCODING *, const char *, int);
-#endif /* XML_MIN_SIZE */
- int(PTRFASTCALL *isName2)(const ENCODING *, const char *);
- int(PTRFASTCALL *isName3)(const ENCODING *, const char *);
- int(PTRFASTCALL *isName4)(const ENCODING *, const char *);
- int(PTRFASTCALL *isNmstrt2)(const ENCODING *, const char *);
- int(PTRFASTCALL *isNmstrt3)(const ENCODING *, const char *);
- int(PTRFASTCALL *isNmstrt4)(const ENCODING *, const char *);
- int(PTRFASTCALL *isInvalid2)(const ENCODING *, const char *);
- int(PTRFASTCALL *isInvalid3)(const ENCODING *, const char *);
- int(PTRFASTCALL *isInvalid4)(const ENCODING *, const char *);
+ int (*isName2)(const ENCODING *, const char *);
+ int (*isName3)(const ENCODING *, const char *);
+ int (*isName4)(const ENCODING *, const char *);
+ int (*isNmstrt2)(const ENCODING *, const char *);
+ int (*isNmstrt3)(const ENCODING *, const char *);
+ int (*isNmstrt4)(const ENCODING *, const char *);
+ int (*isInvalid2)(const ENCODING *, const char *);
+ int (*isInvalid3)(const ENCODING *, const char *);
+ int (*isInvalid4)(const ENCODING *, const char *);
};
#define AS_NORMAL_ENCODING(enc) ((const struct normal_encoding *)(enc))
-#ifdef XML_MIN_SIZE
-
-# define STANDARD_VTABLE(E) \
- E##byteType, E##isNameMin, E##isNmstrtMin, E##byteToAscii, E##charMatches,
-
-#else
-
-# define STANDARD_VTABLE(E) /* as nothing */
-
-#endif
+#define STANDARD_VTABLE(E) /* as nothing */
#define NORMAL_VTABLE(E) \
E##isName2, E##isName3, E##isName4, E##isNmstrt2, E##isNmstrt3, \
@@ -227,80 +217,33 @@ struct normal_encoding {
/* isNmstrt2 */ NULL, /* isNmstrt3 */ NULL, /* isNmstrt4 */ NULL, \
/* isInvalid2 */ NULL, /* isInvalid3 */ NULL, /* isInvalid4 */ NULL
-static int FASTCALL checkCharRefNumber(int result);
+#define UTF16_NULL_VTABLE(E) NULL_VTABLE
+
+static int checkCharRefNumber(int result);
#include "xmltok_impl.h"
#include "ascii.h"
-#ifdef XML_MIN_SIZE
-# define sb_isNameMin isNever
-# define sb_isNmstrtMin isNever
-#endif
-
-#ifdef XML_MIN_SIZE
-# define MINBPC(enc) ((enc)->minBytesPerChar)
-#else
/* minimum bytes per character */
-# define MINBPC(enc) 1
-#endif
+#define MINBPC(enc) 1
#define SB_BYTE_TYPE(enc, p) \
(((const struct normal_encoding *)(enc))->type[(unsigned char)*(p)])
-#ifdef XML_MIN_SIZE
-static int PTRFASTCALL
-sb_byteType(const ENCODING *enc, const char *p) {
- return SB_BYTE_TYPE(enc, p);
-}
-# define BYTE_TYPE(enc, p) (AS_NORMAL_ENCODING(enc)->byteType(enc, p))
-#else
-# define BYTE_TYPE(enc, p) SB_BYTE_TYPE(enc, p)
-#endif
+#define BYTE_TYPE(enc, p) SB_BYTE_TYPE(enc, p)
-#ifdef XML_MIN_SIZE
-# define BYTE_TO_ASCII(enc, p) (AS_NORMAL_ENCODING(enc)->byteToAscii(enc, p))
-static int PTRFASTCALL
-sb_byteToAscii(const ENCODING *enc, const char *p) {
- UNUSED_P(enc);
- return *p;
-}
-#else
-# define BYTE_TO_ASCII(enc, p) (*(p))
-#endif
+#define BYTE_TO_ASCII(enc, p) (*(p))
#define IS_NAME_CHAR(enc, p, n) (AS_NORMAL_ENCODING(enc)->isName##n(enc, p))
#define IS_NMSTRT_CHAR(enc, p, n) (AS_NORMAL_ENCODING(enc)->isNmstrt##n(enc, p))
-#ifdef XML_MIN_SIZE
-# define IS_INVALID_CHAR(enc, p, n) \
- (AS_NORMAL_ENCODING(enc)->isInvalid##n \
- && AS_NORMAL_ENCODING(enc)->isInvalid##n(enc, p))
-#else
-# define IS_INVALID_CHAR(enc, p, n) \
- (AS_NORMAL_ENCODING(enc)->isInvalid##n(enc, p))
-#endif
+#define IS_INVALID_CHAR(enc, p, n) \
+ (AS_NORMAL_ENCODING(enc)->isInvalid##n(enc, p))
-#ifdef XML_MIN_SIZE
-# define IS_NAME_CHAR_MINBPC(enc, p) \
- (AS_NORMAL_ENCODING(enc)->isNameMin(enc, p))
-# define IS_NMSTRT_CHAR_MINBPC(enc, p) \
- (AS_NORMAL_ENCODING(enc)->isNmstrtMin(enc, p))
-#else
-# define IS_NAME_CHAR_MINBPC(enc, p) (0)
-# define IS_NMSTRT_CHAR_MINBPC(enc, p) (0)
-#endif
+#define IS_NAME_CHAR_MINBPC(enc, p) (0)
+#define IS_NMSTRT_CHAR_MINBPC(enc, p) (0)
-#ifdef XML_MIN_SIZE
-# define CHAR_MATCHES(enc, p, c) \
- (AS_NORMAL_ENCODING(enc)->charMatches(enc, p, c))
-static int PTRCALL
-sb_charMatches(const ENCODING *enc, const char *p, int c) {
- UNUSED_P(enc);
- return *p == c;
-}
-#else
/* c is an ASCII character */
-# define CHAR_MATCHES(enc, p, c) (*(p) == (c))
-#endif
+#define CHAR_MATCHES(enc, p, c) (*(p) == (c))
#define PREFIX(ident) normal_##ident
#define XML_TOK_IMPL_C
@@ -363,7 +306,7 @@ _INTERNAL_trim_to_complete_utf8_characters(const char *from,
*fromLimRef = fromLim;
}
-static enum XML_Convert_Result PTRCALL
+static enum XML_Convert_Result
utf8_toUtf8(const ENCODING *enc, const char **fromP, const char *fromLim,
char **toP, const char *toLim) {
bool input_incomplete = false;
@@ -402,7 +345,7 @@ utf8_toUtf8(const ENCODING *enc, const char **fromP, const char *fromLim,
return XML_CONVERT_COMPLETED;
}
-static enum XML_Convert_Result PTRCALL
+static enum XML_Convert_Result
utf8_toUtf16(const ENCODING *enc, const char **fromP, const char *fromLim,
unsigned short **toP, const unsigned short *toLim) {
enum XML_Convert_Result res = XML_CONVERT_COMPLETED;
@@ -500,7 +443,7 @@ static const struct normal_encoding internal_utf8_encoding
},
STANDARD_VTABLE(sb_) NORMAL_VTABLE(utf8_)};
-static enum XML_Convert_Result PTRCALL
+static enum XML_Convert_Result
latin1_toUtf8(const ENCODING *enc, const char **fromP, const char *fromLim,
char **toP, const char *toLim) {
UNUSED_P(enc);
@@ -523,7 +466,7 @@ latin1_toUtf8(const ENCODING *enc, const char **fromP, const char *fromLim,
}
}
-static enum XML_Convert_Result PTRCALL
+static enum XML_Convert_Result
latin1_toUtf16(const ENCODING *enc, const char **fromP, const char *fromLim,
unsigned short **toP, const unsigned short *toLim) {
UNUSED_P(enc);
@@ -558,7 +501,7 @@ static const struct normal_encoding latin1_encoding
},
STANDARD_VTABLE(sb_) NULL_VTABLE};
-static enum XML_Convert_Result PTRCALL
+static enum XML_Convert_Result
ascii_toUtf8(const ENCODING *enc, const char **fromP, const char *fromLim,
char **toP, const char *toLim) {
UNUSED_P(enc);
@@ -593,7 +536,7 @@ static const struct normal_encoding ascii_encoding
},
STANDARD_VTABLE(sb_) NULL_VTABLE};
-static int PTRFASTCALL
+static int
unicode_byte_type(char hi, char lo) {
switch ((unsigned char)hi) {
/* 0xD800-0xDBFF first 16-bit code unit or high surrogate (W1) */
@@ -620,7 +563,7 @@ unicode_byte_type(char hi, char lo) {
}
#define DEFINE_UTF16_TO_UTF8(E) \
- static enum XML_Convert_Result PTRCALL E##toUtf8( \
+ static enum XML_Convert_Result E##toUtf8( \
const ENCODING *enc, const char **fromP, const char *fromLim, \
char **toP, const char *toLim) { \
const char *from = *fromP; \
@@ -641,7 +584,7 @@ unicode_byte_type(char hi, char lo) {
*(*toP)++ = lo; \
break; \
} \
- /* fall through */ \
+ EXPAT_FALLTHROUGH; \
case 0x1: \
case 0x2: \
case 0x3: \
@@ -697,15 +640,16 @@ unicode_byte_type(char hi, char lo) {
}
#define DEFINE_UTF16_TO_UTF16(E) \
- static enum XML_Convert_Result PTRCALL E##toUtf16( \
+ static enum XML_Convert_Result E##toUtf16( \
const ENCODING *enc, const char **fromP, const char *fromLim, \
unsigned short **toP, const unsigned short *toLim) { \
enum XML_Convert_Result res = XML_CONVERT_COMPLETED; \
UNUSED_P(enc); \
fromLim = *fromP + (((fromLim - *fromP) >> 1) << 1); /* shrink to even */ \
- /* Avoid copying first half only of surrogate */ \
+ /* Avoid copying the first half (2 bytes) of surrogate pairs (4 bytes) */ \
if (fromLim - *fromP > ((toLim - *toP) << 1) \
- && (GET_HI(fromLim - 2) & 0xF8) == 0xD8) { \
+ && /* are the last two bytes a high surrogate (0xD800-0xDBFF)? */ \
+ (GET_HI(fromLim - 2) & 0xFC) == 0xD8) { \
fromLim -= 2; \
res = XML_CONVERT_INPUT_INCOMPLETE; \
} \
@@ -743,70 +687,38 @@ DEFINE_UTF16_TO_UTF16(big2_)
UCS2_GET_NAMING(namePages, (unsigned char)p[1], (unsigned char)p[0])
#define LITTLE2_IS_NMSTRT_CHAR_MINBPC(p) \
UCS2_GET_NAMING(nmstrtPages, (unsigned char)p[1], (unsigned char)p[0])
+/* A 4-byte UTF-16 character is a surrogate pair; byteType only reports BT_LEAD4
+ for a high surrogate, so the pair is invalid unless the second unit is a low
+ surrogate (U+DC00..U+DFFF, i.e. high byte 0xDC..0xDF). */
+#define LITTLE2_IS_INVALID_CHAR(p, n) \
+ ((n) == 4 && ((unsigned char)(p)[3] & 0xFC) != 0xDC)
-#ifdef XML_MIN_SIZE
-
-static int PTRFASTCALL
-little2_byteType(const ENCODING *enc, const char *p) {
- return LITTLE2_BYTE_TYPE(enc, p);
-}
-
-static int PTRFASTCALL
-little2_byteToAscii(const ENCODING *enc, const char *p) {
- UNUSED_P(enc);
- return LITTLE2_BYTE_TO_ASCII(p);
-}
-
-static int PTRCALL
-little2_charMatches(const ENCODING *enc, const char *p, int c) {
- UNUSED_P(enc);
- return LITTLE2_CHAR_MATCHES(p, c);
-}
-
-static int PTRFASTCALL
-little2_isNameMin(const ENCODING *enc, const char *p) {
- UNUSED_P(enc);
- return LITTLE2_IS_NAME_CHAR_MINBPC(p);
-}
-
-static int PTRFASTCALL
-little2_isNmstrtMin(const ENCODING *enc, const char *p) {
- UNUSED_P(enc);
- return LITTLE2_IS_NMSTRT_CHAR_MINBPC(p);
-}
-
-# undef VTABLE
-# define VTABLE VTABLE1, little2_toUtf8, little2_toUtf16
+#undef PREFIX
+#define PREFIX(ident) little2_##ident
+#define MINBPC(enc) 2
+/* CHAR_MATCHES is guaranteed to have MINBPC bytes available. */
+#define BYTE_TYPE(enc, p) LITTLE2_BYTE_TYPE(enc, p)
+#define BYTE_TO_ASCII(enc, p) LITTLE2_BYTE_TO_ASCII(p)
+#define CHAR_MATCHES(enc, p, c) LITTLE2_CHAR_MATCHES(p, c)
+#define IS_NAME_CHAR(enc, p, n) 0
+#define IS_NAME_CHAR_MINBPC(enc, p) LITTLE2_IS_NAME_CHAR_MINBPC(p)
+#define IS_NMSTRT_CHAR(enc, p, n) (0)
+#define IS_NMSTRT_CHAR_MINBPC(enc, p) LITTLE2_IS_NMSTRT_CHAR_MINBPC(p)
+#define IS_INVALID_CHAR(enc, p, n) LITTLE2_IS_INVALID_CHAR(p, n)
-#else /* not XML_MIN_SIZE */
+#define XML_TOK_IMPL_C
+#include "xmltok_impl.c"
+#undef XML_TOK_IMPL_C
-# undef PREFIX
-# define PREFIX(ident) little2_##ident
-# define MINBPC(enc) 2
-/* CHAR_MATCHES is guaranteed to have MINBPC bytes available. */
-# define BYTE_TYPE(enc, p) LITTLE2_BYTE_TYPE(enc, p)
-# define BYTE_TO_ASCII(enc, p) LITTLE2_BYTE_TO_ASCII(p)
-# define CHAR_MATCHES(enc, p, c) LITTLE2_CHAR_MATCHES(p, c)
-# define IS_NAME_CHAR(enc, p, n) 0
-# define IS_NAME_CHAR_MINBPC(enc, p) LITTLE2_IS_NAME_CHAR_MINBPC(p)
-# define IS_NMSTRT_CHAR(enc, p, n) (0)
-# define IS_NMSTRT_CHAR_MINBPC(enc, p) LITTLE2_IS_NMSTRT_CHAR_MINBPC(p)
-
-# define XML_TOK_IMPL_C
-# include "xmltok_impl.c"
-# undef XML_TOK_IMPL_C
-
-# undef MINBPC
-# undef BYTE_TYPE
-# undef BYTE_TO_ASCII
-# undef CHAR_MATCHES
-# undef IS_NAME_CHAR
-# undef IS_NAME_CHAR_MINBPC
-# undef IS_NMSTRT_CHAR
-# undef IS_NMSTRT_CHAR_MINBPC
-# undef IS_INVALID_CHAR
-
-#endif /* not XML_MIN_SIZE */
+#undef MINBPC
+#undef BYTE_TYPE
+#undef BYTE_TO_ASCII
+#undef CHAR_MATCHES
+#undef IS_NAME_CHAR
+#undef IS_NAME_CHAR_MINBPC
+#undef IS_NMSTRT_CHAR
+#undef IS_NMSTRT_CHAR_MINBPC
+#undef IS_INVALID_CHAR
#ifdef XML_NS
@@ -822,7 +734,7 @@ static const struct normal_encoding little2_encoding_ns
# include "asciitab.h"
# include "latin1tab.h"
},
- STANDARD_VTABLE(little2_) NULL_VTABLE};
+ STANDARD_VTABLE(little2_) UTF16_NULL_VTABLE(little2_)};
#endif
@@ -840,7 +752,7 @@ static const struct normal_encoding little2_encoding
#undef BT_COLON
#include "latin1tab.h"
},
- STANDARD_VTABLE(little2_) NULL_VTABLE};
+ STANDARD_VTABLE(little2_) UTF16_NULL_VTABLE(little2_)};
#if BYTEORDER != 4321
@@ -852,7 +764,7 @@ static const struct normal_encoding internal_little2_encoding_ns
# include "iasciitab.h"
# include "latin1tab.h"
},
- STANDARD_VTABLE(little2_) NULL_VTABLE};
+ STANDARD_VTABLE(little2_) UTF16_NULL_VTABLE(little2_)};
# endif
@@ -864,7 +776,7 @@ static const struct normal_encoding internal_little2_encoding
# undef BT_COLON
# include "latin1tab.h"
},
- STANDARD_VTABLE(little2_) NULL_VTABLE};
+ STANDARD_VTABLE(little2_) UTF16_NULL_VTABLE(little2_)};
#endif
@@ -876,70 +788,38 @@ static const struct normal_encoding internal_little2_encoding
UCS2_GET_NAMING(namePages, (unsigned char)p[0], (unsigned char)p[1])
#define BIG2_IS_NMSTRT_CHAR_MINBPC(p) \
UCS2_GET_NAMING(nmstrtPages, (unsigned char)p[0], (unsigned char)p[1])
+/* A 4-byte UTF-16 character is a surrogate pair; byteType only reports BT_LEAD4
+ for a high surrogate, so the pair is invalid unless the second unit is a low
+ surrogate (U+DC00..U+DFFF, i.e. high byte 0xDC..0xDF). */
+#define BIG2_IS_INVALID_CHAR(p, n) \
+ ((n) == 4 && ((unsigned char)(p)[2] & 0xFC) != 0xDC)
-#ifdef XML_MIN_SIZE
-
-static int PTRFASTCALL
-big2_byteType(const ENCODING *enc, const char *p) {
- return BIG2_BYTE_TYPE(enc, p);
-}
-
-static int PTRFASTCALL
-big2_byteToAscii(const ENCODING *enc, const char *p) {
- UNUSED_P(enc);
- return BIG2_BYTE_TO_ASCII(p);
-}
-
-static int PTRCALL
-big2_charMatches(const ENCODING *enc, const char *p, int c) {
- UNUSED_P(enc);
- return BIG2_CHAR_MATCHES(p, c);
-}
-
-static int PTRFASTCALL
-big2_isNameMin(const ENCODING *enc, const char *p) {
- UNUSED_P(enc);
- return BIG2_IS_NAME_CHAR_MINBPC(p);
-}
-
-static int PTRFASTCALL
-big2_isNmstrtMin(const ENCODING *enc, const char *p) {
- UNUSED_P(enc);
- return BIG2_IS_NMSTRT_CHAR_MINBPC(p);
-}
-
-# undef VTABLE
-# define VTABLE VTABLE1, big2_toUtf8, big2_toUtf16
+#undef PREFIX
+#define PREFIX(ident) big2_##ident
+#define MINBPC(enc) 2
+/* CHAR_MATCHES is guaranteed to have MINBPC bytes available. */
+#define BYTE_TYPE(enc, p) BIG2_BYTE_TYPE(enc, p)
+#define BYTE_TO_ASCII(enc, p) BIG2_BYTE_TO_ASCII(p)
+#define CHAR_MATCHES(enc, p, c) BIG2_CHAR_MATCHES(p, c)
+#define IS_NAME_CHAR(enc, p, n) 0
+#define IS_NAME_CHAR_MINBPC(enc, p) BIG2_IS_NAME_CHAR_MINBPC(p)
+#define IS_NMSTRT_CHAR(enc, p, n) (0)
+#define IS_NMSTRT_CHAR_MINBPC(enc, p) BIG2_IS_NMSTRT_CHAR_MINBPC(p)
+#define IS_INVALID_CHAR(enc, p, n) BIG2_IS_INVALID_CHAR(p, n)
-#else /* not XML_MIN_SIZE */
+#define XML_TOK_IMPL_C
+#include "xmltok_impl.c"
+#undef XML_TOK_IMPL_C
-# undef PREFIX
-# define PREFIX(ident) big2_##ident
-# define MINBPC(enc) 2
-/* CHAR_MATCHES is guaranteed to have MINBPC bytes available. */
-# define BYTE_TYPE(enc, p) BIG2_BYTE_TYPE(enc, p)
-# define BYTE_TO_ASCII(enc, p) BIG2_BYTE_TO_ASCII(p)
-# define CHAR_MATCHES(enc, p, c) BIG2_CHAR_MATCHES(p, c)
-# define IS_NAME_CHAR(enc, p, n) 0
-# define IS_NAME_CHAR_MINBPC(enc, p) BIG2_IS_NAME_CHAR_MINBPC(p)
-# define IS_NMSTRT_CHAR(enc, p, n) (0)
-# define IS_NMSTRT_CHAR_MINBPC(enc, p) BIG2_IS_NMSTRT_CHAR_MINBPC(p)
-
-# define XML_TOK_IMPL_C
-# include "xmltok_impl.c"
-# undef XML_TOK_IMPL_C
-
-# undef MINBPC
-# undef BYTE_TYPE
-# undef BYTE_TO_ASCII
-# undef CHAR_MATCHES
-# undef IS_NAME_CHAR
-# undef IS_NAME_CHAR_MINBPC
-# undef IS_NMSTRT_CHAR
-# undef IS_NMSTRT_CHAR_MINBPC
-# undef IS_INVALID_CHAR
-
-#endif /* not XML_MIN_SIZE */
+#undef MINBPC
+#undef BYTE_TYPE
+#undef BYTE_TO_ASCII
+#undef CHAR_MATCHES
+#undef IS_NAME_CHAR
+#undef IS_NAME_CHAR_MINBPC
+#undef IS_NMSTRT_CHAR
+#undef IS_NMSTRT_CHAR_MINBPC
+#undef IS_INVALID_CHAR
#ifdef XML_NS
@@ -955,7 +835,7 @@ static const struct normal_encoding big2_encoding_ns
# include "asciitab.h"
# include "latin1tab.h"
},
- STANDARD_VTABLE(big2_) NULL_VTABLE};
+ STANDARD_VTABLE(big2_) UTF16_NULL_VTABLE(big2_)};
#endif
@@ -973,7 +853,7 @@ static const struct normal_encoding big2_encoding
#undef BT_COLON
#include "latin1tab.h"
},
- STANDARD_VTABLE(big2_) NULL_VTABLE};
+ STANDARD_VTABLE(big2_) UTF16_NULL_VTABLE(big2_)};
#if BYTEORDER != 1234
@@ -985,7 +865,7 @@ static const struct normal_encoding internal_big2_encoding_ns
# include "iasciitab.h"
# include "latin1tab.h"
},
- STANDARD_VTABLE(big2_) NULL_VTABLE};
+ STANDARD_VTABLE(big2_) UTF16_NULL_VTABLE(big2_)};
# endif
@@ -997,13 +877,13 @@ static const struct normal_encoding internal_big2_encoding
# undef BT_COLON
# include "latin1tab.h"
},
- STANDARD_VTABLE(big2_) NULL_VTABLE};
+ STANDARD_VTABLE(big2_) UTF16_NULL_VTABLE(big2_)};
#endif
#undef PREFIX
-static int FASTCALL
+static int
streqci(const char *s1, const char *s2) {
for (;;) {
char c1 = *s1++;
@@ -1024,7 +904,7 @@ streqci(const char *s1, const char *s2) {
return 1;
}
-static void PTRCALL
+static void
initUpdatePosition(const ENCODING *enc, const char *ptr, const char *end,
POSITION *pos) {
UNUSED_P(enc);
@@ -1042,7 +922,7 @@ toAscii(const ENCODING *enc, const char *ptr, const char *end) {
return buf[0];
}
-static int FASTCALL
+static int
isSpace(int c) {
switch (c) {
case 0x20:
@@ -1148,6 +1028,34 @@ static const char KW_yes[] = {ASCII_y, ASCII_e, ASCII_s, '\0'};
static const char KW_no[] = {ASCII_n, ASCII_o, '\0'};
+static const char KW_1_dot[] = {ASCII_1, ASCII_PERIOD, '\0'};
+
+/* Checks a version pseudo-attribute value against the VersionNum production.
+ XML 1.0 Fourth Edition only allows the literal "1.0", but the Fifth
+ Edition relaxed this to "1." followed by one or more digits, since Expat
+ only implements 1.0 itself but plans to track the Fifth Edition's laxer
+ grammar here so that "1.1" and similar aren't rejected only to have that
+ rejection reverted later. Returns true for a value matching "1.[0-9]+".
+ val/valEnd bound the value itself; valEnd is the upper bound used when
+ decoding the individual characters between them. */
+static bool
+checkXmlDeclVersionNum(const ENCODING *enc, const char *val,
+ const char *valEnd) {
+ if (valEnd - val < 2 * enc->minBytesPerChar
+ || ! XmlNameMatchesAscii(enc, val, val + 2 * enc->minBytesPerChar,
+ KW_1_dot))
+ return false;
+ val += 2 * enc->minBytesPerChar;
+ if (val == valEnd)
+ return false;
+ for (; val != valEnd; val += enc->minBytesPerChar) {
+ int c = toAscii(enc, val, valEnd);
+ if (c < ASCII_0 || c > ASCII_9)
+ return false;
+ }
+ return true;
+}
+
static int
doParseXmlDecl(const ENCODING *(*encodingFinder)(const ENCODING *, const char *,
const char *),
@@ -1175,6 +1083,22 @@ doParseXmlDecl(const ENCODING *(*encodingFinder)(const ENCODING *, const char *,
*versionPtr = val;
if (versionEndPtr)
*versionEndPtr = ptr;
+ /* The version number must not be empty; VersionNum requires at least
+ one character. The encoding and standalone pseudo-attributes below
+ already reject an empty value, so keep version consistent. */
+ if (val == ptr - enc->minBytesPerChar) {
+ *badPtr = val;
+ return 0;
+ }
+ /* Expat implements XML 1.0 only, so any version outside the "1.0"/"1.x"
+ family is rejected. Following the Fifth Edition's VersionNum
+ production (rather than the Fourth Edition's exact "1.0") avoids
+ rejecting "1.1" now only to have to revert that once Expat tracks
+ the newer edition. */
+ if (! checkXmlDeclVersionNum(enc, val, ptr - enc->minBytesPerChar)) {
+ *badPtr = val;
+ return 0;
+ }
if (! parsePseudoAttribute(enc, ptr, end, &name, &nameEnd, &val, &ptr)) {
*badPtr = ptr;
return 0;
@@ -1229,7 +1153,7 @@ doParseXmlDecl(const ENCODING *(*encodingFinder)(const ENCODING *, const char *,
return 1;
}
-static int FASTCALL
+static int
checkCharRefNumber(int result) {
switch (result >> 8) {
case 0xD8:
@@ -1253,7 +1177,7 @@ checkCharRefNumber(int result) {
return result;
}
-int FASTCALL
+int
XmlUtf8Encode(int c, char *buf) {
enum {
/* minN is minimum legal resulting value for N byte sequence */
@@ -1289,7 +1213,7 @@ XmlUtf8Encode(int c, char *buf) {
return 0; /* LCOV_EXCL_LINE: this case too is eliminated before calling */
}
-int FASTCALL
+int
XmlUtf16Encode(int charNum, unsigned short *buf) {
if (charNum < 0)
return 0;
@@ -1321,7 +1245,7 @@ XmlSizeOfUnknownEncoding(void) {
return sizeof(struct unknown_encoding);
}
-static int PTRFASTCALL
+static int
unknown_isName(const ENCODING *enc, const char *p) {
const struct unknown_encoding *uenc = AS_UNKNOWN_ENCODING(enc);
int c = uenc->convert(uenc->userData, p);
@@ -1330,7 +1254,7 @@ unknown_isName(const ENCODING *enc, const char *p) {
return UCS2_GET_NAMING(namePages, c >> 8, c & 0xFF);
}
-static int PTRFASTCALL
+static int
unknown_isNmstrt(const ENCODING *enc, const char *p) {
const struct unknown_encoding *uenc = AS_UNKNOWN_ENCODING(enc);
int c = uenc->convert(uenc->userData, p);
@@ -1339,14 +1263,14 @@ unknown_isNmstrt(const ENCODING *enc, const char *p) {
return UCS2_GET_NAMING(nmstrtPages, c >> 8, c & 0xFF);
}
-static int PTRFASTCALL
+static int
unknown_isInvalid(const ENCODING *enc, const char *p) {
const struct unknown_encoding *uenc = AS_UNKNOWN_ENCODING(enc);
int c = uenc->convert(uenc->userData, p);
return (c & ~0xFFFF) || checkCharRefNumber(c) < 0;
}
-static enum XML_Convert_Result PTRCALL
+static enum XML_Convert_Result
unknown_toUtf8(const ENCODING *enc, const char **fromP, const char *fromLim,
char **toP, const char *toLim) {
const struct unknown_encoding *uenc = AS_UNKNOWN_ENCODING(enc);
@@ -1376,7 +1300,7 @@ unknown_toUtf8(const ENCODING *enc, const char **fromP, const char *fromLim,
}
}
-static enum XML_Convert_Result PTRCALL
+static enum XML_Convert_Result
unknown_toUtf16(const ENCODING *enc, const char **fromP, const char *fromLim,
unsigned short **toP, const unsigned short *toLim) {
const struct unknown_encoding *uenc = AS_UNKNOWN_ENCODING(enc);
@@ -1398,7 +1322,7 @@ unknown_toUtf16(const ENCODING *enc, const char **fromP, const char *fromLim,
}
ENCODING *
-XmlInitUnknownEncoding(void *mem, int *table, CONVERTER convert,
+XmlInitUnknownEncoding(void *mem, const int *table, CONVERTER convert,
void *userData) {
int i;
struct unknown_encoding *e = (struct unknown_encoding *)mem;
@@ -1500,7 +1424,7 @@ static const char KW_UTF_16LE[]
= {ASCII_U, ASCII_T, ASCII_F, ASCII_MINUS, ASCII_1,
ASCII_6, ASCII_L, ASCII_E, '\0'};
-static int FASTCALL
+static int
getEncodingIndex(const char *name) {
static const char *const encodingNames[] = {
KW_ISO_8859_1, KW_US_ASCII, KW_UTF_8, KW_UTF_16, KW_UTF_16BE, KW_UTF_16LE,
@@ -1557,7 +1481,7 @@ initScan(const ENCODING *const *encodingTable, const INIT_ENCODING *enc,
case 0xEF: /* possibly first byte of UTF-8 BOM */
if (INIT_ENC_INDEX(enc) == ISO_8859_1_ENC && state == XML_CONTENT_STATE)
break;
- /* fall through */
+ EXPAT_FALLTHROUGH;
case 0x00:
case 0x3C:
return XML_TOK_PARTIAL;
@@ -1661,7 +1585,7 @@ initScan(const ENCODING *const *encodingTable, const INIT_ENCODING *enc,
# undef ns
ENCODING *
-XmlInitUnknownEncodingNS(void *mem, int *table, CONVERTER convert,
+XmlInitUnknownEncodingNS(void *mem, const int *table, CONVERTER convert,
void *userData) {
ENCODING *enc = XmlInitUnknownEncoding(mem, table, convert, userData);
if (enc)
diff --git a/Modules/expat/xmltok.h b/Modules/expat/xmltok.h
index c51fce1ec1518be..bf641af6d4f77f0 100644
--- a/Modules/expat/xmltok.h
+++ b/Modules/expat/xmltok.h
@@ -10,7 +10,7 @@
Copyright (c) 2000 Clark Cooper <coopercc@users.sourceforge.net>
Copyright (c) 2002 Fred L. Drake, Jr. <fdrake@users.sourceforge.net>
Copyright (c) 2002-2005 Karl Waclawek <karl@waclawek.net>
- Copyright (c) 2016-2024 Sebastian Pipping <sebastian@pipping.org>
+ Copyright (c) 2016-2026 Sebastian Pipping <sebastian@pipping.org>
Copyright (c) 2017 Rhodri James <rhodri@wildebeest.org.uk>
Licensed under the MIT license:
@@ -32,121 +32,121 @@
DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+ SPDX-License-Identifier: MIT
*/
#ifndef XmlTok_INCLUDED
-#define XmlTok_INCLUDED 1
+# define XmlTok_INCLUDED 1
-#ifdef __cplusplus
-extern "C" {
-#endif
+# include <stdint.h> // uint64_t
/* The following token may be returned by XmlContentTok */
-#define XML_TOK_TRAILING_RSQB \
- -5 /* ] or ]] at the end of the scan; might be \
- start of illegal ]]]]><![CDATA[> sequence */
+# define XML_TOK_TRAILING_RSQB \
+ -5 /* ] or ]] at the end of the scan; might be \
+ start of illegal ]]]]><![CDATA[> sequence */
/* The following tokens may be returned by both XmlPrologTok and
XmlContentTok.
*/
-#define XML_TOK_NONE -4 /* The string to be scanned is empty */
-#define XML_TOK_TRAILING_CR \
- -3 /* A CR at the end of the scan; \
- might be part of CRLF sequence */
-#define XML_TOK_PARTIAL_CHAR -2 /* only part of a multibyte sequence */
-#define XML_TOK_PARTIAL -1 /* only part of a token */
-#define XML_TOK_INVALID 0
+# define XML_TOK_NONE -4 /* The string to be scanned is empty */
+# define XML_TOK_TRAILING_CR \
+ -3 /* A CR at the end of the scan; \
+ might be part of CRLF sequence */
+# define XML_TOK_PARTIAL_CHAR -2 /* only part of a multibyte sequence */
+# define XML_TOK_PARTIAL -1 /* only part of a token */
+# define XML_TOK_INVALID 0
/* The following tokens are returned by XmlContentTok; some are also
returned by XmlAttributeValueTok, XmlEntityTok, XmlCdataSectionTok.
*/
-#define XML_TOK_START_TAG_WITH_ATTS 1
-#define XML_TOK_START_TAG_NO_ATTS 2
-#define XML_TOK_EMPTY_ELEMENT_WITH_ATTS 3 /* empty element tag <e/> */
-#define XML_TOK_EMPTY_ELEMENT_NO_ATTS 4
-#define XML_TOK_END_TAG 5
-#define XML_TOK_DATA_CHARS 6
-#define XML_TOK_DATA_NEWLINE 7
-#define XML_TOK_CDATA_SECT_OPEN 8
-#define XML_TOK_ENTITY_REF 9
-#define XML_TOK_CHAR_REF 10 /* numeric character reference */
+# define XML_TOK_START_TAG_WITH_ATTS 1
+# define XML_TOK_START_TAG_NO_ATTS 2
+# define XML_TOK_EMPTY_ELEMENT_WITH_ATTS 3 /* empty element tag <e/> */
+# define XML_TOK_EMPTY_ELEMENT_NO_ATTS 4
+# define XML_TOK_END_TAG 5
+# define XML_TOK_DATA_CHARS 6
+# define XML_TOK_DATA_NEWLINE 7
+# define XML_TOK_CDATA_SECT_OPEN 8
+# define XML_TOK_ENTITY_REF 9
+# define XML_TOK_CHAR_REF 10 /* numeric character reference */
/* The following tokens may be returned by both XmlPrologTok and
XmlContentTok.
*/
-#define XML_TOK_PI 11 /* processing instruction */
-#define XML_TOK_XML_DECL 12 /* XML decl or text decl */
-#define XML_TOK_COMMENT 13
-#define XML_TOK_BOM 14 /* Byte order mark */
+# define XML_TOK_PI 11 /* processing instruction */
+# define XML_TOK_XML_DECL 12 /* XML decl or text decl */
+# define XML_TOK_COMMENT 13
+# define XML_TOK_BOM 14 /* Byte order mark */
/* The following tokens are returned only by XmlPrologTok */
-#define XML_TOK_PROLOG_S 15
-#define XML_TOK_DECL_OPEN 16 /* <!foo */
-#define XML_TOK_DECL_CLOSE 17 /* > */
-#define XML_TOK_NAME 18
-#define XML_TOK_NMTOKEN 19
-#define XML_TOK_POUND_NAME 20 /* #name */
-#define XML_TOK_OR 21 /* | */
-#define XML_TOK_PERCENT 22
-#define XML_TOK_OPEN_PAREN 23
-#define XML_TOK_CLOSE_PAREN 24
-#define XML_TOK_OPEN_BRACKET 25
-#define XML_TOK_CLOSE_BRACKET 26
-#define XML_TOK_LITERAL 27
-#define XML_TOK_PARAM_ENTITY_REF 28
-#define XML_TOK_INSTANCE_START 29
+# define XML_TOK_PROLOG_S 15
+# define XML_TOK_DECL_OPEN 16 /* <!foo */
+# define XML_TOK_DECL_CLOSE 17 /* > */
+# define XML_TOK_NAME 18
+# define XML_TOK_NMTOKEN 19
+# define XML_TOK_POUND_NAME 20 /* #name */
+# define XML_TOK_OR 21 /* | */
+# define XML_TOK_PERCENT 22
+# define XML_TOK_OPEN_PAREN 23
+# define XML_TOK_CLOSE_PAREN 24
+# define XML_TOK_OPEN_BRACKET 25
+# define XML_TOK_CLOSE_BRACKET 26
+# define XML_TOK_LITERAL 27
+# define XML_TOK_PARAM_ENTITY_REF 28
+# define XML_TOK_INSTANCE_START 29
/* The following occur only in element type declarations */
-#define XML_TOK_NAME_QUESTION 30 /* name? */
-#define XML_TOK_NAME_ASTERISK 31 /* name* */
-#define XML_TOK_NAME_PLUS 32 /* name+ */
-#define XML_TOK_COND_SECT_OPEN 33 /* <![ */
-#define XML_TOK_COND_SECT_CLOSE 34 /* ]]]]><![CDATA[> */
-#define XML_TOK_CLOSE_PAREN_QUESTION 35 /* )? */
-#define XML_TOK_CLOSE_PAREN_ASTERISK 36 /* )* */
-#define XML_TOK_CLOSE_PAREN_PLUS 37 /* )+ */
-#define XML_TOK_COMMA 38
+# define XML_TOK_NAME_QUESTION 30 /* name? */
+# define XML_TOK_NAME_ASTERISK 31 /* name* */
+# define XML_TOK_NAME_PLUS 32 /* name+ */
+# define XML_TOK_COND_SECT_OPEN 33 /* <![ */
+# define XML_TOK_COND_SECT_CLOSE 34 /* ]]]]><![CDATA[> */
+# define XML_TOK_CLOSE_PAREN_QUESTION 35 /* )? */
+# define XML_TOK_CLOSE_PAREN_ASTERISK 36 /* )* */
+# define XML_TOK_CLOSE_PAREN_PLUS 37 /* )+ */
+# define XML_TOK_COMMA 38
/* The following token is returned only by XmlAttributeValueTok */
-#define XML_TOK_ATTRIBUTE_VALUE_S 39
+# define XML_TOK_ATTRIBUTE_VALUE_S 39
/* The following token is returned only by XmlCdataSectionTok */
-#define XML_TOK_CDATA_SECT_CLOSE 40
+# define XML_TOK_CDATA_SECT_CLOSE 40
/* With namespace processing this is returned by XmlPrologTok for a
name with a colon.
*/
-#define XML_TOK_PREFIXED_NAME 41
+# define XML_TOK_PREFIXED_NAME 41
-#ifdef XML_DTD
-# define XML_TOK_IGNORE_SECT 42
-#endif /* XML_DTD */
+# ifdef XML_DTD
+# define XML_TOK_IGNORE_SECT 42
+# endif /* XML_DTD */
-#ifdef XML_DTD
-# define XML_N_STATES 4
-#else /* not XML_DTD */
-# define XML_N_STATES 3
-#endif /* not XML_DTD */
+# ifdef XML_DTD
+# define XML_N_STATES 4
+# else /* not XML_DTD */
+# define XML_N_STATES 3
+# endif /* not XML_DTD */
-#define XML_PROLOG_STATE 0
-#define XML_CONTENT_STATE 1
-#define XML_CDATA_SECTION_STATE 2
-#ifdef XML_DTD
-# define XML_IGNORE_SECTION_STATE 3
-#endif /* XML_DTD */
+# define XML_PROLOG_STATE 0
+# define XML_CONTENT_STATE 1
+# define XML_CDATA_SECTION_STATE 2
+# ifdef XML_DTD
+# define XML_IGNORE_SECTION_STATE 3
+# endif /* XML_DTD */
-#define XML_N_LITERAL_TYPES 2
-#define XML_ATTRIBUTE_VALUE_LITERAL 0
-#define XML_ENTITY_VALUE_LITERAL 1
+# define XML_N_LITERAL_TYPES 2
+# define XML_ATTRIBUTE_VALUE_LITERAL 0
+# define XML_ENTITY_VALUE_LITERAL 1
/* The size of the buffer passed to XmlUtf8Encode must be at least this. */
-#define XML_UTF8_ENCODE_MAX 4
+# define XML_UTF8_ENCODE_MAX 4
/* The size of the buffer passed to XmlUtf16Encode must be at least this. */
-#define XML_UTF16_ENCODE_MAX 2
+# define XML_UTF16_ENCODE_MAX 2
typedef struct position {
/* first line and first column are 0 not 1 */
- XML_Size lineNumber;
- XML_Size columnNumber;
+ uint64_t lineNumber;
+ uint64_t columnNumber;
} POSITION;
typedef struct {
@@ -159,41 +159,40 @@ typedef struct {
struct encoding;
typedef struct encoding ENCODING;
-typedef int(PTRCALL *SCANNER)(const ENCODING *, const char *, const char *,
- const char **);
+typedef int (*SCANNER)(const ENCODING *, const char *, const char *,
+ const char **);
enum XML_Convert_Result {
XML_CONVERT_COMPLETED = 0,
XML_CONVERT_INPUT_INCOMPLETE = 1,
- XML_CONVERT_OUTPUT_EXHAUSTED
- = 2 /* and therefore potentially input remaining as well */
+ XML_CONVERT_OUTPUT_EXHAUSTED = 2 /* and therefore potentially input remaining
+ as well */
};
struct encoding {
SCANNER scanners[XML_N_STATES];
SCANNER literalScanners[XML_N_LITERAL_TYPES];
- int(PTRCALL *nameMatchesAscii)(const ENCODING *, const char *, const char *,
- const char *);
- int(PTRFASTCALL *nameLength)(const ENCODING *, const char *);
- const char *(PTRFASTCALL *skipS)(const ENCODING *, const char *);
- int(PTRCALL *getAtts)(const ENCODING *enc, const char *ptr, int attsMax,
- ATTRIBUTE *atts);
- int(PTRFASTCALL *charRefNumber)(const ENCODING *enc, const char *ptr);
- int(PTRCALL *predefinedEntityName)(const ENCODING *, const char *,
- const char *);
- void(PTRCALL *updatePosition)(const ENCODING *, const char *ptr,
- const char *end, POSITION *);
- int(PTRCALL *isPublicId)(const ENCODING *enc, const char *ptr,
- const char *end, const char **badPtr);
- enum XML_Convert_Result(PTRCALL *utf8Convert)(const ENCODING *enc,
- const char **fromP,
- const char *fromLim, char **toP,
- const char *toLim);
- enum XML_Convert_Result(PTRCALL *utf16Convert)(const ENCODING *enc,
- const char **fromP,
- const char *fromLim,
- unsigned short **toP,
- const unsigned short *toLim);
+ int (*nameMatchesAscii)(const ENCODING *, const char *, const char *,
+ const char *);
+ int (*nameLength)(const ENCODING *, const char *);
+ const char *(*skipS)(const ENCODING *, const char *);
+ int (*getAtts)(const ENCODING *enc, const char *ptr, int attsMax,
+ ATTRIBUTE *atts);
+ int (*charRefNumber)(const ENCODING *enc, const char *ptr);
+ int (*predefinedEntityName)(const ENCODING *, const char *, const char *);
+ void (*updatePosition)(const ENCODING *, const char *ptr, const char *end,
+ POSITION *);
+ int (*isPublicId)(const ENCODING *enc, const char *ptr, const char *end,
+ const char **badPtr);
+ enum XML_Convert_Result (*utf8Convert)(const ENCODING *enc,
+ const char **fromP,
+ const char *fromLim, char **toP,
+ const char *toLim);
+ enum XML_Convert_Result (*utf16Convert)(const ENCODING *enc,
+ const char **fromP,
+ const char *fromLim,
+ unsigned short **toP,
+ const unsigned short *toLim);
int minBytesPerChar;
char isUtf8;
char isUtf16;
@@ -220,63 +219,63 @@ struct encoding {
the prolog outside literals, comments and processing instructions.
*/
-#define XmlTok(enc, state, ptr, end, nextTokPtr) \
- (((enc)->scanners[state])(enc, ptr, end, nextTokPtr))
+# define XmlTok(enc, state, ptr, end, nextTokPtr) \
+ (((enc)->scanners[state])(enc, ptr, end, nextTokPtr))
-#define XmlPrologTok(enc, ptr, end, nextTokPtr) \
- XmlTok(enc, XML_PROLOG_STATE, ptr, end, nextTokPtr)
+# define XmlPrologTok(enc, ptr, end, nextTokPtr) \
+ XmlTok(enc, XML_PROLOG_STATE, ptr, end, nextTokPtr)
-#define XmlContentTok(enc, ptr, end, nextTokPtr) \
- XmlTok(enc, XML_CONTENT_STATE, ptr, end, nextTokPtr)
+# define XmlContentTok(enc, ptr, end, nextTokPtr) \
+ XmlTok(enc, XML_CONTENT_STATE, ptr, end, nextTokPtr)
-#define XmlCdataSectionTok(enc, ptr, end, nextTokPtr) \
- XmlTok(enc, XML_CDATA_SECTION_STATE, ptr, end, nextTokPtr)
+# define XmlCdataSectionTok(enc, ptr, end, nextTokPtr) \
+ XmlTok(enc, XML_CDATA_SECTION_STATE, ptr, end, nextTokPtr)
-#ifdef XML_DTD
+# ifdef XML_DTD
-# define XmlIgnoreSectionTok(enc, ptr, end, nextTokPtr) \
- XmlTok(enc, XML_IGNORE_SECTION_STATE, ptr, end, nextTokPtr)
+# define XmlIgnoreSectionTok(enc, ptr, end, nextTokPtr) \
+ XmlTok(enc, XML_IGNORE_SECTION_STATE, ptr, end, nextTokPtr)
-#endif /* XML_DTD */
+# endif /* XML_DTD */
/* This is used for performing a 2nd-level tokenization on the content
of a literal that has already been returned by XmlTok.
*/
-#define XmlLiteralTok(enc, literalType, ptr, end, nextTokPtr) \
- (((enc)->literalScanners[literalType])(enc, ptr, end, nextTokPtr))
+# define XmlLiteralTok(enc, literalType, ptr, end, nextTokPtr) \
+ (((enc)->literalScanners[literalType])(enc, ptr, end, nextTokPtr))
-#define XmlAttributeValueTok(enc, ptr, end, nextTokPtr) \
- XmlLiteralTok(enc, XML_ATTRIBUTE_VALUE_LITERAL, ptr, end, nextTokPtr)
+# define XmlAttributeValueTok(enc, ptr, end, nextTokPtr) \
+ XmlLiteralTok(enc, XML_ATTRIBUTE_VALUE_LITERAL, ptr, end, nextTokPtr)
-#define XmlEntityValueTok(enc, ptr, end, nextTokPtr) \
- XmlLiteralTok(enc, XML_ENTITY_VALUE_LITERAL, ptr, end, nextTokPtr)
+# define XmlEntityValueTok(enc, ptr, end, nextTokPtr) \
+ XmlLiteralTok(enc, XML_ENTITY_VALUE_LITERAL, ptr, end, nextTokPtr)
-#define XmlNameMatchesAscii(enc, ptr1, end1, ptr2) \
- (((enc)->nameMatchesAscii)(enc, ptr1, end1, ptr2))
+# define XmlNameMatchesAscii(enc, ptr1, end1, ptr2) \
+ (((enc)->nameMatchesAscii)(enc, ptr1, end1, ptr2))
-#define XmlNameLength(enc, ptr) (((enc)->nameLength)(enc, ptr))
+# define XmlNameLength(enc, ptr) (((enc)->nameLength)(enc, ptr))
-#define XmlSkipS(enc, ptr) (((enc)->skipS)(enc, ptr))
+# define XmlSkipS(enc, ptr) (((enc)->skipS)(enc, ptr))
-#define XmlGetAttributes(enc, ptr, attsMax, atts) \
- (((enc)->getAtts)(enc, ptr, attsMax, atts))
+# define XmlGetAttributes(enc, ptr, attsMax, atts) \
+ (((enc)->getAtts)(enc, ptr, attsMax, atts))
-#define XmlCharRefNumber(enc, ptr) (((enc)->charRefNumber)(enc, ptr))
+# define XmlCharRefNumber(enc, ptr) (((enc)->charRefNumber)(enc, ptr))
-#define XmlPredefinedEntityName(enc, ptr, end) \
- (((enc)->predefinedEntityName)(enc, ptr, end))
+# define XmlPredefinedEntityName(enc, ptr, end) \
+ (((enc)->predefinedEntityName)(enc, ptr, end))
-#define XmlUpdatePosition(enc, ptr, end, pos) \
- (((enc)->updatePosition)(enc, ptr, end, pos))
+# define XmlUpdatePosition(enc, ptr, end, pos) \
+ (((enc)->updatePosition)(enc, ptr, end, pos))
-#define XmlIsPublicId(enc, ptr, end, badPtr) \
- (((enc)->isPublicId)(enc, ptr, end, badPtr))
+# define XmlIsPublicId(enc, ptr, end, badPtr) \
+ (((enc)->isPublicId)(enc, ptr, end, badPtr))
-#define XmlUtf8Convert(enc, fromP, fromLim, toP, toLim) \
- (((enc)->utf8Convert)(enc, fromP, fromLim, toP, toLim))
+# define XmlUtf8Convert(enc, fromP, fromLim, toP, toLim) \
+ (((enc)->utf8Convert)(enc, fromP, fromLim, toP, toLim))
-#define XmlUtf16Convert(enc, fromP, fromLim, toP, toLim) \
- (((enc)->utf16Convert)(enc, fromP, fromLim, toP, toLim))
+# define XmlUtf16Convert(enc, fromP, fromLim, toP, toLim) \
+ (((enc)->utf16Convert)(enc, fromP, fromLim, toP, toLim))
typedef struct {
ENCODING initEnc;
@@ -293,13 +292,13 @@ int XmlInitEncoding(INIT_ENCODING *p, const ENCODING **encPtr,
const char *name);
const ENCODING *XmlGetUtf8InternalEncoding(void);
const ENCODING *XmlGetUtf16InternalEncoding(void);
-int FASTCALL XmlUtf8Encode(int charNumber, char *buf);
-int FASTCALL XmlUtf16Encode(int charNumber, unsigned short *buf);
+int XmlUtf8Encode(int charNumber, char *buf);
+int XmlUtf16Encode(int charNumber, unsigned short *buf);
int XmlSizeOfUnknownEncoding(void);
typedef int(XMLCALL *CONVERTER)(void *userData, const char *p);
-ENCODING *XmlInitUnknownEncoding(void *mem, int *table, CONVERTER convert,
+ENCODING *XmlInitUnknownEncoding(void *mem, const int *table, CONVERTER convert,
void *userData);
int XmlParseXmlDeclNS(int isGeneralTextEntity, const ENCODING *enc,
@@ -312,10 +311,7 @@ int XmlInitEncodingNS(INIT_ENCODING *p, const ENCODING **encPtr,
const char *name);
const ENCODING *XmlGetUtf8InternalEncodingNS(void);
const ENCODING *XmlGetUtf16InternalEncodingNS(void);
-ENCODING *XmlInitUnknownEncodingNS(void *mem, int *table, CONVERTER convert,
- void *userData);
-#ifdef __cplusplus
-}
-#endif
+ENCODING *XmlInitUnknownEncodingNS(void *mem, const int *table,
+ CONVERTER convert, void *userData);
#endif /* not XmlTok_INCLUDED */
diff --git a/Modules/expat/xmltok_impl.c b/Modules/expat/xmltok_impl.c
index 239a2d06c4512ce..7900e94701e2768 100644
--- a/Modules/expat/xmltok_impl.c
+++ b/Modules/expat/xmltok_impl.c
@@ -1,4 +1,4 @@
-/* This file is included (from xmltok.c, 1-3 times depending on XML_MIN_SIZE)!
+/* This file is included (from xmltok.c, 3 times)!
__ __ _
___\ \/ /_ __ __ _| |_
/ _ \\ /| '_ \ / _` | __|
@@ -10,13 +10,15 @@
Copyright (c) 2000 Clark Cooper <coopercc@users.sourceforge.net>
Copyright (c) 2002 Fred L. Drake, Jr. <fdrake@users.sourceforge.net>
Copyright (c) 2002-2016 Karl Waclawek <karl@waclawek.net>
- Copyright (c) 2016-2022 Sebastian Pipping <sebastian@pipping.org>
+ Copyright (c) 2016-2026 Sebastian Pipping <sebastian@pipping.org>
Copyright (c) 2017 Rhodri James <rhodri@wildebeest.org.uk>
Copyright (c) 2018 Benjamin Peterson <benjamin@python.org>
Copyright (c) 2018 Anton Maklakov <antmak.pub@gmail.com>
Copyright (c) 2019 David Loffredo <loffredo@steptools.com>
Copyright (c) 2020 Boris Kolpackov <boris@codesynthesis.com>
Copyright (c) 2022 Martin Ettl <ettl.martin78@googlemail.com>
+ Copyright (c) 2026 Nick Begg <nick@stunttruck.net>
+ Copyright (c) 2026 Kartik Kenchi <netliomax25@gmail.com>
Licensed under the MIT license:
Permission is hereby granted, free of charge, to any person obtaining
@@ -37,14 +39,12 @@
DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+ SPDX-License-Identifier: MIT
*/
#ifdef XML_TOK_IMPL_C
-# ifndef IS_INVALID_CHAR // i.e. for UTF-16 and XML_MIN_SIZE not defined
-# define IS_INVALID_CHAR(enc, ptr, n) (0)
-# endif
-
# define INVALID_LEAD_CASE(n, ptr, nextTokPtr) \
case BT_LEAD##n: \
if (end - ptr < n) \
@@ -83,7 +83,7 @@
*nextTokPtr = ptr; \
return XML_TOK_INVALID; \
} \
- /* fall through */ \
+ EXPAT_FALLTHROUGH; \
case BT_NMSTRT: \
case BT_HEX: \
case BT_DIGIT: \
@@ -112,7 +112,7 @@
*nextTokPtr = ptr; \
return XML_TOK_INVALID; \
} \
- /* fall through */ \
+ EXPAT_FALLTHROUGH; \
case BT_NMSTRT: \
case BT_HEX: \
ptr += MINBPC(enc); \
@@ -141,7 +141,7 @@
/* ptr points to character following "<!-" */
-static int PTRCALL
+static int
PREFIX(scanComment)(const ENCODING *enc, const char *ptr, const char *end,
const char **nextTokPtr) {
if (HAS_CHAR(enc, ptr, end)) {
@@ -178,7 +178,7 @@ PREFIX(scanComment)(const ENCODING *enc, const char *ptr, const char *end,
/* ptr points to character following "<!" */
-static int PTRCALL
+static int
PREFIX(scanDecl)(const ENCODING *enc, const char *ptr, const char *end,
const char **nextTokPtr) {
REQUIRE_CHAR(enc, ptr, end);
@@ -209,7 +209,7 @@ PREFIX(scanDecl)(const ENCODING *enc, const char *ptr, const char *end,
*nextTokPtr = ptr;
return XML_TOK_INVALID;
}
- /* fall through */
+ EXPAT_FALLTHROUGH;
case BT_S:
case BT_CR:
case BT_LF:
@@ -227,7 +227,7 @@ PREFIX(scanDecl)(const ENCODING *enc, const char *ptr, const char *end,
return XML_TOK_PARTIAL;
}
-static int PTRCALL
+static int
PREFIX(checkPiTarget)(const ENCODING *enc, const char *ptr, const char *end,
int *tokPtr) {
int upper = 0;
@@ -272,7 +272,7 @@ PREFIX(checkPiTarget)(const ENCODING *enc, const char *ptr, const char *end,
/* ptr points to character following "<?" */
-static int PTRCALL
+static int
PREFIX(scanPi)(const ENCODING *enc, const char *ptr, const char *end,
const char **nextTokPtr) {
int tok;
@@ -323,7 +323,7 @@ PREFIX(scanPi)(const ENCODING *enc, const char *ptr, const char *end,
*nextTokPtr = ptr + MINBPC(enc);
return tok;
}
- /* fall through */
+ EXPAT_FALLTHROUGH;
default:
*nextTokPtr = ptr;
return XML_TOK_INVALID;
@@ -332,7 +332,7 @@ PREFIX(scanPi)(const ENCODING *enc, const char *ptr, const char *end,
return XML_TOK_PARTIAL;
}
-static int PTRCALL
+static int
PREFIX(scanCdataSection)(const ENCODING *enc, const char *ptr, const char *end,
const char **nextTokPtr) {
static const char CDATA_LSQB[]
@@ -351,7 +351,7 @@ PREFIX(scanCdataSection)(const ENCODING *enc, const char *ptr, const char *end,
return XML_TOK_CDATA_SECT_OPEN;
}
-static int PTRCALL
+static int
PREFIX(cdataSectionTok)(const ENCODING *enc, const char *ptr, const char *end,
const char **nextTokPtr) {
if (ptr >= end)
@@ -427,7 +427,7 @@ PREFIX(cdataSectionTok)(const ENCODING *enc, const char *ptr, const char *end,
/* ptr points to character following "</" */
-static int PTRCALL
+static int
PREFIX(scanEndTag)(const ENCODING *enc, const char *ptr, const char *end,
const char **nextTokPtr) {
REQUIRE_CHAR(enc, ptr, end);
@@ -478,7 +478,7 @@ PREFIX(scanEndTag)(const ENCODING *enc, const char *ptr, const char *end,
/* ptr points to character following "&#X" */
-static int PTRCALL
+static int
PREFIX(scanHexCharRef)(const ENCODING *enc, const char *ptr, const char *end,
const char **nextTokPtr) {
if (HAS_CHAR(enc, ptr, end)) {
@@ -509,7 +509,7 @@ PREFIX(scanHexCharRef)(const ENCODING *enc, const char *ptr, const char *end,
/* ptr points to character following "&#" */
-static int PTRCALL
+static int
PREFIX(scanCharRef)(const ENCODING *enc, const char *ptr, const char *end,
const char **nextTokPtr) {
if (HAS_CHAR(enc, ptr, end)) {
@@ -540,7 +540,7 @@ PREFIX(scanCharRef)(const ENCODING *enc, const char *ptr, const char *end,
/* ptr points to character following "&" */
-static int PTRCALL
+static int
PREFIX(scanRef)(const ENCODING *enc, const char *ptr, const char *end,
const char **nextTokPtr) {
REQUIRE_CHAR(enc, ptr, end);
@@ -568,7 +568,7 @@ PREFIX(scanRef)(const ENCODING *enc, const char *ptr, const char *end,
/* ptr points to character following first character of attribute name */
-static int PTRCALL
+static int
PREFIX(scanAtts)(const ENCODING *enc, const char *ptr, const char *end,
const char **nextTokPtr) {
# ifdef XML_NS
@@ -615,7 +615,7 @@ PREFIX(scanAtts)(const ENCODING *enc, const char *ptr, const char *end,
return XML_TOK_INVALID;
}
}
- /* fall through */
+ EXPAT_FALLTHROUGH;
case BT_EQUALS: {
int open;
# ifdef XML_NS
@@ -721,7 +721,7 @@ PREFIX(scanAtts)(const ENCODING *enc, const char *ptr, const char *end,
/* ptr points to character following "<" */
-static int PTRCALL
+static int
PREFIX(scanLt)(const ENCODING *enc, const char *ptr, const char *end,
const char **nextTokPtr) {
# ifdef XML_NS
@@ -819,7 +819,7 @@ PREFIX(scanLt)(const ENCODING *enc, const char *ptr, const char *end,
return XML_TOK_PARTIAL;
}
-static int PTRCALL
+static int
PREFIX(contentTok)(const ENCODING *enc, const char *ptr, const char *end,
const char **nextTokPtr) {
if (ptr >= end)
@@ -898,7 +898,7 @@ PREFIX(contentTok)(const ENCODING *enc, const char *ptr, const char *end,
return XML_TOK_INVALID;
}
}
- /* fall through */
+ EXPAT_FALLTHROUGH;
case BT_AMP:
case BT_LT:
case BT_NONXML:
@@ -919,7 +919,7 @@ PREFIX(contentTok)(const ENCODING *enc, const char *ptr, const char *end,
/* ptr points to character following "%" */
-static int PTRCALL
+static int
PREFIX(scanPercent)(const ENCODING *enc, const char *ptr, const char *end,
const char **nextTokPtr) {
REQUIRE_CHAR(enc, ptr, end);
@@ -949,7 +949,7 @@ PREFIX(scanPercent)(const ENCODING *enc, const char *ptr, const char *end,
return XML_TOK_PARTIAL;
}
-static int PTRCALL
+static int
PREFIX(scanPoundName)(const ENCODING *enc, const char *ptr, const char *end,
const char **nextTokPtr) {
REQUIRE_CHAR(enc, ptr, end);
@@ -979,7 +979,7 @@ PREFIX(scanPoundName)(const ENCODING *enc, const char *ptr, const char *end,
return -XML_TOK_POUND_NAME;
}
-static int PTRCALL
+static int
PREFIX(scanLit)(int open, const ENCODING *enc, const char *ptr, const char *end,
const char **nextTokPtr) {
while (HAS_CHAR(enc, ptr, end)) {
@@ -1013,7 +1013,7 @@ PREFIX(scanLit)(int open, const ENCODING *enc, const char *ptr, const char *end,
return XML_TOK_PARTIAL;
}
-static int PTRCALL
+static int
PREFIX(prologTok)(const ENCODING *enc, const char *ptr, const char *end,
const char **nextTokPtr) {
int tok;
@@ -1059,7 +1059,7 @@ PREFIX(prologTok)(const ENCODING *enc, const char *ptr, const char *end,
/* indicate that this might be part of a CR/LF pair */
return -XML_TOK_PROLOG_S;
}
- /* fall through */
+ EXPAT_FALLTHROUGH;
case BT_S:
case BT_LF:
for (;;) {
@@ -1074,7 +1074,7 @@ PREFIX(prologTok)(const ENCODING *enc, const char *ptr, const char *end,
/* don't split CR/LF pair */
if (ptr + MINBPC(enc) != end)
break;
- /* fall through */
+ EXPAT_FALLTHROUGH;
default:
*nextTokPtr = ptr;
return XML_TOK_PROLOG_S;
@@ -1189,7 +1189,7 @@ PREFIX(prologTok)(const ENCODING *enc, const char *ptr, const char *end,
tok = XML_TOK_NMTOKEN;
break;
}
- /* fall through */
+ EXPAT_FALLTHROUGH;
default:
*nextTokPtr = ptr;
return XML_TOK_INVALID;
@@ -1257,7 +1257,7 @@ PREFIX(prologTok)(const ENCODING *enc, const char *ptr, const char *end,
return -tok;
}
-static int PTRCALL
+static int
PREFIX(attributeValueTok)(const ENCODING *enc, const char *ptr, const char *end,
const char **nextTokPtr) {
const char *start;
@@ -1326,7 +1326,7 @@ PREFIX(attributeValueTok)(const ENCODING *enc, const char *ptr, const char *end,
return XML_TOK_DATA_CHARS;
}
-static int PTRCALL
+static int
PREFIX(entityValueTok)(const ENCODING *enc, const char *ptr, const char *end,
const char **nextTokPtr) {
const char *start;
@@ -1393,7 +1393,7 @@ PREFIX(entityValueTok)(const ENCODING *enc, const char *ptr, const char *end,
# ifdef XML_DTD
-static int PTRCALL
+static int
PREFIX(ignoreSectionTok)(const ENCODING *enc, const char *ptr, const char *end,
const char **nextTokPtr) {
int level = 0;
@@ -1445,7 +1445,7 @@ PREFIX(ignoreSectionTok)(const ENCODING *enc, const char *ptr, const char *end,
# endif /* XML_DTD */
-static int PTRCALL
+static int
PREFIX(isPublicId)(const ENCODING *enc, const char *ptr, const char *end,
const char **badPtr) {
ptr += MINBPC(enc);
@@ -1484,7 +1484,7 @@ PREFIX(isPublicId)(const ENCODING *enc, const char *ptr, const char *end,
case BT_NMSTRT:
if (! (BYTE_TO_ASCII(enc, ptr) & ~0x7f))
break;
- /* fall through */
+ EXPAT_FALLTHROUGH;
default:
switch (BYTE_TO_ASCII(enc, ptr)) {
case 0x24: /* $ */
@@ -1505,7 +1505,7 @@ PREFIX(isPublicId)(const ENCODING *enc, const char *ptr, const char *end,
first attsMax attributes are stored in atts.
*/
-static int PTRCALL
+static int
PREFIX(getAtts)(const ENCODING *enc, const char *ptr, int attsMax,
ATTRIBUTE *atts) {
enum { other, inName, inValue } state = inName;
@@ -1598,7 +1598,7 @@ PREFIX(getAtts)(const ENCODING *enc, const char *ptr, int attsMax,
/* not reached */
}
-static int PTRFASTCALL
+static int
PREFIX(charRefNumber)(const ENCODING *enc, const char *ptr) {
int result = 0;
/* skip &# */
@@ -1656,7 +1656,7 @@ PREFIX(charRefNumber)(const ENCODING *enc, const char *ptr) {
return checkCharRefNumber(result);
}
-static int PTRCALL
+static int
PREFIX(predefinedEntityName)(const ENCODING *enc, const char *ptr,
const char *end) {
UNUSED_P(enc);
@@ -1710,7 +1710,7 @@ PREFIX(predefinedEntityName)(const ENCODING *enc, const char *ptr,
return 0;
}
-static int PTRCALL
+static int
PREFIX(nameMatchesAscii)(const ENCODING *enc, const char *ptr1,
const char *end1, const char *ptr2) {
UNUSED_P(enc);
@@ -1729,7 +1729,7 @@ PREFIX(nameMatchesAscii)(const ENCODING *enc, const char *ptr1,
return ptr1 == end1;
}
-static int PTRFASTCALL
+static int
PREFIX(nameLength)(const ENCODING *enc, const char *ptr) {
const char *start = ptr;
for (;;) {
@@ -1759,7 +1759,7 @@ PREFIX(nameLength)(const ENCODING *enc, const char *ptr) {
}
}
-static const char *PTRFASTCALL
+static const char *
PREFIX(skipS)(const ENCODING *enc, const char *ptr) {
for (;;) {
switch (BYTE_TYPE(enc, ptr)) {
@@ -1774,7 +1774,7 @@ PREFIX(skipS)(const ENCODING *enc, const char *ptr) {
}
}
-static void PTRCALL
+static void
PREFIX(updatePosition)(const ENCODING *enc, const char *ptr, const char *end,
POSITION *pos) {
while (HAS_CHAR(enc, ptr, end)) {
diff --git a/Modules/expat/xmltok_impl.h b/Modules/expat/xmltok_impl.h
index 3469c4ae138c95a..13a08899da3b1a5 100644
--- a/Modules/expat/xmltok_impl.h
+++ b/Modules/expat/xmltok_impl.h
@@ -29,6 +29,8 @@
DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+ SPDX-License-Identifier: MIT
*/
enum {
diff --git a/Modules/expat/xmltok_ns.c b/Modules/expat/xmltok_ns.c
index fbdd3e3c7b79996..d9f69abb92ef39f 100644
--- a/Modules/expat/xmltok_ns.c
+++ b/Modules/expat/xmltok_ns.c
@@ -11,7 +11,8 @@
Copyright (c) 2002 Greg Stein <gstein@users.sourceforge.net>
Copyright (c) 2002 Fred L. Drake, Jr. <fdrake@users.sourceforge.net>
Copyright (c) 2002-2006 Karl Waclawek <karl@waclawek.net>
- Copyright (c) 2017-2021 Sebastian Pipping <sebastian@pipping.org>
+ Copyright (c) 2017-2026 Sebastian Pipping <sebastian@pipping.org>
+ Copyright (c) 2025 Alfonso Gregory <gfunni234@gmail.com>
Licensed under the MIT license:
Permission is hereby granted, free of charge, to any person obtaining
@@ -32,6 +33,8 @@
DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+ SPDX-License-Identifier: MIT
*/
#ifdef XML_TOK_NS_C
@@ -61,14 +64,14 @@ static const ENCODING *const NS(encodings)[] = {
&ns(utf8_encoding).enc /* NO_ENC */
};
-static int PTRCALL
+static int
NS(initScanProlog)(const ENCODING *enc, const char *ptr, const char *end,
const char **nextTokPtr) {
return initScan(NS(encodings), (const INIT_ENCODING *)enc, XML_PROLOG_STATE,
ptr, end, nextTokPtr);
}
-static int PTRCALL
+static int
NS(initScanContent)(const ENCODING *enc, const char *ptr, const char *end,
const char **nextTokPtr) {
return initScan(NS(encodings), (const INIT_ENCODING *)enc, XML_CONTENT_STATE,
@@ -98,13 +101,13 @@ NS(findEncoding)(const ENCODING *enc, const char *ptr, const char *end) {
int i;
XmlUtf8Convert(enc, &ptr, end, &p, p + ENCODING_MAX - 1);
if (ptr != end)
- return 0;
+ return NULL;
*p = 0;
if (streqci(buf, KW_UTF_16) && enc->minBytesPerChar == 2)
return enc;
i = getEncodingIndex(buf);
if (i == UNKNOWN_ENC)
- return 0;
+ return NULL;
return NS(encodings)[i];
}
diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c
index 43d4302b92de756..dd421a9eb70cfbf 100644
--- a/Modules/posixmodule.c
+++ b/Modules/posixmodule.c
@@ -27,6 +27,11 @@
#include "Python.h"
#include "pythread.h"
#include "structmember.h"
+
+#ifdef MS_WINDOWS
+# include <aclapi.h> // SetEntriesInAcl
+# include <sddl.h> // SDDL_REVISION_1
+#endif
#ifndef MS_WINDOWS
#include "posixmodule.h"
#else
@@ -3908,22 +3913,55 @@ If dir_fd is not None, it should be a file descriptor open to a directory,
dir_fd may not be implemented on your platform.
If it is unavailable, using it will raise a NotImplementedError.
-The mode argument is ignored on Windows.
+On Windows, a mode of 0o700 restricts access to the new directory to the
+current user and administrators; other values of mode are ignored.
[clinic start generated code]*/
static PyObject *
os_mkdir_impl(PyObject *module, path_t *path, int mode, int dir_fd)
-/*[clinic end generated code: output=a70446903abe821f input=e965f68377e9b1ce]*/
+/*[clinic end generated code: output=a70446903abe821f input=a50163eb81c6870b]*/
{
int result;
+#ifdef MS_WINDOWS
+ int error = 0;
+ SECURITY_ATTRIBUTES secAttr = { sizeof(secAttr) };
+ SECURITY_ATTRIBUTES *pSecAttr = NULL;
+#endif
#ifdef MS_WINDOWS
Py_BEGIN_ALLOW_THREADS
- result = CreateDirectoryW(path->wide, NULL);
+ if (mode == 0700 /* 0o700 */) {
+ ULONG sdSize;
+ pSecAttr = &secAttr;
+ // Set a discretionary ACL (D) that is protected (P) and includes
+ // inheritable (OICI) entries that allow (A) full control (FA) to
+ // SYSTEM (SY), Administrators (BA), and the owner (OW).
+ if (!ConvertStringSecurityDescriptorToSecurityDescriptorW(
+ L"D:P(A;OICI;FA;;;SY)(A;OICI;FA;;;BA)(A;OICI;FA;;;OW)",
+ SDDL_REVISION_1,
+ &secAttr.lpSecurityDescriptor,
+ &sdSize
+ )) {
+ error = GetLastError();
+ }
+ }
+ if (!error) {
+ result = CreateDirectoryW(path->wide, pSecAttr);
+ if (secAttr.lpSecurityDescriptor &&
+ // uncommonly, LocalFree returns non-zero on error, but still uses
+ // GetLastError() to see what the error code is
+ LocalFree(secAttr.lpSecurityDescriptor)) {
+ error = GetLastError();
+ }
+ }
Py_END_ALLOW_THREADS
- if (!result)
+ if (error) {
+ return PyErr_SetFromWindowsErr(error);
+ }
+ if (!result) {
return path_error(path);
+ }
#else
Py_BEGIN_ALLOW_THREADS
#if HAVE_MKDIRAT
diff --git a/Modules/pyexpat.c b/Modules/pyexpat.c
index ab3dac6db168404..2dcc50e889829bb 100644
--- a/Modules/pyexpat.c
+++ b/Modules/pyexpat.c
@@ -1,6 +1,7 @@
#include "Python.h"
#include <ctype.h>
+#include <stdbool.h>
#include "frameobject.h"
#include "expat.h"
@@ -15,6 +16,23 @@ module pyexpat
#define XML_COMBINED_VERSION (10000*XML_MAJOR_VERSION+100*XML_MINOR_VERSION+XML_MICRO_VERSION)
+/* 3.7 has no pycore compiler-diagnostic macros (added in 3.10). */
+#ifndef _Py_COMP_DIAG_PUSH
+# define _Py_COMP_DIAG_PUSH
+# define _Py_COMP_DIAG_IGNORE_DEPR_DECLS
+# define _Py_COMP_DIAG_POP
+#endif
+
+#if XML_COMBINED_VERSION >= 20900
+# define EXPAT_GetCurrentLineNumber XML_GetCurrentLineNumber64
+# define EXPAT_GetCurrentColumnNumber XML_GetCurrentColumnNumber64
+# define EXPAT_GetCurrentByteIndex XML_GetCurrentByteIndex64
+#else
+# define EXPAT_GetCurrentLineNumber XML_GetCurrentLineNumber
+# define EXPAT_GetCurrentColumnNumber XML_GetCurrentColumnNumber
+# define EXPAT_GetCurrentByteIndex XML_GetCurrentByteIndex
+#endif
+
static XML_Memory_Handling_Suite ExpatMemoryHandler = {
PyObject_Malloc, PyObject_Realloc, PyObject_Free};
@@ -56,6 +74,15 @@ typedef struct {
PyObject_HEAD
XML_Parser itself;
+ /*
+ * Strong reference to a parent `xmlparseobject` if this parser
+ * is a child parser. Set to NULL if this parser is a root parser.
+ * This is needed to keep the parent parser alive as long as it has
+ * at least one child parser.
+ *
+ * See https://github.com/python/cpython/issues/139400 for details.
+ */
+ PyObject *parent;
int ordered_attributes; /* Return attributes as a list. */
int specified_attributes; /* Report only specified attributes. */
int in_callback; /* Is a callback active? */
@@ -64,6 +91,12 @@ typedef struct {
/* NULL if not enabled */
int buffer_size; /* Size of buffer, in XML_Char units */
int buffer_used; /* Buffer units in use */
+ bool reparse_deferral_enabled; /* Whether to defer reparsing of
+ unfinished XML tokens; a de-facto cache of
+ what Expat has the authority on, for lack
+ of a getter API function
+ "XML_GetReparseDeferralEnabled" in Expat
+ 2.6.0 */
PyObject *intern; /* Dictionary to intern strings */
PyObject **handlers;
} xmlparseobject;
@@ -103,6 +136,16 @@ set_error_attr(PyObject *err, const char *name, int value)
return 1;
}
+static int
+set_error_attr_location(PyObject *err, const char *name,
+ unsigned long long value)
+{
+ PyObject *v = PyLong_FromUnsignedLongLong(value);
+ int ok = v != NULL && PyObject_SetAttrString(err, name, v) != -1;
+ Py_XDECREF(v);
+ return ok;
+}
+
/* Build and set an Expat exception, including positioning
* information. Always returns NULL.
*/
@@ -112,10 +155,10 @@ set_error(xmlparseobject *self, enum XML_Error code)
PyObject *err;
PyObject *buffer;
XML_Parser parser = self->itself;
- int lineno = XML_GetErrorLineNumber(parser);
- int column = XML_GetErrorColumnNumber(parser);
+ unsigned long long lineno = EXPAT_GetCurrentLineNumber(parser);
+ unsigned long long column = EXPAT_GetCurrentColumnNumber(parser);
- buffer = PyUnicode_FromFormat("%s: line %i, column %i",
+ buffer = PyUnicode_FromFormat("%s: line %llu, column %llu",
XML_ErrorString(code), lineno, column);
if (buffer == NULL)
return NULL;
@@ -123,8 +166,8 @@ set_error(xmlparseobject *self, enum XML_Error code)
Py_DECREF(buffer);
if ( err != NULL
&& set_error_attr(err, "code", code)
- && set_error_attr(err, "offset", column)
- && set_error_attr(err, "lineno", lineno)) {
+ && set_error_attr_location(err, "offset", column)
+ && set_error_attr_location(err, "lineno", lineno)) {
PyErr_SetObject(ErrorObject, err);
}
Py_XDECREF(err);
@@ -518,6 +561,10 @@ static PyObject *
conv_content_model(XML_Content * const model,
PyObject *(*conv_string)(const XML_Char *))
{
+ if (Py_EnterRecursiveCall(" in conv_content_model")) {
+ return NULL;
+ }
+
PyObject *result = NULL;
PyObject *children = PyTuple_New(model->numchildren);
int i;
@@ -529,7 +576,7 @@ conv_content_model(XML_Content * const model,
conv_string);
if (child == NULL) {
Py_XDECREF(children);
- return NULL;
+ goto done;
}
PyTuple_SET_ITEM(children, i, child);
}
@@ -537,6 +584,8 @@ conv_content_model(XML_Content * const model,
model->type, model->quant,
conv_string,model->name, children);
}
+done:
+ Py_LeaveRecursiveCall();
return result;
}
@@ -705,6 +754,40 @@ get_parse_result(xmlparseobject *self, int rv)
#define MAX_CHUNK_SIZE (1 << 20)
+/*[clinic input]
+pyexpat.xmlparser.SetReparseDeferralEnabled
+
+ enabled: bool
+ /
+
+Enable/Disable reparse deferral; enabled by default with Expat >=2.6.0.
+[clinic start generated code]*/
+
+static PyObject *
+pyexpat_xmlparser_SetReparseDeferralEnabled_impl(xmlparseobject *self,
+ int enabled)
+/*[clinic end generated code: output=5ec539e3b63c8c49 input=021eb9e0bafc32c5]*/
+{
+#if XML_COMBINED_VERSION >= 20600
+ XML_SetReparseDeferralEnabled(self->itself, enabled ? XML_TRUE : XML_FALSE);
+ self->reparse_deferral_enabled = (bool)enabled;
+#endif
+ Py_RETURN_NONE;
+}
+
+/*[clinic input]
+pyexpat.xmlparser.GetReparseDeferralEnabled
+
+Retrieve reparse deferral enabled status; always returns false with Expat <2.6.0.
+[clinic start generated code]*/
+
+static PyObject *
+pyexpat_xmlparser_GetReparseDeferralEnabled_impl(xmlparseobject *self)
+/*[clinic end generated code: output=4e91312e88a595a8 input=54b5f11d32b20f3e]*/
+{
+ return PyBool_FromLong(self->reparse_deferral_enabled);
+}
+
/*[clinic input]
pyexpat.xmlparser.Parse
@@ -897,13 +980,20 @@ pyexpat_xmlparser_GetInputContext_impl(xmlparseobject *self)
/*[clinic end generated code: output=a88026d683fc22cc input=034df8712db68379]*/
{
if (self->in_callback) {
+#if XML_COMBINED_VERSION >= 20900
+ int64_t offset;
+ uint64_t size;
+ const char *buffer
+ = XML_GetInputContext64(self->itself, &offset, &size);
+#else
int offset, size;
const char *buffer
= XML_GetInputContext(self->itself, &offset, &size);
+#endif
if (buffer != NULL)
return PyBytes_FromStringAndSize(buffer + offset,
- size - offset);
+ (Py_ssize_t)(size - offset));
else
Py_RETURN_NONE;
}
@@ -933,6 +1023,12 @@ pyexpat_xmlparser_ExternalEntityParserCreate_impl(xmlparseobject *self,
new_parser = PyObject_GC_New(xmlparseobject, &Xmlparsetype);
if (new_parser == NULL)
return NULL;
+
+ // The new subparser will make use of the parent XML_Parser inside of Expat.
+ // So we need to take subparsers into account with the reference counting
+ // of their parent parser.
+ Py_INCREF(self);
+
new_parser->buffer_size = self->buffer_size;
new_parser->buffer_used = 0;
new_parser->buffer = NULL;
@@ -942,6 +1038,7 @@ pyexpat_xmlparser_ExternalEntityParserCreate_impl(xmlparseobject *self,
new_parser->ns_prefixes = self->ns_prefixes;
new_parser->itself = XML_ExternalEntityParserCreate(self->itself, context,
encoding);
+ new_parser->parent = (PyObject *)self;
new_parser->handlers = 0;
new_parser->intern = self->intern;
Py_XINCREF(new_parser->intern);
@@ -951,11 +1048,13 @@ pyexpat_xmlparser_ExternalEntityParserCreate_impl(xmlparseobject *self,
new_parser->buffer = PyMem_Malloc(new_parser->buffer_size);
if (new_parser->buffer == NULL) {
Py_DECREF(new_parser);
+ Py_DECREF(self);
return PyErr_NoMemory();
}
}
if (!new_parser->itself) {
Py_DECREF(new_parser);
+ Py_DECREF(self);
return PyErr_NoMemory();
}
@@ -968,6 +1067,7 @@ pyexpat_xmlparser_ExternalEntityParserCreate_impl(xmlparseobject *self,
new_parser->handlers = PyMem_New(PyObject *, i);
if (!new_parser->handlers) {
Py_DECREF(new_parser);
+ Py_DECREF(self);
return PyErr_NoMemory();
}
clear_handlers(new_parser, 1);
@@ -1099,6 +1199,8 @@ static struct PyMethodDef xmlparse_methods[] = {
PYEXPAT_XMLPARSER_USEFOREIGNDTD_METHODDEF
#endif
PYEXPAT_XMLPARSER___DIR___METHODDEF
+ PYEXPAT_XMLPARSER_SETREPARSEDEFERRALENABLED_METHODDEF
+ PYEXPAT_XMLPARSER_GETREPARSEDEFERRALENABLED_METHODDEF
{NULL, NULL} /* sentinel */
};
@@ -1181,18 +1283,27 @@ newxmlparseobject(const char *encoding, const char *namespace_separator, PyObjec
self->handlers = NULL;
self->intern = intern;
Py_XINCREF(self->intern);
+#if XML_COMBINED_VERSION >= 20600
+ self->reparse_deferral_enabled = true;
+#else
+ self->reparse_deferral_enabled = false;
+#endif
PyObject_GC_Track(self);
/* namespace_separator is either NULL or contains one char + \0 */
self->itself = XML_ParserCreate_MM(encoding, &ExpatMemoryHandler,
namespace_separator);
+ self->parent = NULL;
if (self->itself == NULL) {
PyErr_SetString(PyExc_RuntimeError,
"XML_ParserCreate failed");
Py_DECREF(self);
return NULL;
}
-#if XML_COMBINED_VERSION >= 20100
+#if XML_COMBINED_VERSION >= 20800
+ /* This feature was added upstream in libexpat 2.8.0. */
+ XML_SetHashSalt16Bytes(self->itself, _Py_HashSecret.expat.hashsalt16);
+#elif XML_COMBINED_VERSION >= 20100
/* This feature was added upstream in libexpat 2.1.0. */
XML_SetHashSalt(self->itself,
(unsigned long)_Py_HashSecret.expat.hashsalt);
@@ -1223,6 +1334,7 @@ xmlparse_dealloc(xmlparseobject *self)
if (self->itself != NULL)
XML_ParserFree(self->itself);
self->itself = NULL;
+ Py_CLEAR(self->parent);
if (self->handlers != NULL) {
for (i = 0; handler_info[i].name != NULL; i++)
@@ -1285,25 +1397,25 @@ xmlparse_getattro(xmlparseobject *self, PyObject *nameobj)
return PyLong_FromLong((long)
XML_GetErrorCode(self->itself));
if (_PyUnicode_EqualToASCIIString(nameobj, "ErrorLineNumber"))
- return PyLong_FromLong((long)
- XML_GetErrorLineNumber(self->itself));
+ return PyLong_FromLongLong((long long)
+ EXPAT_GetCurrentLineNumber(self->itself));
if (_PyUnicode_EqualToASCIIString(nameobj, "ErrorColumnNumber"))
- return PyLong_FromLong((long)
- XML_GetErrorColumnNumber(self->itself));
+ return PyLong_FromLongLong((long long)
+ EXPAT_GetCurrentColumnNumber(self->itself));
if (_PyUnicode_EqualToASCIIString(nameobj, "ErrorByteIndex"))
- return PyLong_FromLong((long)
- XML_GetErrorByteIndex(self->itself));
+ return PyLong_FromLongLong((long long)
+ EXPAT_GetCurrentByteIndex(self->itself));
}
if (first_char == 'C') {
if (_PyUnicode_EqualToASCIIString(nameobj, "CurrentLineNumber"))
- return PyLong_FromLong((long)
- XML_GetCurrentLineNumber(self->itself));
+ return PyLong_FromLongLong((long long)
+ EXPAT_GetCurrentLineNumber(self->itself));
if (_PyUnicode_EqualToASCIIString(nameobj, "CurrentColumnNumber"))
- return PyLong_FromLong((long)
- XML_GetCurrentColumnNumber(self->itself));
+ return PyLong_FromLongLong((long long)
+ EXPAT_GetCurrentColumnNumber(self->itself));
if (_PyUnicode_EqualToASCIIString(nameobj, "CurrentByteIndex"))
- return PyLong_FromLong((long)
- XML_GetCurrentByteIndex(self->itself));
+ return PyLong_FromLongLong((long long)
+ EXPAT_GetCurrentByteIndex(self->itself));
}
if (first_char == 'b') {
if (_PyUnicode_EqualToASCIIString(nameobj, "buffer_size"))
@@ -1493,6 +1605,7 @@ xmlparse_traverse(xmlparseobject *op, visitproc visit, void *arg)
int i;
for (i = 0; handler_info[i].name != NULL; i++)
Py_VISIT(op->handlers[i]);
+ Py_VISIT(op->parent);
return 0;
}
@@ -1501,6 +1614,10 @@ xmlparse_clear(xmlparseobject *op)
{
clear_handlers(op, 0);
Py_CLEAR(op->intern);
+ // NOTE: We cannot call Py_CLEAR(op->parent) prior to calling
+ // XML_ParserFree(op->itself), or a subparser could lose its parent
+ // XML_Parser while still making use of it internally.
+ // https://github.com/python/cpython/issues/139400
return 0;
}
@@ -1865,8 +1982,11 @@ MODULE_INITFUNC(void)
capi.MICRO_VERSION = XML_MICRO_VERSION;
capi.ErrorString = XML_ErrorString;
capi.GetErrorCode = XML_GetErrorCode;
+_Py_COMP_DIAG_PUSH
+_Py_COMP_DIAG_IGNORE_DEPR_DECLS
capi.GetErrorColumnNumber = XML_GetErrorColumnNumber;
capi.GetErrorLineNumber = XML_GetErrorLineNumber;
+_Py_COMP_DIAG_POP
capi.Parse = XML_Parse;
capi.ParserCreate_MM = XML_ParserCreate_MM;
capi.ParserFree = XML_ParserFree;
@@ -1882,10 +2002,23 @@ MODULE_INITFUNC(void)
capi.SetEncoding = XML_SetEncoding;
capi.DefaultUnknownEncodingHandler = PyUnknownEncodingHandler;
#if XML_COMBINED_VERSION >= 20100
+_Py_COMP_DIAG_PUSH
+_Py_COMP_DIAG_IGNORE_DEPR_DECLS
capi.SetHashSalt = XML_SetHashSalt;
+_Py_COMP_DIAG_POP
#else
capi.SetHashSalt = NULL;
#endif
+#if XML_COMBINED_VERSION >= 20800
+ capi.SetHashSalt16Bytes = XML_SetHashSalt16Bytes;
+#else
+ capi.SetHashSalt16Bytes = NULL;
+#endif
+#if XML_COMBINED_VERSION >= 20600
+ capi.SetReparseDeferralEnabled = XML_SetReparseDeferralEnabled;
+#else
+ capi.SetReparseDeferralEnabled = NULL;
+#endif
/* export using capsule */
capi_object = PyCapsule_New(&capi, PyExpat_CAPSULE_NAME, NULL);
diff --git a/Modules/unicodedata.c b/Modules/unicodedata.c
index e8788f5036ddefe..b69f8a9c87283ed 100644
--- a/Modules/unicodedata.c
+++ b/Modules/unicodedata.c
@@ -482,19 +482,80 @@ get_decomp_record(PyObject *self, Py_UCS4 code, int *index, int *prefix, int *co
#define NCount (VCount*TCount)
#define SCount (LCount*NCount)
+/* Small combining runs are usually cheaper with insertion sort. */
+#define CANONICAL_ORDERING_COUNTING_SORT_THRESHOLD 20
+
+static void
+canonical_ordering_sort_insertion(int kind, void *data,
+ Py_ssize_t start, Py_ssize_t end)
+{
+ for (Py_ssize_t i = start + 1; i < end; i++) {
+ Py_UCS4 code = PyUnicode_READ(kind, data, i);
+ unsigned char combining = _getrecord_ex(code)->combining;
+ Py_ssize_t j = i;
+
+ while (j > start) {
+ Py_UCS4 previous = PyUnicode_READ(kind, data, j - 1);
+ if (_getrecord_ex(previous)->combining <= combining) {
+ break;
+ }
+ PyUnicode_WRITE(kind, data, j, previous);
+ j--;
+ }
+ if (j != i) {
+ PyUnicode_WRITE(kind, data, j, code);
+ }
+ }
+}
+
+static void
+canonical_ordering_sort_counting(int kind, void *data,
+ Py_ssize_t start, Py_ssize_t end,
+ Py_UCS4 *sortbuf)
+{
+ Py_ssize_t counts[256] = {0};
+ Py_ssize_t run_length = end - start;
+ Py_ssize_t total = 0;
+
+ for (Py_ssize_t i = start; i < end; i++) {
+ Py_UCS4 code = PyUnicode_READ(kind, data, i);
+ unsigned char combining = _getrecord_ex(code)->combining;
+ counts[combining]++;
+ }
+
+ for (size_t i = 0; i < Py_ARRAY_LENGTH(counts); i++) {
+ Py_ssize_t count = counts[i];
+ counts[i] = total;
+ total += count;
+ }
+
+ /* Reuse counts[] as the next output slot for each CCC. */
+ for (Py_ssize_t i = start; i < end; i++) {
+ Py_UCS4 code = PyUnicode_READ(kind, data, i);
+ unsigned char combining = _getrecord_ex(code)->combining;
+ sortbuf[counts[combining]++] = code;
+ }
+ for (Py_ssize_t i = 0; i < run_length; i++) {
+ PyUnicode_WRITE(kind, data, start + i, sortbuf[i]);
+ }
+}
+
static PyObject*
nfd_nfkd(PyObject *self, PyObject *input, int k)
{
PyObject *result;
Py_UCS4 *output;
Py_ssize_t i, o, osize;
- int kind;
- void *data;
+ int input_kind, result_kind;
+ const void *input_data;
+ void *result_data;
/* Longest decomposition in Unicode 3.2: U+FDFA */
Py_UCS4 stack[20];
Py_ssize_t space, isize;
int index, prefix, count, stackptr;
unsigned char prev, cur;
+ Py_UCS4 *sortbuf = NULL;
+ Py_ssize_t sortbuflen = 0;
stackptr = 0;
isize = PyUnicode_GET_LENGTH(input);
@@ -514,11 +575,11 @@ nfd_nfkd(PyObject *self, PyObject *input, int k)
return NULL;
}
i = o = 0;
- kind = PyUnicode_KIND(input);
- data = PyUnicode_DATA(input);
+ input_kind = PyUnicode_KIND(input);
+ input_data = PyUnicode_DATA(input);
while (i < isize) {
- stack[stackptr++] = PyUnicode_READ(kind, data, i++);
+ stack[stackptr++] = PyUnicode_READ(input_kind, input_data, i++);
while(stackptr) {
Py_UCS4 code = stack[--stackptr];
/* Hangul Decomposition adds three characters in
@@ -583,35 +644,66 @@ nfd_nfkd(PyObject *self, PyObject *input, int k)
PyMem_Free(output);
if (!result)
return NULL;
+
/* result is guaranteed to be ready, as it is compact. */
- kind = PyUnicode_KIND(result);
- data = PyUnicode_DATA(result);
+ result_kind = PyUnicode_KIND(result);
+ result_data = PyUnicode_DATA(result);
- /* Sort canonically. */
+ /* Sort each consecutive combining-character run canonically. */
i = 0;
- prev = _getrecord_ex(PyUnicode_READ(kind, data, i))->combining;
- for (i++; i < PyUnicode_GET_LENGTH(result); i++) {
- cur = _getrecord_ex(PyUnicode_READ(kind, data, i))->combining;
- if (prev == 0 || cur == 0 || prev <= cur) {
- prev = cur;
+ while (i < o) {
+ Py_ssize_t run_length, run_start;
+ int needs_sort = 0;
+
+ Py_UCS4 ch = PyUnicode_READ(result_kind, result_data, i);
+ prev = _getrecord_ex(ch)->combining;
+ if (prev == 0) {
+ i++;
continue;
}
- /* Non-canonical order. Need to switch *i with previous. */
- o = i - 1;
- while (1) {
- Py_UCS4 tmp = PyUnicode_READ(kind, data, o+1);
- PyUnicode_WRITE(kind, data, o+1,
- PyUnicode_READ(kind, data, o));
- PyUnicode_WRITE(kind, data, o, tmp);
- o--;
- if (o < 0)
- break;
- prev = _getrecord_ex(PyUnicode_READ(kind, data, o))->combining;
- if (prev == 0 || prev <= cur)
+
+ run_start = i++;
+ while (i < o) {
+ Py_UCS4 ch = PyUnicode_READ(result_kind, result_data, i);
+ cur = _getrecord_ex(ch)->combining;
+ if (cur == 0) {
break;
+ }
+ if (prev > cur) {
+ needs_sort = 1;
+ }
+ prev = cur;
+ i++;
}
- prev = _getrecord_ex(PyUnicode_READ(kind, data, i))->combining;
+ if (!needs_sort) {
+ continue;
+ }
+
+ run_length = i - run_start;
+ if (run_length < CANONICAL_ORDERING_COUNTING_SORT_THRESHOLD) {
+ canonical_ordering_sort_insertion(result_kind, result_data,
+ run_start, i);
+ continue;
+ }
+
+ if (run_length > sortbuflen) {
+ Py_UCS4 *new_sortbuf = PyMem_Resize(sortbuf,
+ Py_UCS4,
+ run_length);
+ if (new_sortbuf == NULL) {
+ PyErr_NoMemory();
+ PyMem_Free(sortbuf);
+ Py_DECREF(result);
+ return NULL;
+ }
+ sortbuf = new_sortbuf;
+ sortbuflen = run_length;
+ }
+
+ canonical_ordering_sort_counting(result_kind, result_data,
+ run_start, i, sortbuf);
}
+ PyMem_Free(sortbuf);
return result;
}
diff --git a/Objects/listobject.c b/Objects/listobject.c
index 60837fcf1e1e066..c2ea753a4486d8d 100644
--- a/Objects/listobject.c
+++ b/Objects/listobject.c
@@ -2661,7 +2661,14 @@ list_richcompare(PyObject *v, PyObject *w, int op)
}
/* Compare the final item again using the proper operator */
- return PyObject_RichCompare(vl->ob_item[i], wl->ob_item[i], op);
+ PyObject *vitem = vl->ob_item[i];
+ PyObject *witem = wl->ob_item[i];
+ Py_INCREF(vitem);
+ Py_INCREF(witem);
+ PyObject *result = PyObject_RichCompare(vl->ob_item[i], wl->ob_item[i], op);
+ Py_DECREF(vitem);
+ Py_DECREF(witem);
+ return result;
}
/*[clinic input]
@@ -2804,6 +2811,23 @@ list_subscript(PyListObject* self, PyObject* item)
}
}
+static Py_ssize_t
+adjust_slice_indexes(PyListObject *lst,
+ Py_ssize_t *start, Py_ssize_t *stop,
+ Py_ssize_t step)
+{
+ Py_ssize_t slicelength = PySlice_AdjustIndices(Py_SIZE(lst), start, stop,
+ step);
+
+ /* Make sure s[5:2] = [..] inserts at the right place:
+ before 5, not before 2. */
+ if ((step < 0 && *start < *stop) ||
+ (step > 0 && *start > *stop))
+ *stop = *start;
+
+ return slicelength;
+}
+
static int
list_ass_subscript(PyListObject* self, PyObject* item, PyObject* value)
{
@@ -2816,22 +2840,11 @@ list_ass_subscript(PyListObject* self, PyObject* item, PyObject* value)
return list_ass_item(self, i, value);
}
else if (PySlice_Check(item)) {
- Py_ssize_t start, stop, step, slicelength;
+ Py_ssize_t start, stop, step;
if (PySlice_Unpack(item, &start, &stop, &step) < 0) {
return -1;
}
- slicelength = PySlice_AdjustIndices(Py_SIZE(self), &start, &stop,
- step);
-
- if (step == 1)
- return list_ass_slice(self, start, stop, value);
-
- /* Make sure s[5:2] = [..] inserts at the right place:
- before 5, not before 2. */
- if ((step < 0 && start < stop) ||
- (step > 0 && start > stop))
- stop = start;
if (value == NULL) {
/* delete slice */
@@ -2840,6 +2853,12 @@ list_ass_subscript(PyListObject* self, PyObject* item, PyObject* value)
Py_ssize_t i;
int res;
+ Py_ssize_t slicelength = adjust_slice_indexes(self, &start, &stop,
+ step);
+
+ if (step == 1)
+ return list_ass_slice(self, start, stop, value);
+
if (slicelength <= 0)
return 0;
@@ -2915,6 +2934,15 @@ list_ass_subscript(PyListObject* self, PyObject* item, PyObject* value)
if (!seq)
return -1;
+ Py_ssize_t slicelength = adjust_slice_indexes(self, &start, &stop,
+ step);
+
+ if (step == 1) {
+ int res = list_ass_slice(self, start, stop, seq);
+ Py_DECREF(seq);
+ return res;
+ }
+
if (PySequence_Fast_GET_SIZE(seq) != slicelength) {
PyErr_Format(PyExc_ValueError,
"attempt to assign sequence of "
diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c
index d7e5a9d9cc98f2e..9c6d34eab09797a 100644
--- a/Objects/unicodeobject.c
+++ b/Objects/unicodeobject.c
@@ -5918,19 +5918,22 @@ PyUnicode_AsUTF16String(PyObject *unicode)
static _PyUnicode_Name_CAPI *ucnhash_CAPI = NULL;
PyObject *
-_PyUnicode_DecodeUnicodeEscape(const char *s,
+_PyUnicode_DecodeUnicodeEscape2(const char *s,
Py_ssize_t size,
const char *errors,
- const char **first_invalid_escape)
+ int *first_invalid_escape_char,
+ const char **first_invalid_escape_ptr)
{
const char *starts = s;
+ const char *initial_starts = starts;
_PyUnicodeWriter writer;
const char *end;
PyObject *errorHandler = NULL;
PyObject *exc = NULL;
// so we can remember if we've seen an invalid escape char or not
- *first_invalid_escape = NULL;
+ *first_invalid_escape_char = -1;
+ *first_invalid_escape_ptr = NULL;
if (size == 0) {
_Py_RETURN_UNICODE_EMPTY();
@@ -6106,9 +6109,12 @@ _PyUnicode_DecodeUnicodeEscape(const char *s,
goto error;
default:
- if (*first_invalid_escape == NULL) {
- *first_invalid_escape = s-1; /* Back up one char, since we've
- already incremented s. */
+ if (*first_invalid_escape_char == -1) {
+ *first_invalid_escape_char = c;
+ if (starts == initial_starts) {
+ /* Back up one char, since we've already incremented s. */
+ *first_invalid_escape_ptr = s - 1;
+ }
}
WRITE_ASCII_CHAR('\\');
WRITE_CHAR(c);
@@ -6142,20 +6148,36 @@ _PyUnicode_DecodeUnicodeEscape(const char *s,
return NULL;
}
+// Export for binary compatibility.
+PyObject *
+_PyUnicode_DecodeUnicodeEscape(const char *s,
+ Py_ssize_t size,
+ const char *errors,
+ const char **first_invalid_escape)
+{
+ int first_invalid_escape_char;
+ return _PyUnicode_DecodeUnicodeEscape2(
+ s, size, errors,
+ &first_invalid_escape_char,
+ first_invalid_escape);
+}
+
PyObject *
PyUnicode_DecodeUnicodeEscape(const char *s,
Py_ssize_t size,
const char *errors)
{
- const char *first_invalid_escape;
- PyObject *result = _PyUnicode_DecodeUnicodeEscape(s, size, errors,
- &first_invalid_escape);
+ int first_invalid_escape_char;
+ const char *first_invalid_escape_ptr;
+ PyObject *result = _PyUnicode_DecodeUnicodeEscape2(s, size, errors,
+ &first_invalid_escape_char,
+ &first_invalid_escape_ptr);
if (result == NULL)
return NULL;
- if (first_invalid_escape != NULL) {
+ if (first_invalid_escape_char != -1) {
if (PyErr_WarnFormat(PyExc_DeprecationWarning, 1,
"invalid escape sequence '\\%c'",
- (unsigned char)*first_invalid_escape) < 0) {
+ first_invalid_escape_char) < 0) {
Py_DECREF(result);
return NULL;
}
diff --git a/PCbuild/_elementtree.vcxproj b/PCbuild/_elementtree.vcxproj
index e430a520a2a13a4..7e216763694b0b5 100644
--- a/PCbuild/_elementtree.vcxproj
+++ b/PCbuild/_elementtree.vcxproj
@@ -62,7 +62,7 @@
<ItemDefinitionGroup>
<ClCompile>
<AdditionalIncludeDirectories>..\Modules\expat;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
- <PreprocessorDefinitions>_CRT_SECURE_NO_WARNINGS;USE_PYEXPAT_CAPI;XML_STATIC;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+ <PreprocessorDefinitions>_CRT_SECURE_NO_WARNINGS;USE_PYEXPAT_CAPI;XML_POOR_ENTROPY;XML_STATIC;%(PreprocessorDefinitions)</PreprocessorDefinitions>
</ClCompile>
</ItemDefinitionGroup>
<ItemGroup>
@@ -71,19 +71,23 @@
<ClInclude Include="..\Modules\expat\expat.h" />
<ClInclude Include="..\Modules\expat\expat_config.h" />
<ClInclude Include="..\Modules\expat\expat_external.h" />
+ <ClInclude Include="..\Modules\expat\fallthrough.h" />
+ <ClInclude Include="..\Modules\expat\hash_table.h" />
<ClInclude Include="..\Modules\expat\iasciitab.h" />
<ClInclude Include="..\Modules\expat\internal.h" />
<ClInclude Include="..\Modules\expat\latin1tab.h" />
- <ClInclude Include="..\Modules\expat\macconfig.h" />
+ <ClInclude Include="..\Modules\expat\memory_sanitizer.h" />
<ClInclude Include="..\Modules\expat\nametab.h" />
<ClInclude Include="..\Modules\expat\pyexpatns.h" />
<ClInclude Include="..\Modules\expat\utf8tab.h" />
<ClInclude Include="..\Modules\expat\winconfig.h" />
+ <ClInclude Include="..\Modules\expat\xcs.h" />
<ClInclude Include="..\Modules\expat\xmlrole.h" />
<ClInclude Include="..\Modules\expat\xmltok.h" />
</ItemGroup>
<ItemGroup>
<ClCompile Include="..\Modules\_elementtree.c" />
+ <ClCompile Include="..\Modules\expat\xcs.c" />
<ClCompile Include="..\Modules\expat\xmlparse.c" />
<ClCompile Include="..\Modules\expat\xmlrole.c" />
<ClCompile Include="..\Modules\expat\xmltok.c" />
diff --git a/PCbuild/_elementtree.vcxproj.filters b/PCbuild/_elementtree.vcxproj.filters
index 6acdf35846ab149..2c6533f60b997dc 100644
--- a/PCbuild/_elementtree.vcxproj.filters
+++ b/PCbuild/_elementtree.vcxproj.filters
@@ -24,6 +24,12 @@
<ClInclude Include="..\Modules\expat\expat_external.h">
<Filter>Header Files</Filter>
</ClInclude>
+ <ClInclude Include="..\Modules\expat\fallthrough.h">
+ <Filter>Header Files</Filter>
+ </ClInclude>
+ <ClInclude Include="..\Modules\expat\hash_table.h">
+ <Filter>Header Files\expat</Filter>
+ </ClInclude>
<ClInclude Include="..\Modules\expat\iasciitab.h">
<Filter>Header Files</Filter>
</ClInclude>
@@ -36,6 +42,9 @@
<ClInclude Include="..\Modules\expat\macconfig.h">
<Filter>Header Files</Filter>
</ClInclude>
+ <ClInclude Include="..\Modules\expat\memory_sanitizer.h">
+ <Filter>Header Files</Filter>
+ </ClInclude>
<ClInclude Include="..\Modules\expat\nametab.h">
<Filter>Header Files</Filter>
</ClInclude>
@@ -48,6 +57,9 @@
<ClInclude Include="..\Modules\expat\winconfig.h">
<Filter>Header Files</Filter>
</ClInclude>
+ <ClInclude Include="..\Modules\expat\xcs.h">
+ <Filter>Header Files\expat</Filter>
+ </ClInclude>
<ClInclude Include="..\Modules\expat\xmlrole.h">
<Filter>Header Files</Filter>
</ClInclude>
@@ -59,6 +71,9 @@
<ClCompile Include="..\Modules\_elementtree.c">
<Filter>Source Files</Filter>
</ClCompile>
+ <ClCompile Include="..\Modules\expat\xcs.c">
+ <Filter>Source Files\expat</Filter>
+ </ClCompile>
<ClCompile Include="..\Modules\expat\xmlparse.c">
<Filter>Source Files</Filter>
</ClCompile>
diff --git a/PCbuild/_ssl.vcxproj b/PCbuild/_ssl.vcxproj
index aaf95a361c589f9..32413fdfffc13bd 100644
--- a/PCbuild/_ssl.vcxproj
+++ b/PCbuild/_ssl.vcxproj
@@ -65,6 +65,9 @@
<AdditionalDependencies>ws2_32.lib;crypt32.lib;%(AdditionalDependencies)</AdditionalDependencies>
</Link>
</ItemDefinitionGroup>
+ <ItemGroup>
+ <ClInclude Include="..\Modules\_ssl_distrusted_spki.h" />
+ </ItemGroup>
<ItemGroup>
<ClCompile Include="..\Modules\_ssl.c" />
<ClCompile Include="$(opensslIncludeDir)\applink.c">
diff --git a/PCbuild/_ssl.vcxproj.filters b/PCbuild/_ssl.vcxproj.filters
index bd46b609840b2d2..b8988eedcdc019e 100644
--- a/PCbuild/_ssl.vcxproj.filters
+++ b/PCbuild/_ssl.vcxproj.filters
@@ -4,6 +4,9 @@
<Filter Include="Source Files">
<UniqueIdentifier>{695348f7-e9f6-4fe1-bc03-5f08ffc8095b}</UniqueIdentifier>
</Filter>
+ <Filter Include="Header Files">
+ <UniqueIdentifier>{7c1bd5da-8912-4107-b6ac-f3930f2d90c7}</UniqueIdentifier>
+ </Filter>
</ItemGroup>
<ItemGroup>
<ClCompile Include="..\Modules\_ssl.c">
@@ -13,6 +16,11 @@
<Filter>Source Files</Filter>
</ClCompile>
</ItemGroup>
+ <ItemGroup>
+ <ClInclude Include="..\Modules\_ssl_distrusted_spki.h">
+ <Filter>Header Files</Filter>
+ </ClInclude>
+ </ItemGroup>
<ItemGroup>
<ResourceCompile Include="..\PC\python_nt.rc" />
</ItemGroup>
diff --git a/PCbuild/find_python.bat b/PCbuild/find_python.bat
index 40937a67c33ac7a..1064f106f8da1cb 100644
--- a/PCbuild/find_python.bat
+++ b/PCbuild/find_python.bat
@@ -45,7 +45,7 @@
@set _Py_HOST_PYTHON=%HOST_PYTHON%
@if "%_Py_HOST_PYTHON%"=="" set _Py_HOST_PYTHON=py
@if "%_Py_NUGET%"=="" (set _Py_NUGET=%_Py_EXTERNALS_DIR%\nuget.exe)
-@if "%_Py_NUGET_URL%"=="" (set _Py_NUGET_URL=https://aka.ms/nugetclidl)
+@if "%_Py_NUGET_URL%"=="" (set _Py_NUGET_URL=https://dist.nuget.org/win-x86-commandline/latest/nuget.exe)
@if NOT exist "%_Py_NUGET%" (
@echo Downloading nuget...
@rem NB: Must use single quotes around NUGET here, NOT double!
diff --git a/PCbuild/pyexpat.vcxproj b/PCbuild/pyexpat.vcxproj
index 9a765ff07bde884..37cb1019ff32fe6 100644
--- a/PCbuild/pyexpat.vcxproj
+++ b/PCbuild/pyexpat.vcxproj
@@ -59,15 +59,20 @@
<ItemDefinitionGroup>
<ClCompile>
<AdditionalIncludeDirectories>$(PySourcePath)Modules\expat;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
- <PreprocessorDefinitions>_CRT_SECURE_NO_WARNINGS;PYEXPAT_EXPORTS;HAVE_EXPAT_H;XML_STATIC;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+ <PreprocessorDefinitions>_CRT_SECURE_NO_WARNINGS;PYEXPAT_EXPORTS;HAVE_EXPAT_H;XML_POOR_ENTROPY;XML_STATIC;%(PreprocessorDefinitions)</PreprocessorDefinitions>
</ClCompile>
</ItemDefinitionGroup>
<ItemGroup>
+ <ClInclude Include="..\Modules\expat\fallthrough.h" />
+ <ClInclude Include="..\Modules\expat\hash_table.h" />
+ <ClInclude Include="..\Modules\expat\memory_sanitizer.h" />
+ <ClInclude Include="..\Modules\expat\xcs.h" />
<ClInclude Include="..\Modules\expat\xmlrole.h" />
<ClInclude Include="..\Modules\expat\xmltok.h" />
</ItemGroup>
<ItemGroup>
<ClCompile Include="..\Modules\pyexpat.c" />
+ <ClCompile Include="..\Modules\expat\xcs.c" />
<ClCompile Include="..\Modules\expat\xmlparse.c" />
<ClCompile Include="..\Modules\expat\xmlrole.c" />
<ClCompile Include="..\Modules\expat\xmltok.c" />
diff --git a/PCbuild/pyexpat.vcxproj.filters b/PCbuild/pyexpat.vcxproj.filters
index f8d46026c9c284d..8ae5b22ce6e6638 100644
--- a/PCbuild/pyexpat.vcxproj.filters
+++ b/PCbuild/pyexpat.vcxproj.filters
@@ -9,6 +9,18 @@
</Filter>
</ItemGroup>
<ItemGroup>
+ <ClInclude Include="..\Modules\expat\fallthrough.h">
+ <Filter>Header Files</Filter>
+ </ClInclude>
+ <ClInclude Include="..\Modules\expat\hash_table.h">
+ <Filter>Header Files</Filter>
+ </ClInclude>
+ <ClInclude Include="..\Modules\expat\memory_sanitizer.h">
+ <Filter>Header Files</Filter>
+ </ClInclude>
+ <ClInclude Include="..\Modules\expat\xcs.h">
+ <Filter>Header Files</Filter>
+ </ClInclude>
<ClInclude Include="..\Modules\expat\xmlrole.h">
<Filter>Header Files</Filter>
</ClInclude>
@@ -20,6 +32,9 @@
<ClCompile Include="..\Modules\pyexpat.c">
<Filter>Source Files</Filter>
</ClCompile>
+ <ClCompile Include="..\Modules\expat\xcs.c">
+ <Filter>Source Files</Filter>
+ </ClCompile>
<ClCompile Include="..\Modules\expat\xmlparse.c">
<Filter>Source Files</Filter>
</ClCompile>
diff --git a/PCbuild/pyproject.props b/PCbuild/pyproject.props
index 0f7561744249f55..5dd43227a7eab81 100644
--- a/PCbuild/pyproject.props
+++ b/PCbuild/pyproject.props
@@ -66,6 +66,7 @@
<LinkTimeCodeGeneration Condition="$(Configuration) == 'Release'">UseLinkTimeCodeGeneration</LinkTimeCodeGeneration>
<LinkTimeCodeGeneration Condition="$(SupportPGO) and $(Configuration) == 'PGInstrument'">PGInstrument</LinkTimeCodeGeneration>
<LinkTimeCodeGeneration Condition="$(SupportPGO) and $(Configuration) == 'PGUpdate'">PGUpdate</LinkTimeCodeGeneration>
+ <AdditionalOptions Condition="$(Configuration) != 'Debug'">/OPT:REF,NOICF %(AdditionalOptions)</AdditionalOptions>
</Link>
<Lib>
<LinkTimeCodeGeneration Condition="$(Configuration) == 'Release'">true</LinkTimeCodeGeneration>
diff --git a/PCbuild/python.props b/PCbuild/python.props
index c28dc3197fe4c21..65897cae6ef20bd 100644
--- a/PCbuild/python.props
+++ b/PCbuild/python.props
@@ -81,6 +81,9 @@
<zlibDir Condition="!HasTrailingSlash($(zlibDir))">$(zlibDir)\</zlibDir>
<zlibIncludeDir>$(zlibDir)include</zlibIncludeDir>
<zlibLibDir>$(zlibDir)lib</zlibLibDir>
+ <!-- Platform shared/zlib (1.3.2 and later) installs the import library as z.lib -->
+ <zlibLibName Condition="$(zlibLibName) == '' and Exists('$(zlibLibDir)\z.lib')">z.lib</zlibLibName>
+ <zlibLibName Condition="$(zlibLibName) == ''">zlib.lib</zlibLibName>
</PropertyGroup>
<PropertyGroup>
diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj
index a3d5a267b98b730..25b4d5d01539ac1 100644
--- a/PCbuild/pythoncore.vcxproj
+++ b/PCbuild/pythoncore.vcxproj
@@ -73,7 +73,7 @@
<PreprocessorDefinitions>_USRDLL;Py_BUILD_CORE;Py_BUILD_CORE_BUILTIN;Py_ENABLE_SHARED;_Py_HAVE_ZLIB;MS_DLL_ID="$(SysWinVer)";%(PreprocessorDefinitions)</PreprocessorDefinitions>
</ClCompile>
<Link>
- <AdditionalDependencies>version.lib;shlwapi.lib;ws2_32.lib;pathcch.lib;zlib.lib;%(AdditionalDependencies)</AdditionalDependencies>
+ <AdditionalDependencies>version.lib;shlwapi.lib;ws2_32.lib;pathcch.lib;$(zlibLibName);%(AdditionalDependencies)</AdditionalDependencies>
<AdditionalLibraryDirectories Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">$(OutDir);%(AdditionalLibraryDirectories);$(zlibLibDir)</AdditionalLibraryDirectories>
<AdditionalLibraryDirectories Condition="'$(Configuration)|$(Platform)'=='PGInstrument|x64'">$(OutDir);%(AdditionalLibraryDirectories);$(zlibLibDir)</AdditionalLibraryDirectories>
<AdditionalLibraryDirectories Condition="'$(Configuration)|$(Platform)'=='PGUpdate|x64'">$(OutDir);%(AdditionalLibraryDirectories);$(zlibLibDir)</AdditionalLibraryDirectories>
diff --git a/Python/dtoa.c b/Python/dtoa.c
index b7bb7acfb6c215a..23efd67eb2ac667 100644
--- a/Python/dtoa.c
+++ b/Python/dtoa.c
@@ -64,6 +64,10 @@
* 7. _Py_dg_strtod has been modified so that it doesn't accept strings with
* leading whitespace.
*
+ * 8. _Py_dg_dtoa clamps ndigits in modes 3 and 5 so that its buffer size
+ * arithmetic cannot exceed the int range, and rv_alloc's size doubling
+ * uses size_t. (gh-158446)
+ *
***************************************************************/
/* Please send bug reports for the original dtoa.c code to David M. Gay (dmg
@@ -2158,7 +2162,8 @@ _Py_dg_strtod(const char *s00, char **se)
static char *
rv_alloc(int i)
{
- int j, k, *r;
+ int k, *r;
+ size_t j; /* size_t so that j <<= 1 cannot overflow for i near INT_MAX */
j = sizeof(ULong);
for(k = 0;
@@ -2422,6 +2427,15 @@ _Py_dg_dtoa(double dd, int mode, int ndigits,
leftright = 0;
/* fall through */
case 5:
+ /* -330 < k < 330 for any finite nonzero double. Clamp ndigits so
+ that ndigits + k + 1 stays within int range; no double has
+ anywhere near this many decimal digits so the digits returned
+ are unaffected (*decpt saturates in the no_digits case). Same
+ bound as DOUBLE_TO_STRING_PRECISION_MAX in pystrtod.c. */
+ if (ndigits > INT_MAX - 1024)
+ ndigits = INT_MAX - 1024;
+ else if (ndigits < -(INT_MAX - 1024))
+ ndigits = -(INT_MAX - 1024);
i = ndigits + k + 1;
ilim = i;
ilim1 = i - 1;
diff --git a/Python/pystrtod.c b/Python/pystrtod.c
index fea7e45c3b6247f..0e8c23a1a8e16c0 100644
--- a/Python/pystrtod.c
+++ b/Python/pystrtod.c
@@ -441,6 +441,15 @@ _Py_string_to_number_with_underscores(
return NULL;
}
+/* Largest precision magnitude accepted by PyOS_double_to_string(). The
+ output buffer sizes computed below and within _Py_dg_dtoa() use int and
+ Py_ssize_t arithmetic on roughly precision + (digits before the point, at
+ most DBL_MAX_10_EXP + 1 == 309) + a few bytes of sign, point and exponent.
+ Staying this far inside the int range keeps all of those sums in range.
+ (Only C callers can pass a negative precision.) _Py_dg_dtoa() applies
+ the same bound to its ndigits argument. */
+#define DOUBLE_TO_STRING_PRECISION_MAX (INT_MAX - 1024)
+
#ifdef PY_NO_SHORT_FLOAT_REPR
/* Given a string that may have a decimal point in the current
@@ -806,6 +815,13 @@ PyAPI_FUNC(char *) PyOS_double_to_string(double val,
int t, exp;
int upper = 0;
+ if (precision > DOUBLE_TO_STRING_PRECISION_MAX
+ || precision < -DOUBLE_TO_STRING_PRECISION_MAX)
+ {
+ PyErr_SetString(PyExc_ValueError, "precision too big");
+ return NULL;
+ }
+
/* Validate format_code, and map upper and lower case */
switch (format_code) {
case 'e': /* exponent */
@@ -1250,6 +1266,13 @@ PyAPI_FUNC(char *) PyOS_double_to_string(double val,
const char * const *float_strings = lc_float_strings;
int mode;
+ if (precision > DOUBLE_TO_STRING_PRECISION_MAX
+ || precision < -DOUBLE_TO_STRING_PRECISION_MAX)
+ {
+ PyErr_SetString(PyExc_ValueError, "precision too big");
+ return NULL;
+ }
+
/* Validate format_code, and map upper and lower case. Compute the
mode and make any adjustments as needed. */
switch (format_code) {
diff --git a/Tools/msi/get_externals.bat b/Tools/msi/get_externals.bat
index d66781d7c42ec49..e76ba701171c989 100644
--- a/Tools/msi/get_externals.bat
+++ b/Tools/msi/get_externals.bat
@@ -6,7 +6,7 @@ set HERE=%~dp0
if "%PCBUILD%"=="" (set PCBUILD=%HERE%..\..\PCbuild\)
if "%EXTERNALS_DIR%"=="" (set EXTERNALS_DIR=%HERE%..\..\externals\windows-installer)
if "%NUGET%"=="" (set NUGET=%EXTERNALS_DIR%\..\nuget.exe)
-if "%NUGET_URL%"=="" (set NUGET_URL=https://aka.ms/nugetclidl)
+if "%NUGET_URL%"=="" (set NUGET_URL=https://dist.nuget.org/win-x86-commandline/latest/nuget.exe)
set DO_FETCH=true
set DO_CLEAN=false
diff --git a/Tools/unicode/makeunicodedata.py b/Tools/unicode/makeunicodedata.py
index 009767183904e89..de7d2ea3026c943 100644
--- a/Tools/unicode/makeunicodedata.py
+++ b/Tools/unicode/makeunicodedata.py
@@ -27,12 +27,13 @@
#
import os
+import subprocess
import sys
import zipfile
from textwrap import dedent
-SCRIPT = sys.argv[0]
+SCRIPT = os.path.normpath(sys.argv[0])
VERSION = "3.3"
# The Unicode Database
@@ -124,6 +125,7 @@ def maketables(trace=0):
makeunicodename(unicode, trace)
makeunicodedata(unicode, trace)
makeunicodetype(unicode, trace)
+ makestringprep()
# --------------------------------------------------------------------
# unicode character properties
@@ -786,6 +788,19 @@ def word_key(a):
fp.close()
+
+def makestringprep():
+ FILE = "Lib/stringprep.py"
+
+ print("--- Preparing", FILE, "...")
+
+ MKSTRINGPREP = "Tools/unicode/mkstringprep.py"
+
+ with open(FILE, "w") as f:
+ f.truncate()
+ subprocess.check_call([sys.executable, MKSTRINGPREP], stdout=f)
+
+
def merge_old_version(version, new, old):
# Changes to exclusion file not implemented yet
if old.exclusions != new.exclusions:
diff --git a/Tools/unicode/mkstringprep.py b/Tools/unicode/mkstringprep.py
index ead020c3a7a4bbc..9740338fef9d3d4 100644
--- a/Tools/unicode/mkstringprep.py
+++ b/Tools/unicode/mkstringprep.py
@@ -1,18 +1,20 @@
-import re, sys
-from unicodedata import ucd_3_2_0 as unicodedata
+import re
+import os
+import unicodedata as unicodedata_current
+from unicodedata import ucd_3_2_0 as unicodedata_320
-if sys.maxunicode == 65535:
- raise RuntimeError("need UCS-4 Python")
+FILENAME = "Tools/unicode/data/rfc3454.txt"
+URL = "https://www.rfc-editor.org/rfc/rfc3454.txt"
def gen_category(cats):
for i in range(0, 0x110000):
- if unicodedata.category(chr(i)) in cats:
- yield(i)
+ if unicodedata_320.category(chr(i)) in cats:
+ yield i
def gen_bidirectional(cats):
for i in range(0, 0x110000):
- if unicodedata.bidirectional(chr(i)) in cats:
- yield(i)
+ if unicodedata_320.bidirectional(chr(i)) in cats:
+ yield i
def compact_set(l):
single = []
@@ -50,8 +52,16 @@ def compact_set(l):
############## Read the tables in the RFC #######################
-with open("rfc3454.txt") as f:
- data = f.readlines()
+try:
+ data_file = open(FILENAME, encoding='utf-8')
+except FileNotFoundError:
+ import urllib.request
+ os.makedirs(os.path.dirname(FILENAME), exist_ok=True)
+ urllib.request.urlretrieve(URL, filename=FILENAME)
+ data_file = open(FILENAME, encoding='utf-8')
+
+with data_file:
+ data = data_file.readlines()
tables = []
curname = None
@@ -119,10 +129,18 @@ def compact_set(l):
and mappings, for which a mapping function is provided.
\"\"\"
-from unicodedata import ucd_3_2_0 as unicodedata
+# This check asserts that mkstringprep.py has been run
+# when unicodedata is modified to ensure conformant behavior.
+import unicodedata
""")
-print("assert unicodedata.unidata_version == %r" % (unicodedata.unidata_version,))
+print("assert unicodedata.unidata_version == %r" % (unicodedata_current.unidata_version,))
+
+print("""
+from unicodedata import ucd_3_2_0 as unicodedata_320
+""")
+
+print("assert unicodedata_320.unidata_version == %r" % (unicodedata_320.unidata_version,))
# A.1 is the table of unassigned characters
# XXX Plane 15 PUA is listed as unassigned in Python.
@@ -142,7 +160,7 @@ def compact_set(l):
print("""
def in_table_a1(code):
- if unicodedata.category(code) != 'Cn': return False
+ if unicodedata_320.category(code) != 'Cn': return False
c = ord(code)
if 0xFDD0 <= c < 0xFDF0: return False
return (c & 0xFFFF) not in (0xFFFE, 0xFFFF)
@@ -175,21 +193,33 @@ def in_table_b1(code):
# B.3 is mostly Python's .lower, except for a number
# of special cases, e.g. considering canonical forms.
+# To enforce Unicode 3.2.0 behavior of .lower instead of
+# whatever Unicode version is included with Python we
+# add unassigned or newly case-folding codepoints to
+# the exception map, too.
b3_exceptions = {}
for k,v in table_b2.items():
if list(map(ord, chr(k).lower())) != v:
b3_exceptions[k] = "".join(map(chr,v))
+for cp in range(0x110000):
+ ch = chr(cp)
+ # Assigned in current Unicode version
+ # and supports case folding, but not
+ # explicitly in B.2 or B.3 tables.
+ if (unicodedata_current.category(ch) != "Cn"
+ and ch.lower() != ch
+ and cp not in table_b2
+ and cp not in table_b3):
+ b3_exceptions[cp] = ch # Identity.
b3 = sorted(b3_exceptions.items())
print("""
b3_exceptions = {""")
for i, kv in enumerate(b3):
- print("0x%x:%a," % kv, end=' ')
- if i % 4 == 3:
- print()
+ print("0x%x:%a," % kv, end='\n' if i % 4 == 3 else ' ')
print("}")
print("""
@@ -210,9 +240,9 @@ def map_table_b3(code):
def map_table_b2(a):
al = map_table_b3(a)
- b = unicodedata.normalize("NFKC", al)
+ b = unicodedata_320.normalize("NFKC", al)
bl = "".join([map_table_b3(ch) for ch in b])
- c = unicodedata.normalize("NFKC", bl)
+ c = unicodedata_320.normalize("NFKC", bl)
if b != c:
return c
else:
@@ -229,9 +259,9 @@ def map_table_b2(a):
print("""
def map_table_b2(a):
al = map_table_b3(a)
- b = unicodedata.normalize("NFKC", al)
+ b = unicodedata_320.normalize("NFKC", al)
bl = "".join([map_table_b3(ch) for ch in b])
- c = unicodedata.normalize("NFKC", bl)
+ c = unicodedata_320.normalize("NFKC", bl)
if b != c:
return c
else:
@@ -254,16 +284,16 @@ def in_table_c11(code):
del tables[0]
assert name == "C.1.2"
-# table = set(table.keys())
-# Zs = set(gen_category(["Zs"])) - {0x20}
-# assert Zs == table
+table = set(table.keys())
+Zs = set(gen_category(["Zs"])) - {0x20}
+assert Zs == table
print("""
def in_table_c12(code):
- return unicodedata.category(code) == "Zs" and code != " "
+ return unicodedata_320.category(code) == "Zs" and code != " "
def in_table_c11_c12(code):
- return unicodedata.category(code) == "Zs"
+ return unicodedata_320.category(code) == "Zs"
""")
# C.2.1 ASCII control characters
@@ -278,7 +308,7 @@ def in_table_c11_c12(code):
print("""
def in_table_c21(code):
- return ord(code) < 128 and unicodedata.category(code) == "Cc"
+ return ord(code) < 128 and unicodedata_320.category(code) == "Cc"
""")
# C.2.2 Non-ASCII control characters. It also includes
@@ -298,11 +328,11 @@ def in_table_c21(code):
def in_table_c22(code):
c = ord(code)
if c < 128: return False
- if unicodedata.category(code) == "Cc": return True
+ if unicodedata_320.category(code) == "Cc": return True
return c in c22_specials
def in_table_c21_c22(code):
- return unicodedata.category(code) == "Cc" or \\
+ return unicodedata_320.category(code) == "Cc" or \\
ord(code) in c22_specials
""")
@@ -316,7 +346,7 @@ def in_table_c21_c22(code):
print("""
def in_table_c3(code):
- return unicodedata.category(code) == "Co"
+ return unicodedata_320.category(code) == "Co"
""")
# C.4 Non-character code points, xFFFE, xFFFF
@@ -349,7 +379,7 @@ def in_table_c4(code):
print("""
def in_table_c5(code):
- return unicodedata.category(code) == "Cs"
+ return unicodedata_320.category(code) == "Cs"
""")
# C.6 Inappropriate for plain text
@@ -414,7 +444,7 @@ def in_table_c9(code):
print("""
def in_table_d1(code):
- return unicodedata.bidirectional(code) in ("R","AL")
+ return unicodedata_320.bidirectional(code) in ("R","AL")
""")
# D.2 Characters with bidirectional property "L"
@@ -427,5 +457,5 @@ def in_table_d1(code):
print("""
def in_table_d2(code):
- return unicodedata.bidirectional(code) == "L"
-""")
+ return unicodedata_320.bidirectional(code) == "L"
+""", end="")
diff --git a/setup.py b/setup.py
index 38c487f2a8f3ec7..6c3a2891bf0bae5 100644
--- a/setup.py
+++ b/setup.py
@@ -1536,8 +1536,11 @@ class db_found(Exception): pass
('XML_POOR_ENTROPY', '1'),
]
extra_compile_args = []
- expat_lib = []
- expat_sources = ['expat/xmlparse.c',
+ # bpo-44394: libexpat uses isnan() of math.h and needs linkage
+ # against the libm
+ expat_lib = ['m']
+ expat_sources = ['expat/xcs.c',
+ 'expat/xmlparse.c',
'expat/xmlrole.c',
'expat/xmltok.c']
expat_depends = ['expat/ascii.h',
@@ -1548,6 +1551,7 @@ class db_found(Exception): pass
'expat/internal.h',
'expat/latin1tab.h',
'expat/utf8tab.h',
+ 'expat/xcs.h',
'expat/xmlrole.h',
'expat/xmltok.h',
'expat/xmltok_impl.h'
@@ -2205,7 +2209,7 @@ def split_var(name, sep):
include_dirs=openssl_includes,
library_dirs=openssl_libdirs,
libraries=openssl_libs,
- depends=['socketmodule.h']
+ depends=['socketmodule.h', '_ssl_distrusted_spki.h']
)
else:
ssl_ext = None
]]></plaintext></not></p></not></div>