Skip to content

gh-53950: Clear MESSAGES in msgfmt #133316

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 4 commits into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 0 additions & 10 deletions Lib/test/test_tools/test_msgfmt.py
Original file line number Diff line number Diff line change
Expand Up @@ -145,12 +145,6 @@ def test_generic_syntax_error(self):


class POParserTest(unittest.TestCase):
@classmethod
def tearDownClass(cls):
# msgfmt uses a global variable to store messages,
# clear it after the tests.
msgfmt.MESSAGES.clear()

def test_strings(self):
# Test that the PO parser correctly handles and unescape
# strings in the PO file.
Expand Down Expand Up @@ -202,8 +196,6 @@ def test_strings(self):
# check the result.
po = f'msgid {po_string}\nmsgstr "translation"'
Path('messages.po').write_text(po)
# Reset the global MESSAGES dictionary
msgfmt.MESSAGES.clear()
msgfmt.make('messages.po', 'messages.mo')

with open('messages.mo', 'rb') as f:
Expand Down Expand Up @@ -235,8 +227,6 @@ def test_strings(self):
with self.subTest(string=invalid_string):
po = f'msgid {invalid_string}\nmsgstr "translation"'
Path('messages.po').write_text(po)
# Reset the global MESSAGES dictionary
msgfmt.MESSAGES.clear()
with self.assertRaises(Exception):
msgfmt.make('messages.po', 'messages.mo')

Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Fix bug in :program:`msgfmt` where ``MESSAGES`` was not cleared.
30 changes: 13 additions & 17 deletions Tools/i18n/msgfmt.py
Original file line number Diff line number Diff line change
Expand Up @@ -37,39 +37,34 @@
__version__ = "1.2"


MESSAGES = {}


def usage(code, msg=''):
print(__doc__, file=sys.stderr)
if msg:
print(msg, file=sys.stderr)
sys.exit(code)


def add(ctxt, id, str, fuzzy):
def add(messages, ctxt, id, str, fuzzy):
"Add a non-fuzzy translation to the dictionary."
global MESSAGES
if not fuzzy and str:
if ctxt is None:
MESSAGES[id] = str
messages[id] = str
else:
MESSAGES[b"%b\x04%b" % (ctxt, id)] = str
messages[b"%b\x04%b" % (ctxt, id)] = str


def generate():
def generate(messages):
"Return the generated output."
global MESSAGES
# the keys are sorted in the .mo file
keys = sorted(MESSAGES.keys())
keys = sorted(messages.keys())
offsets = []
ids = strs = b''
for id in keys:
# For each string, we need size and file offset. Each string is NUL
# terminated; the NUL does not count into the size.
offsets.append((len(ids), len(id), len(strs), len(MESSAGES[id])))
offsets.append((len(ids), len(id), len(strs), len(messages[id])))
ids += id + b'\0'
strs += MESSAGES[id] + b'\0'
strs += messages[id] + b'\0'
output = ''
# The header is 7 32-bit unsigned integers. We don't use hash tables, so
# the keys start right after the index tables.
Expand Down Expand Up @@ -99,6 +94,7 @@ def generate():


def make(filename, outfile):
messages = {}
ID = 1
STR = 2
CTXT = 3
Expand Down Expand Up @@ -140,7 +136,7 @@ def make(filename, outfile):
lno += 1
# If we get a comment line after a msgstr, this is a new entry
if l[0] == '#' and section == STR:
add(msgctxt, msgid, msgstr, fuzzy)
add(messages, msgctxt, msgid, msgstr, fuzzy)
section = msgctxt = None
fuzzy = 0
# Record a fuzzy mark
Expand All @@ -152,7 +148,7 @@ def make(filename, outfile):
# Now we are in a msgid or msgctxt section, output previous section
if l.startswith('msgctxt'):
if section == STR:
add(msgctxt, msgid, msgstr, fuzzy)
add(messages, msgctxt, msgid, msgstr, fuzzy)
section = CTXT
l = l[7:]
msgctxt = b''
Expand All @@ -169,7 +165,7 @@ def make(filename, outfile):
charset = p.parsestr(msgstr.decode(encoding)).get_content_charset()
if charset:
encoding = charset
add(msgctxt, msgid, msgstr, fuzzy)
add(messages, msgctxt, msgid, msgstr, fuzzy)
msgctxt = None
section = ID
l = l[5:]
Expand Down Expand Up @@ -219,10 +215,10 @@ def make(filename, outfile):
sys.exit(1)
# Add last entry
if section == STR:
add(msgctxt, msgid, msgstr, fuzzy)
add(messages, msgctxt, msgid, msgstr, fuzzy)

# Compute output
output = generate()
output = generate(messages)

try:
with open(outfile,"wb") as f:
Expand Down
Loading