From 32585804f8263a5acc46676fc8eb2b29ca87be45 Mon Sep 17 00:00:00 2001 From: hoangduytranuk Date: Fri, 22 Apr 2022 20:12:18 +0100 Subject: [PATCH 1/4] Addressing problems raised in sphinx_issues #10104 --- babel/messages/catalog.py | 4 ++-- babel/messages/pofile.py | 42 +++++++++++++++++++++++++-------------- 2 files changed, 29 insertions(+), 17 deletions(-) diff --git a/babel/messages/catalog.py b/babel/messages/catalog.py index 228b10b71..850f68a67 100644 --- a/babel/messages/catalog.py +++ b/babel/messages/catalog.py @@ -35,7 +35,7 @@ (?:\.(?:\*|[\d]+))? [hlL]? ) - ([diouxXeEfFgGcrs%]) + ((?\?]|$)) ''', re.VERBOSE) @@ -94,7 +94,7 @@ def __init__(self, id, string=u'', locations=(), flags=(), auto_comments=(), if not string and self.pluralizable: string = (u'', u'') self.string = string - self.locations = list(distinct(locations)) + self.locations = list(set(locations)) self.flags = set(flags) if id and self.python_format: self.flags.add('python-format') diff --git a/babel/messages/pofile.py b/babel/messages/pofile.py index bd29e731a..5c04a6972 100644 --- a/babel/messages/pofile.py +++ b/babel/messages/pofile.py @@ -276,6 +276,7 @@ def _process_comment(self, line): self.locations.append((location[:pos], lineno)) else: self.locations.append((location, None)) + self.locations = list(set(self.locations)) elif line[1:].startswith(','): for flag in line[2:].lstrip().split(','): self.flags.append(flag.strip()) @@ -521,14 +522,23 @@ def _write(text): text = text.encode(catalog.charset, 'backslashreplace') fileobj.write(text) - def _write_comment(comment, prefix=''): - # xgettext always wraps comments even if --no-wrap is passed; - # provide the same behaviour - if width and width > 0: - _width = width - else: - _width = 76 - for line in wraptext(comment, _width): + def _write_comment(comment, prefix='', comment_determinator=None): + # NEVER wrap comments, this observation: "xgettext always wraps comments even if --no-wrap is passed;" is FALSE. There seemed to be a bug in the xgettext code, because wrapping doesn't always occur + # Make sure comments are unique and sorted alphabetically so locations can be easily searched and identify + has_comment = (bool(comment) and len(comment) > 0) + if not has_comment: + return + + try: + has_comment_split = (comment_determinator is not None) + comment_list = (comment.split(comment_determinator) if has_comment_split else comment) + except Exception as e: + # sometimes comment came as a list + comment_list = comment + + comment_list = list(set(comment)) + comment_list.sort() + for line in comment_list: _write('#%s %s\n' % (prefix, line.strip())) def _write_message(message, prefix=''): @@ -577,10 +587,11 @@ def _write_message(message, prefix=''): comment_header = u'\n'.join(lines) _write(comment_header + u'\n') - for comment in message.user_comments: - _write_comment(comment) - for comment in message.auto_comments: - _write_comment(comment, prefix='.') + comment_list = '\n'.join(message.user_comments) + _write_comment(comment_list, comment_determinator='\n') + + comment_list = '\n'.join(message.auto_comments) + _write_comment(comment_list, comment_determinator='\n') if not no_location: locs = [] @@ -602,7 +613,8 @@ def _write_message(message, prefix=''): location = u'%s' % filename.replace(os.sep, '/') if location not in locs: locs.append(location) - _write_comment(' '.join(locs), prefix=':') + loc_list = '\n'.join(locs) + _write_comment(loc_list, prefix=':', comment_determinator='\n') if message.flags: _write('#%s\n' % ', '.join([''] + sorted(message.flags))) @@ -622,8 +634,8 @@ def _write_message(message, prefix=''): catalog.obsolete.values(), sort_by=sort_by ): - for comment in message.user_comments: - _write_comment(comment) + comment_list = '\n'.join(message.user_comments) + _write_comment(comment_list, comment_determinator='\n') _write_message(message, prefix='#~ ') _write('\n') From 67aa652a1c10bfdc92587cdae87fdab65d633b23 Mon Sep 17 00:00:00 2001 From: hoangduytranuk Date: Sun, 24 Apr 2022 02:02:54 +0100 Subject: [PATCH 2/4] Corrected the unnecesary codes that was introduced in the last fix to the issue 10104 by simlify in down to checking for list instance of the input and making sure that they are. --- babel/messages/pofile.py | 25 +++++++------------------ 1 file changed, 7 insertions(+), 18 deletions(-) diff --git a/babel/messages/pofile.py b/babel/messages/pofile.py index 5c04a6972..40f105ff7 100644 --- a/babel/messages/pofile.py +++ b/babel/messages/pofile.py @@ -522,21 +522,15 @@ def _write(text): text = text.encode(catalog.charset, 'backslashreplace') fileobj.write(text) - def _write_comment(comment, prefix='', comment_determinator=None): + def _write_comment(comment, prefix=''): # NEVER wrap comments, this observation: "xgettext always wraps comments even if --no-wrap is passed;" is FALSE. There seemed to be a bug in the xgettext code, because wrapping doesn't always occur # Make sure comments are unique and sorted alphabetically so locations can be easily searched and identify has_comment = (bool(comment) and len(comment) > 0) if not has_comment: return - try: - has_comment_split = (comment_determinator is not None) - comment_list = (comment.split(comment_determinator) if has_comment_split else comment) - except Exception as e: - # sometimes comment came as a list - comment_list = comment - - comment_list = list(set(comment)) + is_list_type = isinstance(comment, list) + comment_list = (list(comment) if not is_list_type else comment) comment_list.sort() for line in comment_list: _write('#%s %s\n' % (prefix, line.strip())) @@ -587,11 +581,8 @@ def _write_message(message, prefix=''): comment_header = u'\n'.join(lines) _write(comment_header + u'\n') - comment_list = '\n'.join(message.user_comments) - _write_comment(comment_list, comment_determinator='\n') - - comment_list = '\n'.join(message.auto_comments) - _write_comment(comment_list, comment_determinator='\n') + _write_comment(message.user_comments) + _write_comment(message.auto_comments) if not no_location: locs = [] @@ -613,8 +604,7 @@ def _write_message(message, prefix=''): location = u'%s' % filename.replace(os.sep, '/') if location not in locs: locs.append(location) - loc_list = '\n'.join(locs) - _write_comment(loc_list, prefix=':', comment_determinator='\n') + _write_comment(locs, prefix=':') if message.flags: _write('#%s\n' % ', '.join([''] + sorted(message.flags))) @@ -634,8 +624,7 @@ def _write_message(message, prefix=''): catalog.obsolete.values(), sort_by=sort_by ): - comment_list = '\n'.join(message.user_comments) - _write_comment(comment_list, comment_determinator='\n') + _write_comment(message.user_comments) _write_message(message, prefix='#~ ') _write('\n') From 53c89f446366c168ed42bd82e297b38cf3b041bd Mon Sep 17 00:00:00 2001 From: hoangduytranuk Date: Sun, 24 Apr 2022 02:45:53 +0100 Subject: [PATCH 3/4] Removed redundant codes in the last fix, making it more clean and simpler --- babel/messages/pofile.py | 25 +++++++------------------ 1 file changed, 7 insertions(+), 18 deletions(-) diff --git a/babel/messages/pofile.py b/babel/messages/pofile.py index 5c04a6972..7ccca4da6 100644 --- a/babel/messages/pofile.py +++ b/babel/messages/pofile.py @@ -522,21 +522,15 @@ def _write(text): text = text.encode(catalog.charset, 'backslashreplace') fileobj.write(text) - def _write_comment(comment, prefix='', comment_determinator=None): + def _write_comment(comment, prefix='': # NEVER wrap comments, this observation: "xgettext always wraps comments even if --no-wrap is passed;" is FALSE. There seemed to be a bug in the xgettext code, because wrapping doesn't always occur # Make sure comments are unique and sorted alphabetically so locations can be easily searched and identify has_comment = (bool(comment) and len(comment) > 0) if not has_comment: return - try: - has_comment_split = (comment_determinator is not None) - comment_list = (comment.split(comment_determinator) if has_comment_split else comment) - except Exception as e: - # sometimes comment came as a list - comment_list = comment - - comment_list = list(set(comment)) + is_list_type = isinstance(comment, list) + comment_list = (list(comment) if not is_list_type else comment) comment_list.sort() for line in comment_list: _write('#%s %s\n' % (prefix, line.strip())) @@ -587,11 +581,8 @@ def _write_message(message, prefix=''): comment_header = u'\n'.join(lines) _write(comment_header + u'\n') - comment_list = '\n'.join(message.user_comments) - _write_comment(comment_list, comment_determinator='\n') - - comment_list = '\n'.join(message.auto_comments) - _write_comment(comment_list, comment_determinator='\n') + _write_comment(message.user_comments) + _write_comment(message.auto_comments) if not no_location: locs = [] @@ -613,8 +604,7 @@ def _write_message(message, prefix=''): location = u'%s' % filename.replace(os.sep, '/') if location not in locs: locs.append(location) - loc_list = '\n'.join(locs) - _write_comment(loc_list, prefix=':', comment_determinator='\n') + _write_comment(locs, prefix=':') if message.flags: _write('#%s\n' % ', '.join([''] + sorted(message.flags))) @@ -634,8 +624,7 @@ def _write_message(message, prefix=''): catalog.obsolete.values(), sort_by=sort_by ): - comment_list = '\n'.join(message.user_comments) - _write_comment(comment_list, comment_determinator='\n') + _write_comment(message.user_comments) _write_message(message, prefix='#~ ') _write('\n') From 314b6487dd763d7e16bf6cf07d8002e74be547f0 Mon Sep 17 00:00:00 2001 From: hoangduytranuk Date: Sun, 15 May 2022 15:20:17 +0100 Subject: [PATCH 4/4] This fix propose following changes which can be significant to the NORMAL behaviour user expected: 1. PYTHON_FORMAT will correctly, with some degrees of limitation (haven't found with sampled test file of Blender UI translation file, but that doesn't mean it will not be broken again) identify '%s' '%d' flags in the messages and flag it as 'python-format', which removed mistaken recognitions of percentage sign, ie. '%50' as a python-format, which POEditor will flag up as an ERROR in the message. 2. Locations and comments are sorted Alphabetically and removed duplicated redundant lines. 3. Comment lines are NO LONGER wrapped with provided line_width. There appeared to be an error in the xgettext code when running with --no-wrap flag. Message bodies (ie. msgid, msgstr) should be wrapped but NOT comments. The wrapping can cause unnecessary version changes appeared in the repository submissions. --- babel/messages/pofile.py | 32 +++++++++++--------------------- 1 file changed, 11 insertions(+), 21 deletions(-) diff --git a/babel/messages/pofile.py b/babel/messages/pofile.py index 5c04a6972..cdcb99657 100644 --- a/babel/messages/pofile.py +++ b/babel/messages/pofile.py @@ -275,8 +275,7 @@ def _process_comment(self, line): continue self.locations.append((location[:pos], lineno)) else: - self.locations.append((location, None)) - self.locations = list(set(self.locations)) + self.locations.append((location, None)) elif line[1:].startswith(','): for flag in line[2:].lstrip().split(','): self.flags.append(flag.strip()) @@ -522,21 +521,14 @@ def _write(text): text = text.encode(catalog.charset, 'backslashreplace') fileobj.write(text) - def _write_comment(comment, prefix='', comment_determinator=None): + def _write_comment(comment, prefix=''): # NEVER wrap comments, this observation: "xgettext always wraps comments even if --no-wrap is passed;" is FALSE. There seemed to be a bug in the xgettext code, because wrapping doesn't always occur # Make sure comments are unique and sorted alphabetically so locations can be easily searched and identify - has_comment = (bool(comment) and len(comment) > 0) - if not has_comment: + if not comment: return - try: - has_comment_split = (comment_determinator is not None) - comment_list = (comment.split(comment_determinator) if has_comment_split else comment) - except Exception as e: - # sometimes comment came as a list - comment_list = comment - - comment_list = list(set(comment)) + is_list_type = isinstance(comment, list) + comment_list = (list(comment) if not is_list_type else comment) comment_list.sort() for line in comment_list: _write('#%s %s\n' % (prefix, line.strip())) @@ -587,11 +579,9 @@ def _write_message(message, prefix=''): comment_header = u'\n'.join(lines) _write(comment_header + u'\n') - comment_list = '\n'.join(message.user_comments) - _write_comment(comment_list, comment_determinator='\n') + _write_comment(message.user_comments) + _write_comment(message.auto_comments, prefix='.') - comment_list = '\n'.join(message.auto_comments) - _write_comment(comment_list, comment_determinator='\n') if not no_location: locs = [] @@ -613,8 +603,9 @@ def _write_message(message, prefix=''): location = u'%s' % filename.replace(os.sep, '/') if location not in locs: locs.append(location) - loc_list = '\n'.join(locs) - _write_comment(loc_list, prefix=':', comment_determinator='\n') + + _write_comment(locs, prefix=':') + if message.flags: _write('#%s\n' % ', '.join([''] + sorted(message.flags))) @@ -634,8 +625,7 @@ def _write_message(message, prefix=''): catalog.obsolete.values(), sort_by=sort_by ): - comment_list = '\n'.join(message.user_comments) - _write_comment(comment_list, comment_determinator='\n') + _write_comment(message.user_comments) _write_message(message, prefix='#~ ') _write('\n')