Big refactor of bl_i18n_utils, switching to an object-based model. Still a bit wip.
Notes: * Most tools here have been updated, only a few (unused :p ) features should be missing. But some updates are rough for now, and performances are globally worse currently, will address this later (this is only really noticeable when you work over the whole repo, anyway ;) ). * This update breaks "edit translations" addon, will fix it later, once core (i.e. this module) is considered stable again!
This commit is contained in:
Bastien Montagne
@@ -23,27 +23,41 @@
|
||||
# You should not directly use this script, rather use update_msg.py!
|
||||
|
||||
import os
|
||||
import re
|
||||
import collections
|
||||
import copy
|
||||
|
||||
# XXX Relative import does not work here when used from Blender...
|
||||
from bl_i18n_utils import settings
|
||||
|
||||
import bpy
|
||||
|
||||
#classes = set()
|
||||
|
||||
print(dir(settings))
|
||||
|
||||
SOURCE_DIR = settings.SOURCE_DIR
|
||||
|
||||
CUSTOM_PY_UI_FILES = [os.path.abspath(os.path.join(SOURCE_DIR, p)) for p in settings.CUSTOM_PY_UI_FILES]
|
||||
FILE_NAME_MESSAGES = settings.FILE_NAME_MESSAGES
|
||||
COMMENT_PREFIX = settings.COMMENT_PREFIX
|
||||
CONTEXT_PREFIX = settings.CONTEXT_PREFIX
|
||||
MSG_COMMENT_PREFIX = settings.MSG_COMMENT_PREFIX
|
||||
MSG_CONTEXT_PREFIX = settings.MSG_CONTEXT_PREFIX
|
||||
CONTEXT_DEFAULT = settings.CONTEXT_DEFAULT
|
||||
#CONTEXT_DEFAULT = bpy.app.i18n.contexts.default # XXX Not yet! :)
|
||||
UNDOC_OPS_STR = settings.UNDOC_OPS_STR
|
||||
|
||||
NC_ALLOWED = settings.WARN_MSGID_NOT_CAPITALIZED_ALLOWED
|
||||
|
||||
|
||||
##### Utils #####
|
||||
|
||||
# check for strings like ": %d"
|
||||
ignore_reg = re.compile(r"^(?:[-*.()/\\+:%xWXYZ0-9]|%d|%f|%s|%r|\s)*$")
|
||||
filter_message = ignore_reg.match
|
||||
|
||||
|
||||
def check(check_ctxt, messages, key, msgsrc):
|
||||
"""
|
||||
Performs a set of checks over the given key (context, message)...
|
||||
"""
|
||||
if check_ctxt is None:
|
||||
return
|
||||
multi_rnatip = check_ctxt.get("multi_rnatip")
|
||||
@@ -73,7 +87,79 @@ def check(check_ctxt, messages, key, msgsrc):
|
||||
undoc_ops.add(key)
|
||||
|
||||
|
||||
def print_warnings(check_ctxt, messages):
|
||||
if check_ctxt is not None:
|
||||
print("WARNINGS:")
|
||||
keys = set()
|
||||
for c in check_ctxt.values():
|
||||
keys |= c
|
||||
# XXX Temp, see below
|
||||
keys -= check_ctxt["multi_rnatip"]
|
||||
for key in keys:
|
||||
if key in check_ctxt["undoc_ops"]:
|
||||
print("\tThe following operators are undocumented:")
|
||||
else:
|
||||
print("\t“{}”|“{}”:".format(*key))
|
||||
if key in check_ctxt["multi_lines"]:
|
||||
print("\t\t-> newline in this message!")
|
||||
if key in check_ctxt["not_capitalized"]:
|
||||
print("\t\t-> message not capitalized!")
|
||||
if key in check_ctxt["end_point"]:
|
||||
print("\t\t-> message with endpoint!")
|
||||
# XXX Hide this one for now, too much false positives.
|
||||
# if key in check_ctxt["multi_rnatip"]:
|
||||
# print("\t\t-> tip used in several RNA items")
|
||||
if key in check_ctxt["py_in_rna"]:
|
||||
print("\t\t-> RNA message also used in py UI code:")
|
||||
print("\t\t{}".format("\n\t\t".join(messages[key])))
|
||||
|
||||
|
||||
def enable_addons(addons={}, support={}, disable=False):
|
||||
"""
|
||||
Enable (or disable) addons based either on a set of names, or a set of 'support' types.
|
||||
Returns the list of all affected addons (as fake modules)!
|
||||
"""
|
||||
import addon_utils
|
||||
import bpy
|
||||
|
||||
userpref = bpy.context.user_preferences
|
||||
used_ext = {ext.module for ext in userpref.addons}
|
||||
|
||||
ret = [mod for mod in addon_utils.modules(addon_utils.addons_fake_modules)
|
||||
if ((addons and mod.__name__ in addons) or
|
||||
(not addons and addon_utils.module_bl_info(mod)["support"] in support))]
|
||||
|
||||
for mod in ret:
|
||||
module_name = mod.__name__
|
||||
if disable:
|
||||
if module_name not in used_ext:
|
||||
continue
|
||||
print(" Disabling module ", module_name)
|
||||
bpy.ops.wm.addon_disable(module=module_name)
|
||||
else:
|
||||
if module_name in used_ext:
|
||||
continue
|
||||
print(" Enabling module ", module_name)
|
||||
bpy.ops.wm.addon_enable(module=module_name)
|
||||
|
||||
# XXX There are currently some problems with bpy/rna...
|
||||
# *Very* tricky to solve!
|
||||
# So this is a hack to make all newly added operator visible by
|
||||
# bpy.types.OperatorProperties.__subclasses__()
|
||||
for cat in dir(bpy.ops):
|
||||
cat = getattr(bpy.ops, cat)
|
||||
for op in dir(cat):
|
||||
getattr(cat, op).get_rna()
|
||||
|
||||
return ret
|
||||
|
||||
|
||||
##### RNA #####
|
||||
|
||||
def dump_messages_rna(messages, check_ctxt):
|
||||
"""
|
||||
Dump into messages dict all RNA-defined UI messages (labels en tooltips).
|
||||
"""
|
||||
import bpy
|
||||
|
||||
def classBlackList():
|
||||
@@ -257,20 +343,17 @@ def dump_messages_rna(messages, check_ctxt):
|
||||
# Parse everything (recursively parsing from bpy_struct "class"...).
|
||||
processed = process_cls_list(type(bpy.context).__base__.__subclasses__())
|
||||
print("{} classes processed!".format(processed))
|
||||
# import pickle
|
||||
# global classes
|
||||
# classes = {str(c) for c in classes}
|
||||
# with open("/home/i7deb64/Bureau/tpck_2", "wb") as f:
|
||||
# pickle.dump(classes, f, protocol=0)
|
||||
|
||||
from bpy_extras.keyconfig_utils import KM_HIERARCHY
|
||||
|
||||
walk_keymap_hierarchy(KM_HIERARCHY, "KM_HIERARCHY")
|
||||
|
||||
|
||||
def dump_messages_pytext(messages, check_ctxt):
|
||||
""" dumps text inlined in the python user interface: eg.
|
||||
##### Python source code #####
|
||||
|
||||
def dump_py_messages_from_files(messages, check_ctxt, files):
|
||||
"""
|
||||
Dump text inlined in the python files given, e.g. 'My Name' in:
|
||||
layout.prop("someprop", text="My Name")
|
||||
"""
|
||||
import ast
|
||||
@@ -278,7 +361,6 @@ def dump_messages_pytext(messages, check_ctxt):
|
||||
# -------------------------------------------------------------------------
|
||||
# Gather function names
|
||||
|
||||
import bpy
|
||||
# key: func_id
|
||||
# val: [(arg_kw, arg_pos), (arg_kw, arg_pos), ...]
|
||||
func_translate_args = {}
|
||||
@@ -290,15 +372,12 @@ def dump_messages_pytext(messages, check_ctxt):
|
||||
# E.g. we don’t want to get strings inside subscripts (blah["foo"])!
|
||||
stopper_nodes = {ast.Subscript, }
|
||||
|
||||
# For now only consider functions from UILayout...
|
||||
for func_id, func in bpy.types.UILayout.bl_rna.functions.items():
|
||||
# check it has a 'text' argument
|
||||
# check it has one or more arguments as defined in translate_kw
|
||||
for (arg_pos, (arg_kw, arg)) in enumerate(func.parameters.items()):
|
||||
if ((arg_kw in translate_kw) and
|
||||
(arg.is_output is False) and
|
||||
(arg.type == 'STRING')):
|
||||
|
||||
func_translate_args.setdefault(func_id, []).append((arg_kw,
|
||||
arg_pos))
|
||||
if ((arg_kw in translate_kw) and (arg.is_output is False) and (arg.type == 'STRING')):
|
||||
func_translate_args.setdefault(func_id, []).append((arg_kw, arg_pos))
|
||||
# print(func_translate_args)
|
||||
|
||||
check_ctxt_py = None
|
||||
@@ -308,19 +387,20 @@ def dump_messages_pytext(messages, check_ctxt):
|
||||
"not_capitalized": check_ctxt["not_capitalized"],
|
||||
"end_point": check_ctxt["end_point"]}
|
||||
|
||||
# -------------------------------------------------------------------------
|
||||
# Function definitions
|
||||
|
||||
# Helper function
|
||||
def extract_strings(fp_rel, node):
|
||||
""" Recursively get strings, needed in case we have "Blah" + "Blah",
|
||||
passed as an argument in that case it wont evaluate to a string.
|
||||
However, break on some kind of stopper nodes, like e.g. Subscript.
|
||||
"""
|
||||
|
||||
Recursively get strings, needed in case we have "Blah" + "Blah", passed as an argument in that case it won't
|
||||
evaluate to a string. However, break on some kind of stopper nodes, like e.g. Subscript.
|
||||
"""
|
||||
if type(node) == ast.Str:
|
||||
eval_str = ast.literal_eval(node)
|
||||
if eval_str:
|
||||
key = (CONTEXT_DEFAULT, eval_str)
|
||||
# Parse optional context included in string!
|
||||
if bpy.app.i18n.context_sep in eval_str:
|
||||
key = eval_str.split(bpy.app.i18n.context_sep, 1)
|
||||
else:
|
||||
key = (CONTEXT_DEFAULT, eval_str)
|
||||
msgsrc = "{}:{}".format(fp_rel, node.lineno)
|
||||
check(check_ctxt_py, messages, key, msgsrc)
|
||||
messages.setdefault(key, []).append(msgsrc)
|
||||
@@ -330,10 +410,9 @@ def dump_messages_pytext(messages, check_ctxt):
|
||||
if type(nd) not in stopper_nodes:
|
||||
extract_strings(fp_rel, nd)
|
||||
|
||||
def extract_strings_from_file(fp):
|
||||
filedata = open(fp, 'r', encoding="utf8")
|
||||
root_node = ast.parse(filedata.read(), fp, 'exec')
|
||||
filedata.close()
|
||||
for fp in files:
|
||||
with open(fp, 'r', encoding="utf8") as filedata:
|
||||
root_node = ast.parse(filedata.read(), fp, 'exec')
|
||||
|
||||
fp_rel = os.path.relpath(fp, SOURCE_DIR)
|
||||
|
||||
@@ -361,70 +440,39 @@ def dump_messages_pytext(messages, check_ctxt):
|
||||
if kw.arg == arg_kw:
|
||||
extract_strings(fp_rel, kw.value)
|
||||
|
||||
# -------------------------------------------------------------------------
|
||||
# Dump Messages
|
||||
|
||||
mod_dir = os.path.join(SOURCE_DIR,
|
||||
"release",
|
||||
"scripts",
|
||||
"startup",
|
||||
"bl_ui")
|
||||
def dump_py_messages(messages, check_ctxt, addons):
|
||||
mod_dir = os.path.join(SOURCE_DIR, "release", "scripts", "startup", "bl_ui")
|
||||
|
||||
files = [os.path.join(mod_dir, fn)
|
||||
for fn in sorted(os.listdir(mod_dir))
|
||||
if not fn.startswith("_")
|
||||
if fn.endswith("py")
|
||||
]
|
||||
files = [os.path.join(mod_dir, fn) for fn in sorted(os.listdir(mod_dir))
|
||||
if not fn.startswith("_") if fn.endswith("py")]
|
||||
|
||||
# Dummy Cycles has its py addon in its own dir!
|
||||
files += CUSTOM_PY_UI_FILES
|
||||
|
||||
for fp in files:
|
||||
extract_strings_from_file(fp)
|
||||
# Add all addons we support in main translation file!
|
||||
for mod in addons:
|
||||
fn = mod.__file__
|
||||
if os.path.basename(fn) == "__init__.py":
|
||||
mod_dir = os.path.dirname(fn)
|
||||
files += [fn for fn in sorted(os.listdir(mod_dir))
|
||||
if os.path.isfile(fn) and os.path.splitext(fn)[1] == ".py"]
|
||||
else:
|
||||
files.append(fn)
|
||||
|
||||
dump_py_messages_from_files(messages, check_ctxt, files)
|
||||
|
||||
|
||||
##### Main functions! #####
|
||||
|
||||
def dump_messages(do_messages, do_checks):
|
||||
import collections
|
||||
import re
|
||||
|
||||
def enable_addons():
|
||||
"""For now, enable all official addons, before extracting msgids."""
|
||||
import addon_utils
|
||||
import bpy
|
||||
|
||||
userpref = bpy.context.user_preferences
|
||||
used_ext = {ext.module for ext in userpref.addons}
|
||||
support = {"OFFICIAL"}
|
||||
# collect the categories that can be filtered on
|
||||
addons = [(mod, addon_utils.module_bl_info(mod)) for mod in
|
||||
addon_utils.modules(addon_utils.addons_fake_modules)]
|
||||
|
||||
for mod, info in addons:
|
||||
module_name = mod.__name__
|
||||
if module_name in used_ext or info["support"] not in support:
|
||||
continue
|
||||
print(" Enabling module ", module_name)
|
||||
bpy.ops.wm.addon_enable(module=module_name)
|
||||
|
||||
# XXX There are currently some problems with bpy/rna...
|
||||
# *Very* tricky to solve!
|
||||
# So this is a hack to make all newly added operator visible by
|
||||
# bpy.types.OperatorProperties.__subclasses__()
|
||||
for cat in dir(bpy.ops):
|
||||
cat = getattr(bpy.ops, cat)
|
||||
for op in dir(cat):
|
||||
getattr(cat, op).get_rna()
|
||||
|
||||
# check for strings like ": %d"
|
||||
ignore_reg = re.compile(r"^(?:[-*.()/\\+:%xWXYZ0-9]|%d|%f|%s|%r|\s)*$")
|
||||
filter_message = ignore_reg.match
|
||||
|
||||
messages = getattr(collections, 'OrderedDict', dict)()
|
||||
|
||||
messages[(CONTEXT_DEFAULT, "")] = []
|
||||
|
||||
# Enable all wanted addons.
|
||||
enable_addons()
|
||||
# For now, enable all official addons, before extracting msgids.
|
||||
addons = enable_addons(support={"OFFICIAL"})
|
||||
|
||||
check_ctxt = None
|
||||
if do_checks:
|
||||
@@ -438,35 +486,89 @@ def dump_messages(do_messages, do_checks):
|
||||
# get strings from RNA
|
||||
dump_messages_rna(messages, check_ctxt)
|
||||
|
||||
# get strings from UI layout definitions text="..." args
|
||||
dump_py_messages(messages, check_ctxt, addons)
|
||||
|
||||
del messages[(CONTEXT_DEFAULT, "")]
|
||||
|
||||
print_warnings(check_ctxt, messages)
|
||||
|
||||
if do_messages:
|
||||
print("Writing messages…")
|
||||
num_written = 0
|
||||
num_filtered = 0
|
||||
with open(FILE_NAME_MESSAGES, 'w', encoding="utf8") as message_file:
|
||||
for (ctx, key), value in messages.items():
|
||||
# filter out junk values
|
||||
if filter_message(key):
|
||||
num_filtered += 1
|
||||
continue
|
||||
|
||||
# Remove newlines in key and values!
|
||||
message_file.write("\n".join(MSG_COMMENT_PREFIX + msgsrc.replace("\n", "") for msgsrc in value))
|
||||
message_file.write("\n")
|
||||
if ctx:
|
||||
message_file.write(MSG_CONTEXT_PREFIX + ctx.replace("\n", "") + "\n")
|
||||
message_file.write(key.replace("\n", "") + "\n")
|
||||
num_written += 1
|
||||
|
||||
print("Written {} messages to: {} ({} were filtered out)."
|
||||
"".format(num_written, FILE_NAME_MESSAGES, num_filtered))
|
||||
|
||||
|
||||
def dump_addon_messages(module_name, messages_formats, do_checks):
|
||||
messages = getattr(collections, 'OrderedDict', dict)()
|
||||
|
||||
messages[(CONTEXT_DEFAULT, "")] = []
|
||||
minus_messages = copy.deepcopy(messages)
|
||||
|
||||
check_ctxt = None
|
||||
minus_check_ctxt = None
|
||||
if do_checks:
|
||||
check_ctxt = {"multi_rnatip": set(),
|
||||
"multi_lines": set(),
|
||||
"py_in_rna": set(),
|
||||
"not_capitalized": set(),
|
||||
"end_point": set(),
|
||||
"undoc_ops": set()}
|
||||
minus_check_ctxt = copy.deepcopy(check_ctxt)
|
||||
|
||||
# Get current addon state (loaded or not):
|
||||
was_loaded = addon_utils.check(module_name)[1]
|
||||
|
||||
# Enable our addon and get strings from RNA.
|
||||
enable_addons(addons={module_name})
|
||||
dump_messages_rna(messages, check_ctxt)
|
||||
|
||||
# Now disable our addon, and rescan RNA.
|
||||
enable_addons(addons={module_name}, disable=True)
|
||||
dump_messages_rna(minus_messages, minus_check_ctxt)
|
||||
|
||||
# Restore previous state if needed!
|
||||
if was_loaded:
|
||||
enable_addons(addons={module_name})
|
||||
|
||||
# and make the diff!
|
||||
for key in minus_messages:
|
||||
if k == (CONTEXT_DEFAULT, ""):
|
||||
continue
|
||||
del messages[k]
|
||||
|
||||
if check_ctxt:
|
||||
for key in check_ctxt:
|
||||
for warning in minus_check_ctxt[key]:
|
||||
check_ctxt[key].remove(warning)
|
||||
|
||||
# and we are done with those!
|
||||
del minus_messages
|
||||
del minus_check_ctxt
|
||||
|
||||
# get strings from UI layout definitions text="..." args
|
||||
dump_messages_pytext(messages, check_ctxt)
|
||||
|
||||
del messages[(CONTEXT_DEFAULT, "")]
|
||||
|
||||
if do_checks:
|
||||
print("WARNINGS:")
|
||||
keys = set()
|
||||
for c in check_ctxt.values():
|
||||
keys |= c
|
||||
# XXX Temp, see below
|
||||
keys -= check_ctxt["multi_rnatip"]
|
||||
for key in keys:
|
||||
if key in check_ctxt["undoc_ops"]:
|
||||
print("\tThe following operators are undocumented:")
|
||||
else:
|
||||
print("\t“{}”|“{}”:".format(*key))
|
||||
if key in check_ctxt["multi_lines"]:
|
||||
print("\t\t-> newline in this message!")
|
||||
if key in check_ctxt["not_capitalized"]:
|
||||
print("\t\t-> message not capitalized!")
|
||||
if key in check_ctxt["end_point"]:
|
||||
print("\t\t-> message with endpoint!")
|
||||
# XXX Hide this one for now, too much false positives.
|
||||
# if key in check_ctxt["multi_rnatip"]:
|
||||
# print("\t\t-> tip used in several RNA items")
|
||||
if key in check_ctxt["py_in_rna"]:
|
||||
print("\t\t-> RNA message also used in py UI code:")
|
||||
print("\t\t{}".format("\n\t\t".join(messages[key])))
|
||||
print_warnings
|
||||
|
||||
if do_messages:
|
||||
print("Writing messages…")
|
||||
@@ -491,6 +593,7 @@ def dump_messages(do_messages, do_checks):
|
||||
"".format(num_written, FILE_NAME_MESSAGES, num_filtered))
|
||||
|
||||
|
||||
|
||||
def main():
|
||||
try:
|
||||
import bpy
|
||||
|
||||
@@ -38,45 +38,47 @@ BRANCHES_DIR = settings.BRANCHES_DIR
|
||||
FILE_NAME_POT = settings.FILE_NAME_POT
|
||||
|
||||
|
||||
def print_diff(ref_messages, messages, states):
|
||||
def print_diff(ref_msgs, msgs):
|
||||
# Remove comments from messages list!
|
||||
messages = set(messages.keys()) - states["comm_msg"]
|
||||
unneeded = (messages - ref_messages)
|
||||
for msgid in unneeded:
|
||||
print('\tUnneeded message id "{}"'.format(msgid))
|
||||
messages = set(msgs.msgs.keys()) - msgs.comm_msgs
|
||||
unneeded = (messages - ref_msgs.msgs.keys())
|
||||
for msgkey in unneeded:
|
||||
print('\tUnneeded message context/id "{}"'.format(msgkey))
|
||||
|
||||
missing = (ref_messages - messages)
|
||||
for msgid in missing:
|
||||
print('\tMissing message id "{}"'.format(msgid))
|
||||
missing = (ref_msgs.msgs.keys() - messages)
|
||||
for msgkey in missing:
|
||||
print('\tMissing message context/id "{}"'.format(msgkey))
|
||||
|
||||
for msgid in states["comm_msg"]:
|
||||
print('\tCommented message id "{}"'.format(msgid))
|
||||
for msgid in msgs.comm_msgs:
|
||||
print('\tCommented message context/id "{}"'.format(msgkey))
|
||||
|
||||
print("\t{} unneeded messages, {} missing messages, {} commented messages." \
|
||||
"".format(len(unneeded), len(missing), len(states["comm_msg"])))
|
||||
print("\t{} unneeded messages, {} missing messages, {} commented messages."
|
||||
"".format(len(unneeded), len(missing), len(msgs.comm_msgs)))
|
||||
return 0
|
||||
|
||||
|
||||
def process_po(ref_messages, po, glob_stats, do_stats, do_messages):
|
||||
#def process_po(ref_messages, po, glob_stats, do_stats, do_messages):
|
||||
def process_po(ref_messages, po, do_stats, do_messages):
|
||||
print("Checking {}...".format(po))
|
||||
ret = 0
|
||||
|
||||
messages, states, stats = utils.parse_messages(po)
|
||||
messages = utils.I18nMessages(kind='PO', src=po)
|
||||
if do_messages:
|
||||
t = print_diff(ref_messages, messages, states)
|
||||
t = print_diff(ref_messages, messages)
|
||||
if t:
|
||||
ret = t
|
||||
if do_stats:
|
||||
print("\tStats:")
|
||||
t = utils.print_stats(stats, glob_stats, prefix=" ")
|
||||
t = messages.print_stats(prefix=" ")
|
||||
if t:
|
||||
ret = t
|
||||
if states["is_broken"]:
|
||||
if messages.parsing_errors:
|
||||
print("\tERROR! This .po is broken!")
|
||||
ret = 1
|
||||
return ret
|
||||
|
||||
|
||||
# XXX Quick update for new I18Nfoo objects, need rework!
|
||||
def main():
|
||||
import argparse
|
||||
parser = argparse.ArgumentParser(description="Check po’s in branches " \
|
||||
@@ -97,22 +99,21 @@ def main():
|
||||
if args.pot:
|
||||
global FILE_NAME_POT
|
||||
FILE_NAME_POT = args.pot
|
||||
glob_stats = {"nbr" : 0.0,
|
||||
"lvl" : 0.0,
|
||||
"lvl_ttips" : 0.0,
|
||||
"lvl_trans_ttips" : 0.0,
|
||||
"lvl_ttips_in_trans": 0.0,
|
||||
"lvl_comm" : 0.0,
|
||||
"nbr_signs" : 0,
|
||||
"nbr_trans_signs" : 0,
|
||||
"contexts" : set()}
|
||||
#glob_stats = {"nbr" : 0.0,
|
||||
#"lvl" : 0.0,
|
||||
#"lvl_ttips" : 0.0,
|
||||
#"lvl_trans_ttips" : 0.0,
|
||||
#"lvl_ttips_in_trans": 0.0,
|
||||
#"lvl_comm" : 0.0,
|
||||
#"nbr_signs" : 0,
|
||||
#"nbr_trans_signs" : 0,
|
||||
#"contexts" : set()}
|
||||
ret = 0
|
||||
|
||||
pot_messages = None
|
||||
if args.messages:
|
||||
pot_messages, u1, pot_stats = utils.parse_messages(FILE_NAME_POT)
|
||||
pot_messages = set(pot_messages.keys())
|
||||
glob_stats["nbr_signs"] = pot_stats["nbr_signs"]
|
||||
pot_messages = utils.I18nMessages(kind='PO', src=FILE_NAME_POT)
|
||||
#glob_stats["nbr_signs"] = pot_stats["nbr_signs"]
|
||||
|
||||
if args.langs:
|
||||
for lang in args.langs:
|
||||
@@ -121,16 +122,16 @@ def main():
|
||||
else:
|
||||
po = os.path.join(BRANCHES_DIR, lang, ".".join((lang, "po")))
|
||||
if os.path.exists(po):
|
||||
t = process_po(pot_messages, po, glob_stats,
|
||||
args.stats, args.messages)
|
||||
#t = process_po(pot_messages, po, glob_stats, args.stats, args.messages)
|
||||
t = process_po(pot_messages, po, args.stats, args.messages)
|
||||
if t:
|
||||
ret = t
|
||||
elif args.trunk:
|
||||
for po in os.listdir(TRUNK_PO_DIR):
|
||||
if po.endswith(".po"):
|
||||
po = os.path.join(TRUNK_PO_DIR, po)
|
||||
t = process_po(pot_messages, po, glob_stats,
|
||||
args.stats, args.messages)
|
||||
#t = process_po(pot_messages, po, glob_stats, args.stats, args.messages)
|
||||
t = process_po(pot_messages, po, args.stats, args.messages)
|
||||
if t:
|
||||
ret = t
|
||||
else:
|
||||
@@ -138,35 +139,35 @@ def main():
|
||||
for po in os.listdir(os.path.join(BRANCHES_DIR, lang)):
|
||||
if po.endswith(".po"):
|
||||
po = os.path.join(BRANCHES_DIR, lang, po)
|
||||
t = process_po(pot_messages, po, glob_stats,
|
||||
args.stats, args.messages)
|
||||
#t = process_po(pot_messages, po, glob_stats, args.stats, args.messages)
|
||||
t = process_po(pot_messages, po, args.stats, args.messages)
|
||||
if t:
|
||||
ret = t
|
||||
|
||||
if args.stats and glob_stats["nbr"] != 0.0:
|
||||
nbr_contexts = len(glob_stats["contexts"] - {""})
|
||||
if nbr_contexts != 1:
|
||||
if nbr_contexts == 0:
|
||||
nbr_contexts = "No"
|
||||
_ctx_txt = "s are"
|
||||
else:
|
||||
_ctx_txt = " is"
|
||||
print("\nAverage stats for all {:.0f} processed files:\n"
|
||||
" {:>6.1%} done!\n"
|
||||
" {:>6.1%} of messages are tooltips.\n"
|
||||
" {:>6.1%} of tooltips are translated.\n"
|
||||
" {:>6.1%} of translated messages are tooltips.\n"
|
||||
" {:>6.1%} of messages are commented.\n"
|
||||
" The org msgids are currently made of {} signs.\n"
|
||||
" All processed translations are currently made of {} signs.\n"
|
||||
" {} specific context{} present:\n {}\n"
|
||||
"".format(glob_stats["nbr"], glob_stats["lvl"] / glob_stats["nbr"],
|
||||
glob_stats["lvl_ttips"] / glob_stats["nbr"],
|
||||
glob_stats["lvl_trans_ttips"] / glob_stats["nbr"],
|
||||
glob_stats["lvl_ttips_in_trans"] / glob_stats["nbr"],
|
||||
glob_stats["lvl_comm"] / glob_stats["nbr"], glob_stats["nbr_signs"],
|
||||
glob_stats["nbr_trans_signs"], nbr_contexts, _ctx_txt,
|
||||
"\n ".join(glob_stats["contexts"]-{""})))
|
||||
#if args.stats and glob_stats["nbr"] != 0.0:
|
||||
#nbr_contexts = len(glob_stats["contexts"] - {""})
|
||||
#if nbr_contexts != 1:
|
||||
#if nbr_contexts == 0:
|
||||
#nbr_contexts = "No"
|
||||
#_ctx_txt = "s are"
|
||||
#else:
|
||||
#_ctx_txt = " is"
|
||||
#print("\nAverage stats for all {:.0f} processed files:\n"
|
||||
#" {:>6.1%} done!\n"
|
||||
#" {:>6.1%} of messages are tooltips.\n"
|
||||
#" {:>6.1%} of tooltips are translated.\n"
|
||||
#" {:>6.1%} of translated messages are tooltips.\n"
|
||||
#" {:>6.1%} of messages are commented.\n"
|
||||
#" The org msgids are currently made of {} signs.\n"
|
||||
#" All processed translations are currently made of {} signs.\n"
|
||||
#" {} specific context{} present:\n {}\n"
|
||||
#"".format(glob_stats["nbr"], glob_stats["lvl"] / glob_stats["nbr"],
|
||||
#glob_stats["lvl_ttips"] / glob_stats["nbr"],
|
||||
#glob_stats["lvl_trans_ttips"] / glob_stats["nbr"],
|
||||
#glob_stats["lvl_ttips_in_trans"] / glob_stats["nbr"],
|
||||
#glob_stats["lvl_comm"] / glob_stats["nbr"], glob_stats["nbr_signs"],
|
||||
#glob_stats["nbr_trans_signs"], nbr_contexts, _ctx_txt,
|
||||
#"\n ".join(glob_stats["contexts"]-{""})))
|
||||
|
||||
return ret
|
||||
|
||||
|
||||
@@ -39,30 +39,27 @@ BRANCHES_DIR = settings.BRANCHES_DIR
|
||||
|
||||
def do_clean(po, strict):
|
||||
print("Cleaning {}...".format(po))
|
||||
messages, states, u1 = utils.parse_messages(po)
|
||||
msgs = utils.I18nMessages(kind='PO', src=po)
|
||||
|
||||
if strict and states["is_broken"]:
|
||||
if strict and msgs.parsing_errors:
|
||||
print("ERROR! This .po file is broken!")
|
||||
return 1
|
||||
|
||||
for msgkey in states["comm_msg"]:
|
||||
del messages[msgkey]
|
||||
utils.write_messages(po, messages, states["comm_msg"], states["fuzzy_msg"])
|
||||
print("Removed {} commented messages.".format(len(states["comm_msg"])))
|
||||
nbr_rem = len(msgs.comm_msgs)
|
||||
for msgkey in msgs.comm_msgs:
|
||||
del msgs.msgs[msgkey]
|
||||
msgs.write(kind='PO', dest=po)
|
||||
print("Removed {} commented messages.".format(nbr_rem))
|
||||
return 0
|
||||
|
||||
|
||||
def main():
|
||||
import argparse
|
||||
parser = argparse.ArgumentParser(description="Clean po’s in branches " \
|
||||
"or trunk (i.e. remove " \
|
||||
"all commented messages).")
|
||||
parser.add_argument('-t', '--trunk', action="store_true",
|
||||
help="Clean po’s in trunk rather than branches.")
|
||||
parser.add_argument('-s', '--strict', action="store_true",
|
||||
help="Raise an error if a po is broken.")
|
||||
parser.add_argument('langs', metavar='ISO_code', nargs='*',
|
||||
help="Restrict processed languages to those.")
|
||||
parser = argparse.ArgumentParser(description="Clean po’s in branches or trunk (i.e. remove all commented "
|
||||
"messages).")
|
||||
parser.add_argument('-t', '--trunk', action="store_true", help="Clean po’s in trunk rather than branches.")
|
||||
parser.add_argument('-s', '--strict', action="store_true", help="Raise an error if a po is broken.")
|
||||
parser.add_argument('langs', metavar='ISO_code', nargs='*', help="Restrict processed languages to those.")
|
||||
args = parser.parse_args()
|
||||
|
||||
ret = 0
|
||||
|
||||
@@ -26,7 +26,6 @@ import os
|
||||
import shutil
|
||||
import sys
|
||||
import subprocess
|
||||
from codecs import open
|
||||
|
||||
try:
|
||||
import settings
|
||||
@@ -48,14 +47,10 @@ PY3 = settings.PYTHON3_EXEC
|
||||
|
||||
def main():
|
||||
import argparse
|
||||
parser = argparse.ArgumentParser(description="Import advanced enough po’s " \
|
||||
"from branches to trunk.")
|
||||
parser.add_argument('-t', '--threshold', type=int,
|
||||
help="Import threshold, as a percentage.")
|
||||
parser.add_argument('-s', '--strict', action="store_true",
|
||||
help="Raise an error if a po is broken.")
|
||||
parser.add_argument('langs', metavar='ISO_code', nargs='*',
|
||||
help="Restrict processed languages to those.")
|
||||
parser = argparse.ArgumentParser(description="Import advanced enough po’s from branches to trunk.")
|
||||
parser.add_argument('-t', '--threshold', type=float, help="Import threshold, as a percentage.")
|
||||
parser.add_argument('-s', '--strict', action="store_true", help="Raise an error if a po is broken.")
|
||||
parser.add_argument('langs', metavar='ISO_code', nargs='*', help="Restrict processed languages to those.")
|
||||
args = parser.parse_args()
|
||||
|
||||
ret = 0
|
||||
@@ -70,51 +65,40 @@ def main():
|
||||
po = os.path.join(BRANCHES_DIR, lang, ".".join((lang, "po")))
|
||||
if os.path.exists(po):
|
||||
po_is_rtl = os.path.join(BRANCHES_DIR, lang, RTL_PREPROCESS_FILE)
|
||||
msgs, state, stats = utils.parse_messages(po)
|
||||
tot_msgs = stats["tot_msg"]
|
||||
trans_msgs = stats["trans_msg"]
|
||||
msgs = utils.I18nMessages(iso=lang, kind='PO', src=po)
|
||||
lvl = 0.0
|
||||
if tot_msgs:
|
||||
lvl = float(trans_msgs) / float(tot_msgs)
|
||||
if msgs.nbr_msgs:
|
||||
lvl = msgs.nbr_trans_msgs / msgs.nbr_msgs
|
||||
if lvl > threshold:
|
||||
if state["is_broken"] and args.strict:
|
||||
print("{:<10}: {:>6.1%} done, but BROKEN, skipped." \
|
||||
"".format(lang, lvl))
|
||||
if msgs.parsing_errors and args.strict:
|
||||
print("{:<10}: {:>6.1%} done, but BROKEN, skipped.".format(lang, lvl))
|
||||
ret = 1
|
||||
else:
|
||||
if os.path.exists(po_is_rtl):
|
||||
out_po = os.path.join(TRUNK_PO_DIR,
|
||||
".".join((lang, "po")))
|
||||
out_raw_po = os.path.join(TRUNK_PO_DIR,
|
||||
"_".join((lang, "raw.po")))
|
||||
out_po = os.path.join(TRUNK_PO_DIR, ".".join((lang, "po")))
|
||||
out_raw_po = os.path.join(TRUNK_PO_DIR, "_".join((lang, "raw.po")))
|
||||
keys = []
|
||||
trans = []
|
||||
for k, m in msgs.items():
|
||||
for k, m in msgs.msgs.items():
|
||||
keys.append(k)
|
||||
trans.append("".join(m["msgstr_lines"]))
|
||||
trans.append(m.msgstr)
|
||||
trans = rtl_preprocess.log2vis(trans)
|
||||
for k, t in zip(keys, trans):
|
||||
# Mono-line for now...
|
||||
msgs[k]["msgstr_lines"] = [t]
|
||||
utils.write_messages(out_po, msgs, state["comm_msg"],
|
||||
state["fuzzy_msg"])
|
||||
msgs[k].msgstr = t
|
||||
msgs.write(kinf='PO', dest=out_po)
|
||||
# Also copies org po!
|
||||
shutil.copy(po, out_raw_po)
|
||||
print("{:<10}: {:>6.1%} done, enough translated " \
|
||||
"messages, processed and copied to trunk." \
|
||||
print("{:<10}: {:>6.1%} done, enough translated messages, processed and copied to trunk."
|
||||
"".format(lang, lvl))
|
||||
else:
|
||||
shutil.copy(po, TRUNK_PO_DIR)
|
||||
print("{:<10}: {:>6.1%} done, enough translated " \
|
||||
"messages, copied to trunk.".format(lang, lvl))
|
||||
print("{:<10}: {:>6.1%} done, enough translated messages, copied to trunk.".format(lang, lvl))
|
||||
else:
|
||||
if state["is_broken"] and args.strict:
|
||||
print("{:<10}: {:>6.1%} done, BROKEN and not enough " \
|
||||
"translated messages, skipped".format(lang, lvl))
|
||||
if msgs.parsing_errors and args.strict:
|
||||
print("{:<10}: {:>6.1%} done, BROKEN and not enough translated messages, skipped".format(lang, lvl))
|
||||
ret = 1
|
||||
else:
|
||||
print("{:<10}: {:>6.1%} done, not enough translated " \
|
||||
"messages, skipped.".format(lang, lvl))
|
||||
print("{:<10}: {:>6.1%} done, not enough translated messages, skipped.".format(lang, lvl))
|
||||
return ret
|
||||
|
||||
|
||||
|
||||
@@ -37,26 +37,22 @@ except:
|
||||
from . import (settings, utils)
|
||||
|
||||
|
||||
# XXX This is a quick hack to make it work with new I18n... objects! To be reworked!
|
||||
def main():
|
||||
import argparse
|
||||
parser = argparse.ArgumentParser(description="" \
|
||||
"Merge one or more .po files into the first dest one.\n" \
|
||||
"If a msgkey (msgctxt, msgid) is present in more than " \
|
||||
"one merged po, the one in the first file wins, unless " \
|
||||
"it’s marked as fuzzy and one later is not.\n" \
|
||||
"The fuzzy flag is removed if necessary.\n" \
|
||||
"All other comments are never modified.\n" \
|
||||
"Commented messages in dst will always remain " \
|
||||
"commented, and commented messages are never merged " \
|
||||
parser = argparse.ArgumentParser(description=""
|
||||
"Merge one or more .po files into the first dest one.\n"
|
||||
"If a msgkey (msgctxt, msgid) is present in more than one merged po, the one in the first file "
|
||||
"wins, unless it’s marked as fuzzy and one later is not.\n"
|
||||
"The fuzzy flag is removed if necessary.\n"
|
||||
"All other comments are never modified.\n"
|
||||
"Commented messages in dst will always remain commented, and commented messages are never merged "
|
||||
"from sources.")
|
||||
parser.add_argument('-s', '--stats', action="store_true",
|
||||
help="Show statistics info.")
|
||||
parser.add_argument('-s', '--stats', action="store_true", help="Show statistics info.")
|
||||
parser.add_argument('-r', '--replace', action="store_true",
|
||||
help="Replace existing messages of same \"level\" already in dest po.")
|
||||
parser.add_argument('dst', metavar='dst.po',
|
||||
help="The dest po into which merge the others.")
|
||||
parser.add_argument('src', metavar='src.po', nargs='+',
|
||||
help="The po's to merge into the dst.po one.")
|
||||
parser.add_argument('dst', metavar='dst.po', help="The dest po into which merge the others.")
|
||||
parser.add_argument('src', metavar='src.po', nargs='+', help="The po's to merge into the dst.po one.")
|
||||
args = parser.parse_args()
|
||||
|
||||
ret = 0
|
||||
@@ -67,89 +63,78 @@ def main():
|
||||
nbr_added = 0
|
||||
nbr_unfuzzied = 0
|
||||
|
||||
dst_messages, dst_states, dst_stats = utils.parse_messages(args.dst)
|
||||
if dst_states["is_broken"]:
|
||||
dst_msgs = utils.I18nMessages(kind='PO', src=args.dst)
|
||||
if dst_msgs.parsing_errors:
|
||||
print("Dest po is BROKEN, aborting.")
|
||||
return 1
|
||||
if args.stats:
|
||||
print("Dest po, before merging:")
|
||||
utils.print_stats(dst_stats, prefix="\t")
|
||||
# If we don’t want to replace existing valid translations, pre-populate
|
||||
# done_msgkeys and done_fuzzy_msgkeys.
|
||||
dst_msgs.print_stats(prefix="\t")
|
||||
# If we don’t want to replace existing valid translations, pre-populate done_msgkeys and done_fuzzy_msgkeys.
|
||||
if not args.replace:
|
||||
done_msgkeys = dst_states["trans_msg"].copy()
|
||||
done_fuzzy_msgkeys = dst_states["fuzzy_msg"].copy()
|
||||
done_msgkeys = dst_msgs.trans_msgs.copy()
|
||||
done_fuzzy_msgkeys = dst_msgs.fuzzy_msgs.copy()
|
||||
for po in args.src:
|
||||
messages, states, stats = utils.parse_messages(po)
|
||||
if states["is_broken"]:
|
||||
msgs = utils.I18nMessages(kind='PO', src=po)
|
||||
if msgs.parsing_errors:
|
||||
print("\tSrc po {} is BROKEN, skipping.".format(po))
|
||||
ret = 1
|
||||
continue
|
||||
print("\tMerging {}...".format(po))
|
||||
if args.stats:
|
||||
print("\t\tMerged po stats:")
|
||||
utils.print_stats(stats, prefix="\t\t\t")
|
||||
for msgkey, val in messages.items():
|
||||
msgs.print_stats(prefix="\t\t\t")
|
||||
for msgkey, msg in msgs.msgs.items():
|
||||
msgctxt, msgid = msgkey
|
||||
# This msgkey has already been completely merged, or is a commented one,
|
||||
# or the new message is commented, skip it.
|
||||
if msgkey in (done_msgkeys | dst_states["comm_msg"] | states["comm_msg"]):
|
||||
if msgkey in (done_msgkeys | dst_msgs.comm_msgs | msgs.comm_msgs):
|
||||
continue
|
||||
is_ttip = utils.is_tooltip(msgid)
|
||||
is_ttip = msg.is_tooltip
|
||||
# New messages does not yet exists in dest.
|
||||
if msgkey not in dst_messages:
|
||||
dst_messages[msgkey] = messages[msgkey]
|
||||
if msgkey in states["fuzzy_msg"]:
|
||||
if msgkey not in dst_msgs.msgs:
|
||||
dst_msgs[msgkey] = msgs.msgs[msgkey]
|
||||
if msgkey in msgs.fuzzy_msgs:
|
||||
done_fuzzy_msgkeys.add(msgkey)
|
||||
dst_states["fuzzy_msg"].add(msgkey)
|
||||
elif msgkey in states["trans_msg"]:
|
||||
dst_msgs.fuzzy_msgs.add(msgkey)
|
||||
elif msgkey in msgs.trans_msgs:
|
||||
done_msgkeys.add(msgkey)
|
||||
dst_states["trans_msg"].add(msgkey)
|
||||
dst_stats["trans_msg"] += 1
|
||||
if is_ttip:
|
||||
dst_stats["trans_ttips"] += 1
|
||||
dst_msgs.trans_msgs.add(msgkey)
|
||||
nbr_added += 1
|
||||
dst_stats["tot_msg"] += 1
|
||||
if is_ttip:
|
||||
dst_stats["tot_ttips"] += 1
|
||||
# From now on, the new messages is already in dst.
|
||||
# New message is neither translated nor fuzzy, skip it.
|
||||
elif msgkey not in (states["trans_msg"] | states["fuzzy_msg"]):
|
||||
elif msgkey not in (msgs.trans_msgs | msgs.fuzzy_msgs):
|
||||
continue
|
||||
# From now on, the new message is either translated or fuzzy!
|
||||
# The new message is translated.
|
||||
elif msgkey in states["trans_msg"]:
|
||||
dst_messages[msgkey]["msgstr_lines"] = messages[msgkey]["msgstr_lines"]
|
||||
elif msgkey in msgs.trans_msgs:
|
||||
dst_msgs.msgs[msgkey].msgstr = msg.msgstr
|
||||
done_msgkeys.add(msgkey)
|
||||
done_fuzzy_msgkeys.discard(msgkey)
|
||||
if msgkey in dst_states["fuzzy_msg"]:
|
||||
dst_states["fuzzy_msg"].remove(msgkey)
|
||||
if msgkey in dst_msgs.fuzzy_msgs:
|
||||
dst_msgs.fuzzy_msgs.remove(msgkey)
|
||||
nbr_unfuzzied += 1
|
||||
if msgkey not in dst_states["trans_msg"]:
|
||||
dst_states["trans_msg"].add(msgkey)
|
||||
dst_stats["trans_msg"] += 1
|
||||
if is_ttip:
|
||||
dst_stats["trans_ttips"] += 1
|
||||
if msgkey not in dst_msgs.trans_msgs:
|
||||
dst_msgs.trans_msgs.add(msgkey)
|
||||
else:
|
||||
nbr_replaced += 1
|
||||
nbr_merged += 1
|
||||
# The new message is fuzzy, org one is fuzzy too,
|
||||
# and this msgkey has not yet been merged.
|
||||
elif msgkey not in (dst_states["trans_msg"] | done_fuzzy_msgkeys):
|
||||
dst_messages[msgkey]["msgstr_lines"] = messages[msgkey]["msgstr_lines"]
|
||||
# The new message is fuzzy, org one is fuzzy too, and this msgkey has not yet been merged.
|
||||
elif msgkey not in (dst_msgs.trans_msgs | done_fuzzy_msgkeys):
|
||||
dst_msgs[msgkey].msgstr = msg.msgstr
|
||||
done_fuzzy_msgkeys.add(msgkey)
|
||||
dst_states["fuzzy_msg"].add(msgkey)
|
||||
dst_msgs.fuzzy_msgs.add(msgkey)
|
||||
nbr_merged += 1
|
||||
nbr_replaced += 1
|
||||
|
||||
utils.write_messages(args.dst, dst_messages, dst_states["comm_msg"], dst_states["fuzzy_msg"])
|
||||
dst_msgs.write(kind='PO', dest=args.dst)
|
||||
|
||||
print("Merged completed. {} messages were merged (among which {} were replaced), " \
|
||||
"{} were added, {} were \"un-fuzzied\"." \
|
||||
"".format(nbr_merged, nbr_replaced, nbr_added, nbr_unfuzzied))
|
||||
print("Merged completed. {} messages were merged (among which {} were replaced), {} were added, "
|
||||
"{} were \"un-fuzzied\".".format(nbr_merged, nbr_replaced, nbr_added, nbr_unfuzzied))
|
||||
if args.stats:
|
||||
dst_msgs.update_info()
|
||||
print("Final merged po stats:")
|
||||
utils.print_stats(dst_stats, prefix="\t")
|
||||
dst_msgs.print_stats(prefix="\t")
|
||||
return ret
|
||||
|
||||
|
||||
|
||||
@@ -96,13 +96,62 @@ IMPORT_MIN_LEVEL = -1
|
||||
IMPORT_LANGUAGES_SKIP = {'am', 'bg', 'fi', 'el', 'et', 'ko', 'ne', 'pl', 'ro', 'uz', 'uz@cyrillic'}
|
||||
|
||||
# The comment prefix used in generated messages.txt file.
|
||||
COMMENT_PREFIX = "#~ "
|
||||
|
||||
# The comment prefix used to mark sources of msgids, in po's.
|
||||
COMMENT_PREFIX_SOURCE = "#: "
|
||||
MSG_COMMENT_PREFIX = "#~ "
|
||||
|
||||
# The comment prefix used in generated messages.txt file.
|
||||
CONTEXT_PREFIX = "MSGCTXT:"
|
||||
MSG_CONTEXT_PREFIX = "MSGCTXT:"
|
||||
|
||||
# The default comment prefix used in po's.
|
||||
PO_COMMENT_PREFIX= "# "
|
||||
|
||||
# The comment prefix used to mark sources of msgids, in po's.
|
||||
PO_COMMENT_PREFIX_SOURCE = "#: "
|
||||
|
||||
# The comment prefix used to mark sources of msgids, in po's.
|
||||
PO_COMMENT_PREFIX_SOURCE_CUSTOM = "#. :src: "
|
||||
|
||||
# The comment prefix used to comment entries in po's.
|
||||
PO_COMMENT_PREFIX_MSG= "#~ "
|
||||
|
||||
# The comment prefix used to mark fuzzy msgids, in po's.
|
||||
PO_COMMENT_FUZZY = "#, fuzzy"
|
||||
|
||||
# The prefix used to define context, in po's.
|
||||
PO_MSGCTXT = "msgctxt "
|
||||
|
||||
# The prefix used to define msgid, in po's.
|
||||
PO_MSGID = "msgid "
|
||||
|
||||
# The prefix used to define msgstr, in po's.
|
||||
PO_MSGSTR = "msgstr "
|
||||
|
||||
# The 'header' key of po files.
|
||||
PO_HEADER_KEY = ("", "")
|
||||
|
||||
PO_HEADER_MSGSTR = (
|
||||
"Project-Id-Version: Blender {blender_ver} (r{blender_rev})\\n\n"
|
||||
"Report-Msgid-Bugs-To: \\n\n"
|
||||
"POT-Creation-Date: {time}\\n\n"
|
||||
"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\\n\n"
|
||||
"Last-Translator: FULL NAME <EMAIL@ADDRESS>\\n\n"
|
||||
"Language-Team: LANGUAGE <LL@li.org>\\n\n"
|
||||
"Language: {iso}\\n\n"
|
||||
"MIME-Version: 1.0\\n\n"
|
||||
"Content-Type: text/plain; charset=UTF-8\\n\n"
|
||||
"Content-Transfer-Encoding: 8bit\n"
|
||||
)
|
||||
PO_HEADER_COMMENT_COPYRIGHT = (
|
||||
"# Blender's translation file (po format).\n"
|
||||
"# Copyright (C) {year} The Blender Foundation.\n"
|
||||
"# This file is distributed under the same license as the Blender package.\n"
|
||||
"#\n"
|
||||
)
|
||||
PO_HEADER_COMMENT = (
|
||||
"# FIRST AUTHOR <EMAIL@ADDRESS>, YEAR.\n"
|
||||
"#"
|
||||
)
|
||||
|
||||
TEMPLATE_ISO_ID = "__TEMPLATE__"
|
||||
|
||||
# Default context.
|
||||
CONTEXT_DEFAULT = ""
|
||||
@@ -121,8 +170,7 @@ PYGETTEXT_ALLOWED_EXTS = {".c", ".cpp", ".cxx", ".hpp", ".hxx", ".h"}
|
||||
PYGETTEXT_MAX_MULTI_CTXT = 16
|
||||
|
||||
# Where to search contexts definitions, relative to SOURCE_DIR (defined below).
|
||||
PYGETTEXT_CONTEXTS_DEFSRC = os.path.join("source", "blender", "blenfont",
|
||||
"BLF_translation.h")
|
||||
PYGETTEXT_CONTEXTS_DEFSRC = os.path.join("source", "blender", "blenfont", "BLF_translation.h")
|
||||
|
||||
# Regex to extract contexts defined in BLF_translation.h
|
||||
# XXX Not full-proof, but should be enough here!
|
||||
@@ -146,11 +194,21 @@ _str_base = (
|
||||
"(?P={_}2)" # And closing quote.
|
||||
)
|
||||
str_clean_re = _str_base.format(_="g", capt="P<clean>")
|
||||
_inbetween_str_re = (
|
||||
# XXX Strings may have comments between their pieces too, not only spaces!
|
||||
r"(?:\s*(?:"
|
||||
# A C comment
|
||||
r"/\*.*(?!\*/).\*/|"
|
||||
# Or a C++ one!
|
||||
r"//[^\n]*\n"
|
||||
# And we are done!
|
||||
r")?)*"
|
||||
)
|
||||
# Here we have to consider two different cases (empty string and other).
|
||||
_str_whole_re = (
|
||||
_str_base.format(_="{_}1_", capt=":") +
|
||||
# Optional loop start, this handles "split" strings...
|
||||
"(?:(?<=[\"'])\\s*(?=[\"'])(?:"
|
||||
"(?:(?<=[\"'])" + _inbetween_str_re + "(?=[\"'])(?:"
|
||||
+ _str_base.format(_="{_}2_", capt=":") +
|
||||
# End of loop.
|
||||
"))*"
|
||||
@@ -231,6 +289,8 @@ WARN_MSGID_NOT_CAPITALIZED_ALLOWED = {
|
||||
}
|
||||
WARN_MSGID_NOT_CAPITALIZED_ALLOWED |= set(lng[2] for lng in LANGUAGES)
|
||||
|
||||
PARSER_CACHE_HASH = 'sha1'
|
||||
|
||||
|
||||
###############################################################################
|
||||
# PATHS
|
||||
|
||||
@@ -43,42 +43,31 @@ FILE_NAME_POT = settings.FILE_NAME_POT
|
||||
|
||||
def main():
|
||||
import argparse
|
||||
parser = argparse.ArgumentParser(description="" \
|
||||
"Update all branches:\n" \
|
||||
"* Generate a temp messages.txt file.\n" \
|
||||
"* Use it to generate a temp .pot file.\n" \
|
||||
"* Use it to update all .po’s in /branches.")
|
||||
parser.add_argument('--pproc-contexts', action="store_true",
|
||||
help="Pre-process po’s to avoid having plenty of "
|
||||
"fuzzy msgids just because a context was "
|
||||
"added/changed!")
|
||||
parser.add_argument('-c', '--no_checks', default=True,
|
||||
action="store_false",
|
||||
help="No checks over UI messages.")
|
||||
parser = argparse.ArgumentParser(description="Update all branches:\n"
|
||||
"* Generate a temp messages.txt file.\n"
|
||||
"* Use it to generate a blender.pot file.\n"
|
||||
"* Use it to update all .po’s in /branches.")
|
||||
#parser.add_argument('--pproc-contexts', action="store_true",
|
||||
#help="Pre-process po’s to avoid having plenty of fuzzy msgids just because a context was "
|
||||
#"added/changed!")
|
||||
parser.add_argument('-c', '--no_checks', default=True, action="store_false", help="No checks over UI messages.")
|
||||
parser.add_argument('-a', '--add', action="store_true",
|
||||
help="Add missing po’s (useful only when one or " \
|
||||
"more languages are given!).")
|
||||
parser.add_argument('langs', metavar='ISO_code', nargs='*',
|
||||
help="Restrict processed languages to those.")
|
||||
help="Add missing po’s (useful only when one or more languages are given!).")
|
||||
parser.add_argument('langs', metavar='ISO_code', nargs='*', help="Restrict processed languages to those.")
|
||||
args = parser.parse_args()
|
||||
|
||||
ret = 0
|
||||
|
||||
# Generate a temp messages file.
|
||||
dummy, msgfile = tempfile.mkstemp(suffix=".txt",
|
||||
prefix="blender_messages_")
|
||||
dummy, msgfile = tempfile.mkstemp(suffix=".txt", prefix="blender_messages_")
|
||||
os.close(dummy)
|
||||
cmd = (PY3, "./update_msg.py", "-o", msgfile)
|
||||
t = subprocess.call(cmd)
|
||||
if t:
|
||||
ret = t
|
||||
|
||||
# Generate a temp pot file.
|
||||
# Back to having a pot file in trunk/po. It's quite useful for translators that want to start
|
||||
# Generate blender.pot file in trunk/po. It's quite useful for translators that want to start
|
||||
# a new translation and not not want to bother generating their own po from scratch!
|
||||
# dummy, potfile = tempfile.mkstemp(suffix=".pot",
|
||||
# prefix="blender_pot_")
|
||||
# os.close(dummy)
|
||||
potfile = FILE_NAME_POT
|
||||
cmd = [PY3, "./update_pot.py", "-i", msgfile, "-o", potfile]
|
||||
if not args.no_checks:
|
||||
@@ -93,8 +82,8 @@ def main():
|
||||
if args.add:
|
||||
cmd.append("-a")
|
||||
cmd += args.langs
|
||||
if args.pproc_contexts:
|
||||
cmd.append("--pproc-contexts")
|
||||
#if args.pproc_contexts:
|
||||
#cmd.append("--pproc-contexts")
|
||||
t = subprocess.call(cmd)
|
||||
if t:
|
||||
ret = t
|
||||
|
||||
@@ -73,11 +73,11 @@ def find_matching_po(languages, stats, forbidden):
|
||||
|
||||
def main():
|
||||
import argparse
|
||||
parser = argparse.ArgumentParser(description=""
|
||||
"Update 'languages' text file used by Blender at runtime to build translations menu.")
|
||||
parser = argparse.ArgumentParser(description="Update 'languages' text file used by Blender at runtime to build "
|
||||
"translations menu.")
|
||||
parser.add_argument('-m', '--min_translation', type=int, default=-100,
|
||||
help="Minimum level of translation, as a percentage "
|
||||
"(translations below this are commented out in menu).")
|
||||
help="Minimum level of translation, as a percentage (translations below this are commented out "
|
||||
"in menu).")
|
||||
parser.add_argument('langs', metavar='ISO_code', nargs='*',
|
||||
help="Unconditionally exclude those languages from the menu.")
|
||||
args = parser.parse_args()
|
||||
@@ -92,8 +92,8 @@ def main():
|
||||
for po in os.listdir(TRUNK_PO_DIR):
|
||||
if po.endswith(".po") and not po.endswith("_raw.po"):
|
||||
lang = os.path.basename(po)[:-3]
|
||||
u1, u2, _stats = utils.parse_messages(os.path.join(TRUNK_PO_DIR, po))
|
||||
stats[lang] = _stats["trans_msg"] / _stats["tot_msg"]
|
||||
msgs = utils.I18nMessages(kind='PO', src=os.path.join(TRUNK_PO_DIR, po))
|
||||
stats[lang] = msgs.nbr_trans_msgs / msgs.nbr_msgs
|
||||
|
||||
# Generate languages file used by Blender's i18n system.
|
||||
# First, match all entries in LANGUAGES to a lang in stats, if possible!
|
||||
|
||||
@@ -51,11 +51,11 @@ def process_po(po, lang, mo=None):
|
||||
|
||||
# show stats
|
||||
cmd = (GETTEXT_MSGFMT_EXECUTABLE,
|
||||
"--statistics",
|
||||
po,
|
||||
"-o",
|
||||
mo or os.path.join(mo_dir, ".".join((DOMAIN, "mo"))),
|
||||
)
|
||||
"--statistics",
|
||||
po,
|
||||
"-o",
|
||||
mo or os.path.join(mo_dir, ".".join((DOMAIN, "mo"))),
|
||||
)
|
||||
|
||||
print("Running ", " ".join(cmd))
|
||||
ret = subprocess.call(cmd)
|
||||
|
||||
@@ -41,6 +41,7 @@ TRUNK_PO_DIR = settings.TRUNK_PO_DIR
|
||||
FILE_NAME_POT = settings.FILE_NAME_POT
|
||||
|
||||
|
||||
# XXX Not updated, not sure it's that much useful...
|
||||
def pproc_newcontext_po(po, pot_messages, pot_stats):
|
||||
print("Adding new contexts to {}...".format(po))
|
||||
messages, state, stats = utils.parse_messages(po)
|
||||
@@ -73,41 +74,39 @@ def pproc_newcontext_po(po, pot_messages, pot_stats):
|
||||
|
||||
def process_po(po, lang):
|
||||
# update po file
|
||||
cmd = (GETTEXT_MSGMERGE_EXECUTABLE,
|
||||
"--update",
|
||||
"-w", "1", # XXX Ugly hack to prevent msgmerge merging
|
||||
# short source comments together!
|
||||
"--no-wrap",
|
||||
"--backup=none",
|
||||
"--lang={}".format(lang),
|
||||
po,
|
||||
FILE_NAME_POT,
|
||||
)
|
||||
#cmd = (GETTEXT_MSGMERGE_EXECUTABLE,
|
||||
#"--update",
|
||||
#"-w", "1", # XXX Ugly hack to prevent msgmerge merging short source comments together!
|
||||
#"--no-wrap",
|
||||
#"--backup=none",
|
||||
#"--lang={}".format(lang),
|
||||
#po,
|
||||
#FILE_NAME_POT,
|
||||
#)
|
||||
|
||||
pot = utils.I18nMessages(kind='PO', src=FILE_NAME_POT)
|
||||
msg = utils.I18nMessages(iso=lang, kind='PO', src=po)
|
||||
print("Updating {}...".format(po))
|
||||
print("Running ", " ".join(cmd))
|
||||
ret = subprocess.call(cmd)
|
||||
msg.update(pot)
|
||||
msg.write(kind='PO', dest=po)
|
||||
#print("Running ", " ".join(cmd))
|
||||
#ret = subprocess.call(cmd)
|
||||
print("Finished!\n")
|
||||
return ret
|
||||
return 0
|
||||
|
||||
|
||||
def main():
|
||||
import argparse
|
||||
parser = argparse.ArgumentParser(description="Write out messages.txt "
|
||||
"from Blender.")
|
||||
parser.add_argument('-t', '--trunk', action="store_true",
|
||||
help="Update po’s in /trunk/po rather than /branches.")
|
||||
parser.add_argument('-i', '--input', metavar="File",
|
||||
help="Input pot file path.")
|
||||
parser.add_argument('--pproc-contexts', action="store_true",
|
||||
help="Pre-process po’s to avoid having plenty of "
|
||||
"fuzzy msgids just because a context was "
|
||||
"added/changed!")
|
||||
parser.add_argument('-t', '--trunk', action="store_true", help="Update po’s in /trunk/po rather than /branches.")
|
||||
parser.add_argument('-i', '--input', metavar="File", help="Input pot file path.")
|
||||
#parser.add_argument('--pproc-contexts', action="store_true",
|
||||
#help="Pre-process po’s to avoid having plenty of fuzzy msgids just because a context was "
|
||||
#"added/changed!")
|
||||
parser.add_argument('-a', '--add', action="store_true",
|
||||
help="Add missing po’s (useful only when one or "
|
||||
"more languages are given!).")
|
||||
parser.add_argument('langs', metavar='ISO_code', nargs='*',
|
||||
help="Restrict processed languages to those.")
|
||||
help="Add missing po’s (useful only when one or more languages are given!).")
|
||||
parser.add_argument('langs', metavar='ISO_code', nargs='*', help="Restrict processed languages to those.")
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.input:
|
||||
@@ -115,9 +114,11 @@ def main():
|
||||
FILE_NAME_POT = args.input
|
||||
ret = 0
|
||||
|
||||
if args.pproc_contexts:
|
||||
_ctxt_proc = pproc_newcontext_po
|
||||
pot_messages, _a, pot_stats = utils.parse_messages(FILE_NAME_POT)
|
||||
#if args.pproc_contexts:
|
||||
#_ctxt_proc = pproc_newcontext_po
|
||||
#pot_messages, _a, pot_stats = utils.parse_messages(FILE_NAME_POT)
|
||||
if 0:
|
||||
pass
|
||||
else:
|
||||
_ctxt_proc = lambda a, b, c: 0
|
||||
pot_messages, pot_stats = None, None
|
||||
|
||||
@@ -23,6 +23,7 @@
|
||||
# Update blender.pot file from messages.txt
|
||||
|
||||
import subprocess
|
||||
import collections
|
||||
import os
|
||||
import sys
|
||||
import re
|
||||
@@ -41,9 +42,11 @@ except:
|
||||
LANGUAGES_CATEGORIES = settings.LANGUAGES_CATEGORIES
|
||||
LANGUAGES = settings.LANGUAGES
|
||||
|
||||
COMMENT_PREFIX = settings.COMMENT_PREFIX
|
||||
COMMENT_PREFIX_SOURCE = settings.COMMENT_PREFIX_SOURCE
|
||||
CONTEXT_PREFIX = settings.CONTEXT_PREFIX
|
||||
PO_COMMENT_PREFIX = settings.PO_COMMENT_PREFIX
|
||||
PO_COMMENT_PREFIX_SOURCE = settings.PO_COMMENT_PREFIX_SOURCE
|
||||
PO_COMMENT_PREFIX_SOURCE_CUSTOM = settings.PO_COMMENT_PREFIX_SOURCE_CUSTOM
|
||||
MSG_COMMENT_PREFIX = settings.MSG_COMMENT_PREFIX
|
||||
MSG_CONTEXT_PREFIX = settings.MSG_CONTEXT_PREFIX
|
||||
FILE_NAME_MESSAGES = settings.FILE_NAME_MESSAGES
|
||||
FILE_NAME_POT = settings.FILE_NAME_POT
|
||||
SOURCE_DIR = settings.SOURCE_DIR
|
||||
@@ -79,6 +82,10 @@ _clean_str = re.compile(settings.str_clean_re).finditer
|
||||
clean_str = lambda s: "".join(m.group("clean") for m in _clean_str(s))
|
||||
|
||||
|
||||
def _new_messages():
|
||||
return getattr(collections, "OrderedDict", dict)()
|
||||
|
||||
|
||||
def check_file(path, rel_path, messages):
|
||||
def process_entry(ctxt, msg):
|
||||
# Context.
|
||||
@@ -207,20 +214,22 @@ def get_svnrev():
|
||||
|
||||
|
||||
def gen_empty_pot():
|
||||
blender_ver = ""
|
||||
blender_rev = get_svnrev().decode()
|
||||
utctime = time.gmtime()
|
||||
time_str = time.strftime("%Y-%m-%d %H:%M+0000", utctime)
|
||||
year_str = time.strftime("%Y", utctime)
|
||||
|
||||
return utils.gen_empty_messages(blender_rev, time_str, year_str)
|
||||
return utils.I18nMessages.gen_empty_messages("__POT__", blender_ver, blender_rev, time_str, year_str)
|
||||
|
||||
|
||||
escape_re = tuple(re.compile(r[0]) for r in settings.ESCAPE_RE)
|
||||
escape = lambda s, n: escape_re[n].sub(settings.ESCAPE_RE[n][1], s)
|
||||
|
||||
|
||||
def merge_messages(msgs, states, messages, do_checks, spell_cache):
|
||||
num_added = num_present = 0
|
||||
def merge_messages(msgs, messages, do_checks, spell_cache):
|
||||
num_added = 0
|
||||
num_present = msgs.nbr_msgs
|
||||
for (context, msgid), srcs in messages.items():
|
||||
if do_checks:
|
||||
err = spell_check(msgid, spell_cache)
|
||||
@@ -233,19 +242,15 @@ def merge_messages(msgs, states, messages, do_checks, spell_cache):
|
||||
for n in range(len(escape_re)):
|
||||
msgid = escape(msgid, n)
|
||||
|
||||
srcs = [COMMENT_PREFIX_SOURCE + s for s in srcs]
|
||||
|
||||
key = (context, msgid)
|
||||
if key not in msgs:
|
||||
msgs[key] = {"msgid_lines": [msgid],
|
||||
"msgstr_lines": [""],
|
||||
"comment_lines": srcs,
|
||||
"msgctxt_lines": [context]}
|
||||
if key not in msgs.msgs:
|
||||
msg = utils.I18nMessage([context], [msgid], [""], [])
|
||||
msg.sources = srcs
|
||||
msgs.msgs[key] = msg
|
||||
num_added += 1
|
||||
else:
|
||||
# We need to merge comments!
|
||||
msgs[key]["comment_lines"].extend(srcs)
|
||||
num_present += 1
|
||||
# We need to merge sources!
|
||||
msgs.msgs[key].sources += srcs
|
||||
|
||||
return num_added, num_present
|
||||
|
||||
@@ -270,7 +275,7 @@ def main():
|
||||
|
||||
print("Running fake py gettext…")
|
||||
# Not using any more xgettext, simpler to do it ourself!
|
||||
messages = utils.new_messages()
|
||||
messages = _new_messages()
|
||||
py_xgettext(messages)
|
||||
print("Finished, found {} messages.".format(len(messages)))
|
||||
|
||||
@@ -281,55 +286,49 @@ def main():
|
||||
spell_cache = set()
|
||||
|
||||
print("Generating POT file {}…".format(FILE_NAME_POT))
|
||||
msgs, states = gen_empty_pot()
|
||||
tot_messages, _a = merge_messages(msgs, states, messages,
|
||||
True, spell_cache)
|
||||
msgs = gen_empty_pot()
|
||||
tot_messages, _a = merge_messages(msgs, messages, True, spell_cache)
|
||||
|
||||
# add messages collected automatically from RNA
|
||||
print("\tMerging RNA messages from {}…".format(FILE_NAME_MESSAGES))
|
||||
messages = utils.new_messages()
|
||||
messages.clear()
|
||||
with open(FILE_NAME_MESSAGES, encoding="utf-8") as f:
|
||||
srcs = []
|
||||
context = ""
|
||||
for line in f:
|
||||
line = utils.stripeol(line)
|
||||
|
||||
if line.startswith(COMMENT_PREFIX):
|
||||
srcs.append(line[len(COMMENT_PREFIX):].strip())
|
||||
elif line.startswith(CONTEXT_PREFIX):
|
||||
context = line[len(CONTEXT_PREFIX):].strip()
|
||||
if line.startswith(MSG_COMMENT_PREFIX):
|
||||
srcs.append(line[len(MSG_COMMENT_PREFIX):].strip())
|
||||
elif line.startswith(MSG_CONTEXT_PREFIX):
|
||||
context = line[len(MSG_CONTEXT_PREFIX):].strip()
|
||||
else:
|
||||
key = (context, line)
|
||||
messages[key] = srcs
|
||||
srcs = []
|
||||
context = ""
|
||||
num_added, num_present = merge_messages(msgs, states, messages,
|
||||
True, spell_cache)
|
||||
num_added, num_present = merge_messages(msgs, messages, True, spell_cache)
|
||||
tot_messages += num_added
|
||||
print("\tMerged {} messages ({} were already present)."
|
||||
"".format(num_added, num_present))
|
||||
print("\tMerged {} messages ({} were already present).".format(num_added, num_present))
|
||||
|
||||
print("\tAdding languages labels...")
|
||||
messages = {(CONTEXT_DEFAULT, lng[1]):
|
||||
("Languages’ labels from bl_i18n_utils/settings.py",)
|
||||
for lng in LANGUAGES}
|
||||
messages.update({(CONTEXT_DEFAULT, cat[1]):
|
||||
("Language categories’ labels from bl_i18n_utils/settings.py",)
|
||||
for cat in LANGUAGES_CATEGORIES})
|
||||
num_added, num_present = merge_messages(msgs, states, messages,
|
||||
True, spell_cache)
|
||||
messages.clear()
|
||||
messages.update(((CONTEXT_DEFAULT, lng[1]), ("Languages’ labels from bl_i18n_utils/settings.py",))
|
||||
for lng in LANGUAGES)
|
||||
messages.update(((CONTEXT_DEFAULT, cat[1]), ("Language categories’ labels from bl_i18n_utils/settings.py",))
|
||||
for cat in LANGUAGES_CATEGORIES)
|
||||
num_added, num_present = merge_messages(msgs, messages, True, spell_cache)
|
||||
tot_messages += num_added
|
||||
print("\tAdded {} language messages.".format(num_added))
|
||||
|
||||
# Write back all messages into blender.pot.
|
||||
utils.write_messages(FILE_NAME_POT, msgs, states["comm_msg"],
|
||||
states["fuzzy_msg"])
|
||||
msgs.write('PO', FILE_NAME_POT)
|
||||
|
||||
if SPELL_CACHE and spell_cache:
|
||||
with open(SPELL_CACHE, 'wb') as f:
|
||||
pickle.dump(spell_cache, f)
|
||||
|
||||
print("Finished, total: {} messages!".format(tot_messages - 1))
|
||||
print("Finished, total: {} messages!".format(tot_messages))
|
||||
|
||||
return 0
|
||||
|
||||
|
||||
@@ -25,8 +25,7 @@
|
||||
# * Copy po’s from branches advanced enough.
|
||||
# * Clean po’s in trunk.
|
||||
# * Compile po’s in trunk in mo’s, keeping track of those failing.
|
||||
# * Remove po’s, mo’s (and their dir’s) that failed to compile or
|
||||
# are no more present in trunk.
|
||||
# * Remove po’s, mo’s (and their dir’s) that failed to compile or are no more present in trunk.
|
||||
|
||||
import subprocess
|
||||
import os
|
||||
@@ -50,25 +49,6 @@ LANGUAGES_FILE = settings.LANGUAGES_FILE
|
||||
PY3 = settings.PYTHON3_EXEC
|
||||
|
||||
|
||||
def find_matching_po(languages, stats):
|
||||
"""Match languages defined in LANGUAGES setting to relevant po, if possible!"""
|
||||
ret = []
|
||||
for uid, label, org_key in languages:
|
||||
key = org_key
|
||||
if key not in stats:
|
||||
# Try to simplify the key (eg from es_ES to es).
|
||||
if '_' in org_key:
|
||||
key = org_key[0:org_key.index('_')]
|
||||
if '@' in org_key:
|
||||
key = key + org_key[org_key.index('@'):]
|
||||
if key in stats:
|
||||
ret.append((stats[key], uid, label, org_key))
|
||||
else:
|
||||
# Mark invalid entries, so that we can put them in the languages file,
|
||||
# but commented!
|
||||
ret.append((0.0, -uid, label, org_key))
|
||||
return ret
|
||||
|
||||
def main():
|
||||
import argparse
|
||||
parser = argparse.ArgumentParser(description=""
|
||||
@@ -99,7 +79,8 @@ def main():
|
||||
os.remove(po)
|
||||
|
||||
# Copy po’s from branches.
|
||||
cmd = [PY3, "./import_po_from_branches.py", "-s"]
|
||||
#cmd = [PY3, "./import_po_from_branches.py", "-s"]
|
||||
cmd = [PY3, "./import_po_from_branches.py"]
|
||||
if args.threshold is not None:
|
||||
cmd += ["-t", str(args.threshold)]
|
||||
if args.langs:
|
||||
@@ -124,7 +105,8 @@ def main():
|
||||
if args.langs and lang not in args.langs:
|
||||
continue
|
||||
|
||||
cmd = [PY3, "./clean_po.py", "-t", "-s", lang]
|
||||
#cmd = [PY3, "./clean_po.py", "-t", "-s", lang]
|
||||
cmd = [PY3, "./clean_po.py", "-t", lang]
|
||||
t = subprocess.call(cmd)
|
||||
if t:
|
||||
ret = t
|
||||
|
||||
@@ -20,357 +20,660 @@
|
||||
|
||||
# Some misc utilities...
|
||||
|
||||
import os
|
||||
import sys
|
||||
import collections
|
||||
import copy
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
|
||||
from bl_i18n_utils import settings
|
||||
|
||||
|
||||
COMMENT_PREFIX = settings.COMMENT_PREFIX
|
||||
PO_COMMENT_PREFIX = settings.PO_COMMENT_PREFIX
|
||||
PO_COMMENT_PREFIX_MSG = settings.PO_COMMENT_PREFIX_MSG
|
||||
PO_COMMENT_PREFIX_SOURCE = settings.PO_COMMENT_PREFIX_SOURCE
|
||||
PO_COMMENT_PREFIX_SOURCE_CUSTOM = settings.PO_COMMENT_PREFIX_SOURCE_CUSTOM
|
||||
PO_COMMENT_FUZZY = settings.PO_COMMENT_FUZZY
|
||||
PO_MSGCTXT = settings.PO_MSGCTXT
|
||||
PO_MSGID = settings.PO_MSGID
|
||||
PO_MSGSTR = settings.PO_MSGSTR
|
||||
|
||||
PO_HEADER_KEY = settings.PO_HEADER_KEY
|
||||
PO_HEADER_COMMENT = settings.PO_HEADER_COMMENT
|
||||
PO_HEADER_COMMENT_COPYRIGHT = settings.PO_HEADER_COMMENT_COPYRIGHT
|
||||
PO_HEADER_MSGSTR = settings.PO_HEADER_MSGSTR
|
||||
|
||||
PARSER_CACHE_HASH = settings.PARSER_CACHE_HASH
|
||||
|
||||
WARN_NC = settings.WARN_MSGID_NOT_CAPITALIZED
|
||||
NC_ALLOWED = settings.WARN_MSGID_NOT_CAPITALIZED_ALLOWED
|
||||
PARSER_CACHE_HASH = settings.PARSER_CACHE_HASH
|
||||
|
||||
|
||||
##### Misc Utils #####
|
||||
|
||||
def stripeol(s):
|
||||
return s.rstrip("\n\r")
|
||||
|
||||
|
||||
# XXX For now, we assume that all messages > 30 chars are tooltips!
|
||||
def is_tooltip(msgid):
|
||||
return len(msgid) > 30
|
||||
_valid_po_path_re = re.compile(r"^\S+:[0-9]+$")
|
||||
def is_valid_po_path(path):
|
||||
return bool(_valid_po_path_re.match(path))
|
||||
|
||||
|
||||
def new_messages():
|
||||
return getattr(collections, 'OrderedDict', dict)()
|
||||
|
||||
|
||||
def parse_messages(fname):
|
||||
class I18nMessage:
|
||||
"""
|
||||
Returns a tupple (messages, states, stats).
|
||||
messages is an odereddict of dicts
|
||||
{(ctxt, msgid): {msgid_lines:, msgstr_lines:,
|
||||
comment_lines:, msgctxt_lines:}}.
|
||||
states is a dict of three sets of (msgid, ctxt), and a boolean flag
|
||||
indicating the .po is somewhat broken
|
||||
{trans_msg:, fuzzy_msg:, comm_msg:, is_broken:}.
|
||||
stats is a dict of values
|
||||
{tot_msg:, trans_msg:, tot_ttips:, trans_ttips:, comm_msg:,
|
||||
nbr_signs:, nbr_trans_signs:, contexts: set()}.
|
||||
Note: This function will silently "arrange" mis-formated entries, thus
|
||||
using afterward write_messages() should always produce a po-valid file,
|
||||
though not correct!
|
||||
Internal representation of a message.
|
||||
"""
|
||||
tot_messages = 0
|
||||
tot_tooltips = 0
|
||||
trans_messages = 0
|
||||
trans_tooltips = 0
|
||||
comm_messages = 0
|
||||
nbr_signs = 0
|
||||
nbr_trans_signs = 0
|
||||
contexts = set()
|
||||
reading_msgid = False
|
||||
reading_msgstr = False
|
||||
reading_msgctxt = False
|
||||
reading_comment = False
|
||||
is_translated = False
|
||||
is_fuzzy = False
|
||||
is_commented = False
|
||||
is_broken = False
|
||||
msgid_lines = []
|
||||
msgstr_lines = []
|
||||
msgctxt_lines = []
|
||||
comment_lines = []
|
||||
__slots__ = ("msgctxt_lines", "msgid_lines", "msgstr_lines", "comment_lines", "is_fuzzy", "is_commented")
|
||||
|
||||
messages = new_messages()
|
||||
translated_messages = set()
|
||||
fuzzy_messages = set()
|
||||
commented_messages = set()
|
||||
def __init__(self, msgctxt_lines=[], msgid_lines=[], msgstr_lines=[], comment_lines=[],
|
||||
is_commented=False, is_fuzzy=False):
|
||||
self.msgctxt_lines = msgctxt_lines
|
||||
self.msgid_lines = msgid_lines
|
||||
self.msgstr_lines = msgstr_lines
|
||||
self.comment_lines = comment_lines
|
||||
self.is_fuzzy = is_fuzzy
|
||||
self.is_commented = is_commented
|
||||
|
||||
def clean_vars():
|
||||
nonlocal reading_msgid, reading_msgstr, reading_msgctxt, \
|
||||
reading_comment, is_fuzzy, is_translated, is_commented, \
|
||||
msgid_lines, msgstr_lines, msgctxt_lines, comment_lines
|
||||
reading_msgid = reading_msgstr = reading_msgctxt = \
|
||||
reading_comment = False
|
||||
is_tooltip = is_fuzzy = is_translated = is_commented = False
|
||||
def _get_msgctxt(self):
|
||||
return ("".join(self.msgctxt_lines)).replace("\\n", "\n")
|
||||
def _set_msgctxt(self, ctxt):
|
||||
self.msgctxt_lines = [ctxt]
|
||||
msgctxt = property(_get_msgctxt, _set_msgctxt)
|
||||
|
||||
def _get_msgid(self):
|
||||
return ("".join(self.msgid_lines)).replace("\\n", "\n")
|
||||
def _set_msgid(self, msgid):
|
||||
self.msgid_lines = [msgid]
|
||||
msgid = property(_get_msgid, _set_msgid)
|
||||
|
||||
def _get_msgstr(self):
|
||||
return ("".join(self.msgstr_lines)).replace("\\n", "\n")
|
||||
def _set_msgstr(self, msgstr):
|
||||
self.msgstr_lines = [msgstr]
|
||||
msgstr = property(_get_msgstr, _set_msgstr)
|
||||
|
||||
def _get_sources(self):
|
||||
lstrip1 = len(PO_COMMENT_PREFIX_SOURCE)
|
||||
lstrip2 = len(PO_COMMENT_PREFIX_SOURCE_CUSTOM)
|
||||
return ([l[lstrip1:] for l in self.comment_lines if l.startswith(PO_COMMENT_PREFIX_SOURCE)] +
|
||||
[l[lstrip2:] for l in self.comment_lines if l.startswith(PO_COMMENT_PREFIX_SOURCE_CUSTOM)])
|
||||
def _set_sources(self, sources):
|
||||
# list.copy() is not available in py3.2 ...
|
||||
cmmlines = []
|
||||
cmmlines[:] = self.comment_lines
|
||||
for l in cmmlines:
|
||||
if l.startswith(PO_COMMENT_PREFIX_SOURCE) or l.startswith(PO_COMMENT_PREFIX_SOURCE_CUSTOM):
|
||||
self.comment_lines.remove(l)
|
||||
lines_src = []
|
||||
lines_src_custom = []
|
||||
for src in sources:
|
||||
if is_valid_po_path(src):
|
||||
lines_src.append(PO_COMMENT_PREFIX_SOURCE + src)
|
||||
else:
|
||||
lines_src_custom.append(PO_COMMENT_PREFIX_SOURCE_CUSTOM + src)
|
||||
self.comment_lines += lines_src_custom + lines_src
|
||||
sources = property(_get_sources, _set_sources)
|
||||
|
||||
def _get_is_tooltip(self):
|
||||
# XXX For now, we assume that all messages > 30 chars are tooltips!
|
||||
return len(self.msgid) > 30
|
||||
is_tooltip = property(_get_is_tooltip)
|
||||
|
||||
def normalize(self, max_len=80):
|
||||
"""
|
||||
Normalize this message, call this before exporting it...
|
||||
Currently normalize msgctxt, msgid and msgstr lines to given max_len (if below 1, make them single line).
|
||||
"""
|
||||
max_len -= 2 # The two quotes!
|
||||
# We do not need the full power of textwrap... We just split first at escaped new lines, then into each line
|
||||
# if needed... No word splitting, nor fancy spaces handling!
|
||||
def _wrap(text, max_len, init_len):
|
||||
if len(text) + init_len < max_len:
|
||||
return [text]
|
||||
lines = text.splitlines()
|
||||
ret = []
|
||||
for l in lines:
|
||||
tmp = []
|
||||
cur_len = 0
|
||||
words = l.split(' ')
|
||||
for w in words:
|
||||
cur_len += len(w) + 1
|
||||
if cur_len > (max_len - 1) and tmp:
|
||||
ret.append(" ".join(tmp) + " ")
|
||||
del tmp[:]
|
||||
cur_len = len(w) + 1
|
||||
tmp.append(w)
|
||||
if tmp:
|
||||
ret.append(" ".join(tmp))
|
||||
return ret
|
||||
if max_len < 1:
|
||||
self.msgctxt_lines = self.msgctxt.replace("\n", "\\n\n").splitlines()
|
||||
self.msgid_lines = self.msgid.replace("\n", "\\n\n").splitlines()
|
||||
self.msgstr_lines = self.msgstr.replace("\n", "\\n\n").splitlines()
|
||||
else:
|
||||
init_len = len(PO_MSGCTXT) + 1
|
||||
if self.is_commented:
|
||||
init_len += len(PO_COMMENT_PREFIX_MSG)
|
||||
self.msgctxt_lines = _wrap(self.msgctxt.replace("\n", "\\n\n"), max_len, init_len)
|
||||
|
||||
init_len = len(PO_MSGID) + 1
|
||||
if self.is_commented:
|
||||
init_len += len(PO_COMMENT_PREFIX_MSG)
|
||||
self.msgid_lines = _wrap(self.msgid.replace("\n", "\\n\n"), max_len, init_len)
|
||||
|
||||
init_len = len(PO_MSGSTR) + 1
|
||||
if self.is_commented:
|
||||
init_len += len(PO_COMMENT_PREFIX_MSG)
|
||||
self.msgstr_lines = _wrap(self.msgstr.replace("\n", "\\n\n"), max_len, init_len)
|
||||
|
||||
|
||||
class I18nMessages:
|
||||
"""
|
||||
Internal representation of messages for one language (iso code), with additional stats info.
|
||||
"""
|
||||
|
||||
# Avoid parsing again!
|
||||
# Keys should be (pseudo) file-names, values are tuples (hash, I18nMessages)
|
||||
# Note: only used by po parser currently!
|
||||
_parser_cache = {}
|
||||
|
||||
def __init__(self, iso="__POT__", kind=None, key=None, src=None):
|
||||
self.iso = iso
|
||||
self.msgs = self._new_messages()
|
||||
self.trans_msgs = set()
|
||||
self.fuzzy_msgs = set()
|
||||
self.comm_msgs = set()
|
||||
self.ttip_msgs = set()
|
||||
self.contexts = set()
|
||||
self.nbr_msgs = 0
|
||||
self.nbr_trans_msgs = 0
|
||||
self.nbr_ttips = 0
|
||||
self.nbr_trans_ttips = 0
|
||||
self.nbr_comm_msgs = 0
|
||||
self.nbr_signs = 0
|
||||
self.nbr_trans_signs = 0
|
||||
self.parsing_errors = []
|
||||
if kind and src:
|
||||
self.parse(kind, key, src)
|
||||
self.update_info()
|
||||
|
||||
@staticmethod
|
||||
def _new_messages():
|
||||
return getattr(collections, 'OrderedDict', dict)()
|
||||
|
||||
@classmethod
|
||||
def gen_empty_messages(cls, iso, blender_ver, blender_rev, time, year, default_copyright=True):
|
||||
"""Generate an empty I18nMessages object (only header is present!)."""
|
||||
msgstr = PO_HEADER_MSGSTR.format(blender_ver=str(blender_ver), blender_rev=int(blender_rev),
|
||||
time=str(time), iso=str(iso))
|
||||
comment = ""
|
||||
if default_copyright:
|
||||
comment = PO_HEADER_COMMENT_COPYRIGHT.format(year=str(year))
|
||||
comment = comment + PO_HEADER_COMMENT
|
||||
|
||||
msgs = cls(iso=iso)
|
||||
msgs.msgs[PO_HEADER_KEY] = I18nMessage([], [""], [msgstr], [comment], False, True)
|
||||
msgs.update_info()
|
||||
|
||||
return msgs
|
||||
|
||||
def normalize(self, max_len=80):
|
||||
for msg in self.msgs.values():
|
||||
msg.normalize(max_len)
|
||||
|
||||
def merge(self, replace=False, *args):
|
||||
pass
|
||||
|
||||
def update(self, ref, use_similar=0.8, keep_old_commented=True):
|
||||
"""
|
||||
Update this I18nMessage with the ref one. Translations from ref are never used. Source comments from ref
|
||||
completely replace current ones. If use_similar is not 0.0, it will try to match new messages in ref with an
|
||||
existing one. Messages no more found in ref will be marked as commented if keep_old_commented is True,
|
||||
or removed.
|
||||
"""
|
||||
import difflib
|
||||
similar_pool = {}
|
||||
if use_similar > 0.0:
|
||||
for key in self.msgs:
|
||||
similar_pool.setdefault(key[1], set()).add(key)
|
||||
|
||||
msgs = self._new_messages()
|
||||
for (key, msg) in ref.msgs.items():
|
||||
if key in self.msgs:
|
||||
msgs[key] = self.msgs[key]
|
||||
msgs[key].sources = msg.sources
|
||||
else:
|
||||
skey = None
|
||||
if use_similar > 0.0:
|
||||
# try to find some close key in existing messages...
|
||||
tmp = difflib.get_close_matches(key[1], similar_pool, n=1, cutoff=use_similar)
|
||||
if tmp:
|
||||
tmp = tmp[0]
|
||||
# Try to get the same context, else just get one...
|
||||
skey = (key[0], tmp)
|
||||
if skey not in similar_pool[tmp]:
|
||||
skey = tuple(similar_pool[tmp])[0]
|
||||
msgs[key] = msg
|
||||
if skey:
|
||||
msgs[key].msgstr = self.msgs[skey].msgstr
|
||||
msgs[key].is_fuzzy = True
|
||||
# Add back all "old" and already commented messages as commented ones, if required.
|
||||
if keep_old_commented:
|
||||
for key, msg in self.msgs.items():
|
||||
if key not in msgs:
|
||||
msgs[key] = msg
|
||||
msgs[key].is_commented = True
|
||||
# And finalize the update!
|
||||
self.msgs = msgs
|
||||
|
||||
def update_info(self):
|
||||
self.trans_msgs.clear()
|
||||
self.fuzzy_msgs.clear()
|
||||
self.comm_msgs.clear()
|
||||
self.ttip_msgs.clear()
|
||||
self.contexts.clear()
|
||||
self.nbr_signs = 0
|
||||
self.nbr_trans_signs = 0
|
||||
for key, msg in self.msgs.items():
|
||||
if key == PO_HEADER_KEY:
|
||||
continue
|
||||
if msg.is_commented:
|
||||
self.comm_msgs.add(key)
|
||||
else:
|
||||
if msg.msgstr:
|
||||
self.trans_msgs.add(key)
|
||||
if msg.is_fuzzy:
|
||||
self.fuzzy_msgs.add(key)
|
||||
if msg.is_tooltip:
|
||||
self.ttip_msgs.add(key)
|
||||
self.contexts.add(key[0])
|
||||
self.nbr_signs += len(msg.msgid)
|
||||
self.nbr_trans_signs += len(msg.msgstr)
|
||||
self.nbr_msgs = len(self.msgs)
|
||||
self.nbr_trans_msgs = len(self.trans_msgs)
|
||||
self.nbr_ttips = len(self.ttip_msgs)
|
||||
self.nbr_trans_ttips = len(self.ttip_msgs & self.trans_msgs)
|
||||
self.nbr_comm_msgs = len(self.comm_msgs)
|
||||
|
||||
def print_stats(self, prefix=""):
|
||||
"""
|
||||
Print out some stats about an I18nMessages object.
|
||||
"""
|
||||
lvl = 0.0
|
||||
lvl_ttips = 0.0
|
||||
lvl_comm = 0.0
|
||||
lvl_trans_ttips = 0.0
|
||||
lvl_ttips_in_trans = 0.0
|
||||
if self.nbr_msgs > 0:
|
||||
lvl = float(self.nbr_trans_msgs) / float(self.nbr_msgs)
|
||||
lvl_ttips = float(self.nbr_ttips) / float(self.nbr_msgs)
|
||||
lvl_comm = float(self.nbr_comm_msgs) / float(self.nbr_msgs + self.nbr_comm_msgs)
|
||||
if self.nbr_ttips > 0:
|
||||
lvl_trans_ttips = float(self.nbr_trans_ttips) / float(self.nbr_ttips)
|
||||
if self.nbr_trans_msgs > 0:
|
||||
lvl_ttips_in_trans = float(self.nbr_trans_ttips) / float(self.nbr_trans_msgs)
|
||||
|
||||
lines = ("",
|
||||
"{:>6.1%} done! ({} translated messages over {}).\n"
|
||||
"".format(lvl, self.nbr_trans_msgs, self.nbr_msgs),
|
||||
"{:>6.1%} of messages are tooltips ({} over {}).\n"
|
||||
"".format(lvl_ttips, self.nbr_ttips, self.nbr_msgs),
|
||||
"{:>6.1%} of tooltips are translated ({} over {}).\n"
|
||||
"".format(lvl_trans_ttips, self.nbr_trans_ttips, self.nbr_ttips),
|
||||
"{:>6.1%} of translated messages are tooltips ({} over {}).\n"
|
||||
"".format(lvl_ttips_in_trans, self.nbr_trans_ttips, self.nbr_trans_msgs),
|
||||
"{:>6.1%} of messages are commented ({} over {}).\n"
|
||||
"".format(lvl_comm, self.nbr_comm_msgs, self.nbr_comm_msgs + self.nbr_msgs),
|
||||
"This translation is currently made of {} signs.\n".format(self.nbr_trans_signs))
|
||||
print(prefix.join(lines))
|
||||
|
||||
def parse(self, kind, key, src):
|
||||
del self.parsing_errors[:]
|
||||
self.parsers[kind](self, src, key)
|
||||
if self.parsing_errors:
|
||||
print("WARNING! Errors while parsing {}:".format(key))
|
||||
for line, error in self.parsing_errors:
|
||||
print(" Around line {}: {}".format(line, error))
|
||||
print("The parser solved them as well as it could...")
|
||||
self.update_info()
|
||||
|
||||
def parse_messages_from_po(self, src, key=None):
|
||||
"""
|
||||
Parse a po file.
|
||||
Note: This function will silently "arrange" mis-formated entries, thus using afterward write_messages() should
|
||||
always produce a po-valid file, though not correct!
|
||||
"""
|
||||
reading_msgid = False
|
||||
reading_msgstr = False
|
||||
reading_msgctxt = False
|
||||
reading_comment = False
|
||||
is_commented = False
|
||||
is_fuzzy = False
|
||||
msgctxt_lines = []
|
||||
msgid_lines = []
|
||||
msgstr_lines = []
|
||||
msgctxt_lines = []
|
||||
comment_lines = []
|
||||
|
||||
def finalize_message():
|
||||
nonlocal reading_msgid, reading_msgstr, reading_msgctxt, \
|
||||
reading_comment, is_fuzzy, is_translated, is_commented, \
|
||||
msgid_lines, msgstr_lines, msgctxt_lines, comment_lines, \
|
||||
messages, translated_messages, fuzzy_messages, \
|
||||
commented_messages, \
|
||||
tot_messages, tot_tooltips, trans_messages, trans_tooltips, \
|
||||
comm_messages, nbr_signs, nbr_trans_signs, contexts
|
||||
# Helper function
|
||||
def finalize_message(self, line_nr):
|
||||
nonlocal reading_msgid, reading_msgstr, reading_msgctxt, reading_comment
|
||||
nonlocal is_commented, is_fuzzy, msgid_lines, msgstr_lines, msgctxt_lines, comment_lines
|
||||
|
||||
msgid = "".join(msgid_lines)
|
||||
msgctxt = "".join(msgctxt_lines)
|
||||
msgkey = (msgctxt, msgid)
|
||||
is_ttip = is_tooltip(msgid)
|
||||
msgid = "".join(msgid_lines)
|
||||
msgctxt = "".join(msgctxt_lines)
|
||||
msgkey = (msgctxt, msgid)
|
||||
|
||||
# Never allow overriding existing msgid/msgctxt pairs!
|
||||
if msgkey in messages:
|
||||
clean_vars()
|
||||
return
|
||||
# Never allow overriding existing msgid/msgctxt pairs!
|
||||
if msgkey in self.msgs:
|
||||
self.parsing_errors.append((line_nr, "{} context/msgid is already in current messages!".format(msgkey)))
|
||||
return
|
||||
|
||||
nbr_signs += len(msgid)
|
||||
if is_commented:
|
||||
commented_messages.add(msgkey)
|
||||
elif is_fuzzy:
|
||||
fuzzy_messages.add(msgkey)
|
||||
elif is_translated:
|
||||
translated_messages.add(msgkey)
|
||||
nbr_trans_signs += len("".join(msgstr_lines))
|
||||
messages[msgkey] = {"msgid_lines" : msgid_lines,
|
||||
"msgstr_lines" : msgstr_lines,
|
||||
"comment_lines": comment_lines,
|
||||
"msgctxt_lines": msgctxt_lines}
|
||||
self.msgs[msgkey] = I18nMessage(msgctxt_lines, msgid_lines, msgstr_lines, comment_lines,
|
||||
is_commented, is_fuzzy)
|
||||
|
||||
if is_commented:
|
||||
comm_messages += 1
|
||||
else:
|
||||
tot_messages += 1
|
||||
if is_ttip:
|
||||
tot_tooltips += 1
|
||||
if not is_fuzzy and is_translated:
|
||||
trans_messages += 1
|
||||
if is_ttip:
|
||||
trans_tooltips += 1
|
||||
if msgctxt not in contexts:
|
||||
contexts.add(msgctxt)
|
||||
# Let's clean up and get ready for next message!
|
||||
reading_msgid = reading_msgstr = reading_msgctxt = reading_comment = False
|
||||
is_commented = is_fuzzy = False
|
||||
msgctxt_lines = []
|
||||
msgid_lines = []
|
||||
msgstr_lines = []
|
||||
comment_lines = []
|
||||
|
||||
clean_vars()
|
||||
# try to use src as file name...
|
||||
if os.path.exists(src):
|
||||
if not key:
|
||||
key = src
|
||||
with open(src, 'r', encoding="utf-8") as f:
|
||||
src = f.read()
|
||||
|
||||
with open(fname, 'r', encoding="utf-8") as f:
|
||||
for line_nr, line in enumerate(f):
|
||||
line = stripeol(line)
|
||||
# Try to use values from cache!
|
||||
curr_hash = None
|
||||
if key and key in self._parser_cache:
|
||||
old_hash, msgs = self._parser_cache[key]
|
||||
import hashlib
|
||||
curr_hash = hashlib.new(PARSER_CACHE_HASH, src.encode()).digest()
|
||||
if curr_hash == old_hash:
|
||||
self.msgs = copy.deepcopy(msgs) # we might edit self.msgs!
|
||||
return
|
||||
|
||||
_comm_msgctxt = PO_COMMENT_PREFIX_MSG + PO_MSGCTXT
|
||||
_len_msgctxt = len(PO_MSGCTXT + '"')
|
||||
_len_comm_msgctxt = len(_comm_msgctxt + '"')
|
||||
_comm_msgid = PO_COMMENT_PREFIX_MSG + PO_MSGID
|
||||
_len_msgid = len(PO_MSGID + '"')
|
||||
_len_comm_msgid = len(_comm_msgid + '"')
|
||||
_comm_msgstr = PO_COMMENT_PREFIX_MSG + PO_MSGSTR
|
||||
_len_msgstr = len(PO_MSGSTR + '"')
|
||||
_len_comm_msgstr = len(_comm_msgstr + '"')
|
||||
_len_comm_str = len(PO_COMMENT_PREFIX_MSG + '"')
|
||||
|
||||
# Main loop over all lines in src...
|
||||
for line_nr, line in enumerate(src.splitlines()):
|
||||
if line == "":
|
||||
finalize_message()
|
||||
finalize_message(self, line_nr)
|
||||
|
||||
elif line.startswith("msgctxt") or \
|
||||
line.startswith("".join((COMMENT_PREFIX, "msgctxt"))):
|
||||
elif line.startswith(PO_MSGCTXT) or line.startswith(_comm_msgctxt):
|
||||
reading_comment = False
|
||||
reading_ctxt = True
|
||||
if line.startswith(COMMENT_PREFIX):
|
||||
if line.startswith(PO_COMMENT_PREFIX_MSG):
|
||||
is_commented = True
|
||||
line = line[9 + len(COMMENT_PREFIX):-1]
|
||||
line = line[_len_comm_msgctxt:-1]
|
||||
else:
|
||||
line = line[9:-1]
|
||||
line = line[_len_msgctxt:-1]
|
||||
msgctxt_lines.append(line)
|
||||
|
||||
elif line.startswith("msgid") or \
|
||||
line.startswith("".join((COMMENT_PREFIX, "msgid"))):
|
||||
elif line.startswith(PO_MSGID) or line.startswith(_comm_msgid):
|
||||
reading_comment = False
|
||||
reading_msgid = True
|
||||
if line.startswith(COMMENT_PREFIX):
|
||||
if line.startswith(PO_COMMENT_PREFIX_MSG):
|
||||
if not is_commented and reading_ctxt:
|
||||
self.parsing_errors.append((line_nr, "commented msgid following regular msgctxt"))
|
||||
is_commented = True
|
||||
line = line[7 + len(COMMENT_PREFIX):-1]
|
||||
line = line[_len_comm_msgid:-1]
|
||||
else:
|
||||
line = line[7:-1]
|
||||
line = line[_len_msgid:-1]
|
||||
reading_ctxt = False
|
||||
msgid_lines.append(line)
|
||||
|
||||
elif line.startswith("msgstr") or \
|
||||
line.startswith("".join((COMMENT_PREFIX, "msgstr"))):
|
||||
elif line.startswith(PO_MSGSTR) or line.startswith(_comm_msgstr):
|
||||
if not reading_msgid:
|
||||
is_broken = True
|
||||
self.parsing_errors.append((line_nr, "msgstr without a prior msgid"))
|
||||
else:
|
||||
reading_msgid = False
|
||||
reading_msgstr = True
|
||||
if line.startswith(COMMENT_PREFIX):
|
||||
line = line[8 + len(COMMENT_PREFIX):-1]
|
||||
if line.startswith(PO_COMMENT_PREFIX_MSG):
|
||||
line = line[_len_comm_msgstr:-1]
|
||||
if not is_commented:
|
||||
is_broken = True
|
||||
self.parsing_errors.append((line_nr, "commented msgstr following regular msgid"))
|
||||
else:
|
||||
line = line[8:-1]
|
||||
line = line[_len_msgstr:-1]
|
||||
if is_commented:
|
||||
is_broken = True
|
||||
self.parsing_errors.append((line_nr, "regular msgstr following commented msgid"))
|
||||
msgstr_lines.append(line)
|
||||
if line:
|
||||
is_translated = True
|
||||
|
||||
elif line.startswith("#"):
|
||||
if reading_msgid:
|
||||
if is_commented:
|
||||
msgid_lines.append(line[1 + len(COMMENT_PREFIX):-1])
|
||||
else:
|
||||
msgid_lines.append(line)
|
||||
is_broken = True
|
||||
elif reading_msgstr:
|
||||
if is_commented:
|
||||
msgstr_lines.append(line[1 + len(COMMENT_PREFIX):-1])
|
||||
else:
|
||||
msgstr_lines.append(line)
|
||||
is_broken = True
|
||||
elif line.startswith(PO_COMMENT_PREFIX[0]):
|
||||
if line.startswith(PO_COMMENT_PREFIX_MSG):
|
||||
if reading_msgctxt:
|
||||
if is_commented:
|
||||
msgctxt_lines.append(line[_len_comm_str:-1])
|
||||
else:
|
||||
msgctxt_lines.append(line)
|
||||
self.parsing_errors.append((line_nr, "commented string while reading regular msgctxt"))
|
||||
elif reading_msgid:
|
||||
if is_commented:
|
||||
msgid_lines.append(line[_len_comm_str:-1])
|
||||
else:
|
||||
msgid_lines.append(line)
|
||||
self.parsing_errors.append((line_nr, "commented string while reading regular msgid"))
|
||||
elif reading_msgstr:
|
||||
if is_commented:
|
||||
msgstr_lines.append(line[_len_comm_str:-1])
|
||||
else:
|
||||
msgstr_lines.append(line)
|
||||
self.parsing_errors.append((line_nr, "commented string while reading regular msgstr"))
|
||||
else:
|
||||
if line.startswith("#, fuzzy"):
|
||||
if reading_msgctxt or reading_msgid or reading_msgstr:
|
||||
self.parsing_errors.append((line_nr,
|
||||
"commented string within msgctxt, msgid or msgstr scope, ignored"))
|
||||
elif line.startswith(PO_COMMENT_FUZZY):
|
||||
is_fuzzy = True
|
||||
else:
|
||||
comment_lines.append(line)
|
||||
reading_comment = True
|
||||
|
||||
else:
|
||||
if reading_msgid:
|
||||
if reading_msgctxt:
|
||||
msgctxt_lines.append(line[1:-1])
|
||||
elif reading_msgid:
|
||||
msgid_lines.append(line[1:-1])
|
||||
elif reading_msgstr:
|
||||
line = line[1:-1]
|
||||
msgstr_lines.append(line)
|
||||
if not is_translated and line:
|
||||
is_translated = True
|
||||
else:
|
||||
is_broken = True
|
||||
self.parsing_errors.append((line_nr, "regular string outside msgctxt, msgid or msgstr scope"))
|
||||
print(line)
|
||||
|
||||
# If no final empty line, last message is not finalized!
|
||||
if reading_msgstr:
|
||||
finalize_message()
|
||||
finalize_message(self, line_nr)
|
||||
|
||||
return (messages,
|
||||
{"trans_msg": translated_messages,
|
||||
"fuzzy_msg": fuzzy_messages,
|
||||
"comm_msg" : commented_messages,
|
||||
"is_broken": is_broken},
|
||||
{"tot_msg" : tot_messages,
|
||||
"trans_msg" : trans_messages,
|
||||
"tot_ttips" : tot_tooltips,
|
||||
"trans_ttips" : trans_tooltips,
|
||||
"comm_msg" : comm_messages,
|
||||
"nbr_signs" : nbr_signs,
|
||||
"nbr_trans_signs": nbr_trans_signs,
|
||||
"contexts" : contexts})
|
||||
if key:
|
||||
if not curr_hash:
|
||||
import hashlib
|
||||
curr_hash = hashlib.new(PARSER_CACHE_HASH, src.encode()).digest()
|
||||
self._parser_cache[key] = (curr_hash, self.msgs)
|
||||
|
||||
def write(self, kind, dest):
|
||||
self.writers[kind](self, dest)
|
||||
|
||||
def write_messages(fname, messages, commented, fuzzy):
|
||||
"Write in fname file the content of messages (similar to parse_messages " \
|
||||
"returned values). commented and fuzzy are two sets containing msgid. " \
|
||||
"Returns the number of written messages."
|
||||
num = 0
|
||||
with open(fname, 'w', encoding="utf-8") as f:
|
||||
for msgkey, val in messages.items():
|
||||
msgctxt, msgid = msgkey
|
||||
f.write("\n".join(val["comment_lines"]))
|
||||
# Only mark as fuzzy if msgstr is not empty!
|
||||
if msgkey in fuzzy and "".join(val["msgstr_lines"]):
|
||||
f.write("\n#, fuzzy")
|
||||
if msgkey in commented:
|
||||
if msgctxt:
|
||||
f.write("\n{}msgctxt \"".format(COMMENT_PREFIX))
|
||||
f.write("\"\n{}\"".format(COMMENT_PREFIX).join(
|
||||
val["msgctxt_lines"]))
|
||||
f.write("\"")
|
||||
f.write("\n{}msgid \"".format(COMMENT_PREFIX))
|
||||
f.write("\"\n{}\"".format(COMMENT_PREFIX).join(
|
||||
val["msgid_lines"]))
|
||||
f.write("\"\n{}msgstr \"".format(COMMENT_PREFIX))
|
||||
f.write("\"\n{}\"".format(COMMENT_PREFIX).join(
|
||||
val["msgstr_lines"]))
|
||||
f.write("\"\n\n")
|
||||
else:
|
||||
if msgctxt:
|
||||
f.write("\nmsgctxt \"")
|
||||
f.write("\"\n\"".join(val["msgctxt_lines"]))
|
||||
f.write("\"")
|
||||
f.write("\nmsgid \"")
|
||||
f.write("\"\n\"".join(val["msgid_lines"]))
|
||||
f.write("\"\nmsgstr \"")
|
||||
f.write("\"\n\"".join(val["msgstr_lines"]))
|
||||
f.write("\"\n\n")
|
||||
num += 1
|
||||
return num
|
||||
def write_messages_to_po(self, fname):
|
||||
"""
|
||||
Write messages in fname po file.
|
||||
"""
|
||||
self.normalize(max_len=0) # No wrapping for now...
|
||||
with open(fname, 'w', encoding="utf-8") as f:
|
||||
for msg in self.msgs.values():
|
||||
f.write("\n".join(msg.comment_lines))
|
||||
# Only mark as fuzzy if msgstr is not empty!
|
||||
if msg.is_fuzzy and msg.msgstr:
|
||||
f.write("\n" + PO_COMMENT_FUZZY)
|
||||
_p = PO_COMMENT_PREFIX_MSG if msg.is_commented else ""
|
||||
_pmsgctxt = _p + PO_MSGCTXT
|
||||
_pmsgid = _p + PO_MSGID
|
||||
_pmsgstr = _p + PO_MSGSTR
|
||||
chunks = []
|
||||
if msg.msgctxt:
|
||||
if len(msg.msgctxt_lines) > 1:
|
||||
chunks += [
|
||||
"\n" + _pmsgctxt + "\"\"\n" + _p + "\"",
|
||||
("\"\n" + _p + "\"").join(msg.msgctxt_lines),
|
||||
"\"",
|
||||
]
|
||||
else:
|
||||
chunks += ["\n" + _pmsgctxt + "\"" + msg.msgctxt + "\""]
|
||||
if len(msg.msgid_lines) > 1:
|
||||
print(msg.msgid_lines)
|
||||
chunks += [
|
||||
"\n" + _pmsgid + "\"\"\n" + _p + "\"",
|
||||
("\"\n" + _p + "\"").join(msg.msgid_lines),
|
||||
"\"",
|
||||
]
|
||||
else:
|
||||
chunks += ["\n" + _pmsgid + "\"" + msg.msgid + "\""]
|
||||
if len(msg.msgstr_lines) > 1:
|
||||
chunks += [
|
||||
"\n" + _pmsgstr + "\"\"\n" + _p + "\"",
|
||||
("\"\n" + _p + "\"").join(msg.msgstr_lines),
|
||||
"\"",
|
||||
]
|
||||
else:
|
||||
chunks += ["\n" + _pmsgstr + "\"" + msg.msgstr + "\""]
|
||||
chunks += ["\n\n"]
|
||||
f.write("".join(chunks))
|
||||
|
||||
|
||||
def gen_empty_messages(blender_rev, time_str, year_str):
|
||||
"""Generate an empty messages & state data (only header if present!)."""
|
||||
header_key = ("", "")
|
||||
|
||||
messages = new_messages()
|
||||
messages[header_key] = {
|
||||
"msgid_lines": [""],
|
||||
"msgctxt_lines": [],
|
||||
"msgstr_lines": [
|
||||
"Project-Id-Version: Blender r{}\\n"
|
||||
"".format(blender_rev),
|
||||
"Report-Msgid-Bugs-To: \\n",
|
||||
"POT-Creation-Date: {}\\n"
|
||||
"".format(time_str),
|
||||
"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\\n",
|
||||
"Last-Translator: FULL NAME <EMAIL@ADDRESS>\\n",
|
||||
"Language-Team: LANGUAGE <LL@li.org>\\n",
|
||||
"Language: \\n",
|
||||
"MIME-Version: 1.0\\n",
|
||||
"Content-Type: text/plain; charset=UTF-8\\n",
|
||||
"Content-Transfer-Encoding: 8bit\\n"
|
||||
],
|
||||
"comment_lines": [
|
||||
"# Blender's translation file (po format).",
|
||||
"# Copyright (C) {} The Blender Foundation."
|
||||
"".format(year_str),
|
||||
"# This file is distributed under the same "
|
||||
"# license as the Blender package.",
|
||||
"# FIRST AUTHOR <EMAIL@ADDRESS>, YEAR.",
|
||||
"#",
|
||||
],
|
||||
parsers = {
|
||||
"PO": parse_messages_from_po,
|
||||
# "PYTUPLE": parse_messages_from_pytuple,
|
||||
}
|
||||
|
||||
states = {"trans_msg": set(),
|
||||
"fuzzy_msg": {header_key},
|
||||
"comm_msg": set(),
|
||||
"is_broken": False}
|
||||
|
||||
return messages, states
|
||||
writers = {
|
||||
"PO": write_messages_to_po,
|
||||
#"PYDICT": write_messages_to_pydict,
|
||||
}
|
||||
|
||||
|
||||
def print_stats(stats, glob_stats=None, prefix=""):
|
||||
class I18n:
|
||||
"""
|
||||
Print out some stats about a po file.
|
||||
glob_stats is for making global stats over several po's.
|
||||
Internal representation of a whole translation set.
|
||||
"""
|
||||
tot_msgs = stats["tot_msg"]
|
||||
trans_msgs = stats["trans_msg"]
|
||||
tot_ttips = stats["tot_ttips"]
|
||||
trans_ttips = stats["trans_ttips"]
|
||||
comm_msgs = stats["comm_msg"]
|
||||
nbr_signs = stats["nbr_signs"]
|
||||
nbr_trans_signs = stats["nbr_trans_signs"]
|
||||
contexts = stats["contexts"]
|
||||
lvl = lvl_ttips = lvl_trans_ttips = lvl_ttips_in_trans = lvl_comm = 0.0
|
||||
|
||||
if tot_msgs > 0:
|
||||
lvl = float(trans_msgs) / float(tot_msgs)
|
||||
lvl_ttips = float(tot_ttips) / float(tot_msgs)
|
||||
lvl_comm = float(comm_msgs) / float(tot_msgs+comm_msgs)
|
||||
if tot_ttips > 0:
|
||||
lvl_trans_ttips = float(trans_ttips) / float(tot_ttips)
|
||||
if trans_msgs > 0:
|
||||
lvl_ttips_in_trans = float(trans_ttips) / float(trans_msgs)
|
||||
def __init__(self, src):
|
||||
self.trans = {}
|
||||
self.update_info()
|
||||
|
||||
if glob_stats:
|
||||
glob_stats["nbr"] += 1.0
|
||||
glob_stats["lvl"] += lvl
|
||||
glob_stats["lvl_ttips"] += lvl_ttips
|
||||
glob_stats["lvl_trans_ttips"] += lvl_trans_ttips
|
||||
glob_stats["lvl_ttips_in_trans"] += lvl_ttips_in_trans
|
||||
glob_stats["lvl_comm"] += lvl_comm
|
||||
glob_stats["nbr_trans_signs"] += nbr_trans_signs
|
||||
if glob_stats["nbr_signs"] == 0:
|
||||
glob_stats["nbr_signs"] = nbr_signs
|
||||
glob_stats["contexts"] |= contexts
|
||||
def update_info(self):
|
||||
self.nbr_trans = 0
|
||||
self.lvl = 0.0
|
||||
self.lvl_ttips = 0.0
|
||||
self.lvl_trans_ttips = 0.0
|
||||
self.lvl_ttips_in_trans = 0.0
|
||||
self.lvl_comm = 0.0
|
||||
self.nbr_signs = 0
|
||||
self.nbr_trans_signs = 0
|
||||
self.contexts = set()
|
||||
|
||||
lines = ("",
|
||||
"{:>6.1%} done! ({} translated messages over {}).\n"
|
||||
"".format(lvl, trans_msgs, tot_msgs),
|
||||
"{:>6.1%} of messages are tooltips ({} over {}).\n"
|
||||
"".format(lvl_ttips, tot_ttips, tot_msgs),
|
||||
"{:>6.1%} of tooltips are translated ({} over {}).\n"
|
||||
"".format(lvl_trans_ttips, trans_ttips, tot_ttips),
|
||||
"{:>6.1%} of translated messages are tooltips ({} over {}).\n"
|
||||
"".format(lvl_ttips_in_trans, trans_ttips, trans_msgs),
|
||||
"{:>6.1%} of messages are commented ({} over {}).\n"
|
||||
"".format(lvl_comm, comm_msgs, comm_msgs + tot_msgs),
|
||||
"This translation is currently made of {} signs.\n"
|
||||
"".format(nbr_trans_signs))
|
||||
print(prefix.join(lines))
|
||||
return 0
|
||||
if TEMPLATE_ISO_ID in self.trans:
|
||||
self.nbr_trans = len(self.trans) - 1
|
||||
self.nbr_signs = self.trans[TEMPLATE_ISO_ID].nbr_signs
|
||||
else:
|
||||
self.nbr_trans = len(self.trans)
|
||||
for iso, msgs in self.trans.items():
|
||||
msgs.update_info()
|
||||
if msgs.nbr_msgs > 0:
|
||||
self.lvl += float(msgs.nbr_trans_msgs) / float(msgs.nbr_msgs)
|
||||
self.lvl_ttips += float(msgs.nbr_ttips) / float(msgs.nbr_msgs)
|
||||
self.lvl_comm += float(msgs.nbr_comm_msgs) / float(msgs.nbr_msgs + msgs.nbr_comm_msgs)
|
||||
if msgs.nbr_ttips > 0:
|
||||
self.lvl_trans_ttips = float(msgs.nbr_trans_ttips) / float(msgs.nbr_ttips)
|
||||
if msgs.nbr_trans_msgs > 0:
|
||||
self.lvl_ttips_in_trans = float(msgs.nbr_trans_ttips) / float(msgs.nbr_trans_msgs)
|
||||
if self.nbr_signs == 0:
|
||||
self.nbr_signs = msgs.nbr_signs
|
||||
self.nbr_trans_signs += msgs.nbr_trans_signs
|
||||
self.contexts |= msgs.contexts
|
||||
|
||||
def print_stats(self, prefix="", print_msgs=True):
|
||||
"""
|
||||
Print out some stats about an I18n object.
|
||||
If print_msgs is True, it will also print all its translations' stats.
|
||||
"""
|
||||
if print_msgs:
|
||||
msgs_prefix = prefix + " "
|
||||
for key, msgs in self.trans:
|
||||
if key == TEMPLATE_ISO_ID:
|
||||
continue
|
||||
print(prefix + key + ":")
|
||||
msgs.print_stats(prefix=msgs_prefix)
|
||||
print(prefix)
|
||||
|
||||
nbr_contexts = len(self.contexts - {CONTEXT_DEFAULT})
|
||||
if nbr_contexts != 1:
|
||||
if nbr_contexts == 0:
|
||||
nbr_contexts = "No"
|
||||
_ctx_txt = "s are"
|
||||
else:
|
||||
_ctx_txt = " is"
|
||||
lines = ("",
|
||||
"Average stats for all {} translations:\n".format(self.nbr_trans),
|
||||
" {:>6.1%} done!\n".format(self.lvl / self.nbr_trans),
|
||||
" {:>6.1%} of messages are tooltips.\n".format(self.lvl_ttips / self.nbr_trans),
|
||||
" {:>6.1%} of tooltips are translated.\n".format(self.lvl_trans_ttips / self.nbr_trans),
|
||||
" {:>6.1%} of translated messages are tooltips.\n".format(self.lvl_ttips_in_trans / self.nbr_trans),
|
||||
" {:>6.1%} of messages are commented.\n".format(self.lvl_comm / self.nbr_trans),
|
||||
" The org msgids are currently made of {} signs.\n".format(self.nbr_signs),
|
||||
" All processed translations are currently made of {} signs.\n".format(self.nbr_trans_signs),
|
||||
" {} specific context{} present:\n {}\n"
|
||||
"".format(self.nbr_contexts, _ctx_txt, "\n ".join(self.contexts - {CONTEXT_DEFAULT})),
|
||||
"\n")
|
||||
print(prefix.join(lines))
|
||||
|
||||
|
||||
##### Parsers #####
|
||||
|
||||
#def parse_messages_from_pytuple(self, src, key=None):
|
||||
#"""
|
||||
#Returns a dict of tuples similar to the one returned by parse_messages_from_po (one per language, plus a 'pot'
|
||||
#one keyed as '__POT__').
|
||||
#"""
|
||||
## src may be either a string to be interpreted as py code, or a real tuple!
|
||||
#if isinstance(src, str):
|
||||
#src = eval(src)
|
||||
#
|
||||
#curr_hash = None
|
||||
#if key and key in _parser_cache:
|
||||
#old_hash, ret = _parser_cache[key]
|
||||
#import hashlib
|
||||
#curr_hash = hashlib.new(PARSER_CACHE_HASH, str(src).encode()).digest()
|
||||
#if curr_hash == old_hash:
|
||||
#return ret
|
||||
#
|
||||
#pot = new_messages()
|
||||
#states = gen_states()
|
||||
#stats = gen_stats()
|
||||
#ret = {"__POT__": (pot, states, stats)}
|
||||
#for msg in src:
|
||||
#key = msg[0]
|
||||
#messages[msgkey] = gen_message(msgid_lines, msgstr_lines, comment_lines, msgctxt_lines)
|
||||
#pot[key] = gen_message(msgid_lines=[key[1]], msgstr_lines=[
|
||||
#for lang, trans, (is_fuzzy, comments) in msg[2:]:
|
||||
#if trans and not is_fuzzy:
|
||||
#i18n_dict.setdefault(lang, dict())[key] = trans
|
||||
#
|
||||
#if key:
|
||||
#if not curr_hash:
|
||||
#import hashlib
|
||||
#curr_hash = hashlib.new(PARSER_CACHE_HASH, str(src).encode()).digest()
|
||||
#_parser_cache[key] = (curr_hash, val)
|
||||
#return ret
|
||||
Reference in New Issue
Block a user