Skip to content

GitLab

Commit ab6c6b8f authored by Carl Schwan's avatar Carl Schwan 🚴
Browse files
Options

Update translations script to support multiple pos per lang

parent 930fe7e9
Hide whitespace changes
Inline Side-by-side
Showing with 122 additions and 88 deletions
StaticMessages.sh
View file @ ab6c6b8f
#!/usr/bin/env bash
# SPDX-FileCopyrightText: 2020 Carl Schwan <carl@carlschwan.eu>
#
# SPDX-LicenseIdentifier: BSD-2-Clause
FILENAME="kde-org"
function export_pot_file # First parameter will be the path of the pot file we have to create, includes $FILENAME
function export_pot_dir # First parameter will be the path of the directory where we have to store the pot files
{
potfile=$1
python3 translations.py extract $potfile
echo "Creating POT files"
potdir=$1
python3 translations.py extract $potdir
}
function import_po_files # First parameter will be a path that will contain several .po files with the format LANG.po
function import_po_dirs # First parameter will be a path that will be a directory to the dirs for each lang and then all the .po files inside
{
podir=$1
python3 translations.py import $podir
python3 translations.py generate-translations
rm -rf locale
echo "Nothing to do: import is done in binary factory"
# podir=$1
# python3 translations.py import $podir
# python3 translations.py generate-translations
# rm -rf locale
}
l10n-fetch-po-files.py
View file @ ab6c6b8f
......@@ -29,9 +29,9 @@ Run me from KDE.org top-level directory.
SVN_PATH = "svn://anonsvn.kde.org/home/kde/trunk/l10n-kf5/"
SOURCE_PO_PATH = "/messages/websites-kde-org/www_www.po"
OUTPUT_PO_PATH = "./t/"
OUTPUT_PO_PATTERN = "%s.po"
SOURCE_PO_PATHS = ["/messages/websites-kde-org/www_www.po", "/messages/websites-kde-org/promo.po", "/messages/websites-kde-org/release_announcements.pot"]
OUTPUT_PO_PATH = "./pos/"
OUTPUT_PO_PATTERN = "%s/%s.po"
fixer = re.compile(r'^#~\| ', re.MULTILINE)
re_empty_msgid = re.compile('^msgid ""$', re.MULTILINE)
......@@ -41,27 +41,31 @@ re_has_qt_contexts = re.compile('X-Qt-Contexts: true\\n')
if not os.path.exists(OUTPUT_PO_PATH):
os.mkdir(OUTPUT_PO_PATH)
all_languages = "af ar as ast az be be@latin bg bn bn_IN br bs ca ca@valencia crh cs csb cy da de el en en_GB eo es et eu fa fi fr fy ga gd gl gu ha he hi hne hr hsb hu hy ia id is it ja ka kk km kn ko ku lb lt lv mai mk ml mr ms mt nb nds ne nl nn nso oc or pa pl ps pt pt_BR README ro ru rw scripts se si sk sl sq sr sr@ijekavian sr@ijekavianlatin sr@latin sv ta te tg th tn tr tt ug uk uz uz@cyrillic vi wa xh zh_CN zh_HK zh_TW"
all_languages = "af ar as ast az be be@latin bg bn bn_IN br bs ca ca@valencia crh cs csb cy da de el en en_GB eo es et eu fa fi fr fy ga gd gl gu ha he hi hne hr hsb hu hy ia id is it ja ka kk km kn ko ku lb lt lv mai mk ml mr ms mt nb nds ne nl nn nso oc or pa pl ps pt pt_BR ro ru rw scripts se si sk sl sq sr sr@ijekavian sr@ijekavianlatin sr@latin sv ta te tg th tn tr tt ug uk uz uz@cyrillic vi wa xh zh_CN zh_HK zh_TW"
all_languages = [x.strip() for x in all_languages.split(" ") if len(x)]
for lang in all_languages:
print(lang)
try:
raw_data = subprocess.check_output(['svn', 'cat', SVN_PATH + lang + SOURCE_PO_PATH],
stderr=subprocess.PIPE)
(transformed, subs) = fixer.subn('# ~| ', raw_data.decode('utf-8'))
pos1 = re_empty_msgid.search(transformed).start()
pos2 = re_empty_line.search(transformed).start()
if re_has_qt_contexts.search(transformed, pos1, pos2) is None:
transformed = transformed[:pos2] + \
'"X-Qt-Contexts: true\\n"\n' + \
transformed[pos2:]
subs = subs + 1
if (subs > 0):
print("Fetched {} (and performed {} cleanups)".format(lang, subs))
else:
print("Fetched {}".format(lang))
with open(OUTPUT_PO_PATH + OUTPUT_PO_PATTERN % lang, "wb") as f:
f.write(transformed.encode())
except subprocess.CalledProcessError:
print("No data for {}".format(lang))
if not os.path.exists(OUTPUT_PO_PATH + '/' + lang):
os.mkdir(OUTPUT_PO_PATH + '/' + lang)
for SOURCE_PO_PATH in SOURCE_PO_PATHS:
try:
raw_data = subprocess.check_output(['svn', 'cat', SVN_PATH + lang + SOURCE_PO_PATH],
stderr=subprocess.PIPE)
(transformed, subs) = fixer.subn('# ~| ', raw_data.decode('utf-8'))
pos1 = re_empty_msgid.search(transformed).start()
pos2 = re_empty_line.search(transformed).start()
if re_has_qt_contexts.search(transformed, pos1, pos2) is None:
transformed = transformed[:pos2] + \
'"X-Qt-Contexts: true\\n"\n' + \
transformed[pos2:]
subs = subs + 1
if (subs > 0):
print("Fetched {} (and performed {} cleanups)".format(lang, subs))
else:
print("Fetched {}".format(lang))
name = os.path.splitext(os.path.basename(SOURCE_PO_PATH))[0]
with open(OUTPUT_PO_PATH + OUTPUT_PO_PATTERN % (lang, name), "wb") as f:
f.write(transformed.encode())
except subprocess.CalledProcessError:
print("No data for {}".format(lang))
translations.py
View file @ ab6c6b8f
......@@ -21,7 +21,7 @@ try:
except ImportError:
from yaml import Loader, Dumper
def webpage_list():
def webpage_list() -> tp.Dict[str, tp.List[str]]:
"""
Generate a list of webpage that need to be translated
"""
......@@ -30,26 +30,32 @@ def webpage_list():
application_announcements = glob.glob('./content/announcements/announce-applications-*')
plasma_announcements_all = glob.glob('./content/announcements/plasma-5.*')
kde_announcements = glob.glob('./content/announcements/4.1[0-3]/*') + glob.glob('./content/announcements/announce-4.11.*')
release_announcements = glob.glob('./content/announcements/releases/*.md')
plasma_annoucements = []
for plasma in plasma_announcements_all:
if "changelog" not in plasma:
plasma_annoucements.append(plasma)
#return [
# 'content/announcements/plasma-5.20.0.md',
#]
return [
'content/announcements/plasma5.0.md',
'content/fundraisers/yearend2016/thanks_paypal.md',
'content/fundraisers/yearend2014/thanks_paypal.md',
'content/announcements/plasma5.0-beta2.md',
'content/announcements/plasma2tp.md',
] + frameworks_announcements + plasma_annoucements + application_announcements + kde_announcements + [
'content/products.md',
'content/_index.html',
'content/community/donations/_index.md',
]
return {
'test': ['./content/announcements/announce-applications-17.08.3.md']
}
return {
'www_www': [
'content/announcements/plasma5.0.md',
'content/fundraisers/yearend2016/thanks_paypal.md',
'content/fundraisers/yearend2014/thanks_paypal.md',
'content/announcements/plasma5.0-beta2.md',
'content/announcements/plasma2tp.md',
] + frameworks_announcements + application_announcements + kde_announcements + plasma_annoucements,
'release_announcements': release_announcements,
'promo': [
'content/products.md',
'content/_index.html',
'content/community/donations/_index.md',
]
}
def import_frontmatter(data, _):
......@@ -231,11 +237,12 @@ def import_content(post, _) -> tp.Tuple[str, float]:
is_translated = False
if not is_translated:
translated += sectionContent + '\n' if list_item else '\n\n'
translated += sectionContent + ('\n' if list_item else '\n\n')
else:
translated += processed_line[2].format(*tuple([_(x) for x in processed_line[0]])) + '\n' if list_item else '\n\n'
totalTranslated += 1
totalString += 1
sectionContent = ""
......@@ -269,6 +276,29 @@ def extract(args):
"""
First parameter will be the path of the pot file we have to create
"""
if not os.path.exists(args.pot):
os.makedirs(args.pot)
for category, webpages in webpage_list().items():
pot = polib.POFile(check_for_duplicates=True)
pot.metadata = {
'Project-Id-Version': '1.0',
'Report-Msgid-Bugs-To': 'kde-www@kde.org',
'Last-Translator': 'you <you@example.com>',
'Language-Team': 'English <yourteam@example.com>',
'MIME-Version': '1.0',
'Content-Type': 'text/plain; charset=utf-8',
'Content-Transfer-Encoding': '8bit',
}
for webpage in webpages:
print(webpage)
post = frontmatter.load(webpage)
extract_frontmatter(post.metadata, pot, webpage)
extract_content(post, pot, webpage)
pot.save(args.pot + '/' + category + '.pot')
pot_file = args.pot
pot = polib.POFile(check_for_duplicates=True)
pot.metadata = {
......@@ -280,12 +310,6 @@ def extract(args):
'Content-Type': 'text/plain; charset=utf-8',
'Content-Transfer-Encoding': '8bit',
}
for webpage in webpage_list():
print(webpage)
post = frontmatter.load(webpage)
extract_frontmatter(post.metadata, pot, webpage)
extract_content(post, pot, webpage)
with open("i18n/en.yaml", 'r') as stream:
en_string_trans = safe_load(stream)
......@@ -325,7 +349,7 @@ def extract(args):
except:
pass
pot.save(pot_file)
pot.save(args.pot + '/' + 'menu_footer_shared.pot')
def import_po(args):
......@@ -334,18 +358,18 @@ def import_po(args):
"""
directory = args.directory
for translation in os.listdir(directory):
lang = os.path.splitext(translation)[0]
lang = translation
if lang == "ca@valencia":
lang = "ca-valencia"
if lang == "pt_BR":
lang = "pt-br"
os.makedirs("locale/" + lang + "/LC_MESSAGES/")
po_file = "locale/" + lang + "/LC_MESSAGES/announcement"
shutil.copyfile(directory + "/" + translation, po_file + ".po")
if not os.path.exists("locale/" + lang + "/LC_MESSAGES/"):
os.makedirs("locale/" + lang + "/LC_MESSAGES/")
for po in os.listdir(directory + "/" + translation):
po_file = "locale/" + lang + "/LC_MESSAGES/" + po
shutil.copyfile(directory + "/" + translation + "/" + po, po_file)
command = "msgfmt " + po_file + " -o " + po_file[:-2] + "mo"
subprocess.run(command, shell=True, check=True)
command = "msgfmt " + po_file + ".po -o " + po_file + ".mo"
subprocess.run(command, shell=True, check=True)
print("Translations files for " + translation + " imported")
......@@ -356,30 +380,31 @@ def generate_translations(args):
for translations in os.listdir('locale'):
print(translations)
os.environ["LANGUAGE"] = translations
gettext.bindtextdomain('announcement', os.path.abspath('locale'))
gettext.textdomain('announcement')
_ = gettext.gettext
for webpage in webpage_list():
translated = re.sub('content/', 'content/' + translations + '/', webpage)
translated = re.sub('content/', 'content-trans/' + translations + '/', webpage)
print(translated)
os.makedirs(os.path.dirname(translated), exist_ok=True)
post = frontmatter.load(webpage)
has_translations = import_frontmatter(post.metadata, _)
content, percent = import_content(post, _)
if percent > 0.5 or has_translations:
with open(translated, 'w+') as translation_file:
translation_file.write('---\n')
translation_file.write(dump(post.metadata, default_flow_style=False, allow_unicode=True))
translation_file.write('---\n')
translation_file.write(content)
with open(translated) as f_input:
data = f_input.read().rstrip('\n')
with open(translated, 'w') as f_output:
f_output.write(data)
for category, webpages in webpage_list().items():
gettext.bindtextdomain(category, os.path.abspath('locale'))
gettext.textdomain(category)
_ = gettext.gettext
for webpage in webpages:
translated = re.sub('content/', 'content/' + translations + '/', webpage)
translated = re.sub('content/', 'content-trans/' + translations + '/', webpage)
print(translated)
os.makedirs(os.path.dirname(translated), exist_ok=True)
post = frontmatter.load(webpage)
has_translations = import_frontmatter(post.metadata, _)
content, percent = import_content(post, _)
if percent > 0.5 or has_translations:
with open(translated, 'w+') as translation_file:
translation_file.write('---\n')
translation_file.write(dump(post.metadata, default_flow_style=False, allow_unicode=True))
translation_file.write('---\n')
translation_file.write(content)
with open(translated) as f_input:
data = f_input.read().rstrip('\n')
with open(translated, 'w') as f_output:
f_output.write(data)
with open("i18n/en.yaml", 'r') as stream:
en_string_trans = safe_load(stream)
......@@ -389,8 +414,8 @@ def generate_translations(args):
for translations in os.listdir('locale'):
os.environ["LANGUAGE"] = translations
gettext.bindtextdomain('announcement', os.path.abspath('locale'))
gettext.textdomain('announcement')
gettext.bindtextdomain('menu_footer_static', os.path.abspath('locale'))
gettext.textdomain('menu_footer_static')
_ = gettext.gettext
trans_content = dict()
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment