From 4ad2c64785fea52ae8844d26fcc11aab32b9b82b Mon Sep 17 00:00:00 2001 From: Julien Palard Date: Sat, 10 Dec 2016 22:27:01 +0100 Subject: [PATCH 1/3] Clone or re-clone repo if not present or from wrong upstream. This will typically be usefull when switching from hg to git or git to hg, as git pull will fail on hg and vice-versa. As a bonus, if repo does not exists, it's created. --- build_docs.py | 61 +++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 44 insertions(+), 17 deletions(-) diff --git a/build_docs.py b/build_docs.py index cbd004d..2b0400a 100644 --- a/build_docs.py +++ b/build_docs.py @@ -25,14 +25,18 @@ import os import subprocess import sys +import shutil +CPYTHON_GIT = "https://github.com/python/cpython.git" +CPYTHON_HG = "https://hg.python.org/cpython" + BRANCHES = [ - # version, isdev - (3.5, False), - (3.6, True), - (3.7, True), - (2.7, False) + # version, branch, isdev + (3.5, '3.5', False), + (3.6, '3.6', True), + (3.7, 'master', True), + (2.7, '2.7', False) ] @@ -80,21 +84,43 @@ def changed_files(directory, other): return changed -def build_one(version, isdev, quick, sphinxbuild, build_root, www_root, +def update_repo(dest, branch, git=False): + logging.info("Updating repository %s", dest) + if git: + shell_out("git -C {} reset --hard HEAD".format(dest)) + shell_out("git -C {} checkout {}".format(dest, branch)) + shell_out("git -C {} pull --ff-only".format(dest)) + else: + shell_out("hg --cwd {} pull -u".format(dest)) + + +def clone_repo(dest, branch, git=False): + """This function will remove a clone if the update fails, and re-clone + it from scratch. + + This mean that switching from hg to git and vice-versa + is now a seamless operation. + """ + vcs = 'git' if git else 'hg' + upstream = CPYTHON_GIT if git else CPYTHON_HG + try: + update_repo(dest, branch, git) + except subprocess.CalledProcessError: + if os.path.exists(dest): + shutil.rmtree(dest) + logging.info("Cloning cpython") + os.makedirs(dest, mode=0o775) + shell_out("{} clone {} {}".format(vcs, upstream, dest)) + shell_out("{} checkout {}".format(vcs, branch)) + +def build_one(version, branch, isdev, quick, sphinxbuild, build_root, www_root, skip_cache_invalidation=False, group='docs', git=False, log_directory='/var/log/docsbuild/'): checkout = build_root + "/python" + str(version).replace('.', '') target = www_root + "/" + str(version) logging.info("Doc autobuild started in %s", checkout) + clone_repo(checkout, branch, git) os.chdir(checkout) - - logging.info("Updating checkout") - if git: - shell_out("git reset --hard HEAD") - shell_out("git pull --ff-only") - else: - shell_out("hg pull -u") - maketarget = "autobuild-" + ("dev" if isdev else "stable") + ("-html" if quick else "") logging.info("Running make %s", maketarget) logname = os.path.basename(checkout) + ".log" @@ -219,13 +245,14 @@ def parse_args(): sphinxbuild = os.path.join(args.build_root, "environment/bin/sphinx-build") try: if args.branch: - build_one(args.branch, args.devel, args.quick, sphinxbuild, + build_one(args.branch, args.branch, args.devel, args.quick, + sphinxbuild, args.build_root, args.www_root, args.skip_cache_invalidation, args.group, args.git, args.log_directory) else: - for version, devel in BRANCHES: - build_one(version, devel, args.quick, sphinxbuild, + for version, branch, devel in BRANCHES: + build_one(version, branch, devel, args.quick, sphinxbuild, args.build_root, args.www_root, args.skip_cache_invalidation, args.group, args.git, args.log_directory) From 41e5b269c309dcc666efa0c7bd58d847f6f5209c Mon Sep 17 00:00:00 2001 From: Julien Palard Date: Sat, 10 Dec 2016 22:46:22 +0100 Subject: [PATCH 2/3] Make script run with temporary paths. So, we automatically create missing folders, and check if the venv is here before running, as it's probably missing with temporary paths. --- build_docs.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/build_docs.py b/build_docs.py index 2b0400a..3638f6f 100644 --- a/build_docs.py +++ b/build_docs.py @@ -116,6 +116,7 @@ def clone_repo(dest, branch, git=False): def build_one(version, branch, isdev, quick, sphinxbuild, build_root, www_root, skip_cache_invalidation=False, group='docs', git=False, log_directory='/var/log/docsbuild/'): + os.makedirs(log_directory, mode=0o750, exist_ok=True) checkout = build_root + "/python" + str(version).replace('.', '') target = www_root + "/" + str(version) logging.info("Doc autobuild started in %s", checkout) @@ -129,6 +130,7 @@ def build_one(version, branch, isdev, quick, sphinxbuild, build_root, www_root, changed = changed_files(os.path.join(checkout, "Doc/build/html"), target) logging.info("Copying HTML files to %s", target) + os.makedirs(target, mode=0o775, exist_ok=True) shell_out("chown -R :{} Doc/build/html/".format(group)) shell_out("chmod -R o+r Doc/build/html/") shell_out("find Doc/build/html/ -type d -exec chmod o+x {} ';'") @@ -232,6 +234,13 @@ def parse_args(): return parser.parse_args() +def check_environment(build_root): + venv_path = os.path.join(build_root, "environment") + if not os.path.isdir(venv_path): + logging.error("venv is missing in %s, salt should have built it.", + venv_path) + exit(1) + if __name__ == '__main__': args = parse_args() if sys.stderr.isatty(): @@ -242,6 +251,7 @@ def parse_args(): filename=os.path.join(args.log_directory, "docsbuild.log")) logging.root.setLevel(logging.DEBUG) + check_environment(args.build_root) sphinxbuild = os.path.join(args.build_root, "environment/bin/sphinx-build") try: if args.branch: From 98528a5682f67438757a01481637c5c72d8c528d Mon Sep 17 00:00:00 2001 From: Julien Palard Date: Sun, 11 Dec 2016 02:05:57 +0100 Subject: [PATCH 3/3] Adding french, japanese, and chinese translations. --- build_docs.py | 86 ++++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 74 insertions(+), 12 deletions(-) diff --git a/build_docs.py b/build_docs.py index 3638f6f..cb961e6 100644 --- a/build_docs.py +++ b/build_docs.py @@ -26,6 +26,7 @@ import subprocess import sys import shutil +from collections import namedtuple CPYTHON_GIT = "https://github.com/python/cpython.git" @@ -39,6 +40,30 @@ (2.7, '2.7', False) ] +Repository = namedtuple('Repository', 'name url') +Translation = namedtuple('Translation', 'lang repo branch po_path') + +AFPY = Repository('afpy', 'https://github.com/AFPy/python_doc_fr.git') +JA_35 = Repository('ja_3.5', 'https://github.com/python-doc-ja/py35-locale.git') +JA_27 = Repository('ja_2.7', 'https://github.com/python-doc-ja/py27-locale.git') +TW = Repository('tw', 'https://github.com/python-doc-tw/cpython-tw.git') + +# Only use lowercased tranlation names, with dash (no underscore) for +# URL consistency, as the name is used in the docs.python.org URL. +I18N_CONF = [ + { # French + 'default': Translation('fr', AFPY, 'master', '3.6'), + 3.5: Translation('fr', AFPY, 'master', '3.5'), + 2.7: Translation('fr', AFPY, 'master', '2.7'), + }, + { # Japanese + 'default': Translation('ja', JA_35, 'master', 'ja/LC_MESSAGES'), + 2.7: Translation('ja', JA_27, 'master', 'ja/LC_MESSAGES'), + }, + { # Chienese as spoken in Taiwan + 'default': Translation('zh', TW, 'tw-3.5', 'Doc/locale/zh_Hant/LC_MESSAGES'), + } +] def _file_unchanged(old, new): with open(old, "rb") as fp1, open(new, "rb") as fp2: @@ -94,7 +119,7 @@ def update_repo(dest, branch, git=False): shell_out("hg --cwd {} pull -u".format(dest)) -def clone_repo(dest, branch, git=False): +def clone_repo(url, dest, branch, git=False): """This function will remove a clone if the update fails, and re-clone it from scratch. @@ -102,32 +127,41 @@ def clone_repo(dest, branch, git=False): is now a seamless operation. """ vcs = 'git' if git else 'hg' - upstream = CPYTHON_GIT if git else CPYTHON_HG try: update_repo(dest, branch, git) except subprocess.CalledProcessError: if os.path.exists(dest): shutil.rmtree(dest) - logging.info("Cloning cpython") + logging.info("Cloning %s", url) os.makedirs(dest, mode=0o775) - shell_out("{} clone {} {}".format(vcs, upstream, dest)) - shell_out("{} checkout {}".format(vcs, branch)) + shell_out("{} clone {} {}".format(vcs, url, dest)) + if git: + shell_out("git -C {} checkout {}".format(dest, branch)) + else: + shell_out("hg --cwd {} checkout {}".format(dest, branch)) + def build_one(version, branch, isdev, quick, sphinxbuild, build_root, www_root, + translation=None, skip_cache_invalidation=False, group='docs', git=False, log_directory='/var/log/docsbuild/'): os.makedirs(log_directory, mode=0o750, exist_ok=True) checkout = build_root + "/python" + str(version).replace('.', '') - target = www_root + "/" + str(version) + target = www_root + "/" + (translation.lang + "/" if translation else "") + str(version) logging.info("Doc autobuild started in %s", checkout) - clone_repo(checkout, branch, git) + clone_repo(CPYTHON_GIT if git else CPYTHON_HG, + checkout, branch, git) os.chdir(checkout) maketarget = "autobuild-" + ("dev" if isdev else "stable") + ("-html" if quick else "") + sphinxopts = "" + if translation: + sphinxopts = 'SPHINXOPTS="-D gettext_compact=0 -D locale_dirs=../locale -D language={}"'.format( + translation.lang) logging.info("Running make %s", maketarget) logname = os.path.basename(checkout) + ".log" - shell_out("cd Doc; make SPHINXBUILD=%s %s >> %s 2>&1" % - (sphinxbuild, maketarget, os.path.join(log_directory, logname))) - + shell_out("cd Doc; make %s SPHINXBUILD=%s %s >> %s 2>&1" % + (sphinxopts, sphinxbuild, maketarget, + os.path.join(log_directory, logname))) changed = changed_files(os.path.join(checkout, "Doc/build/html"), target) logging.info("Copying HTML files to %s", target) os.makedirs(target, mode=0o775, exist_ok=True) @@ -163,6 +197,21 @@ def build_one(version, branch, isdev, quick, sphinxbuild, build_root, www_root, logging.info("Finished %s", checkout) +def copy_po_files(translation, version, build_root): + locale_dir = os.path.join(build_root, + "python" + str(version).replace('.', ''), + 'locale', + translation.lang, + 'LC_MESSAGES') + os.makedirs(locale_dir, mode=0o775, exist_ok=True) + shell_out("rsync -a --include '*.po' --include '*/' --exclude '*' {}/ {}".format( + os.path.join(build_root, + 'i18n', + translation.repo.name, + translation.po_path), + locale_dir)) + + def build_devguide(devguide_checkout, devguide_target, sphinxbuild, skip_cache_invalidation=False): build_directory = os.path.join(devguide_checkout, "build/html") @@ -257,15 +306,28 @@ def check_environment(build_root): if args.branch: build_one(args.branch, args.branch, args.devel, args.quick, sphinxbuild, - args.build_root, args.www_root, + args.build_root, args.www_root, None, args.skip_cache_invalidation, args.group, args.git, args.log_directory) else: for version, branch, devel in BRANCHES: build_one(version, branch, devel, args.quick, sphinxbuild, - args.build_root, args.www_root, + args.build_root, args.www_root, None, args.skip_cache_invalidation, args.group, args.git, args.log_directory) + for translations in I18N_CONF: + translation = translations.get(branch, translations['default']) + clone_repo(translation.repo.url, + os.path.join(args.build_root, + 'i18n', + translation.repo.name), + translation.branch, True) + copy_po_files(translation, version, args.build_root) + build_one(version, branch, devel, args.quick, sphinxbuild, + args.build_root, args.www_root, translation, + args.skip_cache_invalidation, args.group, args.git, + args.log_directory) + build_devguide(args.devguide_checkout, args.devguide_target, sphinxbuild, args.skip_cache_invalidation) except Exception: