From 2e0f8b5a70cb634e5e05c23fee25bfcf1181458f Mon Sep 17 00:00:00 2001 From: Ivan Grokhotkov Date: Mon, 5 Mar 2018 20:12:52 +0800 Subject: [PATCH] docs: speed up incremental builds MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit On each documentation build (‘make html’), doxygen regenerates XML files. In addition to that, gen-dxd.py regenerates API reference files under _build/inc/. This results in Sphinx flagging about half of the input files as modified, and incremental builds taking long time. With this change, XML files generated by Doxygen are copied into docs/xml_in directory only when they are changed. Breathe is pointed to docs/xml_in directory instead of docs/xml. In addition to that, gen-dxd.py is modified to only write to the output file when contents change. Overall, incremental build time (with no source files changed) is reduced from ~7 minutes to ~8 seconds (on a particular OS X computer). Due to the way Breathe includes Doxygen XML files, there is still going to be a massive rebuild every time functions, enums, macros, structures are added or removed from the header files scanned by Doxygen, but at least individual .rst files can be edited at a much faster pace. --- .gitignore | 2 ++ docs/conf.py | 23 ++++++++++++++++---- docs/gen-dxd.py | 19 +++++++++++++---- docs/link-roles.py | 2 +- docs/local_util.py | 53 ++++++++++++++++++++++++++++++++++++++++++++++ docs/repo_util.py | 5 ----- 6 files changed, 90 insertions(+), 14 deletions(-) create mode 100644 docs/local_util.py delete mode 100644 docs/repo_util.py diff --git a/.gitignore b/.gitignore index 3abace88d7..2b5352ff21 100644 --- a/.gitignore +++ b/.gitignore @@ -32,7 +32,9 @@ docs/doxygen-warning-log.txt docs/sphinx-warning-log.txt docs/sphinx-warning-log-sanitized.txt docs/xml/ +docs/xml_in/ docs/man/ +docs/doxygen_sqlite3.db # Unit test app files tools/unit-test-app/sdkconfig diff --git a/docs/conf.py b/docs/conf.py index aee4803d43..a2cafdbf0a 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -22,15 +22,26 @@ import shlex # documentation root, use os.path.abspath to make it absolute, like shown here. sys.path.insert(0, os.path.abspath('.')) -from repo_util import run_cmd_get_output +from local_util import run_cmd_get_output, copy_if_modified + +builddir = '_build' +if 'BUILDDIR' in os.environ: + builddir = os.environ['BUILDDIR'] # Call Doxygen to get XML files from the header files print "Calling Doxygen to generate latest XML files" call('doxygen') +# Doxygen has generated XML files in 'xml' directory. +# Copy them to 'xml_in', only touching the files which have changed. +copy_if_modified('xml/', 'xml_in/') + # Generate 'api_name.inc' files using the XML files by Doxygen -os.system("python gen-dxd.py") +os.system('python gen-dxd.py') + # Generate 'kconfig.inc' file from components' Kconfig files -os.system("python gen-kconfig-doc.py > _build/inc/kconfig.inc") +kconfig_inc_path = '{}/inc/kconfig.inc'.format(builddir) +os.system('python gen-kconfig-doc.py > ' + kconfig_inc_path + '.in') +copy_if_modified(kconfig_inc_path + '.in', kconfig_inc_path) # http://stackoverflow.com/questions/12772927/specifying-an-online-image-in-sphinx-restructuredtext-format # @@ -63,7 +74,11 @@ rackdiag_fontpath = '_static/DejaVuSans.ttf' packetdiag_fontpath = '_static/DejaVuSans.ttf' # Breathe extension variables -breathe_projects = { "esp32-idf": "xml/" } + +# Doxygen regenerates files in 'xml/' directory every time, +# but we copy files to 'xml_in/' only when they change, to speed up +# incremental builds. +breathe_projects = { "esp32-idf": "xml_in/" } breathe_default_project = "esp32-idf" # Add any paths that contain templates here, relative to this directory. diff --git a/docs/gen-dxd.py b/docs/gen-dxd.py index 4be75915b7..4ebf06b43d 100644 --- a/docs/gen-dxd.py +++ b/docs/gen-dxd.py @@ -10,6 +10,11 @@ import sys import os import re +# Determime build directory +builddir = '_build' +if 'BUILDDIR' in os.environ: + builddir = os.environ['BUILDDIR'] + # Script configuration header_file_path_prefix = "../components/" """string: path prefix for header files. @@ -20,7 +25,7 @@ doxyfile_path = "Doxyfile" xml_directory_path = "xml" """string: path to directory with XML files by Doxygen. """ -inc_directory_path = "_build/inc" +inc_directory_path = os.path.join(builddir, 'inc') """string: path prefix for header files. """ all_kinds = [ @@ -263,9 +268,15 @@ def generate_api_inc_files(): api_name = get_api_name(header_file_path) inc_file_path = inc_directory_path + "/" + api_name + ".inc" rst_output = generate_directives(header_file_path) - inc_file = open(inc_file_path, "w") - inc_file.write(rst_output) - inc_file.close() + + previous_rst_output = '' + if os.path.isfile(inc_file_path): + with open(inc_file_path, "r") as inc_file_old: + previous_rst_output = inc_file_old.read() + + if previous_rst_output != rst_output: + with open(inc_file_path, "w") as inc_file: + inc_file.write(rst_output) if __name__ == "__main__": diff --git a/docs/link-roles.py b/docs/link-roles.py index a9e45145ec..7bfe353921 100644 --- a/docs/link-roles.py +++ b/docs/link-roles.py @@ -2,7 +2,7 @@ import re from docutils import nodes -from repo_util import run_cmd_get_output +from local_util import run_cmd_get_output def get_github_rev(): path = run_cmd_get_output('git rev-parse --short HEAD') diff --git a/docs/local_util.py b/docs/local_util.py new file mode 100644 index 0000000000..d85ab38025 --- /dev/null +++ b/docs/local_util.py @@ -0,0 +1,53 @@ +# Utility functions used in conf.py +# +# Copyright 2017 Espressif Systems (Shanghai) PTE LTD +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http:#www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import re +import os +import shutil + +def run_cmd_get_output(cmd): + return os.popen(cmd).read().strip() + +def files_equal(path_1, path_2): + if not os.path.exists(path_1) or not os.path.exists(path_2): + return False + file_1_contents = '' + with open(path_1, "r") as f_1: + file_1_contents = f_1.read() + file_2_contents = '' + with open(path_2, "r") as f_2: + file_2_contents = f_2.read() + return file_1_contents == file_2_contents + +def copy_file_if_modified(src_file_path, dst_file_path): + if not files_equal(src_file_path, dst_file_path): + dst_dir_name = os.path.dirname(dst_file_path) + if not os.path.isdir(dst_dir_name): + os.makedirs(dst_dir_name) + shutil.copy(src_file_path, dst_file_path) + +def copy_if_modified(src_path, dst_path): + if os.path.isfile(src_path): + copy_file_if_modified(src_path, dst_path) + return + + src_path_len = len(src_path) + for root, dirs, files in os.walk(src_path): + for src_file_name in files: + src_file_path = os.path.join(root, src_file_name) + dst_file_path = os.path.join(dst_path + root[src_path_len:], src_file_name) + copy_file_if_modified(src_file_path, dst_file_path) + diff --git a/docs/repo_util.py b/docs/repo_util.py deleted file mode 100644 index 6249c11df1..0000000000 --- a/docs/repo_util.py +++ /dev/null @@ -1,5 +0,0 @@ -import re -import os - -def run_cmd_get_output(cmd): - return os.popen(cmd).read().strip()