Merge pull request #18884 from seiko2plus/breathe_doxygen

DOC: Add support for documenting C/C++ via Doxygen & Breathe

.circleci/config.yml

@@ -8,7 +8,11 @@ jobs:
     docker:
       # CircleCI maintains a library of pre-built images
       # documented at https://circleci.com/docs/2.0/circleci-images/
-      - image: cimg/python:3.8
+      # circleci/python3.8 images come with old versions of Doxygen(1.6.x),
+      # therefore a new base image chose instead to guarantee to
+      # have a newer version >= 1.8.10 to avoid warnings
+      # that related to the default behaviors or non-exist config options
+      - image: cimg/base:2021.05
 
     working_directory: ~/repo
 
@@ -23,7 +27,8 @@ jobs:
           name: create virtual environment, install dependencies
           command: |
             sudo apt-get update
-            sudo apt-get install -y graphviz texlive-fonts-recommended texlive-latex-recommended texlive-latex-extra latexmk texlive-xetex
+            sudo apt-get install -y python3.8 python3.8-dev python3-venv graphviz texlive-fonts-recommended texlive-latex-recommended \
+              texlive-latex-extra latexmk texlive-xetex doxygen
             python3.8 -m venv venv
             . venv/bin/activate
 
@@ -58,7 +63,7 @@ jobs:
             . venv/bin/activate
             cd doc
             # Don't use -q, show warning summary"
-            SPHINXOPTS="-n" make -e html || echo "ignoring errors for now, see gh-13114"
+            SPHINXOPTS="-j4 -n" make -e html || echo "ignoring errors for now, see gh-13114"
 
       - run:
           name: build devdocs
@@ -67,14 +72,14 @@ jobs:
             . venv/bin/activate
             cd doc
             make clean
-            SPHINXOPTS=-q make -e html
+            SPHINXOPTS="-j4 -q" make -e html
 
       - run:
           name: build neps
           command: |
             . venv/bin/activate
             cd doc/neps
-            SPHINXOPTS=-q make -e html
+            SPHINXOPTS="-j4 -q" make -e html
 
       - store_artifacts:
           path: doc/build/html/

doc/Makefile

@@ -14,6 +14,7 @@ PYTHON = python$(PYVER)
 SPHINXOPTS    ?=
 SPHINXBUILD   ?= LANG=C sphinx-build
 PAPER         ?=
+DOXYGEN       ?= doxygen
 # For merging a documentation archive into a git checkout of numpy/doc
 # Turn a tag like v1.18.0 into 1.18
 # Use sed -n -e 's/patttern/match/p' to return a blank value if no match
@@ -77,7 +78,7 @@ INSTALL_DIR = $(CURDIR)/build/inst-dist
 INSTALL_PPH = $(INSTALL_DIR)/lib/python$(PYVER)/site-packages:$(INSTALL_DIR)/local/lib/python$(PYVER)/site-packages:$(INSTALL_DIR)/lib/python$(PYVER)/dist-packages:$(INSTALL_DIR)/local/lib/python$(PYVER)/dist-packages
 UPLOAD_DIR=/srv/docs_scipy_org/doc/numpy-$(RELEASE)
 
-DIST_VARS=SPHINXBUILD="LANG=C PYTHONPATH=$(INSTALL_PPH) python$(PYVER) `which sphinx-build`" PYTHON="PYTHONPATH=$(INSTALL_PPH) python$(PYVER)" 
+DIST_VARS=SPHINXBUILD="LANG=C PYTHONPATH=$(INSTALL_PPH) python$(PYVER) `which sphinx-build`" PYTHON="PYTHONPATH=$(INSTALL_PPH) python$(PYVER)"
 
 NUMPYVER:=$(shell $(PYTHON) -c "import numpy; print(numpy.version.git_revision[:10])" 2>/dev/null)
 GITVER ?= $(shell cd ..; $(PYTHON) -c "import versioneer as v; print(v.get_versions()['full-revisionid'][:10])")
@@ -176,6 +177,12 @@ build/generate-stamp: $(wildcard source/reference/*.rst)
 html: version-check html-build
 html-build: generate
 	mkdir -p build/html build/doctrees
+	$(PYTHON) preprocess.py
+ifeq (, $(shell which $(DOXYGEN)))
+	@echo "Unable to find 'Doxygen:$(DOXYGEN)', skip generating C/C++ API from comment blocks."
+else
+	$(DOXYGEN) build/doxygen/Doxyfile
+endif
 	$(SPHINXBUILD) -b html $(ALLSPHINXOPTS) build/html $(FILES)
 	$(PYTHON) postprocess.py html build/html/*.html
 	@echo

doc/preprocess.py

@@ -0,0 +1,51 @@
+#!/usr/bin/env python3
+import subprocess
+import os
+import sys
+from string import Template
+
+def main():
+    doxy_gen(os.path.abspath(os.path.join('..')))
+
+def doxy_gen(root_path):
+    """
+    Generate Doxygen configration file.
+    """
+    confs = doxy_config(root_path)
+    build_path = os.path.join(root_path, "doc", "build", "doxygen")
+    gen_path = os.path.join(build_path, "Doxyfile")
+    if not os.path.exists(build_path):
+        os.makedirs(build_path)
+    with open(gen_path, 'w') as fd:
+        fd.write("#Please Don't Edit! This config file was autogenerated by ")
+        fd.write(f"doxy_gen({root_path}) in doc/preprocess.py.\n")
+        for c in confs:
+            fd.write(c)
+
+class DoxyTpl(Template):
+    delimiter = '@'
+
+def doxy_config(root_path):
+    """
+    Fetch all Doxygen sub-config files and gather it with the main config file.
+    """
+    confs = []
+    dsrc_path = os.path.join(root_path, "doc", "source")
+    sub = dict(ROOT_DIR=root_path)
+    with open(os.path.join(dsrc_path, "doxyfile"), "r") as fd:
+        conf = DoxyTpl(fd.read())
+        confs.append(conf.substitute(CUR_DIR=dsrc_path, **sub))
+
+    for dpath, _, files in os.walk(root_path):
+        if ".doxyfile" not in files:
+            continue
+        conf_path = os.path.join(dpath, ".doxyfile")
+        with open(conf_path, "r") as fd:
+            conf = DoxyTpl(fd.read())
+            confs.append(conf.substitute(CUR_DIR=dpath, **sub))
+    return confs
+
+
+if __name__ == "__main__":
+    main()
+

doc/release/upcoming_changes/18884.new_feature.rst

@@ -0,0 +1,7 @@
+Generate C/C++ API reference documentation from comments blocks is now possible
+-------------------------------------------------------------------------------
+This feature depends on Doxygen_ in the generation process and on Breathe_
+to integrate it with Sphinx.
+
+.. _`Doxygen`: https://www.doxygen.nl/index.html
+.. _`Breathe`: https://breathe.readthedocs.io/en/latest/

doc/source/conf.py

@@ -2,6 +2,7 @@
 import os
 import re
 import sys
+import importlib
 
 # Minimum version, enforced by sphinx
 needs_sphinx = '3.2.0'
@@ -86,6 +87,16 @@ class PyTypeObject(ctypes.Structure):
     'sphinx.ext.mathjax',
 ]
 
+skippable_extensions = [
+    ('breathe', 'skip generating C/C++ API from comment blocks.'),
+]
+for ext, warn in skippable_extensions:
+    ext_exist = importlib.util.find_spec(ext) is not None
+    if ext_exist:
+        extensions.append(ext)
+    else:
+        print(f"Unable to find Sphinx extension '{ext}', {warn}.")
+
 # Add any paths that contain templates here, relative to this directory.
 templates_path = ['_templates']
 
@@ -477,3 +488,11 @@ class NumPyLexer(CLexer):
             inherit,
         ],
     }
+
+
+# -----------------------------------------------------------------------------
+# Breathe & Doxygen
+# -----------------------------------------------------------------------------
+breathe_projects = dict(numpy=os.path.join("..", "build", "doxygen", "xml"))
+breathe_default_project = "numpy"
+breathe_default_members = ("members", "undoc-members", "protected-members")

doc/source/dev/examples/.doxyfile

@@ -0,0 +1,2 @@
+INPUT += @CUR_DIR
+INCLUDE_PATH += @CUR_DIR

doc/source/dev/examples/doxy_class.hpp

@@ -0,0 +1,21 @@
+/**
+ *  Template to represent limbo numbers.
+ *
+ *  Specializations for integer types that are part of nowhere.
+ *  It doesn't support with any real types.
+ *
+ *  @param Tp Type of the integer. Required to be an integer type.
+ *  @param N  Number of elements.
+*/
+template<typename Tp, std::size_t N>
+class DoxyLimbo {
+ public:
+    /// Default constructor. Initialize nothing.
+    DoxyLimbo();
+    /// Set Default behavior for copy the limbo.
+    DoxyLimbo(const DoxyLimbo<Tp, N> &l);
+    /// Returns the raw data for the limbo.
+    const Tp *data();
+ protected:
+    Tp p_data[N]; ///< Example for inline comment.
+};

doc/source/dev/examples/doxy_func.h

@@ -0,0 +1,11 @@
+/**
+ * This a simple brief.
+ *
+ * And the details goes here.
+ * Multi lines are welcome.
+ *
+ * @param  num  leave a comment for parameter num.
+ * @param  str  leave a comment for the second parameter.
+ * @return      leave a comment for the returned value.
+ */
+int doxy_javadoc_example(int num, const char *str);

doc/source/dev/examples/doxy_rst.h

@@ -0,0 +1,15 @@
+/**
+ * A comment block contains reST markup.
+ * @rst
+ * .. note::
+ *
+ *   Thanks to Breathe_, we were able to bring it to Doxygen_
+ *
+ * Some code example::
+ *
+ *   int example(int x) {
+ *       return x * 2;
+ *   }
+ * @endrst
+ */
+void doxy_reST_example(void);