DOC: Add support for documenting C/C++ via Doxygen & Breathe

.circleci/config.yml

@@ -8,7 +8,7 @@ jobs:
     docker:
       # CircleCI maintains a library of pre-built images
       # documented at https://circleci.com/docs/2.0/circleci-images/
-      - image: circleci/python:3.8.4
+      - image: cimg/base:2021.05
 
     working_directory: ~/repo
 
@@ -23,7 +23,8 @@ jobs:
           name: create virtual environment, install dependencies
           command: |
             sudo apt-get update
-            sudo apt-get install -y graphviz texlive-fonts-recommended texlive-latex-recommended texlive-latex-extra texlive-generic-extra latexmk texlive-xetex
+            sudo apt-get install -y python3.8 python3.8-dev python3-venv graphviz texlive-fonts-recommended texlive-latex-recommended \
+                                    texlive-latex-extra texlive-formats-extra latexmk texlive-xetex doxygen
             python3.8 -m venv venv
             . venv/bin/activate
 

doc/Makefile

@@ -14,6 +14,7 @@ PYTHON = python$(PYVER)
 SPHINXOPTS    ?=
 SPHINXBUILD   ?= LANG=C sphinx-build
 PAPER         ?=
+DOXYGEN       ?= doxygen
 # For merging a documentation archive into a git checkout of numpy/doc
 # Turn a tag like v1.18.0 into 1.18
 # Use sed -n -e 's/patttern/match/p' to return a blank value if no match
@@ -77,7 +78,7 @@ INSTALL_DIR = $(CURDIR)/build/inst-dist
 INSTALL_PPH = $(INSTALL_DIR)/lib/python$(PYVER)/site-packages:$(INSTALL_DIR)/local/lib/python$(PYVER)/site-packages:$(INSTALL_DIR)/lib/python$(PYVER)/dist-packages:$(INSTALL_DIR)/local/lib/python$(PYVER)/dist-packages
 UPLOAD_DIR=/srv/docs_scipy_org/doc/numpy-$(RELEASE)
 
-DIST_VARS=SPHINXBUILD="LANG=C PYTHONPATH=$(INSTALL_PPH) python$(PYVER) `which sphinx-build`" PYTHON="PYTHONPATH=$(INSTALL_PPH) python$(PYVER)" 
+DIST_VARS=SPHINXBUILD="LANG=C PYTHONPATH=$(INSTALL_PPH) python$(PYVER) `which sphinx-build`" PYTHON="PYTHONPATH=$(INSTALL_PPH) python$(PYVER)"
 
 NUMPYVER:=$(shell $(PYTHON) -c "import numpy; print(numpy.version.git_revision[:10])" 2>/dev/null)
 GITVER ?= $(shell cd ..; $(PYTHON) -c "import versioneer as v; print(v.get_versions()['full-revisionid'][:10])")
@@ -176,6 +177,8 @@ build/generate-stamp: $(wildcard source/reference/*.rst)
 html: version-check html-build
 html-build: generate
 	mkdir -p build/html build/doctrees
+	$(PYTHON) preprocess.py
+	$(DOXYGEN) build/doxygen/Doxyfile
 	$(SPHINXBUILD) -b html $(ALLSPHINXOPTS) build/html $(FILES)
 	$(PYTHON) postprocess.py html build/html/*.html
 	@echo

doc/preprocess.py

@@ -0,0 +1,50 @@
+#!/usr/bin/env python3
+import subprocess
+import os
+import sys
+from string import Template
+
+def main():
+    doxy_gen(os.path.abspath(os.path.join('..')))
+
+def doxy_gen(root_path):
+    """
+    Generate Doxygen configration file.
+    """
+    confs = doxy_config(root_path)
+    build_path = os.path.join(root_path, "doc", "build", "doxygen")
+    gen_path = os.path.join(build_path, "Doxyfile")
+    if not os.path.exists(build_path):
+        os.makedirs(build_path)
+    with open(gen_path, 'w') as fd:
+        fd.write("#Please Don't Edit! This config file was autogenerated.\n")
+        for c in confs:
+            fd.write(c)
+
+class DoxyTpl(Template):
+    delimiter = '@'
+
+def doxy_config(root_path):
+    """
+    Fetch all Doxygen sub-config files and gather it with the main config file.
+    """
+    confs = []
+    dsrc_path = os.path.join(root_path, "doc", "source")
+    sub = dict(ROOT_DIR=root_path)
+    with open(os.path.join(dsrc_path, "doxyfile"), "r") as fd:
+        conf = DoxyTpl(fd.read())
+        confs.append(conf.substitute(CUR_DIR=dsrc_path, **sub))
+
+    for dpath, _, files in os.walk(root_path):
+        if ".doxyfile" not in files:
+            continue
+        conf_path = os.path.join(dpath, ".doxyfile")
+        with open(conf_path, "r") as fd:
+            conf = DoxyTpl(fd.read())
+            confs.append(conf.substitute(CUR_DIR=dpath, **sub))
+    return confs
+
+
+if __name__ == "__main__":
+    main()
+

doc/source/conf.py

@@ -84,6 +84,7 @@ class PyTypeObject(ctypes.Structure):
     'IPython.sphinxext.ipython_console_highlighting',
     'IPython.sphinxext.ipython_directive',
     'sphinx.ext.imgmath',
+    'breathe'
 ]
 
 imgmath_image_format = 'svg'
@@ -466,3 +467,11 @@ class NumPyLexer(CLexer):
             inherit,
         ],
     }
+
+
+# -----------------------------------------------------------------------------
+# Breathe & Doxygen
+# -----------------------------------------------------------------------------
+breathe_projects = dict(numpy=os.path.join("..", "build", "doxygen", "xml"))
+breathe_default_project = "numpy"
+

doc/source/docs/examples/.doxyfile

@@ -0,0 +1 @@
+INPUT += @CUR_DIR

doc/source/docs/examples/c_doxygen_group.h

@@ -0,0 +1,24 @@
+///@addtogroup C_doxygen_group_example
+///@{
+
+/// An example of documented data type.
+typedef struct {
+    int size; ///< Size of allacoted data.
+    void *ptr; ///< Pointer of allacoted data.
+} cdoxy_data1;
+
+enum cdoxy_nowhere_in {
+    cdoxy_earth = 1, ///< A reigon of the earth
+    cdoxy_moon, ///< A reigon of the moon
+    cdoxy_sun ///< A reigon of our star
+};
+
+/**
+ * An imaginary function that allocates a certain reigon of nowhere.
+ *
+ * @param size The size of the reigon.
+ * @param type The located nowhere.
+ */
+cdoxy_data1 *cdoxy_allocate(int size, cdoxy_nowhere_in type);
+
+///@}

doc/source/docs/examples/c_doxygen_rst.h

@@ -0,0 +1,16 @@
+/**
+ * A comment block contains reST markup.
+ * @rst
+ * .. note::
+ *
+ *   Thanks to Breathe_, we were able to bring it to Doxygen_
+ *
+ * Some code example::
+ *
+ *   int example(int x) {
+ *       return x * 2;
+ *   }
+ * @endrst
+ */
+void rst_example1_c_function(void);
+

doc/source/docs/howto_build_docs.rst

@@ -27,7 +27,16 @@ additional parts required for building the documentation::
 
 In addition, building the documentation requires the Sphinx extension
 `plot_directive`, which is shipped with Matplotlib_. This Sphinx extension can
-be installed by installing Matplotlib. You will also need Python>=3.6.
+be installed by installing Matplotlib. You will also need Python>=3.6 and Doxygen.
+
+For installing Doxygen_, please check the official `download <https://www.doxygen.nl/download.html#srcbin>`_
+and `installation <https://www.doxygen.nl/manual/install.html>`_ pages, or if you
+are using Linux then you can install it through your distribution package manager.
+
+.. note::
+
+   Try to install a newer version of Doxygen_ > 1.8.10 otherwise you may get some warnings
+   during the build.
 
 Since large parts of the main documentation are obtained from numpy via
 ``import numpy`` and examining the docstrings, you will need to first build
@@ -91,6 +100,7 @@ on how to update https://numpy.org/doc.
 
 .. _Matplotlib: https://matplotlib.org/
 .. _HOWTO RELEASE: https://github.com/numpy/numpy/blob/main/doc/HOWTO_RELEASE.rst.txt
+.. _Doxygen: https://www.doxygen.nl/index.html
 
 Sphinx extensions
 -----------------

doc/source/docs/howto_document.rst

@@ -73,3 +73,138 @@ Please use the numpydoc `formatting standard`_ as shown in their example_
 
 .. _`formatting standard`: https://numpydoc.readthedocs.io/en/latest/format.html
 .. _example: https://numpydoc.readthedocs.io/en/latest/example.html
+
+.. _doc_c_code:
+
+Documenting C/C++ Code
+**********************
+
+Recently, NumPy headed out to use Doxygen_ along with Sphinx due to the urgent need
+to generate C/C++ API reference documentation from comment blocks, especially with
+the most recent expansion in the use of SIMD, and also due to the future reliance on C++.
+Thanks to Breathe_, we were able to bind both worlds together at the lowest cost.
+
+It takes three steps to complete the documentation process:
+
+Writing the comment blocks
+--------------------------
+
+Although there is still no commenting style set to follow, the Javadoc
+is more preferable than the others due to the similarities with the current
+existing non-indexed comment blocks.
+
+.. note::
+   If you have never used Doxygen_ before, then maybe you need to take a quick
+   look at `"Documenting the code" <https://www.doxygen.nl/manual/docblocks.html>`_.
+
+This is how Javadoc style looks like::
+
+  /**
+   * Your brief is here.
+   */
+  void function(void);
+
+For line comment, insert a triple forward slash::
+
+  /// Your brief is here.
+  void function(void);
+
+And for structured data members, just insert triple forward slash and less-than sign::
+
+  /// Data type brief.
+  typedef struct {
+      int member_1; ///< Member 1 description.
+      int member_2; ///< Member 2 description.
+  } data_type;
+
+
+**Common Doxygen Tags**:
+
+.. note::
+   For more tags/commands, please take a look at https://www.doxygen.nl/manual/commands.html
+
+``@brief``
+
+Starts a paragraph that serves as a brief description. By default the first sentence
+of the documentation block is automatically treated as a brief description, since
+option `JAVADOC_AUTOBRIEF <https://www.doxygen.nl/manual/config.html#cfg_javadoc_autobrief>`_
+is enabled within doxygen configurations.
+
+``@details``
+
+Just like ``@brief`` starts a brief description, ``@details`` starts the detailed description.
+You can also start a new paragraph (blank line) then the ``@details`` command is not needed.
+
+``@param``
+
+Starts a parameter description for a function parameter with name <parameter-name>,
+followed by a description of the parameter. The existence of the parameter is checked
+and a warning is given if the documentation of this (or any other) parameter is missing
+or not present in the function declaration or definition.
+
+``@code/@endcode``
+
+Starts/Ends a block of code. A code block is treated differently from ordinary text.
+It is interpreted as source code.
+
+``@rst/@endrst``
+
+Starts/Ends a block of reST markup. Take a look at the following example:
+
+.. literalinclude:: examples/c_doxygen_rst.h
+
+.. doxygenfunction:: rst_example1_c_function
+
+Feeding Doxygen
+---------------
+
+Not all headers files are collected automatically. You have to add the desired
+C/C++ header paths within the sub-config files of Doxygen.
+
+Sub-config files have the unique name ``.doxyfile``, which you can usually find near
+directories that contain documented headers. You need to create a new config file if
+there's no one located in a path close(2-depth) to the headers you want to add.
+
+Sub-config files can accept any of Doxygen_ `configuration options <https://www.doxygen.nl/manual/config.html>`_,
+but you should be aware of not overriding or re-initializing any configuration option,
+you must only count on the concatenation operator "+=". For example::
+
+   # to specfiy certain headers
+   INPUT += @CUR_DIR/header1.h \
+            @CUR_DIR/header2.h
+   # to add all headers in certain path
+   INPUT += @CUR_DIR/to/headers
+   # to define certain macros
+   PREDEFINED += C_MACRO(X)=X
+   # to enable certain branches
+   PREDEFINED += NPY_HAVE_FEATURE \
+                 NPY_HAVE_FEATURE2
+
+.. note::
+
+    @CUR_DIR is a template constant returns the current
+    dir path of the sub-config file.
+
+Inclusion directives
+--------------------
+
+Breathe_ provides a wide range of custom directives to allow
+combining the generated documents by Doxygen_.
+
+.. note::
+
+   For more information, please check out "`Directives & Config Variables <https://breathe.readthedocs.io/en/latest/directives.html>`_"
+
+
+Tutroial
+
+Assume having a C header contains a group of functions, and types:
+
+.. literalinclude:: examples/c_doxygen_group.h
+.. doxygengroup:: C_doxygen_group_example
+    :members:
+    :undoc-members:
+
+.. _`Doxygen`: https://www.doxygen.nl/index.html
+.. _`Breathe`: https://breathe.readthedocs.io/en/latest/
+