DOC: Add support for documenting C/C++ via Doxygen & Breathe

doc/Makefile

@@ -14,6 +14,7 @@ PYTHON = python$(PYVER)
 SPHINXOPTS    ?=
 SPHINXBUILD   ?= LANG=C sphinx-build
 PAPER         ?=
+DOXYGEN       ?= doxygen
 # For merging a documentation archive into a git checkout of numpy/doc
 # Turn a tag like v1.18.0 into 1.18
 # Use sed -n -e 's/patttern/match/p' to return a blank value if no match
@@ -77,7 +78,7 @@ INSTALL_DIR = $(CURDIR)/build/inst-dist
 INSTALL_PPH = $(INSTALL_DIR)/lib/python$(PYVER)/site-packages:$(INSTALL_DIR)/local/lib/python$(PYVER)/site-packages:$(INSTALL_DIR)/lib/python$(PYVER)/dist-packages:$(INSTALL_DIR)/local/lib/python$(PYVER)/dist-packages
 UPLOAD_DIR=/srv/docs_scipy_org/doc/numpy-$(RELEASE)
 
-DIST_VARS=SPHINXBUILD="LANG=C PYTHONPATH=$(INSTALL_PPH) python$(PYVER) `which sphinx-build`" PYTHON="PYTHONPATH=$(INSTALL_PPH) python$(PYVER)" 
+DIST_VARS=SPHINXBUILD="LANG=C PYTHONPATH=$(INSTALL_PPH) python$(PYVER) `which sphinx-build`" PYTHON="PYTHONPATH=$(INSTALL_PPH) python$(PYVER)"
 
 NUMPYVER:=$(shell $(PYTHON) -c "import numpy; print(numpy.version.git_revision[:10])" 2>/dev/null)
 GITVER ?= $(shell cd ..; $(PYTHON) -c "import versioneer as v; print(v.get_versions()['full-revisionid'][:10])")
@@ -176,6 +177,8 @@ build/generate-stamp: $(wildcard source/reference/*.rst)
 html: version-check html-build
 html-build: generate
 	mkdir -p build/html build/doctrees
+	$(PYTHON) preprocess.py
+	$(DOXYGEN) build/doxygen/Doxyfile
 	$(SPHINXBUILD) -b html $(ALLSPHINXOPTS) build/html $(FILES)
 	$(PYTHON) postprocess.py html build/html/*.html
 	@echo

doc/preprocess.py

@@ -0,0 +1,50 @@
+#!/usr/bin/env python3
+import subprocess
+import os
+import sys
+from string import Template
+
+def main():
+    doxy_gen(os.path.abspath(os.path.join('..')))
+
+def doxy_gen(root_path):
+    """
+    Generate Doxygen configration file.
+    """
+    confs = doxy_config(root_path)
+    build_path = os.path.join(root_path, "doc", "build", "doxygen")
+    gen_path = os.path.join(build_path, "Doxyfile")
+    if not os.path.exists(build_path):
+        os.makedirs(build_path)
+    with open(gen_path, 'w') as fd:
+        fd.write("#Please Don't Edit! This config file was autogenerated.\n")
+        for c in confs:
+            fd.write(c)
+
+class DoxyTpl(Template):
+    delimiter = '@'
+
+def doxy_config(root_path):
+    """
+    Fetch all Doxygen sub-config files and gather it with the main config file.
+    """
+    confs = []
+    dsrc_path = os.path.join(root_path, "doc", "source")
+    sub = dict(ROOT_DIR=root_path)
+    with open(os.path.join(dsrc_path, "doxyfile"), "r") as fd:
+        conf = DoxyTpl(fd.read())
+        confs.append(conf.substitute(CUR_DIR=dsrc_path, **sub))
+
+    for dpath, _, files in os.walk(root_path):
+        if ".doxyfile" not in files:
+            continue
+        conf_path = os.path.join(dpath, ".doxyfile")
+        with open(conf_path, "r") as fd:
+            conf = DoxyTpl(fd.read())
+            confs.append(conf.substitute(CUR_DIR=dpath, **sub))
+    return confs
+
+
+if __name__ == "__main__":
+    main()
+

doc/source/conf.py

@@ -84,6 +84,7 @@ class PyTypeObject(ctypes.Structure):
     'IPython.sphinxext.ipython_console_highlighting',
     'IPython.sphinxext.ipython_directive',
     'sphinx.ext.mathjax',
+    'breathe'
 ]
 
 # Add any paths that contain templates here, relative to this directory.
@@ -473,3 +474,11 @@ class NumPyLexer(CLexer):
             inherit,
         ],
     }
+
+
+# -----------------------------------------------------------------------------
+# Breathe & Doxygen
+# -----------------------------------------------------------------------------
+breathe_projects = dict(numpy=os.path.join("..", "build", "doxygen", "xml"))
+breathe_default_project = "numpy"
+breathe_default_members = ("members", "undoc-members", "protected-members")

doc/source/docs/examples/.doxyfile

@@ -0,0 +1,2 @@
+INPUT += @CUR_DIR
+INCLUDE_PATH += @CUR_DIR

doc/source/docs/examples/doxy_rst.h

@@ -0,0 +1,15 @@
+/**
+ * A comment block contains reST markup.
+ * @rst
+ * .. note::
+ *
+ *   Thanks to Breathe_, we were able to bring it to Doxygen_
+ *
+ * Some code example::
+ *
+ *   int example(int x) {
+ *       return x * 2;
+ *   }
+ * @endrst
+ */
+void doxy_reST_example(void);

doc/source/docs/howto_build_docs.rst

@@ -58,18 +58,28 @@ new virtual environment is recommended.
 Dependencies
 ^^^^^^^^^^^^
 
-All of the necessary dependencies for building the NumPy docs can be installed
-with::
+All of the necessary dependencies for building the NumPy docs except for
+Doxygen_ can be installed with::
 
     pip install -r doc_requirements.txt
 
-We currently use Sphinx_ for generating the API and reference
-documentation for NumPy. In addition, building the documentation requires
-the Sphinx extension `plot_directive`, which is shipped with
+We currently use Sphinx_ along with Doxygen_ for generating the API and
+reference documentation for NumPy. In addition, building the documentation
+requires the Sphinx extension `plot_directive`, which is shipped with
 :doc:`Matplotlib <matplotlib:index>`. We also use numpydoc_ to render docstrings in
 the generated API documentation. :doc:`SciPy <scipy:index>`
 is installed since some parts of the documentation require SciPy functions.
 
+For installing Doxygen_, please check the official
+`download <https://www.doxygen.nl/download.html#srcbin>`_ and
+`installation <https://www.doxygen.nl/manual/install.html>`_ pages, or if you
+are using Linux then you can install it through your distribution package manager.
+
+.. note::
+
+   Try to install a newer version of Doxygen_ > 1.8.10 otherwise you may get some
+   warnings during the build.
+
 Submodules
 ^^^^^^^^^^
 
@@ -80,6 +90,7 @@ additional parts required for building the documentation::
 
 .. _Sphinx: http://www.sphinx-doc.org/
 .. _numpydoc: https://numpydoc.readthedocs.io/en/latest/index.html
+.. _Doxygen: https://www.doxygen.nl/index.html
 
 Instructions
 ------------

doc/source/docs/howto_document.rst

@@ -73,3 +73,219 @@ Please use the numpydoc `formatting standard`_ as shown in their example_
 
 .. _`formatting standard`: https://numpydoc.readthedocs.io/en/latest/format.html
 .. _example: https://numpydoc.readthedocs.io/en/latest/example.html
+
+.. _doc_c_code:
+
+Documenting C/C++ Code
+**********************
+
+Recently, NumPy headed out to use Doxygen_ along with Sphinx due to the urgent need
+to generate C/C++ API reference documentation from comment blocks, especially with
+the most recent expansion in the use of SIMD, and also due to the future reliance on C++.
+Thanks to Breathe_, we were able to bind both worlds together at the lowest cost.
+
+**It takes three steps to complete the documentation process**:
+
+1. Writing the comment blocks
+-----------------------------
+
+Although there is still no commenting style set to follow, the Javadoc
+is more preferable than the others due to the similarities with the current
+existing non-indexed comment blocks.
+
+.. note::
+   If you have never used Doxygen_ before, then maybe you need to take a quick
+   look at `"Documenting the code" <https://www.doxygen.nl/manual/docblocks.html>`_.
+
+This is how Javadoc style looks like::
+
+  /**
+   * Your brief is here.
+   */
+  void function(void);
+
+For line comment, insert a triple forward slash::
+
+  /// Your brief is here.
+  void function(void);
+
+And for structured data members, just insert triple forward slash and less-than sign::
+
+  /// Data type brief.
+  typedef struct {
+      int member_1; ///< Member 1 description.
+      int member_2; ///< Member 2 description.
+  } data_type;
+
+
+Common Doxygen Tags:
+++++++++++++++++++++
+
+.. note::
+   For more tags/commands, please take a look at https://www.doxygen.nl/manual/commands.html
+
+``@brief``
+
+Starts a paragraph that serves as a brief description. By default the first sentence
+of the documentation block is automatically treated as a brief description, since
+option `JAVADOC_AUTOBRIEF <https://www.doxygen.nl/manual/config.html#cfg_javadoc_autobrief>`_
+is enabled within doxygen configurations.
+
+``@details``
+
+Just like ``@brief`` starts a brief description, ``@details`` starts the detailed description.
+You can also start a new paragraph (blank line) then the ``@details`` command is not needed.
+
+``@param``
+
+Starts a parameter description for a function parameter with name <parameter-name>,
+followed by a description of the parameter. The existence of the parameter is checked
+and a warning is given if the documentation of this (or any other) parameter is missing
+or not present in the function declaration or definition.
+
+``@return``
+
+Starts a return value description for a function.
+Multiple adjacent ``@return`` commands will be joined into a single paragraph.
+The ``@return`` description ends when a blank line or some other sectioning command is encountered.
+
+``@code/@endcode``
+
+Starts/Ends a block of code. A code block is treated differently from ordinary text.
+It is interpreted as source code.
+
+``@rst/@endrst``
+
+Starts/Ends a block of reST markup. Take a look at the following example:
+
+.. literalinclude:: examples/doxy_rst.h
+
+.. doxygenfunction:: doxy_reST_example
+
+2. Feeding Doxygen
+------------------
+
+Not all headers files are collected automatically. You have to add the desired
+C/C++ header paths within the sub-config files of Doxygen.
+
+Sub-config files have the unique name ``.doxyfile``, which you can usually find near
+directories that contain documented headers. You need to create a new config file if
+there's no one located in a path close(2-depth) to the headers you want to add.
+
+Sub-config files can accept any of Doxygen_ `configuration options <https://www.doxygen.nl/manual/config.html>`_,
+but you should be aware of not overriding or re-initializing any configuration option,
+you must only count on the concatenation operator "+=". For example::
+
+   # to specfiy certain headers
+   INPUT += @CUR_DIR/header1.h \
+            @CUR_DIR/header2.h
+   # to add all headers in certain path
+   INPUT += @CUR_DIR/to/headers
+   # to define certain macros
+   PREDEFINED += C_MACRO(X)=X
+   # to enable certain branches
+   PREDEFINED += NPY_HAVE_FEATURE \
+                 NPY_HAVE_FEATURE2
+
+.. note::
+    @CUR_DIR is a template constant returns the current
+    dir path of the sub-config file.
+
+3. Inclusion directives
+-----------------------
+
+Breathe_ provides a wide range of custom directives to allow
+including the generated documents by Doxygen_ into reST files.
+
+.. note::
+   For more information, please check out "`Directives & Config Variables <https://breathe.readthedocs.io/en/latest/directives.html>`_"
+
+Common directives:
+++++++++++++++++++
+
+``doxygenfunction``
+
+This directive generates the appropriate output for a single function.
+The function name is required to be unique in the project.
+
+.. code::
+
+   .. doxygenfunction:: <function name>
+       :outline:
+       :no-link:
+
+Checkout the `example <https://breathe.readthedocs.io/en/latest/function.html#function-example>`_
+to see it in action.
+
+
+``doxygenclass``
+
+This directive generates the appropriate output for a single class.
+It takes the standard project, path, outline and no-link options and
+additionally the members, protected-members, private-members, undoc-members,
+membergroups and members-only options:
+
+.. code::
+
+    .. doxygenclass:: <class name>
+       :members: [...]
+       :protected-members:
+       :private-members:
+       :undoc-members:
+       :membergroups: ...
+       :members-only:
+       :outline:
+       :no-link:
+
+Checkout the `doxygenclass documentation <https://breathe.readthedocs.io/en/latest/class.html#class-example>_`
+for more details and to see it in action.
+
+``doxygennamespace``
+
+This directive generates the appropriate output for the contents of a namespace.
+It takes the standard project, path, outline and no-link options and additionally the content-only,
+members, protected-members, private-members and undoc-members options.
+To reference a nested namespace, the full namespaced path must be provided,
+e.g. foo::bar for the bar namespace inside the foo namespace.
+
+.. code::
+
+    .. doxygennamespace:: <namespace>
+       :content-only:
+       :outline:
+       :members:
+       :protected-members:
+       :private-members:
+       :undoc-members:
+       :no-link:
+
+Checkout the `doxygennamespace documentation <https://breathe.readthedocs.io/en/latest/namespace.html#namespace-example>`_
+for more details and to see it in action.
+
+``doxygengroup``
+
+This directive generates the appropriate output for the contents of a doxygen group.
+A doxygen group can be declared with specific doxygen markup in the source comments
+as covered in the doxygen `grouping documentation <https://www.doxygen.nl/manual/grouping.html>`_.
+
+It takes the standard project, path, outline and no-link options and additionally the
+content-only, members, protected-members, private-members and undoc-members options.
+
+.. code::
+
+    .. doxygengroup:: <group name>
+       :content-only:
+       :outline:
+       :members:
+       :protected-members:
+       :private-members:
+       :undoc-members:
+       :no-link:
+       :inner:
+
+Checkout the `doxygengroup documentation <https://breathe.readthedocs.io/en/latest/group.html#group-example>`_
+for more details and to see it in action.
+
+.. _`Doxygen`: https://www.doxygen.nl/index.html
+.. _`Breathe`: https://breathe.readthedocs.io/en/latest/
+