Merge pull request #375 from radis/develop

0.10.3

README.rst

@@ -219,6 +219,7 @@ Links
 
 - Interactive Examples: `radis_examples <https://github.com/radis/radis-examples>`__ |badge_examples| |badge_binder|
 
+- `Fitroom <https://github.com/radis/fitroom>`__ (for advanced multidimensional fitting).
 
 
 

docs/api.rst

@@ -0,0 +1,28 @@
+.. _radis_glr_api:
+
+API
+===
+
+Documentation of each module and each function in RADIS
+
+.. currentmodule:: radis
+
+.. automodule:: radis
+   :no-members:
+   :no-inherited-members:
+
+:py:mod:`radis`:
+
+.. autosummary::
+   :toctree: gen_modules/
+   :template: module.rst
+
+   db
+   io
+   lbl
+   levels
+   los
+   misc
+   phys
+   spectrum
+   tools

docs/conf.py

@@ -75,7 +75,7 @@
         "radis": None,
     },
     # directory where function/class granular galleries are stored
-    "backreferences_dir": "gen_modules/backreferences",
+    "backreferences_dir": "source/backreferences",
     # Modules for which function/class level galleries are created.
     "doc_module": ("radis"),
     "inspect_global_variables": True,
@@ -113,7 +113,7 @@ def run_apidoc(_):
         "-f",
         "-e",
         "-o",
-        "gen_modules",
+        "source",
         "--separate",
         "../radis",
     ]
@@ -151,6 +151,7 @@ def setup(app):
     "astropy": ("https://docs.astropy.org/en/stable/", None),
     "habanero": ("https://habanero.readthedocs.io/en/latest/", None),
     "seaborn": ("https://seaborn.pydata.org/", None),
+    "fitroom": ("https://fitroom.readthedocs.io/en/latest/", None),
 }
 
 napoleon_google_docstring = False
@@ -260,6 +261,7 @@ def setup(app):
     "sidebar_includehidden": True,
     "fixed_sidebar": True,
     "analytics_id": "UA-113616205-1",
+    "link": "#7A306C",
 }
 
 # Add any paths that contain custom themes here, relative to this directory.

docs/lbl/lbl.rst

@@ -882,6 +882,26 @@ methods to generate an interactive profiler in the browser.
 
 .. _label_lbl_precompute_spectra:
 
+Predict Time
+------------
+
+:py:meth:`~radis.lbl.factory.SpectrumFactory.predict_time` function uses the input parameters like `Spectral Range`, `Number of lines`, `wstep`,
+`truncation` to predict the estimated calculation time for the Spectrum
+broadening step(bottleneck step) for the current optimization and broadening_method. The formula
+for predicting time is based on benchmarks performed on various parameters for different optimization,
+broadening_method and deriving its time complexity.
+
+The following Benchmarks were used to derive the time complexity:
+
+  https://anandxkumar.github.io/Benchmark_Visualization_GSoC_2021/
+
+Complexity vs Calculation Time Visualizations for different optimizations and broadening_method:
+
+
+|    LBL>Voigt: `LINK <https://public.tableau.com/app/profile/anand.kumar4841/viz/LegacyComplexityvsCalculationTime/Sheet1>`_
+|    DIT>Voigt: `LINK <https://public.tableau.com/app/profile/anand.kumar4841/viz/2_096e-07lines_calculated7_185e-091wLwGSpectral_PointslogSpectral_Points/Sheet1>`_
+|    DIT>FFT: `LINK <https://public.tableau.com/app/profile/anand.kumar4841/viz/LDMLatestLDMFFTComplexity4_675e-081wLwGSpectralPointslogSpectralPoints/Sheet1>`_
+
 Precompute Spectra
 ------------------
 

examples/plot_1T_fit.py

@@ -15,8 +15,7 @@
 the :ref:`multi-temperature fit example<example_multi_temperature_fit>`
 
 More advanced tools for interactive fitting of multi-dimensional, multi-slabs
-spectra can be found in `Fitroom <https://github.com/radis/fitroom>`__
-(access on request by asking on Slack)
+spectra can be found in :py:mod:`fitroom`.
 
 """
 

examples/plot_multi_temperature_fit.py

@@ -14,8 +14,7 @@
 as in the :ref:`one-temperature fit example <example_one_temperature_fit>`
 
 More advanced tools for interactive fitting of multi-dimensional, multi-slabs
-spectra can be found in `Fitroom <https://github.com/radis/fitroom>`__
-(access on request by asking on Slack)
+spectra can be found in :py:mod:`fitroom`.
 
 Typical output is similar to the
 `radis-examples Multi-temperature fit <https://github.com/radis/radis-examples#1-multi-temperature-fit>`__ :

radis/__version__.txt

@@ -1 +1 @@
-0.10.2
+0.10.3

radis/db/classes.py

@@ -281,6 +281,7 @@ def get_molecule(molecule_id):
 KNOWN_EXOMOL_ISOTOPES_NAMES = {
     ("FeH", 1): "56Fe-1H",
     ("SiO", 1): "28Si-16O",  # Placeholder until all molecules parsed from website TODO
+    ("CN", 1): "12C-14N",  # Placeholder until all molecules parsed from website TODO
 }
 """All :py:data:`~radis.db.classes.HITRAN_MOLECULES` are also converted to their ExoMol full-name format
 in :py:func:`~radis.io.exomol.get_exomol_full_isotope_name`

radis/io/cache_files.py

@@ -7,9 +7,9 @@
 ---------------
 
 
-- :func:`~radis.misc.cache_files.check_cache_file`
-- :func:`~radis.misc.cache_files.check_not_deprecated`
-- :func:`~radis.misc.cache_files.save_to_hdf`
+- :func:`~radis.io.cache_files.check_cache_file`
+- :func:`~radis.io.cache_files.check_not_deprecated`
+- :func:`~radis.io.cache_files.save_to_hdf`
 
 See Also
 --------
@@ -359,7 +359,16 @@ def check_not_deprecated(
 
     # Get metadata :
     manager = HDF5Manager(engine)
-    file_metadata = manager.read_metadata(file)
+
+    try:
+        file_metadata = manager.read_metadata(file)
+    except AttributeError as err:
+        if "Attribute 'metadata' does not exist" in str(err):
+            raise DeprecatedFileWarning(
+                "File {0} is deprecated : ".format(file)
+                + "Metadata is missing. Delete it to regenerate it on next run"
+            )
+        raise
 
     # Raise an error if version is not found
     try:

radis/io/dbmanager.py

@@ -27,6 +27,7 @@
 import numpy as np
 import pandas as pd
 from dateutil.parser import parse as parse_date
+from joblib import Parallel, delayed
 from numpy import DataSource
 
 LAST_VALID_DATE = (
@@ -57,12 +58,41 @@ class DatabaseManager(object):
     molecule: str
     local_databases: str
         path to local database
+
+    Other Parameters
+    ----------------
+    *input for :class:`~joblib.parallel.Parallel` loading of database*
+    parallel: bool
+        if ``True``, use parallel loading.
+        Default ``True``.
+    nJobs: int
+        Number of processors to use to load a database (useful for big
+        databases). BE CAREFUL, no check is done on processor use prior
+        to the execution ! Default ``-2``: use all but 1 processors.
+        Use ``1`` for single processor.
+    batch_size: int or ``'auto'``
+        The number of atomic tasks to dispatch at once to each
+        worker. When individual evaluations are very fast, dispatching
+        calls to workers can be slower than sequential computation because
+        of the overhead. Batching fast computations together can mitigate
+        this. Default: ``'auto'``
+    More information in :class:`joblib.parallel.Parallel`
+
     """
 
     # Should be as a close as possible to the content of the corresponding ~/radis.json entry
     # Essentially a FileManager
 
-    def __init__(self, name, molecule, local_databases, verbose=False):
+    def __init__(
+        self,
+        name,
+        molecule,
+        local_databases,
+        verbose=False,
+        parallel=True,
+        nJobs=-2,
+        batch_size="auto",
+    ):
 
         self.name = name
         self.molecule = molecule
@@ -75,6 +105,13 @@ def __init__(self, name, molecule, local_databases, verbose=False):
 
         self.verbose = verbose
 
+        self.parallel = parallel
+        self.nJobs = nJobs
+        self.batch_size = batch_size
+        self.minimum_nfiles = (
+            4  #: type: int. If there are less files, don't use parallel mode.
+        )
+
     def get_filenames(self, engine):
         """Get names of all files in the database (even if not downloaded yet)
 
@@ -239,6 +276,7 @@ def download_and_parse(self, urlnames, local_files, engine="pytables"):
 
         verbose = self.verbose
         molecule = self.molecule
+        parallel = self.parallel
 
         from time import time
 
@@ -252,10 +290,9 @@ def download_and_parse(self, urlnames, local_files, engine="pytables"):
         else:
             pbar_Ntot_estimate_factor = None
         Nlines_total = 0
-        Ndownload = 1
         Ntotal_downloads = len(local_files)
-        for urlname, local_file in zip(urlnames, local_files):
 
+        def download_and_parse_one_file(urlname, local_file, Ndownload):
             if verbose:
                 inputf = urlname.split("/")[-1]
                 print(
@@ -275,6 +312,7 @@ def download_and_parse(self, urlnames, local_files, engine="pytables"):
                 self.ds,
                 urlname,
                 local_file,
+                pbar_active=(not parallel),
                 pbar_t0=time() - t0,
                 pbar_Ntot_estimate_factor=pbar_Ntot_estimate_factor,
                 pbar_Nlines_already=Nlines_total,
@@ -284,8 +322,29 @@ def download_and_parse(self, urlnames, local_files, engine="pytables"):
             # except Exception as err:
             #     raise IOError("Problem parsing `{0}`. Check the error above. It may arise if the file wasn't properly downloaded. Try to delete it".format(self.ds._findfile(urlname))) from err
 
-            Ndownload += 1
-            Nlines_total += Nlines
+            return Nlines
+
+        if parallel and len(local_files) > self.minimum_nfiles:
+            nJobs = self.nJobs
+            batch_size = self.batch_size
+            if self.verbose:
+                print(
+                    f"Downloading and parsing {urlnames} to {local_files} "
+                    + f"({len(local_files)}) files), in parallel ({nJobs} jobs)"
+                )
+            Nlines_total = sum(
+                Parallel(n_jobs=nJobs, batch_size=batch_size, verbose=self.verbose)(
+                    delayed(download_and_parse_one_file)(urlname, local_file, Ndownload)
+                    for urlname, local_file, Ndownload in zip(
+                        urlnames, local_files, range(1, len(local_files) + 1)
+                    )
+                )
+            )
+        else:
+            for urlname, local_file, Ndownload in zip(
+                urlnames, local_files, range(1, len(local_files) + 1)
+            ):
+                download_and_parse_one_file(urlname, local_file, Ndownload)
 
     def clean_download_files(self):
         """Fully unzipped (and working, as it was reloaded): clean files