.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 3.3.2
+current_version = 3.4.0
 commit = True
 tag = True
 tag_name = v{new_version}

.sonarcloud.properties

@@ -1,2 +1,2 @@
 sonar.sources=jsonargparse
-sonar.projectVersion=3.3.2
+sonar.projectVersion=3.4.0

CHANGELOG.rst

@@ -10,6 +10,24 @@ only be introduced in major versions with advance notice in the **Deprecated**
 section of releases.
 
 
+v3.4.0 (2021-02-01)
+-------------------
+
+Added
+-----
+- Save with multifile=True now creates original jsonnet file for ActionJsonnet.
+- default_config_files is now a property of parser objects.
+- Table in readme to ease understanding of extras requires for optional features #38.
+
+Changed
+-------
+- Save with multifile=True uses file extension to choose json or yaml format.
+
+Fixed
+^^^^^
+- Better exception message when using ActionJsonSchema and jsonschema not installed #38.
+
+
 v3.3.2 (2021-01-22)
 -------------------
 

README.rst

@@ -89,8 +89,31 @@ jsonargparse with extras require is as follows:
 
 .. code-block:: bash
 
-    pip install "jsonargparse[signatures]"    # Enable only signatures feature
-    pip install "jsonargparse[all]"           # Enable all optional features
+    pip install "jsonargparse[signatures,urls]"  # Enable signatures and URLs features
+    pip install "jsonargparse[all]"              # Enable all optional features
+
+The following table references sections that describe optional features and the
+corresponding extras requires that enables them.
+
++----------------------------------+------+-------------+---------+------------+------------+
+|                                  | urls | argcomplete | jsonnet | jsonschema | signatures |
++----------------------------------+------+-------------+---------+------------+------------+
+| :ref:`boolean-arguments`         |      |             |         | ✓          | ✓          |
++----------------------------------+------+-------------+---------+------------+------------+
+| :ref:`type-hints`                |      |             |         | ✓          | ✓          |
++----------------------------------+------+-------------+---------+------------+------------+
+| :ref:`classes-methods-functions` |      |             |         |            | ✓          |
++----------------------------------+------+-------------+---------+------------+------------+
+| :ref:`sub-classes`               |      |             |         |            | ✓          |
++----------------------------------+------+-------------+---------+------------+------------+
+| :ref:`parsing-urls`              | ✓    |             |         |            |            |
++----------------------------------+------+-------------+---------+------------+------------+
+| :ref:`json-schemas`              |      |             |         | ✓          | ✓          |
++----------------------------------+------+-------------+---------+------------+------------+
+| :ref:`jsonnet-files`             |      |             | ✓       |            |            |
++----------------------------------+------+-------------+---------+------------+------------+
+| :ref:`tab-completion`            |      | ✓           |         |            |            |
++----------------------------------+------+-------------+---------+------------+------------+
 
 
 Basic usage
@@ -552,6 +575,8 @@ Some notes about this support are:
   be shown as :code:`type: Union[str, null], default: null`.
 
 
+.. _sub-classes:
+
 Class type and sub-classes
 ==========================
 
@@ -695,6 +720,8 @@ the levels. This is, first call :func:`add_subcommands` and
 second level, and so on.
 
 
+.. _json-schemas:
+
 Json schemas
 ============
 
@@ -734,6 +761,8 @@ argument with the :class:`.ActionJsonSchema` action, you can use "%s" in the
 :code:`help` string so that in that position the schema will be printed.
 
 
+.. _jsonnet-files:
+
 Jsonnet files
 =============
 
@@ -971,6 +1000,8 @@ to some desired values. For example:
     Namespace(op=<MyEnum.choice1: -1>)
 
 
+.. _boolean-arguments:
+
 Boolean arguments
 =================
 
@@ -1057,6 +1088,8 @@ instance of the parser for standalone parsing, and in some other place use the
 function to provide an instance of the parser to :class:`.ActionParser`.
 
 
+.. _tab-completion:
+
 Tab completion
 ==============
 

jsonargparse/__init__.py

@@ -7,7 +7,6 @@
 from .optionals import *
 from .util import *
 from argparse import (
-    ArgumentError,
     Action,
     Namespace,
     HelpFormatter,
@@ -20,4 +19,4 @@
 )
 
 
-__version__ = '3.3.2'
+__version__ = '3.4.0'

jsonargparse/actions.py

@@ -4,7 +4,6 @@
 import re
 import sys
 import yaml
-import inspect
 import argparse
 from enum import Enum
 from argparse import Namespace, Action, SUPPRESS, _StoreAction, _HelpAction, _SubParsersAction

jsonargparse/core.py

@@ -17,18 +17,16 @@
 from .signatures import SignatureArguments
 from .typing import type_in
 from .jsonschema import ActionJsonSchema, supported_types
-from .jsonnet import ActionJsonnet, ActionJsonnetExtVars
+from .jsonnet import ActionJsonnet
 from .optionals import (
     import_jsonnet,
-    jsonschema_support,
     argcomplete_support,
     import_argcomplete,
     get_config_read_mode,
     FilesCompleterMethod,
     TypeCastCompleterMethod,
 )
 from .actions import (
-    ActionYesNo,
     ActionEnum,
     ActionParser,
     ActionConfigFile,
@@ -164,12 +162,10 @@ def __init__(
         super().__init__(*args, **kwargs)
         if self.groups is None:
             self.groups = {}
-        if default_config_files is None:
-            default_config_files = []
         self.required_args = set()  # type: Set[str]
         self._stderr = sys.stderr
-        self._default_config_files = default_config_files
         self._parse_as_dict = parse_as_dict
+        self.default_config_files = default_config_files
         self.default_meta = default_meta
         self.default_env = default_env
         self.env_prefix = env_prefix
@@ -180,11 +176,6 @@ def __init__(
             self.add_argument(print_config, action=_ActionPrintConfig)
         if version is not None:
             self.add_argument('--version', action='version', version='%(prog)s '+version, help='Print version and exit.')
-        if len(default_config_files) > 0:
-            group = _ArgumentGroup(self,
-                                   title='default config file locations',
-                                   description=str(default_config_files))
-            self._action_groups = [group] + self._action_groups  # type: ignore
         if parser_mode not in {'yaml', 'jsonnet'}:
             raise ValueError('The only accepted values for parser_mode are {"yaml", "jsonnet"}.')
         if parser_mode == 'jsonnet':
@@ -769,10 +760,6 @@ def save(
         if not overwrite and os.path.isfile(path):
             raise ValueError('Refusing to overwrite existing file: '+path)
         path = Path(path, mode='fc')
-        if format not in {'parser_mode', 'yaml', 'json_indented', 'json'}:
-            raise ValueError('Unknown output format "'+str(format)+'".')
-        if format == 'parser_mode':
-            format = 'yaml' if self.parser_mode == 'yaml' else 'json_indented'
 
         dump_kwargs = {'format': format, 'skip_none': skip_none, 'skip_check': skip_check}
 
@@ -800,22 +787,22 @@ def save_paths(cfg, base=None):
                         if '__path__' in val:
                             val_path = Path(os.path.join(dirname, os.path.basename(val['__path__']())), mode='fc')
                             if not overwrite and os.path.isfile(val_path()):
-                                raise ValueError('Refusing to overwrite existing file: '+val_path(absolute=False))
+                                raise ValueError('Refusing to overwrite existing file: '+str(val_path))
                             action = _find_action(self, kbase)
                             if isinstance(action, ActionParser):
                                 replace_keys[key] = val_path
                                 action._parser.save(val, val_path(), branch=action.dest, **save_kwargs)
                             elif isinstance(action, (ActionJsonSchema, ActionJsonnet)):
                                 replace_keys[key] = val_path
                                 val_out = strip_meta(val)
-                                if format.startswith('json') or isinstance(action, ActionJsonnet):
+                                if '__orig__' in val:
+                                    val_str = val['__orig__']
+                                elif str(val_path).lower().endswith('.json'):
                                     val_str = json.dumps(val_out, indent=2, sort_keys=True, ensure_ascii=False)+'\n'
                                 else:
                                     val_str = yaml.dump(val_out, default_flow_style=False, allow_unicode=True)
                                 with open(val_path(), 'w') as f:
                                     f.write(val_str)
-                            #else:
-                            #    save_paths(val, kbase)
                         else:
                             save_paths(val, kbase)
                 for key, val in replace_keys.items():
@@ -888,7 +875,7 @@ def get_defaults(self, nested:bool=True) -> Namespace:
         self._logger.info('Loaded default values from parser.')
 
         default_config_files = []  # type: List[str]
-        for pattern in self._default_config_files:
+        for pattern in self.default_config_files:
             default_config_files += glob.glob(os.path.expanduser(pattern))
         if len(default_config_files) > 0:
             default_config_file = Path(default_config_files[0], mode=get_config_read_mode())
@@ -1164,6 +1151,37 @@ def error_handler(self, error_handler):
             raise ValueError('error_handler can be either a Callable or None.')
 
 
+    @property
+    def default_config_files(self):
+        """Default config file locations.
+
+        :getter: Returns the current default config file locations.
+        :setter: Sets new default config file locations, e.g. :code:`['~/.config/myapp/*.yaml']`.
+
+        Raises:
+            ValueError: If an invalid value is given.
+        """
+        return self._default_config_files
+
+
+    @default_config_files.setter
+    def default_config_files(self, default_config_files:Optional[List[str]]):
+        if default_config_files is None:
+            self._default_config_files = []
+        elif isinstance(default_config_files, list) and all(isinstance(x, str) for x in default_config_files):
+            self._default_config_files = default_config_files
+        else:
+            raise ValueError('default_config_files has to be None or List[str].')
+
+        if len(self._default_config_files) > 0:
+            group_title = 'default config file locations'
+            group = next((g for g in self._action_groups if g.title == group_title), None)
+            if group is None:
+                group = _ArgumentGroup(self, title=group_title)
+                self._action_groups = [group] + self._action_groups  # type: ignore
+            group.description = str(self._default_config_files)
+
+
     @property
     def default_env(self):
         """Whether by default environment variables parsing is enabled.

jsonargparse/jsonnet.py

@@ -21,6 +21,8 @@
     ParserError,
     Path,
     _get_key_value,
+    _flat_namespace_to_dict,
+    _dict_to_flat_namespace,
 )
 
 
@@ -105,6 +107,8 @@ def _check_type(self, value, cfg):
         ext_vars = {}
         if self._ext_vars is not None:
             try:
+                if isinstance(cfg, dict):
+                    cfg = _flat_namespace_to_dict(_dict_to_flat_namespace(cfg))
                 ext_vars = _get_key_value(cfg, self._ext_vars)
             except (KeyError, AttributeError):
                 pass
@@ -178,6 +182,8 @@ def parse(
             raise ParserError('Problems evaluating jsonnet "'+fname+'" :: '+str(ex)) from ex
         if self._validator is not None:
             self._validator.validate(values)
-        if with_meta and isinstance(values, dict) and fpath is not None:
-            values['__path__'] = fpath
+        if with_meta:
+            if isinstance(values, dict) and fpath is not None:
+                values['__path__'] = fpath
+            values['__orig__'] = snippet
         return dict_to_namespace(values)

jsonargparse/jsonschema.py

@@ -24,7 +24,6 @@
 from .optionals import (
     ModuleNotFound,
     jsonschemaValidationError,
-    jsonschema_support,
     import_jsonschema,
     files_completer,
     argcomplete_warn_redraw_prompt,
@@ -52,9 +51,6 @@
     Dict, dict,
 }
 
-if jsonschema_support:
-    jsonschema, jsonvalidator = import_jsonschema('jsonschema.py')
-
 
 class ActionJsonSchema(Action):
     """Action to parse option as json validated by a jsonschema."""
@@ -95,6 +91,7 @@ def __init__(
                     schema = yaml.safe_load(schema)
                 except (yamlParserError, yamlScannerError) as ex:
                     raise ValueError('Problems parsing schema :: '+str(ex)) from ex
+            jsonvalidator = import_jsonschema('ActionJsonSchema')[1]
             jsonvalidator.check_schema(schema)
             self._validator = self._extend_jsonvalidator_with_default(jsonvalidator)(schema)
             self._enable_path = enable_path
@@ -171,6 +168,7 @@ def set_defaults(validator, properties, instance, schema):
             for error in validate_properties(validator, properties, instance, schema):
                 yield error
 
+        jsonschema = import_jsonschema('ActionJsonSchema')[0]
         return jsonschema.validators.extend(validator_class, {'properties': set_defaults})
 
 
@@ -295,6 +293,7 @@ def _typing_schema(annotation):
                     'required': ['class_path'],
                     'additionalProperties': False,
                 }
+                jsonvalidator = import_jsonschema('ActionJsonSchema')[1]
                 return schema, [(annotation, jsonvalidator(schema), None)]
             return None, None
 
@@ -306,6 +305,7 @@ def _typing_schema(annotation):
                 if schema is not None:
                     members.append(schema)
                     if arg not in typesmap:
+                        jsonvalidator = import_jsonschema('ActionJsonSchema')[1]
                         union_subschemas.append((arg, jsonvalidator(schema), subschemas))
             if len(members) == 1:
                 return members[0], union_subschemas

jsonargparse/signatures.py

@@ -2,7 +2,7 @@
 
 import inspect
 from enum import Enum
-from typing import Union, Optional, List, Container, Type, Callable
+from typing import Optional, Container, Type, Callable
 
 from .util import _issubclass
 from .actions import ActionEnum, _ActionConfigLoad, _ActionHelpClassPath

jsonargparse/util.py

@@ -8,7 +8,7 @@
 import inspect
 import logging
 from copy import deepcopy
-from typing import Dict, Any, Set, Optional, Union
+from typing import Dict, Any, Optional, Union
 from contextlib import contextmanager, redirect_stderr
 from argparse import Namespace
 from yaml.parser import ParserError as yamlParserError
@@ -39,7 +39,7 @@
 null_logger = logging.Logger('jsonargparse_null_logger')
 null_logger.addHandler(logging.NullHandler())
 
-meta_keys = {'__path__', '__default_config__'}
+meta_keys = {'__default_config__', '__path__', '__orig__'}
 
 
 class ParserError(Exception):

jsonargparse_tests/core_tests.py

@@ -551,17 +551,17 @@ def test_save(self):
         os.mkdir(indir)
         main_file = os.path.join(indir, 'main.yaml')
         parser_file = os.path.join(indir, 'parser.yaml')
-        schema_file = os.path.join(indir, 'schema.yaml')
-        jsonnet_file = os.path.join(indir, 'jsonnet.yaml')
+        schema_file = os.path.join(indir, 'schema.json')
+        jsonnet_file = os.path.join(indir, 'jsonnet.json')
 
         cfg1 = parser.get_defaults()
 
         with open(main_file, 'w') as output_file:
             output_file.write('parser: parser.yaml\n')
             if jsonschema_support:
-                output_file.write('schema: schema.yaml\n')
+                output_file.write('schema: schema.json\n')
             if jsonnet_support:
-                output_file.write('jsonnet: jsonnet.yaml\n')
+                output_file.write('jsonnet: jsonnet.json\n')
         with open(parser_file, 'w') as output_file:
             output_file.write(example_parser().dump(cfg1.parser))
         if jsonschema_support:
@@ -573,18 +573,18 @@ def test_save(self):
 
         cfg2 = parser.parse_path(main_file, with_meta=True)
         self.assertEqual(namespace_to_dict(cfg1), strip_meta(cfg2))
-        self.assertEqual(cfg2.parser.__path__(absolute=False), 'parser.yaml')
+        self.assertEqual(str(cfg2.parser.__path__), 'parser.yaml')
         if jsonschema_support:
-            self.assertEqual(cfg2.schema.__path__(absolute=False), 'schema.yaml')
+            self.assertEqual(str(cfg2.schema.__path__), 'schema.json')
         if jsonnet_support:
-            self.assertEqual(cfg2.jsonnet.__path__(absolute=False), 'jsonnet.yaml')
+            self.assertEqual(str(cfg2.jsonnet.__path__), 'jsonnet.json')
 
         parser.save(cfg2, os.path.join(outdir, 'main.yaml'))
         self.assertTrue(os.path.isfile(os.path.join(outdir, 'parser.yaml')))
         if jsonschema_support:
-            self.assertTrue(os.path.isfile(os.path.join(outdir, 'schema.yaml')))
+            self.assertTrue(os.path.isfile(os.path.join(outdir, 'schema.json')))
         if jsonnet_support:
-            self.assertTrue(os.path.isfile(os.path.join(outdir, 'jsonnet.yaml')))
+            self.assertTrue(os.path.isfile(os.path.join(outdir, 'jsonnet.json')))
 
         cfg3 = parser.parse_path(os.path.join(outdir, 'main.yaml'), with_meta=False)
         self.assertEqual(namespace_to_dict(cfg1), namespace_to_dict(cfg3))
@@ -593,6 +593,11 @@ def test_save(self):
         cfg4 = parser.parse_path(os.path.join(outdir, 'main.yaml'), with_meta=False)
         self.assertEqual(namespace_to_dict(cfg1), namespace_to_dict(cfg4))
 
+        if jsonschema_support:
+            cfg2.schema.__path__ = Path(os.path.join(indir, 'schema.yaml'), mode='fc')
+            parser.save(cfg2, os.path.join(outdir, 'main.yaml'), overwrite=True)
+            self.assertTrue(os.path.isfile(os.path.join(outdir, 'schema.yaml')))
+
 
     def test_save_failures(self):
         parser = ArgumentParser()
@@ -657,6 +662,9 @@ def test_default_config_files(self):
         self.assertIn('default config file locations', out.getvalue())
         self.assertIn(os.path.basename(default_config_file), out.getvalue())
 
+        with self.assertRaises(ValueError):
+            parser.default_config_files = False
+
 
     def test_ActionConfigFile_and_ActionPath(self):
         os.mkdir(os.path.join(self.tmpdir, 'example'))

jsonargparse_tests/jsonnet_tests.py

@@ -99,6 +99,43 @@ def test_ActionJsonnet(self):
         self.assertRaises(ValueError, lambda: ActionJsonnet(schema='.'+json.dumps(example_schema)))
 
 
+    def test_ActionJsonnet_save(self):
+        parser = ArgumentParser(error_handler=None)
+        parser.add_argument('--ext_vars',
+            action=ActionJsonnetExtVars())
+        parser.add_argument('--jsonnet',
+            action=ActionJsonnet(ext_vars='ext_vars'))
+        parser.add_argument('--cfg',
+            action=ActionConfigFile)
+
+        jsonnet_file = os.path.join(self.tmpdir, 'example.jsonnet')
+        with open(jsonnet_file, 'w') as output_file:
+            output_file.write(example_2_jsonnet)
+        outdir = os.path.join(self.tmpdir, 'output')
+        outyaml = os.path.join(outdir, 'main.yaml')
+        outjsonnet = os.path.join(outdir, 'example.jsonnet')
+        os.mkdir(outdir)
+
+        cfg = parser.parse_args(['--ext_vars', '{"param": 123}', '--jsonnet', jsonnet_file])
+        self.assertEqual(str(cfg.jsonnet.__path__), jsonnet_file)
+
+        parser.save(cfg, outyaml)
+        cfg2 = parser.parse_args(['--cfg', outyaml])
+        cfg2.cfg = None
+        self.assertTrue(os.path.isfile(outyaml))
+        self.assertTrue(os.path.isfile(outjsonnet))
+        self.assertEqual(strip_meta(cfg), strip_meta(cfg2))
+
+        os.unlink(outyaml)
+        os.unlink(outjsonnet)
+        parser.save(strip_meta(cfg), outyaml)
+        cfg3 = parser.parse_args(['--cfg', outyaml])
+        cfg3.cfg = None
+        self.assertTrue(os.path.isfile(outyaml))
+        self.assertTrue(not os.path.isfile(outjsonnet))
+        self.assertEqual(strip_meta(cfg), strip_meta(cfg3))
+
+
     def test_ActionJsonnet_help(self):
         parser = ArgumentParser()
         parser.add_argument('--jsonnet',

setup.cfg

@@ -48,7 +48,7 @@ doc =
 
 [metadata]
 name = jsonargparse
-version = 3.3.2
+version = 3.4.0
 description = Parsing of command line options, yaml/jsonnet config files and/or environment variables based on argparse.
 long_description_content_type = text/x-rst
 author = Mauricio Villegas

setup.py

@@ -6,6 +6,7 @@
 
 NAME_TESTS = next(filter(lambda x: x.startswith('test_suite = '), open('setup.cfg').readlines())).strip().split()[-1]
 LONG_DESCRIPTION = re.sub(':class:|:func:|:ref:|:py:meth:', '', open('README.rst').read())
+LONG_DESCRIPTION = re.sub('([+|][- ]{12})[- ]{5}', r'\1', LONG_DESCRIPTION)
 CMDCLASS = {}
 
 

sphinx/conf.py

@@ -45,7 +45,8 @@
 
 autodoc_default_options = {
     'exclude-members': 'groups',
-    'member-order': 'bysource'
+    'member-order': 'bysource',
+    'autosummary_imported_members': False,
 }
 
 def skip(app, what, name, obj, would_skip, options):

sphinx/index.rst

@@ -8,79 +8,69 @@ jsonargparse.cli
 ----------------
 .. automodule:: jsonargparse.cli
     :members:
-    :undoc-members:
     :show-inheritance:
     :autosummary:
 
 jsonargparse.core
 -----------------
 .. automodule:: jsonargparse.core
     :members:
-    :undoc-members:
     :show-inheritance:
     :autosummary:
 
 jsonargparse.signatures
 -----------------------
 .. automodule:: jsonargparse.signatures
     :members:
-    :undoc-members:
     :show-inheritance:
     :autosummary:
 
 jsonargparse.typing
 -------------------
 .. automodule:: jsonargparse.typing
     :members:
-    :undoc-members:
     :show-inheritance:
     :autosummary:
 
 jsonargparse.jsonschema
 -----------------------
 .. automodule:: jsonargparse.jsonschema
     :members:
-    :undoc-members:
     :show-inheritance:
     :autosummary:
 
 jsonargparse.jsonnet
 --------------------
 .. automodule:: jsonargparse.jsonnet
     :members:
-    :undoc-members:
     :show-inheritance:
     :autosummary:
 
 jsonargparse.actions
 --------------------
 .. automodule:: jsonargparse.actions
     :members:
-    :undoc-members:
     :show-inheritance:
     :autosummary:
 
 jsonargparse.formatters
 -----------------------
 .. automodule:: jsonargparse.formatters
     :members:
-    :undoc-members:
     :show-inheritance:
     :autosummary:
 
 jsonargparse.optionals
 ----------------------
 .. automodule:: jsonargparse.optionals
     :members:
-    :undoc-members:
     :show-inheritance:
     :autosummary:
 
 jsonargparse.util
 -----------------
 .. automodule:: jsonargparse.util
     :members:
-    :undoc-members:
     :show-inheritance:
     :autosummary: