Ensure loading of collections and plugins (#1074)

lib/ansiblelint/utils.py

@@ -40,7 +40,7 @@
 from ansible.parsing.yaml.constructor import AnsibleConstructor
 from ansible.parsing.yaml.loader import AnsibleLoader
 from ansible.parsing.yaml.objects import AnsibleSequence
-from ansible.plugins.loader import module_loader
+from ansible.plugins.loader import add_all_plugin_dirs
 from ansible.template import Templar
 from yaml.composer import Composer
 from yaml.representer import RepresenterError
@@ -147,9 +147,27 @@ def func_wrapper(*args, **kwargs):
     return func_wrapper
 
 
+def _set_collections_basedir(basedir: str):
+    # Sets the playbook directory as playbook_paths for the collection loader
+    try:
+        # Ansible 2.10+
+        # noqa: # pylint:disable=cyclic-import,import-outside-toplevel
+        from ansible.utils.collection_loader import AnsibleCollectionConfig
+
+        AnsibleCollectionConfig.playbook_paths = basedir
+    except ImportError:
+        # Ansible 2.8 or 2.9
+        # noqa: # pylint:disable=cyclic-import,import-outside-toplevel
+        from ansible.utils.collection_loader import set_collection_playbook_paths
+
+        set_collection_playbook_paths(basedir)
+
+
 def find_children(playbook: Tuple[str, str], playbook_dir: str) -> List:
     if not os.path.exists(playbook[0]):
         return []
+    _set_collections_basedir(playbook_dir or '.')
+    add_all_plugin_dirs(playbook_dir or '.')
     if playbook[1] == 'role':
         playbook_ds = {'roles': [{'role': playbook[0]}]}
     else:
@@ -205,8 +223,7 @@ def play_children(basedir, item, parent_type, playbook_dir):
         'import_tasks': _include_children,
     }
     (k, v) = item
-    play_library = os.path.join(os.path.abspath(basedir), 'library')
-    _load_library_if_exists(play_library)
+    add_all_plugin_dirs(os.path.abspath(basedir))
 
     if k in delegate_map:
         if v:
@@ -310,11 +327,6 @@ def _roles_children(basedir: str, k, v, parent_type: FileType, main='main') -> l
     return results
 
 
-def _load_library_if_exists(path: str) -> None:
-    if os.path.exists(path):
-        module_loader.add_directory(path)
-
-
 def _rolepath(basedir: str, role: str) -> Optional[str]:
     role_path = None
 
@@ -345,7 +357,7 @@ def _rolepath(basedir: str, role: str) -> Optional[str]:
             break
 
     if role_path:
-        _load_library_if_exists(os.path.join(role_path, 'library'))
+        add_all_plugin_dirs(role_path)
 
     return role_path
 

test/TestLocalContent.py

@@ -0,0 +1,42 @@
+"""Test playbooks with local content."""
+import pytest
+
+from ansiblelint.runner import Runner
+
+
+def test_local_collection(default_rules_collection):
+    """Assures local collections are found."""
+    playbook_path = 'test/local-content/test-collection.yml'
+    runner = Runner(default_rules_collection, playbook_path, [], [], [])
+    results = runner.run()
+
+    assert len(runner.playbooks) == 1
+    assert len(results) == 0
+
+
+def test_roles_local_content(default_rules_collection):
+    """Assures local content in roles is found."""
+    playbook_path = 'test/local-content/test-roles-success/test.yml'
+    runner = Runner(default_rules_collection, playbook_path, [], [], [])
+    results = runner.run()
+
+    assert len(runner.playbooks) == 4
+    assert len(results) == 0
+
+
+def test_roles_local_content_failure(default_rules_collection):
+    """Assures local content in roles is found, even if Ansible itself has trouble."""
+    playbook_path = 'test/local-content/test-roles-failed/test.yml'
+    runner = Runner(default_rules_collection, playbook_path, [], [], [])
+    results = runner.run()
+
+    assert len(runner.playbooks) == 4
+    assert len(results) == 0
+
+
+def test_roles_local_content_failure_complete(default_rules_collection):
+    """Role with local content that is not found."""
+    playbook_path = 'test/local-content/test-roles-failed-complete/test.yml'
+    runner = Runner(default_rules_collection, playbook_path, [], [], [])
+    with pytest.raises(SystemExit, match="^3$"):
+        runner.run()

test/local-content/README.md

@@ -0,0 +1,6 @@
+The reason that every roles test gets its own directory is that while they
+use the same three roles, the way the tests work makes sure that when the
+second one runs, the roles and their local plugins from the first test are
+still known to Ansible. For that reason, their names reflect the directory
+they are in to make sure that tests don't use modules/plugins found by
+other tests.

test/local-content/collections/ansible_collections/testns/testcoll/galaxy.yml

@@ -0,0 +1,3 @@
+namespace: testns
+name: testcoll
+version: 0.1.0

test/local-content/collections/ansible_collections/testns/testcoll/plugins/filter/test_filter.py

@@ -0,0 +1,16 @@
+"""A filter plugin."""
+
+
+def a_test_filter(a, b):
+    """Return a string containing both a and b."""
+    return '{0}:{1}'.format(a, b)
+
+
+class FilterModule(object):
+    """Filter plugin."""
+
+    def filters(self):
+        """Return filters."""
+        return {
+            'test_filter': a_test_filter
+        }

test/local-content/collections/ansible_collections/testns/testcoll/plugins/modules/test_module_2.py

@@ -0,0 +1,14 @@
+#!/usr/bin/python
+"""A module."""
+
+from ansible.module_utils.basic import AnsibleModule
+
+
+def main() -> None:
+    """Execute module."""
+    module = AnsibleModule(dict())
+    module.exit_json(msg="Hello 2!")
+
+
+if __name__ == '__main__':
+    main()

test/local-content/test-collection.yml

@@ -0,0 +1,10 @@
+---
+- name: Use module and filter plugin from local collection
+  hosts: localhost
+  tasks:
+    - name: Use module from local collection
+      testns.testcoll.test_module_2:
+    - name: Use filter from local collection
+      assert:
+        that:
+          - 1 | testns.testcoll.test_filter(2) == '1:2'

test/local-content/test-roles-failed-complete/roles/role1/library/test_module_1_failed_complete.py

@@ -0,0 +1,14 @@
+#!/usr/bin/python
+"""A module."""
+
+from ansible.module_utils.basic import AnsibleModule
+
+
+def main() -> None:
+    """Execute module."""
+    module = AnsibleModule(dict())
+    module.exit_json(msg="Hello 1!")
+
+
+if __name__ == '__main__':
+    main()

test/local-content/test-roles-failed-complete/roles/role1/tasks/main.yml

@@ -0,0 +1,3 @@
+---
+- name: Use local module 1
+  test_module_1_failed_complete:

test/local-content/test-roles-failed-complete/roles/role2/tasks/main.yml

@@ -0,0 +1,11 @@
+---
+- name: Use local module from other role that has been included before this one
+  # If it has not been included before, loading this role fails!
+  test_module_1_failed_complete:
+- name: Use local module from other role that has been included before this one
+  # If it has not been included before, loading this role fails!
+  test_module_3_failed_complete:
+- name: Use local test plugin
+  assert:
+    that:
+      - "'2' is b_test_failed_complete '12345'"

test/local-content/test-roles-failed-complete/roles/role2/test_plugins/b_failed_complete.py

@@ -0,0 +1,16 @@
+"""A test plugin."""
+
+
+def compatibility_in_test(a, b):
+    """Return True when a is contained in b."""
+    return a in b
+
+
+class TestModule:
+    """Test plugin."""
+
+    def tests(self):
+        """Return tests."""
+        return {
+            'b_test_failed_complete': compatibility_in_test,
+        }

test/local-content/test-roles-failed-complete/roles/role3/library/test_module_3_failed_complete.py

@@ -0,0 +1,14 @@
+#!/usr/bin/python
+"""A module."""
+
+from ansible.module_utils.basic import AnsibleModule
+
+
+def main() -> None:
+    """Execute module."""
+    module = AnsibleModule(dict())
+    module.exit_json(msg="Hello 3!")
+
+
+if __name__ == '__main__':
+    main()

test/local-content/test-roles-failed-complete/roles/role3/tasks/main.yml

@@ -0,0 +1,3 @@
+---
+- name: Use local module 3
+  test_module_3_failed_complete:

test/local-content/test-roles-failed-complete/test.yml

@@ -0,0 +1,5 @@
+---
+- name: Include role which expects module that is local to other role which is not loaded
+  hosts: localhost
+  roles:
+    - role2

test/local-content/test-roles-failed/roles/role1/library/test_module_1_failed.py

@@ -0,0 +1,14 @@
+#!/usr/bin/python
+"""A module."""
+
+from ansible.module_utils.basic import AnsibleModule
+
+
+def main() -> None:
+    """Execute module."""
+    module = AnsibleModule(dict())
+    module.exit_json(msg="Hello 1!")
+
+
+if __name__ == '__main__':
+    main()

test/local-content/test-roles-failed/roles/role1/tasks/main.yml

@@ -0,0 +1,3 @@
+---
+- name: Use local module 1
+  test_module_1_failed:

test/local-content/test-roles-failed/roles/role2/tasks/main.yml

@@ -0,0 +1,11 @@
+---
+- name: Use local module from other role that has been included before this one
+  # If it has not been included before, loading this role fails!
+  test_module_1_failed:
+- name: Use local module from other role that has been included before this one
+  # If it has not been included before, loading this role fails!
+  test_module_3_failed:
+- name: Use local test plugin
+  assert:
+    that:
+      - "'2' is b_test_failed '12345'"

test/local-content/test-roles-failed/roles/role2/test_plugins/b_failed.py

@@ -0,0 +1,16 @@
+"""A test plugin."""
+
+
+def compatibility_in_test(a, b):
+    """Return True when a is contained in b."""
+    return a in b
+
+
+class TestModule:
+    """Test plugin."""
+
+    def tests(self):
+        """Return tests."""
+        return {
+            'b_test_failed': compatibility_in_test,
+        }

test/local-content/test-roles-failed/roles/role3/library/test_module_3_failed.py

@@ -0,0 +1,14 @@
+#!/usr/bin/python
+"""A module."""
+
+from ansible.module_utils.basic import AnsibleModule
+
+
+def main() -> None:
+    """Execute module."""
+    module = AnsibleModule(dict())
+    module.exit_json(msg="Hello 3!")
+
+
+if __name__ == '__main__':
+    main()

test/local-content/test-roles-failed/roles/role3/tasks/main.yml

@@ -0,0 +1,3 @@
+---
+- name: Use local module 3
+  test_module_3_failed:

test/local-content/test-roles-failed/test.yml

@@ -0,0 +1,7 @@
+---
+- name: Use roles with local module in wrong order, so that Ansible fails
+  hosts: localhost
+  roles:
+    - role2
+    - role3
+    - role1

test/local-content/test-roles-success/roles/role1/library/test_module_1_success.py

@@ -0,0 +1,14 @@
+#!/usr/bin/python
+"""A module."""
+
+from ansible.module_utils.basic import AnsibleModule
+
+
+def main() -> None:
+    """Execute module."""
+    module = AnsibleModule(dict())
+    module.exit_json(msg="Hello 1!")
+
+
+if __name__ == '__main__':
+    main()

test/local-content/test-roles-success/roles/role1/tasks/main.yml

@@ -0,0 +1,3 @@
+---
+- name: Use local module 1
+  test_module_1_success:

test/local-content/test-roles-success/roles/role2/tasks/main.yml

@@ -0,0 +1,11 @@
+---
+- name: Use local module from other role that has been included before this one
+  # If it has not been included before, loading this role fails!
+  test_module_1_success:
+- name: Use local module from other role that has been included before this one
+  # If it has not been included before, loading this role fails!
+  test_module_3_success:
+- name: Use local test plugin
+  assert:
+    that:
+      - "'2' is b_test_success '12345'"

test/local-content/test-roles-success/roles/role2/test_plugins/b_success.py

@@ -0,0 +1,16 @@
+"""A test plugin."""
+
+
+def compatibility_in_test(a, b):
+    """Return True when a is contained in b."""
+    return a in b
+
+
+class TestModule:
+    """Test plugin."""
+
+    def tests(self):
+        """Return tests."""
+        return {
+            'b_test_success': compatibility_in_test,
+        }

test/local-content/test-roles-success/roles/role3/library/test_module_3_success.py

@@ -0,0 +1,14 @@
+#!/usr/bin/python
+"""A module."""
+
+from ansible.module_utils.basic import AnsibleModule
+
+
+def main() -> None:
+    """Execute module."""
+    module = AnsibleModule(dict())
+    module.exit_json(msg="Hello 3!")
+
+
+if __name__ == '__main__':
+    main()

test/local-content/test-roles-success/roles/role3/tasks/main.yml

@@ -0,0 +1,3 @@
+---
+- name: Use local module 3
+  test_module_3_success:

test/local-content/test-roles-success/test.yml

@@ -0,0 +1,7 @@
+---
+- name: Use roles with local modules and test plugins
+  hosts: localhost
+  roles:
+    - role1
+    - role3
+    - role2