From c7742af87e10d2f762b63e44e9b97a833783f6a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gabriel=20Falc=C3=A3o?= Date: Sat, 20 Jan 2024 04:00:35 +0000 Subject: [PATCH] presents more test coverage --- Makefile | 1 + sure/__init__.py | 16 +- sure/core.py | 3 +- sure/doubles/dummies.py | 2 +- sure/errors.py | 5 +- sure/original.py | 2 +- sure/runtime.py | 212 +++--------------- tests/functional/test_runner.py | 20 +- ...est_assertion_builder_assertion_methods.py | 41 +++- ..._assertion_builder_assertion_properties.py | 1 + tests/test_original_api.py | 36 ++- tests/test_runtime/test_base_result.py | 57 +++++ tests/test_runtime/test_feature.py | 57 ++++- tests/test_runtime/test_feature_result.py | 87 +++++++ tests/test_runtime/test_scenario.py | 148 ++++++++++++ tests/test_runtime/test_scenario_result.py | 51 +++++ .../test_runtime/test_scenario_result_set.py | 96 ++++++++ 17 files changed, 637 insertions(+), 198 deletions(-) create mode 100644 tests/test_runtime/test_base_result.py create mode 100644 tests/test_runtime/test_feature_result.py create mode 100644 tests/test_runtime/test_scenario.py create mode 100644 tests/test_runtime/test_scenario_result.py create mode 100644 tests/test_runtime/test_scenario_result_set.py diff --git a/Makefile b/Makefile index 50dcc9d..58f6a24 100644 --- a/Makefile +++ b/Makefile @@ -55,6 +55,7 @@ docs: html-docs $(OPEN_COMMAND) docs/build/html/index.html test tests: + @$(VENV)/bin/pytest --cov=sure tests/test_runtime/test_scenario_result.py @$(VENV)/bin/pytest --cov=sure tests # runs main command-line tool diff --git a/sure/__init__.py b/sure/__init__.py index 51af5fd..b469405 100644 --- a/sure/__init__.py +++ b/sure/__init__.py @@ -753,7 +753,7 @@ def contains(self, expectation): if expectation in self.actual: return True else: - raise Explanation(f'{expectation} should be in {self.actual}').as_assertion(self.actual, expectation, "Content Verification Error") + raise Explanation(f"`{expectation}' should be in `{self.actual}'").as_assertion(self.actual, expectation, "Content Verification Error") contain = contains to_contain = contains @@ -763,7 +763,7 @@ def does_not_contain(self, expectation): if expectation not in self.actual: return True else: - raise Explanation(f'{expectation} should not be in {self.actual}').as_assertion(self.actual, expectation, "Content Verification Error") + raise Explanation(f"`{expectation}' should not be in `{self.actual}'").as_assertion(self.actual, expectation, "Content Verification Error") doesnt_contain = does_not_contain to_not_contain = does_not_contain @@ -805,22 +805,22 @@ def within(self, first, *rest): # instead of hardcoding ``.should_not.be.within`` and ``.should.be.within`` in the # variable assignments below if self.negative: - ppath = "{0}.should_not.be.within".format(self.actual) + ppath = f"({self.actual}).should_not.be.within" else: - ppath = "{0}.should.be.within".format(self.actual) + ppath = f"({self.actual}).should.be.within".format(self.actual) raise AssertionError( ( - "{0}({1}, {2}) must be called with either a iterable:\n" + "{0}({1}, {2}) must be called with either an iterable:\n" "{0}([1, 2, 3, 4])\n" - "or with a range of numbers:" - "{0}(1, 3000)" + "or with a range of numbers, i.e.: " + "`{0}(1, 3000)'" ).format(ppath, first, ", ".join([repr(x) for x in rest])) ) @assertionmethod def equal(self, expectation, epsilon=None): - """compares given object ``X'` with an expected '`Y'` object. + """compares given object ``X'` with an expected '`Y'` object. It primarily assures that the compared objects are absolute equal '`=='`. diff --git a/sure/core.py b/sure/core.py index 391ae0a..f0da38d 100644 --- a/sure/core.py +++ b/sure/core.py @@ -16,6 +16,7 @@ # along with this program. If not, see . import os import types +from pprint import pformat from typing import Union, List, Dict, Tuple from collections import OrderedDict from functools import cache @@ -30,7 +31,7 @@ class Explanation(str): def get_header(self, X, Y, suffix): - header = f"X = {repr(X)}\n and\nY = {repr(Y)}\n{str(suffix)}" + header = f"X = {pformat(X, sort_dicts=False, compact=False)}\n and\nY = {pformat(Y, sort_dicts=False, compact=False)}\n{str(suffix)}" return yellow(header).strip() def get_assertion(self, X, Y, prefix=""): diff --git a/sure/doubles/dummies.py b/sure/doubles/dummies.py index 343d540..3a5624c 100644 --- a/sure/doubles/dummies.py +++ b/sure/doubles/dummies.py @@ -77,7 +77,7 @@ def __eq__(self, given: object): given_type = type(given) module_name = given_type.__module__ type_name = given_type.__name__ - return isinstance(given, self.__expected_type__) and super().__eq__(f"typed:{module_name}.{type_name}") + return isinstance(given, (self.__expected_type__, self.__class__)) and super().__eq__(f"typed:{module_name}.{type_name}") def __repr__(self): return f'' diff --git a/sure/errors.py b/sure/errors.py index 5a8f834..9fd4186 100644 --- a/sure/errors.py +++ b/sure/errors.py @@ -112,8 +112,9 @@ def __init__(self, scenario_result: ScenarioResult): class ExitError(ImmediateExit): - def __init__(self, context: RuntimeContext, result: ScenarioResult): - context.reporter.on_error(context, result) + def __init__(self, context: RuntimeContext, result: ScenarioResult, report:bool = True): + if report: + context.reporter.on_error(context, result) return super().__init__(exit_code("ERROR")) diff --git a/sure/original.py b/sure/original.py index 66babe8..6da963e 100644 --- a/sure/original.py +++ b/sure/original.py @@ -134,7 +134,7 @@ def match(self, *args, **kw): def raises(self, exc, msg=None): if not callable(self.actual): - raise TypeError('%r is not callable' % self.actual) + raise TypeError(f'{self.actual} is not callable') try: self.actual(*self._callable_args, **self._callable_kw) diff --git a/sure/runtime.py b/sure/runtime.py index 50bf076..076b67e 100644 --- a/sure/runtime.py +++ b/sure/runtime.py @@ -635,9 +635,8 @@ def read_scenarios(self, suts): def run(self, reporter: Reporter, runtime: RuntimeOptions) -> stypes.FeatureResult: results = [] + context = RuntimeContext(reporter, runtime) for scenario in self.scenarios: - context = RuntimeContext(reporter, runtime) - result = scenario.run(context) results.append(result) @@ -681,14 +680,15 @@ def run(self, context: RuntimeContext): self, ).uncollapse_nested() results = [] + for collector in collectors: collector_results = [] context.reporter.on_scenario(collector.scenario) - for result, role in collector.run(context): if role == RuntimeRole.Unit: collector_results.append(result) - elif not result.is_success: + + if not result.is_success: if result.is_failure: context.reporter.on_failure(result.scenario, result) if context.options.immediate: @@ -697,11 +697,11 @@ def run(self, context: RuntimeContext): elif result.is_error: context.reporter.on_error(result.scenario, result) if context.options.immediate: - raise ExitError(context, result) + raise ExitError(context, result, report=False) - context.reporter.on_scenario_done( - collector.scenario, ScenarioResultSet(collector_results, context) - ) + context.reporter.on_scenario_done( + collector.scenario, result + ) results.extend(collector_results) return ScenarioResultSet(results, context) @@ -711,6 +711,12 @@ class BaseResult: purpose is to allow for distinguishing result-containing objects.""" def __repr__(self): + if not hasattr(self, 'label'): + raise NotImplementedError(f"{self.__class__} MUST define a `label' property or attribute which must be a string") + + if not isinstance(self.label, str): + raise NotImplementedError(f"{self.__class__}.label must be a string but is a {type(self.label)} instead") + return repr(self.label.lower()) @@ -742,9 +748,6 @@ def __init__( else: self.__error__ = treat_error(error, self.location) - def tb(self): - return traceback.format_exception(*self.exc_info) - @property def label(self) -> str: if self.ok: @@ -762,12 +765,7 @@ def __str__(self): ) def printable(self): - prelude = f"{self.location}" - hook = "" - if callable(getattr(self.error, "printable", None)): - hook = self.error.printable() - - return " ".join(filter(bool, [prelude, hook])) + return f"{self.location}" @property def is_error(self): @@ -778,9 +776,6 @@ def error(self) -> Optional[Exception]: if not isinstance(self.__error__, AssertionError): return self.__error__ - def set_error(self, error: Optional[Exception]): - self.__error__ = error - @property def is_failure(self): return isinstance(self.__failure__, AssertionError) @@ -800,9 +795,6 @@ def ok(self): @property def succinct_failure(self) -> str: - if not self.is_failure: - return "" - return self.stack.location_specific_error() @@ -815,17 +807,17 @@ def __init__(self, scenario_results: List[ScenarioResult], context: RuntimeConte self.failed_scenarios = [] self.errored_scenarios = [] - for scenario in scenario_results: - if scenario.is_failure: - self.failed_scenarios.append(scenario) - if scenario.is_error: - self.errored_scenarios.append(scenario) + for result in scenario_results: + if result.is_failure: + self.failed_scenarios.append(result) + if result.is_error: + self.errored_scenarios.append(result) def printable(self): if self.failure is not None: - return self.failure + return f"{self.failure.__class__.__name__}: {self.failure}" if self.error: - return self.error.printable() + return f"{self.error.__class__.__name__}: {self.error}" return "" @@ -839,17 +831,6 @@ def error(self) -> Optional[Exception]: if scenario.is_error: return scenario.error - @property - def stack(self) -> Optional[ErrorStack]: - for scenario in self.errored_scenarios: - return scenario.stack - - def __getattr__(self, attr): - try: - return self.__getattribute__(attr) - except AttributeError: - return getattr(self.scenario_results[-1], attr) - @property def is_failure(self): return len(self.failed_scenarios) > 0 @@ -860,35 +841,13 @@ def failure(self) -> Optional[Exception]: if scenario.is_failure: return scenario.failure - @property - def succinct_failure(self) -> Optional[str]: - for scenario in self.failed_scenarios: - if scenario.is_failure: - return scenario.succinct_failure - - @property - def first_scenario_result_error(self) -> Optional[ScenarioResult]: - for scenario in self.errored_scenarios: - if scenario.is_error: - return scenario - - @property - def first_scenario_result_fail(self) -> Optional[ScenarioResult]: - for scenario in self.failed_scenarios: - if scenario.is_failure: - return scenario - - @property - def first_nonsuccessful_result(self) -> Optional[ScenarioResult]: - return self.first_scenario_result_error or self.first_scenario_result_fail - -class FeatureResult(BaseResult): +class FeatureResult(ScenarioResultSet): scenario_results: ScenarioResultSet error: Optional[Exception] failure: Optional[AssertionError] - def __init__(self, scenario_results, error=None): + def __init__(self, scenario_results): self.scenario_results = scenario_results self.failed_scenarios = [] self.errored_scenarios = [] @@ -899,35 +858,6 @@ def __init__(self, scenario_results, error=None): if scenario.is_failure: self.failed_scenarios.append(scenario) - def printable(self): - if self.failure is not None: - return self.failure - if self.error: - return self.error.printable() - - return "" - - @property - def is_error(self): - return len(self.errored_scenarios) > 0 - - @property - def error(self) -> Optional[Exception]: - for scenario in self.errored_scenarios: - if scenario.is_error: - return scenario.error - - @property - def stack(self) -> Optional[ErrorStack]: - for scenario in self.errored_scenarios: - return scenario.stack - - def __getattr__(self, attr): - try: - return self.__getattribute__(attr) - except AttributeError: - return getattr(self.scenario_results[-1], attr) - @property def is_failure(self): return len(self.failed_scenarios) > 0 @@ -939,103 +869,31 @@ def failure(self) -> Optional[Exception]: return scenario.failure @property - def succinct_failure(self) -> Optional[str]: - for scenario in self.failed_scenarios: - if scenario.is_failure: - return scenario.succinct_failure + def is_error(self): + return len(self.errored_scenarios) > 0 @property - def first_scenario_result_error(self) -> Optional[ScenarioResult]: + def error(self) -> Optional[Exception]: for scenario in self.errored_scenarios: if scenario.is_error: - return scenario - - @property - def first_scenario_result_fail(self) -> Optional[ScenarioResult]: - for scenario in self.failed_scenarios: - if scenario.is_failure: - return scenario - - @property - def first_nonsuccessful_result(self) -> Optional[ScenarioResult]: - return self.first_scenario_result_error or self.first_scenario_result_fail + return scenario.error -class FeatureResultSet(BaseResult): +class FeatureResultSet(FeatureResult): error: Optional[Exception] failure: Optional[AssertionError] - def __init__(self, feature_results, error=None): - self.feature_results = feature_results - self.failed_features = [] - self.errored_features = [] - - for feature in feature_results: - if feature.is_error: - self.errored_features.append(feature) - if feature.is_failure: - self.failed_features.append(feature) - - def printable(self): - if self.failure is not None: - return self.failure - if self.error: - return self.error.printable() - - return "" - - @property - def is_error(self): - return len(self.errored_features) > 0 - - @property - def error(self) -> Optional[Exception]: - for feature in self.errored_features: - if feature.is_error: - return collapse_path(feature.error) - - @property - def stack(self) -> Optional[ErrorStack]: - for feature in self.errored_features: - return feature.stack - - def __getattr__(self, attr): - try: - return self.__getattribute__(attr) - except AttributeError: - return getattr(self.feature_results[-1], attr) - - @property - def is_failure(self): - return len(self.failed_features) > 0 - - @property - def failure(self) -> Optional[Exception]: - for feature in self.failed_features: - if feature.is_failure: - return feature.failure - - @property - def succinct_failure(self) -> Optional[str]: - for feature in self.failed_features: - if feature.is_failure: - return feature.succinct_failure - @property - def first_scenario_result_error(self) -> Optional[FeatureResult]: - for feature_result in self.errored_features: - if feature_result.is_error: - return feature_result.first_nonsuccessful_result + def feature_results(self): + return self.scenario_results @property - def first_scenario_result_fail(self) -> Optional[FeatureResult]: - for feature_result in self.failed_features: - if feature_result.is_failure: - return feature_result.first_nonsuccessful_result + def failed_features(self): + return self.failed_scenarios @property - def first_nonsuccessful_result(self) -> Optional[FeatureResult]: - return self.first_scenario_result_error or self.first_scenario_result_fail + def errored_features(self): + return self.errored_scenarios def stripped(string): diff --git a/tests/functional/test_runner.py b/tests/functional/test_runner.py index 1950b00..7bffb47 100644 --- a/tests/functional/test_runner.py +++ b/tests/functional/test_runner.py @@ -31,7 +31,8 @@ ScenarioResult, ScenarioResultSet, TestLocation, - FeatureResultSet, RuntimeContext + FeatureResultSet, + RuntimeContext, ) from sure.reporters import test @@ -118,7 +119,6 @@ def test_runner_load_features_from_module_path_recursively(): expects(featureB).to.be.a(Feature) expects(featureB).to.have.property("title").being.equal( "tests.functional.modules.success.module_with_members" - ) expects(featureB).to.have.property("ready").being.equal(True) expects(featureB).to.have.property("scenarios").being.a(list) @@ -204,7 +204,7 @@ def test_runner_execute_success_tests(): expects(feature_result_set).to.be.a(FeatureResultSet) expects(feature_result_set).to.have.property("feature_results").being.length_of(4) expects(feature_result_set).to.have.property("failed_features").being.empty - expects(feature_result_set).to.have.property("errored_scenarios").being.empty + expects(feature_result_set).to.have.property("errored_features").being.empty expects(dict(test.events)).to.equal( { @@ -265,6 +265,8 @@ def test_runner_execute_success_tests(): (anything_of_type(float), "test_function_Z", "ok"), (anything_of_type(float), "TestCase", "ok"), (anything_of_type(float), "UnitCase", "ok"), + (anything_of_type(float), "UnitCase", "ok"), + (anything_of_type(float), "UnitCase", "ok"), (anything_of_type(float), "test_function_A", "ok"), (anything_of_type(float), "test_function_B", "ok"), (anything_of_type(float), "test_function_C", "ok"), @@ -272,14 +274,26 @@ def test_runner_execute_success_tests(): (anything_of_type(float), "TestCaseA", "ok"), (anything_of_type(float), "TestCaseA", "ok"), (anything_of_type(float), "TestCaseA", "ok"), + (anything_of_type(float), "TestCaseA", "ok"), + (anything_of_type(float), "TestCaseA", "ok"), + (anything_of_type(float), "TestCaseB", "ok"), (anything_of_type(float), "TestCaseB", "ok"), (anything_of_type(float), "TestCaseB", "ok"), (anything_of_type(float), "TestCaseB", "ok"), (anything_of_type(float), "TestCaseB", "ok"), + (anything_of_type(float), "TestCaseB", "ok"), + (anything_of_type(float), "TestCaseA", "ok"), (anything_of_type(float), "TestCaseA", "ok"), (anything_of_type(float), "TestCaseA", "ok"), (anything_of_type(float), "TestCaseA", "ok"), (anything_of_type(float), "TestCaseA", "ok"), + (anything_of_type(float), "TestCaseA", "ok"), + (anything_of_type(float), "TestCaseA", "ok"), + (anything_of_type(float), "TestCaseA", "ok"), + (anything_of_type(float), "TestCaseB", "ok"), + (anything_of_type(float), "TestCaseB", "ok"), + (anything_of_type(float), "TestCaseB", "ok"), + (anything_of_type(float), "TestCaseB", "ok"), (anything_of_type(float), "TestCaseB", "ok"), (anything_of_type(float), "TestCaseB", "ok"), (anything_of_type(float), "TestCaseB", "ok"), diff --git a/tests/test_assertion_builder_assertion_methods.py b/tests/test_assertion_builder_assertion_methods.py index 0c0e05b..426c937 100644 --- a/tests/test_assertion_builder_assertion_methods.py +++ b/tests/test_assertion_builder_assertion_methods.py @@ -21,9 +21,46 @@ from sure.doubles import anything_of_type -def test_contains(): - "expects.that().contains" +def test_contains_and_to_contain(): + "expects.that().contains and expects().to_contain" expects.that(set(range(8))).contains(7) expects(set(range(13))).to.contain(8) expects(set(range(33))).to_contain(3) + expects(set()).to_contain.when.called_with("art").should.have.raised("`art' should be in `set()'") + + +def test_does_not_contain_and_to_not_contain(): + "expects().contains and expects().to_not_contain" + + expects(list()).to_not_contain("speculations") + expects(set(range(33))).to_not_contain.when.called_with(3).should.have.raised("`3' should not be in `{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32}'") + + +def test_within(): + "expects().to.be.within()" + + expects(3).to.be.within(set(range(33))) + expects(7).to.be.within.when.called_with(0, 0, 6).should.have.raised( + "(7).should.be.within(0, 0, 6) must be called with either an iterable:\n" + "(7).should.be.within([1, 2, 3, 4])\n" + "or with a range of numbers, i.e.: `(7).should.be.within(1, 3000)'" + ) + expects(3).to.be.within(0, 7) + expects(1).to_not.be.within.when.called_with(0, 0, 7).should.have.raised( + "(1).should_not.be.within(0, 0, 7) must be called with either an iterable:\n" + ) + expects(3).to.not_be.within.when.called_with(set(range(33))).should.have.raised( + "`3' should not be in `{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32}'" + ) + expects("art").to.be.within.when.called_with({"set"}).should.have.raised( + "`art' should be in `{'set'}'" + ) + + +def test_different_of(): + "expects().to.be.different_of()" + + expects([]).to.be.different_of.when.called_with([]).should.have.raised( + ".different_of only works for string comparison but in this case is expecting [] () instead" + ) diff --git a/tests/test_assertion_builder_assertion_properties.py b/tests/test_assertion_builder_assertion_properties.py index 6a80110..9b1af1a 100644 --- a/tests/test_assertion_builder_assertion_properties.py +++ b/tests/test_assertion_builder_assertion_properties.py @@ -30,3 +30,4 @@ class WaveFunctionParameters: frequency = anything_of_type(float) expects(WaveFunctionParameters).to.not_have.property("unrequested_phase_change") + expects(WaveFunctionParameters).to.have.property("frequency").which.should.equal(anything_of_type(float)) diff --git a/tests/test_original_api.py b/tests/test_original_api.py index 177c2d1..c804372 100644 --- a/tests/test_original_api.py +++ b/tests/test_original_api.py @@ -1553,9 +1553,41 @@ def test_deep_comparison_sequences_of_sequences(): except AssertionError as e: expects(str(e)).to_not.be.different_of( """Equality Error -X = [('Bootstraping Redis role', []), ('Restart scalarizr', []), ('Rebundle server', ['rebundle']), ('Use new role', ['rebundle']), ('Restart scalarizr after bundling', ['rebundle']), ('Bundling data', []), ('Modifying data', []), ('Reboot server', []), ('Backuping data on Master', []), ('Setup replication', []), ('Restart scalarizr in slave', []), ('Slave force termination', []), ('Slave delete EBS', ['ec2']), ('Setup replication for EBS test', ['ec2']), ('Writing on Master, reading on Slave', []), ('Slave -> Master promotion', []), ('Restart farm', ['restart_farm'])] +X = [('Bootstraping Redis role', []), + ('Restart scalarizr', []), + ('Rebundle server', ['rebundle']), + ('Use new role', ['rebundle']), + ('Restart scalarizr after bundling', ['rebundle']), + ('Bundling data', []), + ('Modifying data', []), + ('Reboot server', []), + ('Backuping data on Master', []), + ('Setup replication', []), + ('Restart scalarizr in slave', []), + ('Slave force termination', []), + ('Slave delete EBS', ['ec2']), + ('Setup replication for EBS test', ['ec2']), + ('Writing on Master, reading on Slave', []), + ('Slave -> Master promotion', []), + ('Restart farm', ['restart_farm'])] and -Y = [('Bootstraping Redis role', ['rebundle', 'rebundle', 'rebundle']), ('Restart scalarizr', []), ('Rebundle server', ['rebundle']), ('Use new role', ['rebundle']), ('Restart scalarizr after bundling', ['rebundle']), ('Bundling data', []), ('Modifying data', []), ('Reboot server', []), ('Backuping data on Master', []), ('Setup replication', []), ('Restart scalarizr in slave', []), ('Slave force termination', []), ('Slave delete EBS', ['ec2']), ('Setup replication for EBS test', ['ec2']), ('Writing on Master, reading on Slave', []), ('Slave -> Master promotion', []), ('Restart farm', ['restart_farm'])] +Y = [('Bootstraping Redis role', ['rebundle', 'rebundle', 'rebundle']), + ('Restart scalarizr', []), + ('Rebundle server', ['rebundle']), + ('Use new role', ['rebundle']), + ('Restart scalarizr after bundling', ['rebundle']), + ('Bundling data', []), + ('Modifying data', []), + ('Reboot server', []), + ('Backuping data on Master', []), + ('Setup replication', []), + ('Restart scalarizr in slave', []), + ('Slave force termination', []), + ('Slave delete EBS', ['ec2']), + ('Setup replication for EBS test', ['ec2']), + ('Writing on Master, reading on Slave', []), + ('Slave -> Master promotion', []), + ('Restart farm', ['restart_farm'])] Y[0][1] has 3 items whereas X[0][1] is empty """.strip() ) diff --git a/tests/test_runtime/test_base_result.py b/tests/test_runtime/test_base_result.py new file mode 100644 index 0000000..fc67128 --- /dev/null +++ b/tests/test_runtime/test_base_result.py @@ -0,0 +1,57 @@ +# -*- coding: utf-8 -*- +# +# Copyright (C) <2010-2024> Gabriel Falcão +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +"""tests for :class:`sure.runtime.BaseResult`""" +from sure import expects +from sure.runtime import BaseResult + + +description = "tests for :class:`sure.runtime.BaseResult`" + + +def test_base_result___repr___not_implemented_error_missing_label_property(): + "BaseResult.__repr__ should raise :exc:`NotImplementederror` when missing a `label' property" + + expects(repr).when.called_with(BaseResult()).to.have.raised( + NotImplementedError, + " MUST define a `label' property or attribute which must be a string" + ) + + +def test_base_result___repr___not_implemented_error_nonstring_label_property(): + "calling :func:`repr` on a subclass of :class:`sure.runtime.BaseResult` whose label property returns something other than a :class:`str` instance should raise :exc:`NotImplementedError`" + + class FakeResultDummyLabelNonString(BaseResult): + @property + def label(self): + return () + + expects(repr).when.called_with(FakeResultDummyLabelNonString()).to.have.raised( + NotImplementedError, + ".FakeResultDummyLabelNonString'>.label must be a string but is a instead" + ) + + +def test_base_result___repr___returns_lowercase_label(): + "the builtin implementation of :meth:`sure.runtime.BaseResult.__repr__` should return the value of its `label' property as a lower-case string" + + class FakeResultDummyLabel(BaseResult): + @property + def label(self): + return "LABEL" + + repr(FakeResultDummyLabel()).should.equal("'label'") diff --git a/tests/test_runtime/test_feature.py b/tests/test_runtime/test_feature.py index 4762925..ce38f50 100644 --- a/tests/test_runtime/test_feature.py +++ b/tests/test_runtime/test_feature.py @@ -17,9 +17,12 @@ """tests for :class:`sure.runtime.Feature`""" +from unittest.mock import patch, call +from unittest.mock import Mock as Spy from sure import expects -from sure.runtime import Feature +from sure.runtime import Feature, RuntimeOptions, ScenarioResult, Scenario from sure.doubles import stub +from sure.errors import ExitFailure, ExitError description = "tests for :class:`sure.runtime.Feature`" @@ -39,3 +42,55 @@ def test_feature_without_description(): feature = stub(Feature, title="title", description=None) expects(repr(feature)).to.equal('') + + +@patch('sure.errors.sys.exit') +@patch('sure.runtime.RuntimeContext') +def test_feature_run_is_failure(RuntimeContext, exit): + 'Feature.run() should raise :class:`sure.errors.ExitFailure` at the occurrence of failure within an "immediate" failure context' + + reporter_spy = Spy(name='Reporter') + scenario_result = stub(ScenarioResult, is_failure=True, __failure__=AssertionError('contrived failure'), __error__=None) + scenario_run_spy = Spy(name="Scenario.run", return_value=scenario_result) + + scenario_stub = stub(Scenario, run=scenario_run_spy) + feature_stub = stub( + Feature, + title="failure feature test", + description=None, + scenarios=[scenario_stub] + ) + + expects(feature_stub.run).when.called_with(reporter=reporter_spy, runtime=RuntimeOptions(immediate=True)).to.have.raised( + ExitFailure, + 'ExitFailure' + ) + expects(reporter_spy.mock_calls).to.equal([ + call.on_failure(scenario_stub, scenario_result) + ]) + + +@patch('sure.errors.sys.exit') +@patch('sure.runtime.RuntimeContext') +def test_feature_run_is_error(RuntimeContext, exit): + 'Feature.run() should raise :class:`sure.errors.ExitError` at the occurrence of error within an "immediate" error context' + + reporter_spy = Spy(name='Reporter') + scenario_run_spy = Spy(name="Scenario.run") + scenario_stub = stub(Scenario, run=scenario_run_spy) + feature_stub = stub( + Feature, + title="error feature test", + description=None, + scenarios=[scenario_stub] + ) + scenario_result = stub(ScenarioResult, is_error=True, __error__=ValueError('contrived error'), __failure__=None, is_failure=False, scenario=scenario_stub) + scenario_run_spy.return_value = scenario_result + + expects(feature_stub.run).when.called_with(reporter=reporter_spy, runtime=RuntimeOptions(immediate=True)).to.have.raised( + ExitError, + 'ExitError' + ) + expects(reporter_spy.mock_calls).to.equal([ + call.on_error(scenario_stub, scenario_result) + ]) diff --git a/tests/test_runtime/test_feature_result.py b/tests/test_runtime/test_feature_result.py new file mode 100644 index 0000000..f99c85e --- /dev/null +++ b/tests/test_runtime/test_feature_result.py @@ -0,0 +1,87 @@ +# -*- coding: utf-8 -*- +# +# Copyright (C) <2010-2024> Gabriel Falcão +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +"""tests for :class:`sure.runtime.FeatureResult`""" + +import sys +from sure import expects +from sure.doubles import stub +from sure.loader import collapse_path +from sure.runtime import ( + ErrorStack, + RuntimeContext, + Scenario, + ScenarioResult, + ScenarioResultSet, + FeatureResult, + TestLocation, +) + +description = "tests for :class:`sure.runtime.FeatureResult`" + + +def test_feature_result(): + "FeatureResult discerns types of :class:`sure.runtime.ScenarioResult` instances" + + context = stub(RuntimeContext) + scenario_result_sets = [ + ScenarioResultSet([stub(ScenarioResult, __error__=None, __failure__=None)], context=context), + ScenarioResultSet([stub(ScenarioResult, __error__=None, __failure__=AssertionError('dummy'))], context=context), + ScenarioResultSet([stub(ScenarioResult, __error__=ValueError('dummy'), __failure__=None)], context=context), + ScenarioResultSet([stub(ScenarioResult, __error__=None, __failure__=AssertionError('dummy'))], context=context), + ScenarioResultSet([stub(ScenarioResult, __error__=ValueError('dummy'), __failure__=None)], context=context), + ScenarioResultSet([stub(ScenarioResult, __error__=None, __failure__=AssertionError('dummy'))], context=context), + ScenarioResultSet([stub(ScenarioResult, __error__=ValueError('dummy'), __failure__=None)], context=context), + ] + + feature_result = FeatureResult(scenario_result_sets) + + expects(feature_result).to.have.property('failed_scenarios').being.length_of(3) + expects(feature_result).to.have.property('errored_scenarios').being.length_of(3) + expects(feature_result).to.have.property('scenario_results').being.length_of(7) + + +def test_feature_result_printable_with_failure(): + "Feature.printable presents reference to first failure occurrence" + + context = stub(RuntimeContext) + scenario_result_sets = [ + ScenarioResultSet([stub(ScenarioResult, __error__=None, __failure__=None)], context=context), + ScenarioResultSet([stub(ScenarioResult, __error__=None, __failure__=AssertionError('dummy'))], context=context), + ScenarioResultSet([stub(ScenarioResult, __error__=ValueError('dummy'), __failure__=None)], context=context), + ScenarioResultSet([stub(ScenarioResult, __error__=None, __failure__=AssertionError('dummy'))], context=context), + ScenarioResultSet([stub(ScenarioResult, __error__=ValueError('dummy'), __failure__=None)], context=context), + ScenarioResultSet([stub(ScenarioResult, __error__=None, __failure__=AssertionError('dummy'))], context=context), + ScenarioResultSet([stub(ScenarioResult, __error__=ValueError('dummy'), __failure__=None)], context=context), + ] + + feature_result = FeatureResult(scenario_result_sets) + + expects(feature_result.printable()).to.equal("AssertionError: dummy") + + +def test_feature_result_printable_with_error(): + "Feature.printable presents reference to first error occurrence" + + context = stub(RuntimeContext) + scenario_result_sets = [ + ScenarioResultSet([stub(ScenarioResult, __error__=None, __failure__=None)], context=context), + ScenarioResultSet([stub(ScenarioResult, __error__=ValueError('dummy'), __failure__=None)], context=context), + ] + + feature_result = FeatureResult(scenario_result_sets) + + expects(feature_result.printable()).to.equal("ValueError: dummy") diff --git a/tests/test_runtime/test_scenario.py b/tests/test_runtime/test_scenario.py new file mode 100644 index 0000000..377335c --- /dev/null +++ b/tests/test_runtime/test_scenario.py @@ -0,0 +1,148 @@ +# -*- coding: utf-8 -*- +# +# Copyright (C) <2010-2024> Gabriel Falcão +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +"""tests for :class:`sure.runtime.Scenario`""" + + +from unittest.mock import patch, call +from unittest.mock import Mock as Spy +from sure import expects +from sure.runtime import ( + Scenario, + RuntimeOptions, + ScenarioResult, + RuntimeRole, + RuntimeContext, + ScenarioResultSet, +) +from sure.doubles import stub, Dummy, anything +from sure.errors import ExitFailure, ExitError + + +description = "tests for :class:`sure.runtime.Scenario`" + + +@patch("sure.errors.sys.exit") +@patch("sure.runtime.ScenarioArrangement") +def test_scenario_run_when_result_is_failure(ScenarioArrangement, exit): + "Scenario.run() should raise :class:`sure.errors.ExitError` when a failure occurs" + + scenario_stub = stub( + Scenario, + object=Dummy("scenario.object"), + ) + reporter_spy = Spy(name="Reporter") + scenario_arrangement = ScenarioArrangement.return_value + scenario_arrangement.scenario = scenario_stub + ScenarioArrangement.from_generic_object.return_value = scenario_arrangement + scenario_arrangement.uncollapse_nested.return_value = [scenario_arrangement] + scenario_result = stub( + ScenarioResult, + is_failure=True, + is_success=False, + is_error=False, + scenario=scenario_stub, + ) + scenario_arrangement.run.return_value = [(scenario_result, RuntimeRole.Unit)] + + context = stub( + RuntimeContext, options=RuntimeOptions(immediate=False), reporter=reporter_spy + ) + scenario_result_set = scenario_stub.run(context) + + expects(scenario_result_set).to.be.a(ScenarioResultSet) + expects(reporter_spy.mock_calls).to.equal( + [ + call.on_scenario(scenario_arrangement.scenario), + call.on_failure(scenario_stub, scenario_result), + call.on_scenario_done(scenario_arrangement.scenario, scenario_result), + ] + ) + + +@patch("sure.errors.sys.exit") +@patch("sure.runtime.ScenarioArrangement") +def test_scenario_run_when_result_is_failure_and_runtime_options_immediate( + ScenarioArrangement, exit +): + 'Scenario.run() should raise :class:`sure.errors.ExitError` when a failure occurs and the runtime context is configured to "fail immediately"' + + reporter_spy = Spy(name="Reporter") + scenario_arrangement = ScenarioArrangement.return_value + ScenarioArrangement.from_generic_object.return_value = scenario_arrangement + scenario_arrangement.uncollapse_nested.return_value = [scenario_arrangement] + scenario_stub = stub( + Scenario, + object=Dummy("scenario.object"), + ) + scenario_result = stub( + ScenarioResult, + is_failure=True, + is_success=False, + is_error=False, + scenario=scenario_stub, + ) + scenario_arrangement.run.return_value = [(scenario_result, RuntimeRole.Unit)] + + context = stub( + RuntimeContext, options=RuntimeOptions(immediate=True), reporter=reporter_spy + ) + expects(scenario_stub.run).when.called_with(context).should.have.raised(ExitFailure) + + expects(reporter_spy.mock_calls).to.equal( + [ + call.on_scenario(scenario_arrangement.scenario), + call.on_failure(scenario_stub, scenario_result), + ] + ) + + +@patch("sure.errors.sys.exit") +@patch("sure.runtime.ScenarioArrangement") +def test_scenario_run_when_result_is_error_and_runtime_options_immediate( + ScenarioArrangement, exit +): + 'Scenario.run() should raise :class:`sure.errors.ExitError` when an error occurs and the runtime context is configured to "error immediately"' + + reporter_spy = Spy(name="Reporter") + scenario_arrangement = ScenarioArrangement.return_value + ScenarioArrangement.from_generic_object.return_value = scenario_arrangement + scenario_arrangement.uncollapse_nested.return_value = [scenario_arrangement] + scenario_stub = stub( + Scenario, + object=Dummy("scenario.object"), + ) + scenario_result = stub( + ScenarioResult, + is_error=True, + is_success=False, + is_failure=False, + scenario=scenario_stub, + ) + scenario_arrangement.run.return_value = [(scenario_result, RuntimeRole.Unit)] + + context = stub( + RuntimeContext, options=RuntimeOptions(immediate=True), reporter=reporter_spy + ) + expects(scenario_stub.run).when.called_with(context).should.have.raised(ExitError) + + expects(reporter_spy.mock_calls).to.equal( + [ + call.on_scenario(scenario_arrangement.scenario), + call.on_error(scenario_stub, scenario_result), + ] + ) diff --git a/tests/test_runtime/test_scenario_result.py b/tests/test_runtime/test_scenario_result.py new file mode 100644 index 0000000..dfcd87e --- /dev/null +++ b/tests/test_runtime/test_scenario_result.py @@ -0,0 +1,51 @@ +# -*- coding: utf-8 -*- +# +# Copyright (C) <2010-2024> Gabriel Falcão +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +"""tests for :class:`sure.runtime.ScenarioResult`""" + +import sys +from sure import expects +from sure.doubles import stub +from sure.loader import collapse_path +from sure.runtime import ( + ScenarioResult, + Scenario, + TestLocation, + RuntimeContext, + ErrorStack, +) + + +description = "tests for :class:`sure.runtime.ScenarioResult`" + + +def test_scenario_result_printable(): + "meth:`ScenarioResult.printable` returns its location as string" + + location = TestLocation(test_scenario_result_printable) + scenario = stub(Scenario) + context = stub(RuntimeContext) + scenario_result = ScenarioResult( + scenario=scenario, location=location, context=context, error=None + ) + + expects(scenario_result.printable()).to.equal( + ( + 'scenario "meth:`ScenarioResult.printable` returns its location as string" \n' + "defined at " + f"{collapse_path(__file__)}:35" + ) + ) diff --git a/tests/test_runtime/test_scenario_result_set.py b/tests/test_runtime/test_scenario_result_set.py new file mode 100644 index 0000000..779674e --- /dev/null +++ b/tests/test_runtime/test_scenario_result_set.py @@ -0,0 +1,96 @@ +# -*- coding: utf-8 -*- +# +# Copyright (C) <2010-2024> Gabriel Falcão +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +"""tests for :class:`sure.runtime.ScenarioResultSet`""" + +import sys +from sure import expects +from sure.doubles import stub +from sure.loader import collapse_path +from sure.runtime import ( + ErrorStack, + RuntimeContext, + Scenario, + ScenarioResult, + ScenarioResultSet, + TestLocation, +) + +description = "tests for :class:`sure.runtime.ScenarioResultSet`" + + +def test_scenario_result_set(): + "ScenarioResultSet discerns types of :class:`sure.runtime.ScenarioResult` instances" + + scenario_results = [ + stub(ScenarioResult, __error__=None, __failure__=None), + stub(ScenarioResult, __error__=None, __failure__=AssertionError('y')), + stub(ScenarioResult, __error__=InterruptedError('x'), __failure__=None), + stub(ScenarioResult, __error__=None, __failure__=AssertionError('Y')), + stub(ScenarioResult, __error__=InterruptedError('X'), __failure__=None), + stub(ScenarioResult, __error__=None, __failure__=None), + ] + + scenario_result_set = ScenarioResultSet(scenario_results, context=stub(RuntimeContext)) + + expects(scenario_result_set).to.have.property('failed_scenarios').being.length_of(2) + expects(scenario_result_set).to.have.property('errored_scenarios').being.length_of(2) + expects(scenario_result_set).to.have.property('scenario_results').being.length_of(6) + + +def test_scenario_result_set_printable_error(): + "ScenarioResultSet.printable presents reference to first error occurrence" + + scenario_results = [ + stub(ScenarioResult, __error__=None, __failure__=None), + stub(ScenarioResult, __error__=InterruptedError('x'), __failure__=None), + stub(ScenarioResult, __error__=InterruptedError('X'), __failure__=None), + stub(ScenarioResult, __error__=None, __failure__=None), + ] + + scenario_result_set = ScenarioResultSet(scenario_results, context=stub(RuntimeContext)) + + expects(scenario_result_set.printable()).to.equal("InterruptedError: x") + + +def test_scenario_result_set_printable_failure(): + "ScenarioResultSet.printable presents reference to first failure occurrence" + + scenario_results = [ + stub(ScenarioResult, __error__=None, __failure__=None), + stub(ScenarioResult, __error__=None, __failure__=AssertionError('Y')), + stub(ScenarioResult, __error__=InterruptedError('x'), __failure__=None), + stub(ScenarioResult, __error__=None, __failure__=AssertionError('y')), + stub(ScenarioResult, __error__=InterruptedError('X'), __failure__=None), + stub(ScenarioResult, __error__=None, __failure__=None), + ] + + scenario_result_set = ScenarioResultSet(scenario_results, context=stub(RuntimeContext)) + + expects(scenario_result_set.printable()).to.equal("AssertionError: Y") + + +def test_scenario_result_set_printable_no_errors_or_failures(): + "ScenarioResultSet.printable presents empty string when there are no errors or failures" + + scenario_results = [ + stub(ScenarioResult, __error__=None, __failure__=None), + ] + + scenario_result_set = ScenarioResultSet(scenario_results, context=stub(RuntimeContext)) + + expects(scenario_result_set.printable()).to.be.a(str) + expects(scenario_result_set.printable()).to.be.empty