From 8b5c149f81657678a96144278ce760afc0ac004c Mon Sep 17 00:00:00 2001 From: Mick Semb Wever Date: Sun, 16 Jul 2023 13:15:43 +0200 Subject: [PATCH] Run generated upgrade_through_versions_test on pytest >7.2.0 pytest-7.2.0 changed how markers were inherited, https://github.com/pytest-dev/pytest/issues/7792 Also fix how internode_ssl changes seeds patch by Mick Semb Wever; reviewed by XXX for CASSANDRA-18499 --- conftest.py | 4 +- run_dtests.py | 2 +- .../upgrade_through_versions_test.py | 37 ++++++++++++------- 3 files changed, 26 insertions(+), 17 deletions(-) diff --git a/conftest.py b/conftest.py index 3eb8a10948..ef3a9d4701 100644 --- a/conftest.py +++ b/conftest.py @@ -113,8 +113,8 @@ def pytest_configure(config): def sufficient_system_resources_for_resource_intensive_tests(): mem = virtual_memory() total_mem_gb = mem.total / 1024 / 1024 / 1024 - logger.info("total available system memory is %dGB" % total_mem_gb) - # todo kjkj: do not hard code our bound.. for now just do 9 instances at 3gb a piece + logger.info("total available system memory is %dGB, require 27GB for resource intensive tests" % total_mem_gb) + # do not hard code our bound.. for now just do 9 instances at 3gb a piece return total_mem_gb >= 9 * 3 diff --git a/run_dtests.py b/run_dtests.py index 34dd5af766..b8eed9c7f8 100755 --- a/run_dtests.py +++ b/run_dtests.py @@ -131,7 +131,7 @@ def run(self, argv): "import sys\n" "sys.exit(pytest.main([{options}]))\n".format(options=original_raw_cmd_args)) temp = NamedTemporaryFile(dir=getcwd()) - logger.debug('Writing the following to {}:'.format(temp.name)) + logger.debug('Writing to {} the following:\n {}'.format(temp.name, to_execute.encode("utf-8"))) temp.write(to_execute.encode("utf-8")) temp.flush() diff --git a/upgrade_tests/upgrade_through_versions_test.py b/upgrade_tests/upgrade_through_versions_test.py index 1fc39d7c7a..f16693aea8 100644 --- a/upgrade_tests/upgrade_through_versions_test.py +++ b/upgrade_tests/upgrade_through_versions_test.py @@ -22,8 +22,8 @@ from tools.misc import generate_ssl_stores, new_node from .upgrade_manifest import (build_upgrade_pairs, current_2_2_x, - current_3_0_x, indev_3_11_x, current_3_11_x, - current_4_0_x, indev_4_1_x, current_4_1_x, + current_3_0_x, current_3_11_x, + current_4_0_x, current_4_1_x, indev_trunk, CASSANDRA_4_0, CASSANDRA_5_0, RUN_STATIC_UPGRADE_MATRIX) @@ -297,7 +297,6 @@ def shutdown_gently(): @pytest.mark.upgrade_test @pytest.mark.resource_intensive -@pytest.mark.skip("Fake skip so that this isn't run outside of a generated class that removes this annotation") class TestUpgrade(Tester): """ Upgrades a 3-node Murmur3Partitioner cluster through versions specified in test_version_metas. @@ -325,6 +324,8 @@ def fixture_add_additional_log_patterns(self, fixture_dtest_setup): ) def prepare(self): + if type(self).__name__ == "TestUpgrade": + pytest.skip("Skip base class, only generated classes run the tests") logger.debug("Upgrade test beginning, setting CASSANDRA_VERSION to {}, and jdk to {}. (Prior values will be restored after test)." .format(self.test_version_metas[0].version, self.test_version_metas[0].java_version)) cluster = self.cluster @@ -419,7 +420,10 @@ def upgrade_scenario(self, populate=True, create_schema=True, rolling=False, aft if version_meta.family > '3.11' and internode_ssl: seeds =[] for seed in cluster.seeds: - seeds.append(seed.ip_addr + ':7001') + if hasattr(seed, 'ip_addr'): + seeds.append(seed.ip_addr + ':7001') + else: + seeds.append(seed + ':7001') logger.debug("Forcing seeds to 7001 for internode ssl") cluster.seeds = seeds @@ -877,19 +881,11 @@ def create_upgrade_class(clsname, version_metas, protocol_version, print(" using protocol: v{}, and parent classes: {}".format(protocol_version, parent_class_names)) print(" to run these tests alone, use `nosetests {}.py:{}`".format(__name__, clsname)) - upgrade_applies_to_env = RUN_STATIC_UPGRADE_MATRIX or version_metas[-1].matches_current_env_version_family newcls = type( clsname, parent_classes, {'test_version_metas': version_metas, '__test__': True, 'protocol_version': protocol_version, 'extra_config': extra_config} ) - # Remove the skip annotation in the superclass we just derived from, we will add it back if we actually intend - # to skip with a better message - newcls.pytestmark = [mark for mark in newcls.pytestmark if not mark.name == "skip"] - #if not upgrade_applies_to_env: - #print("boo") - #newcls.pytestmark.append(pytest.mark.skip("test not applicable to env")) - print(newcls.pytestmark) if clsname in globals(): raise RuntimeError("Class by name already exists!") @@ -898,6 +894,18 @@ def create_upgrade_class(clsname, version_metas, protocol_version, return newcls +def supported_upgrade_paths(version_metas): + metas = [] + for version_meta in version_metas[:-1]: + for path in build_upgrade_pairs(): + if version_meta.family == path.starting_meta.family and version_metas[-1].family == path.upgrade_meta.family: + metas.append(version_meta) + break + + metas.append(version_metas[-1]) + return metas + + MultiUpgrade = namedtuple('MultiUpgrade', ('name', 'version_metas', 'protocol_version', 'extra_config')) MULTI_UPGRADES = ( @@ -942,7 +950,8 @@ def create_upgrade_class(clsname, version_metas, protocol_version, for upgrade in MULTI_UPGRADES: # if any version_metas are None, this means they are versions not to be tested currently if all(upgrade.version_metas): - metas = upgrade.version_metas + # even for RUN_STATIC_UPGRADE_MATRIX we only test upgrade paths compatible with the end "indev_" version + metas = supported_upgrade_paths(upgrade.version_metas) if not RUN_STATIC_UPGRADE_MATRIX: if metas[-1].matches_current_env_version_family: @@ -962,4 +971,4 @@ def create_upgrade_class(clsname, version_metas, protocol_version, [pair.starting_meta, pair.upgrade_meta], protocol_version=pair.starting_meta.max_proto_v, bootstrap_test=True - ) + ) \ No newline at end of file