From 37910b60cf9c900e06a0307de19a88d512921d6c Mon Sep 17 00:00:00 2001 From: Gary Russell Date: Thu, 7 Jul 2022 10:39:40 -0400 Subject: [PATCH] GH-2239: RetryableTopic Refactoring Resolves https://github.com/spring-projects/spring-kafka/issues/2239 * GH-2239: Replace PartitionPausingBackOffManager New back of manager (and factory) that uses a task scheduler to resume the paused partitions. Revert change to deprecated PartitionPausingBackoffManager. Log resume. * Remove legacy code. Also fix unrelated race in EKIT. Only allow one `RetryTemplateConfigurationSupport` bean. * Fix static var. * Docs. * More docs. * Remove more dead/deprecated code. * Address PR Comments. * Fix RetryTopicConfigurer bean retrieval. * Remove unnecessary casts in doc. --- .../src/main/asciidoc/index.adoc | 4 +- .../src/main/asciidoc/kafka.adoc | 2 +- .../src/main/asciidoc/retrytopic.adoc | 50 ++-- ...kaListenerAnnotationBeanPostProcessor.java | 35 +-- .../RetryableTopicAnnotationProcessor.java | 27 +- .../config/KafkaListenerEndpointRegistry.java | 22 +- .../AbstractKafkaBackOffManagerFactory.java | 19 +- .../kafka/listener/BackOffHandler.java | 25 +- ...ntainerPartitionPausingBackOffManager.java | 94 +++++++ ...PartitionPausingBackOffManagerFactory.java | 54 ++++ .../ContainerPausingBackOffHandler.java | 7 + .../listener/KafkaConsumerTimingAdjuster.java | 46 ---- .../KafkaMessageListenerContainer.java | 9 + .../ListenerContainerPauseService.java | 29 +- .../listener/ListenerContainerRegistry.java | 13 +- ...PartitionPausingBackOffManagerFactory.java | 175 ------------ .../PartitionPausingBackoffManager.java | 221 ---------------- .../WakingKafkaConsumerTimingAdjuster.java | 149 ----------- .../ListenerContainerFactoryConfigurer.java | 129 +-------- .../retrytopic/RetryTopicBootstrapper.java | 161 ----------- .../RetryTopicComponentFactory.java | 4 +- .../RetryTopicConfigurationSupport.java | 145 ++-------- .../retrytopic/RetryTopicConfigurer.java | 33 +-- .../RetryTopicInternalBeanNames.java | 117 -------- .../RetryTopicSchedulerWrapper.java | 68 +++++ .../EnableKafkaIntegrationTests.java | 15 +- .../PartitionPausingBackoffManagerTests.java | 211 --------------- ...akingKafkaConsumerTimingAdjusterTests.java | 82 ------ .../AbstractRetryTopicIntegrationTests.java | 41 +++ .../retrytopic/CircularDltHandlerTests.java | 13 +- .../kafka/retrytopic/DltStartupTests.java | 13 +- .../ExistingRetryTopicIntegrationTests.java | 12 +- ...stenerContainerFactoryConfigurerTests.java | 205 +------------- .../RetryTopicBootstrapperTests.java | 250 ------------------ ...tryTopicConfigurationIntegrationTests.java | 11 +- ...ationManualAssignmentIntegrationTests.java | 11 +- .../RetryTopicConfigurationSupportTests.java | 144 ++-------- ...TopicExceptionRoutingIntegrationTests.java | 9 +- .../RetryTopicIntegrationTests.java | 16 +- .../RetryTopicInternalBeanNamesTests.java | 70 ----- ...gacyFactoryConfigurerIntegrationTests.java | 217 --------------- ...cSameContainerFactoryIntegrationTests.java | 13 +- ...etryableTopicAnnotationProcessorTests.java | 31 ++- 43 files changed, 602 insertions(+), 2400 deletions(-) create mode 100644 spring-kafka/src/main/java/org/springframework/kafka/listener/ContainerPartitionPausingBackOffManager.java create mode 100644 spring-kafka/src/main/java/org/springframework/kafka/listener/ContainerPartitionPausingBackOffManagerFactory.java delete mode 100644 spring-kafka/src/main/java/org/springframework/kafka/listener/KafkaConsumerTimingAdjuster.java delete mode 100644 spring-kafka/src/main/java/org/springframework/kafka/listener/PartitionPausingBackOffManagerFactory.java delete mode 100644 spring-kafka/src/main/java/org/springframework/kafka/listener/PartitionPausingBackoffManager.java delete mode 100644 spring-kafka/src/main/java/org/springframework/kafka/listener/WakingKafkaConsumerTimingAdjuster.java delete mode 100644 spring-kafka/src/main/java/org/springframework/kafka/retrytopic/RetryTopicBootstrapper.java delete mode 100644 spring-kafka/src/main/java/org/springframework/kafka/retrytopic/RetryTopicInternalBeanNames.java create mode 100644 spring-kafka/src/main/java/org/springframework/kafka/retrytopic/RetryTopicSchedulerWrapper.java delete mode 100644 spring-kafka/src/test/java/org/springframework/kafka/listener/PartitionPausingBackoffManagerTests.java delete mode 100644 spring-kafka/src/test/java/org/springframework/kafka/listener/WakingKafkaConsumerTimingAdjusterTests.java create mode 100644 spring-kafka/src/test/java/org/springframework/kafka/retrytopic/AbstractRetryTopicIntegrationTests.java delete mode 100644 spring-kafka/src/test/java/org/springframework/kafka/retrytopic/RetryTopicBootstrapperTests.java delete mode 100644 spring-kafka/src/test/java/org/springframework/kafka/retrytopic/RetryTopicInternalBeanNamesTests.java delete mode 100644 spring-kafka/src/test/java/org/springframework/kafka/retrytopic/RetryTopicLegacyFactoryConfigurerIntegrationTests.java diff --git a/spring-kafka-docs/src/main/asciidoc/index.adoc b/spring-kafka-docs/src/main/asciidoc/index.adoc index 8da473fd10..6226ded78c 100644 --- a/spring-kafka-docs/src/main/asciidoc/index.adoc +++ b/spring-kafka-docs/src/main/asciidoc/index.adoc @@ -47,12 +47,12 @@ The <> covers the core classes to develop a Kafka applicatio include::kafka.adoc[] +include::retrytopic.adoc[] + include::streams.adoc[] include::testing.adoc[] -include::retrytopic.adoc[] - [[tips-n-tricks]] == Tips, Tricks and Examples diff --git a/spring-kafka-docs/src/main/asciidoc/kafka.adoc b/spring-kafka-docs/src/main/asciidoc/kafka.adoc index fcc8f1fb04..68d3c7fd16 100644 --- a/spring-kafka-docs/src/main/asciidoc/kafka.adoc +++ b/spring-kafka-docs/src/main/asciidoc/kafka.adoc @@ -5173,7 +5173,7 @@ This new error handler replaces the `SeekToCurrentErrorHandler` and `RecoveringB One difference is that the fallback behavior for batch listeners (when an exception other than a `BatchListenerFailedException` is thrown) is the equivalent of the <>. IMPORTANT: Starting with version 2.9, the `DefaultErrorHandler` can be configured to provide the same semantics as seeking the unprocessed record offsets as discussed below, but without actually seeking. -Instead, the records are retained by the listener container and resubmitted to the listener after the error handler exits (and after performing a single paused `poll()`, to keep the consumer alive). +Instead, the records are retained by the listener container and resubmitted to the listener after the error handler exits (and after performing a single paused `poll()`, to keep the consumer alive; if <> or a `ContainerPausingBackOffHandler` are being used, the pause may extend over multiple polls). The error handler returns a result to the container that indicates whether the current failing record can be resubmitted, or if it was recovered and then it will not be sent to the listener again. To enable this mode, set the property `seekAfterError` to `false`. diff --git a/spring-kafka-docs/src/main/asciidoc/retrytopic.adoc b/spring-kafka-docs/src/main/asciidoc/retrytopic.adoc index e8ca7b3d58..6810ab53a5 100644 --- a/spring-kafka-docs/src/main/asciidoc/retrytopic.adoc +++ b/spring-kafka-docs/src/main/asciidoc/retrytopic.adoc @@ -5,6 +5,10 @@ IMPORTANT: This is an experimental feature and the usual rule of no breaking API Users are encouraged to try out the feature and provide feedback via GitHub Issues or GitHub discussions. This is regarding the API only; the feature is considered to be complete, and robust. +Version 2.9 changed the mechanism to bootstrap infrastructure beans; see <> for the two mechanisms that are now required to bootstrap the feature. + +After these changes, we are intending to remove the experimental designation, probably in version 3.0. + Achieving non-blocking retry / dlt functionality with Kafka usually requires setting up extra topics and creating and configuring the corresponding listeners. Since 2.7 Spring for Apache Kafka offers support for that via the `@RetryableTopic` annotation and `RetryTopicConfiguration` class to simplify that bootstrapping. @@ -33,28 +37,23 @@ If one message's processing takes longer than the next message's back off period Also, for short delays (about 1s or less), the maintenance work the thread has to do, such as committing offsets, may delay the message processing execution. The precision can also be affected if the retry topic's consumer is handling more than one partition, because we rely on waking up the consumer from polling and having full pollTimeouts to make timing adjustments. -That being said, for consumers handling a single partition the message's processing should happen under 100ms after it's exact due time for most situations. +That being said, for consumers handling a single partition the message's processing should occur approximately at its exact due time for most situations. IMPORTANT: It is guaranteed that a message will never be processed before its due time. -===== Tuning the Delay Precision - -The message's processing delay precision relies on two `ContainerProperties`: `ContainerProperties.pollTimeout` and `ContainerProperties.idlePartitionEventInterval`. -Both properties will be automatically set in the retry topic and dlt's `ListenerContainerFactory` to one quarter of the smallest delay value for that topic, with a minimum value of 250ms and a maximum value of 5000ms. -These values will only be set if the property has its default values - if you change either value yourself your change will not be overridden. -This way you can tune the precision and performance for the retry topics if you need to. - -NOTE: You can have separate `ListenerContainerFactory` instances for the main and retry topics - this way you can have different settings to better suit your needs, such as having a higher polling timeout setting for the main topics and a lower one for the retry topics. - +[[retry-config]] ==== Configuration -Starting with version 2.9, the `@EnableKafkaRetryTopic` annotation should be used in a `@Configuration` annotated class. +Starting with version 2.9, for default configuration, the `@EnableKafkaRetryTopic` annotation should be used in a `@Configuration` annotated class. This enables the feature to bootstrap properly and gives access to injecting some of the feature's components to be looked up at runtime. -Also, to configure the feature's components and global features, the `RetryTopicConfigurationSupport` class should be extended in a `@Configuration` class, and the appropriate methods overridden. -For more details refer to <>. NOTE: It is not necessary to also add `@EnableKafka`, if you add this annotation, because `@EnableKafkaRetryTopic` is meta-annotated with `@EnableKafka`. +Also, starting with that version, for more advanced configuration of the feature's components and global features, the `RetryTopicConfigurationSupport` class should be extended in a `@Configuration` class, and the appropriate methods overridden. +For more details refer to <>. + +IMPORTANT: Only one of the above techniques can be used, and only one `@Configuration` class can extend `RetryTopicConfigurationSupport`. + ===== Using the `@RetryableTopic` annotation To configure the retry topic and dlt for a `@KafkaListener` annotated method, you just have to add the `@RetryableTopic` annotation to it and Spring for Apache Kafka will bootstrap all the necessary topics and consumers with the default configurations. @@ -161,9 +160,9 @@ It's best to use a single `RetryTopicConfiguration` bean for configuration of su [[retry-topic-global-settings]] ===== Configuring Global Settings and Features -Since 2.9, the previous bean overriding approach for configuring components has been deprecated. -This does not change the `RetryTopicConfiguration` beans approach - only components' configurations. -Now the `RetryTopicConfigurationSupport` class should be extended in a `@Configuration` class, and the proper methods overridden. +Since 2.9, the previous bean overriding approach for configuring components has been removed (without deprecation, due to the aforementioned experimental nature of the API). +This does not change the `RetryTopicConfiguration` beans approach - only infrastructure components' configurations. +Now the `RetryTopicConfigurationSupport` class should be extended in a (single) `@Configuration` class, and the proper methods overridden. An example follows: ==== @@ -185,6 +184,15 @@ public class MyRetryTopicConfiguration extends RetryTopicConfigurationSupport { protected void manageNonBlockingFatalExceptions(List> nonBlockingFatalExceptions) { nonBlockingFatalExceptions.add(MyNonBlockingException.class); } + + @Override + protected void configureCustomizers(CustomizersConfigurer customizersConfigurer) { + // Use the new 2.9 mechanism to avoid re-fetching the same records after a pause + customizersConfigurer.customizeErrorHandler(eh -> { + eh.setSeekAfterError(false); + }); + } + } ---- ==== @@ -629,7 +637,7 @@ As an example the following implementation, in addition to the standard suffix, ---- public class CustomRetryTopicNamesProviderFactory implements RetryTopicNamesProviderFactory { - @Override + @Override public RetryTopicNamesProvider createRetryTopicNamesProvider( DestinationTopic.Properties properties) { @@ -728,7 +736,7 @@ In the latter the consumer ends the execution without forwarding the message. ---- @RetryableTopic(dltProcessingFailureStrategy = - DltStrategy.FAIL_ON_ERROR) + DltStrategy.FAIL_ON_ERROR) @KafkaListener(topics = "my-annotated-topic") public void processMessage(MyPojo message) { // ... message processing @@ -777,7 +785,7 @@ In this case after retrials are exhausted the processing simply ends. ---- @RetryableTopic(dltProcessingFailureStrategy = - DltStrategy.NO_DLT) + DltStrategy.NO_DLT) @KafkaListener(topics = "my-annotated-topic") public void processMessage(MyPojo message) { // ... message processing @@ -872,8 +880,8 @@ For example, to change the logging level to WARN you might add: ---- @Override protected void configureCustomizers(CustomizersConfigurer customizersConfigurer) { - customizersConfigurer.customizeErrorHandler(commonErrorHandler -> - ((DefaultErrorHandler) commonErrorHandler).setLogLevel(KafkaException.Level.WARN)) + customizersConfigurer.customizeErrorHandler(defaultErrorHandler -> + defaultErrorHandler.setLogLevel(KafkaException.Level.WARN)) } ---- ==== diff --git a/spring-kafka/src/main/java/org/springframework/kafka/annotation/KafkaListenerAnnotationBeanPostProcessor.java b/spring-kafka/src/main/java/org/springframework/kafka/annotation/KafkaListenerAnnotationBeanPostProcessor.java index 6ea157d029..6f4a477b74 100644 --- a/spring-kafka/src/main/java/org/springframework/kafka/annotation/KafkaListenerAnnotationBeanPostProcessor.java +++ b/spring-kafka/src/main/java/org/springframework/kafka/annotation/KafkaListenerAnnotationBeanPostProcessor.java @@ -58,8 +58,6 @@ import org.springframework.beans.factory.config.ConfigurableBeanFactory; import org.springframework.beans.factory.config.ConfigurableListableBeanFactory; import org.springframework.beans.factory.config.Scope; -import org.springframework.beans.factory.support.BeanDefinitionRegistry; -import org.springframework.beans.factory.support.RootBeanDefinition; import org.springframework.context.ApplicationContext; import org.springframework.context.ApplicationContextAware; import org.springframework.context.ConfigurableApplicationContext; @@ -188,6 +186,8 @@ public class KafkaListenerAnnotationBeanPostProcessor private AnnotationEnhancer enhancer; + private RetryTopicConfigurer retryTopicConfigurer; + @Override public int getOrder() { return LOWEST_PRECEDENCE; @@ -510,27 +510,18 @@ private boolean processMainAndRetryListeners(KafkaListener kafkaListener, Object } private RetryTopicConfigurer getRetryTopicConfigurer() { - bootstrapRetryTopicIfNecessary(); - return this.beanFactory.containsBean("internalRetryTopicConfigurer") - ? this.beanFactory.getBean("internalRetryTopicConfigurer", RetryTopicConfigurer.class) - : this.beanFactory.getBean(RetryTopicBeanNames.RETRY_TOPIC_CONFIGURER_BEAN_NAME, RetryTopicConfigurer.class); - } - - @SuppressWarnings("deprecation") - private void bootstrapRetryTopicIfNecessary() { - if (!(this.beanFactory instanceof BeanDefinitionRegistry)) { - throw new IllegalStateException("BeanFactory must be an instance of " - + BeanDefinitionRegistry.class.getSimpleName() - + " to bootstrap the RetryTopic functionality. Provided beanFactory: " - + this.beanFactory.getClass().getSimpleName()); - } - BeanDefinitionRegistry registry = (BeanDefinitionRegistry) this.beanFactory; - if (!registry.containsBeanDefinition("internalRetryTopicBootstrapper")) { - registry.registerBeanDefinition("internalRetryTopicBootstrapper", - new RootBeanDefinition(org.springframework.kafka.retrytopic.RetryTopicBootstrapper.class)); - this.beanFactory.getBean("internalRetryTopicBootstrapper", - org.springframework.kafka.retrytopic.RetryTopicBootstrapper.class).bootstrapRetryTopic(); + if (this.retryTopicConfigurer == null) { + try { + this.retryTopicConfigurer = this.beanFactory + .getBean(RetryTopicBeanNames.RETRY_TOPIC_CONFIGURER_BEAN_NAME, RetryTopicConfigurer.class); + } + catch (NoSuchBeanDefinitionException ex) { + this.logger.error("A 'RetryTopicConfigurer' with name " + + RetryTopicBeanNames.RETRY_TOPIC_CONFIGURER_BEAN_NAME + "is required."); + throw ex; + } } + return this.retryTopicConfigurer; } private Method checkProxy(Method methodArg, Object bean) { diff --git a/spring-kafka/src/main/java/org/springframework/kafka/annotation/RetryableTopicAnnotationProcessor.java b/spring-kafka/src/main/java/org/springframework/kafka/annotation/RetryableTopicAnnotationProcessor.java index dafa562d77..66d26957bc 100644 --- a/spring-kafka/src/main/java/org/springframework/kafka/annotation/RetryableTopicAnnotationProcessor.java +++ b/spring-kafka/src/main/java/org/springframework/kafka/annotation/RetryableTopicAnnotationProcessor.java @@ -76,8 +76,6 @@ public class RetryableTopicAnnotationProcessor { private final BeanExpressionContext expressionContext; - private static final String DEFAULT_SPRING_BOOT_KAFKA_TEMPLATE_NAME = "kafkaTemplate"; - /** * Construct an instance using the provided parameters and default resolver, * expression context. @@ -214,26 +212,15 @@ private EndpointHandlerMethod getDltProcessor(Method listenerMethod, Object bean } } try { - return this.beanFactory.getBean( - org.springframework.kafka.retrytopic.RetryTopicInternalBeanNames.DEFAULT_KAFKA_TEMPLATE_BEAN_NAME, + return this.beanFactory.getBean(RetryTopicBeanNames.DEFAULT_KAFKA_TEMPLATE_BEAN_NAME, KafkaOperations.class); } - catch (NoSuchBeanDefinitionException ex) { - try { - return this.beanFactory.getBean(RetryTopicBeanNames.DEFAULT_KAFKA_TEMPLATE_BEAN_NAME, - KafkaOperations.class); - } - catch (NoSuchBeanDefinitionException ex2) { - try { - return this.beanFactory.getBean(DEFAULT_SPRING_BOOT_KAFKA_TEMPLATE_NAME, KafkaOperations.class); - } - catch (NoSuchBeanDefinitionException exc) { - exc.addSuppressed(ex); - exc.addSuppressed(ex2); - throw new BeanInitializationException("Could not find a KafkaTemplate to configure the retry topics.", // NOSONAR (lost stack trace) - exc); - } - } + catch (NoSuchBeanDefinitionException ex2) { + KafkaOperations kafkaOps = this.beanFactory.getBeanProvider(KafkaOperations.class).getIfUnique(); + Assert.state(kafkaOps != null, () -> "A single KafkaTemplate bean could not be found in the context; " + + " a single instance must exist, or one specifically named " + + RetryTopicBeanNames.DEFAULT_KAFKA_TEMPLATE_BEAN_NAME); + return kafkaOps; } } diff --git a/spring-kafka/src/main/java/org/springframework/kafka/config/KafkaListenerEndpointRegistry.java b/spring-kafka/src/main/java/org/springframework/kafka/config/KafkaListenerEndpointRegistry.java index dd55ab4c55..d112b162b6 100644 --- a/spring-kafka/src/main/java/org/springframework/kafka/config/KafkaListenerEndpointRegistry.java +++ b/spring-kafka/src/main/java/org/springframework/kafka/config/KafkaListenerEndpointRegistry.java @@ -75,6 +75,8 @@ public class KafkaListenerEndpointRegistry implements ListenerContainerRegistry, protected final LogAccessor logger = new LogAccessor(LogFactory.getLog(getClass())); //NOSONAR + private final Map unregisteredContainers = new ConcurrentHashMap<>(); + private final Map listenerContainers = new ConcurrentHashMap<>(); private int phase = AbstractMessageListenerContainer.DEFAULT_PHASE; @@ -109,6 +111,17 @@ public MessageListenerContainer getListenerContainer(String id) { return this.listenerContainers.get(id); } + @Override + @Nullable + public MessageListenerContainer getUnregisteredListenerContainer(String id) { + MessageListenerContainer container = this.unregisteredContainers.get(id); + if (container == null) { + refreshContextContainers(); + return this.unregisteredContainers.get(id); + } + return null; + } + /** * By default, containers registered for endpoints after the context is refreshed * are immediately started, regardless of their autoStartup property, to comply with @@ -156,10 +169,17 @@ public Collection getListenerContainers() { public Collection getAllListenerContainers() { List containers = new ArrayList<>(); containers.addAll(getListenerContainers()); - containers.addAll(this.applicationContext.getBeansOfType(MessageListenerContainer.class, true, false).values()); + refreshContextContainers(); + containers.addAll(this.unregisteredContainers.values()); return containers; } + private void refreshContextContainers() { + this.unregisteredContainers.clear(); + this.applicationContext.getBeansOfType(MessageListenerContainer.class, true, false).values() + .forEach(container -> this.unregisteredContainers.put(container.getListenerId(), container)); + } + /** * Create a message listener container for the given {@link KafkaListenerEndpoint}. *

This create the necessary infrastructure to honor that endpoint diff --git a/spring-kafka/src/main/java/org/springframework/kafka/listener/AbstractKafkaBackOffManagerFactory.java b/spring-kafka/src/main/java/org/springframework/kafka/listener/AbstractKafkaBackOffManagerFactory.java index 6869cc8fcc..98d83e0eae 100644 --- a/spring-kafka/src/main/java/org/springframework/kafka/listener/AbstractKafkaBackOffManagerFactory.java +++ b/spring-kafka/src/main/java/org/springframework/kafka/listener/AbstractKafkaBackOffManagerFactory.java @@ -25,6 +25,7 @@ * Base class for {@link KafkaBackOffManagerFactory} implementations. * * @author Tomaz Fernandes + * @author Gary Russell * @since 2.7 * @see KafkaConsumerBackoffManager */ @@ -35,24 +36,23 @@ public abstract class AbstractKafkaBackOffManagerFactory private ListenerContainerRegistry listenerContainerRegistry; + /** + * Creates an instance that will retrieve the {@link ListenerContainerRegistry} from + * the {@link ApplicationContext}. + */ + public AbstractKafkaBackOffManagerFactory() { + this.listenerContainerRegistry = null; + } + /** * Creates an instance with the provided {@link ListenerContainerRegistry}, * which will be used to fetch the {@link MessageListenerContainer} to back off. - * @param listenerContainerRegistry the listenerContainerRegistry to use. */ public AbstractKafkaBackOffManagerFactory(ListenerContainerRegistry listenerContainerRegistry) { this.listenerContainerRegistry = listenerContainerRegistry; } - /** - * Creates an instance that will retrieve the {@link ListenerContainerRegistry} from - * the {@link ApplicationContext}. - */ - public AbstractKafkaBackOffManagerFactory() { - this.listenerContainerRegistry = null; - } - /** * Sets the {@link ListenerContainerRegistry}, that will be used to fetch the * {@link MessageListenerContainer} to back off. @@ -90,4 +90,5 @@ protected T getBean(String beanName, Class beanClass) { public void setApplicationContext(ApplicationContext applicationContext) { this.applicationContext = applicationContext; } + } diff --git a/spring-kafka/src/main/java/org/springframework/kafka/listener/BackOffHandler.java b/spring-kafka/src/main/java/org/springframework/kafka/listener/BackOffHandler.java index 4b7f3417b9..c735befec2 100644 --- a/spring-kafka/src/main/java/org/springframework/kafka/listener/BackOffHandler.java +++ b/spring-kafka/src/main/java/org/springframework/kafka/listener/BackOffHandler.java @@ -16,15 +16,18 @@ package org.springframework.kafka.listener; +import org.apache.kafka.common.TopicPartition; + import org.springframework.lang.Nullable; /** * Handler for the provided back off time, listener container and exception. + * Also supports back off for individual partitions. * - * @author Jan Marincek - * @since 2.9 + * @author Jan Marincek + * @author Gary Russell + * @since 2.9 */ -@FunctionalInterface public interface BackOffHandler { /** @@ -33,6 +36,20 @@ public interface BackOffHandler { * @param exception the exception. * @param nextBackOff the next back off. */ - void onNextBackOff(@Nullable MessageListenerContainer container, Exception exception, long nextBackOff); + default void onNextBackOff(@Nullable MessageListenerContainer container, Exception exception, long nextBackOff) { + throw new UnsupportedOperationException(); + } + + /** + * Perform the next back off for a partition. + * @param container the container. + * @param partition the partition. + * @param nextBackOff the next back off. + */ + default void onNextBackOff(@Nullable MessageListenerContainer container, TopicPartition partition, + long nextBackOff) { + + throw new UnsupportedOperationException(); + } } diff --git a/spring-kafka/src/main/java/org/springframework/kafka/listener/ContainerPartitionPausingBackOffManager.java b/spring-kafka/src/main/java/org/springframework/kafka/listener/ContainerPartitionPausingBackOffManager.java new file mode 100644 index 0000000000..4b9889afe6 --- /dev/null +++ b/spring-kafka/src/main/java/org/springframework/kafka/listener/ContainerPartitionPausingBackOffManager.java @@ -0,0 +1,94 @@ +/* + * Copyright 2018-2022 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.kafka.listener; + +import org.apache.commons.logging.LogFactory; +import org.apache.kafka.common.TopicPartition; + +import org.springframework.core.log.LogAccessor; +import org.springframework.util.Assert; + +/** + * + * A manager that backs off consumption for a given topic if the timestamp provided is not + * due. Use with {@link DefaultErrorHandler} to guarantee that the message is read + * again after partition consumption is resumed (or seek it manually by other means). + * Note that when a record backs off the partition consumption gets paused for + * approximately that amount of time, so you must have a fixed backoff value per partition. + * + * @author Tomaz Fernandes + * @author Gary Russell + * @since 2.9 + * @see DefaultErrorHandler + */ +public class ContainerPartitionPausingBackOffManager implements KafkaConsumerBackoffManager { + + private static final LogAccessor LOGGER = new LogAccessor(LogFactory.getLog(KafkaConsumerBackoffManager.class)); + + private final ListenerContainerRegistry listenerContainerRegistry; + + private final BackOffHandler backOffHandler; + + /** + * Construct an instance with the provided registry and back off handler. + * @param listenerContainerRegistry the registry. + * @param backOffHandler the handler. + */ + public ContainerPartitionPausingBackOffManager(ListenerContainerRegistry listenerContainerRegistry, + BackOffHandler backOffHandler) { + + Assert.notNull(listenerContainerRegistry, "'listenerContainerRegistry' cannot be null"); + Assert.notNull(backOffHandler, "'backOffHandler' cannot be null"); + this.listenerContainerRegistry = listenerContainerRegistry; + this.backOffHandler = backOffHandler; + } + + /** + * Backs off if the current time is before the dueTimestamp provided + * in the {@link Context} object. + * @param context the back off context for this execution. + */ + @Override + public void backOffIfNecessary(Context context) { + long backoffTime = context.getDueTimestamp() - System.currentTimeMillis(); + LOGGER.debug(() -> "Back off time: " + backoffTime + " Context: " + context); + if (backoffTime > 0) { + pauseConsumptionAndThrow(context, backoffTime); + } + } + + private void pauseConsumptionAndThrow(Context context, Long backOffTime) throws KafkaBackoffException { + TopicPartition topicPartition = context.getTopicPartition(); + MessageListenerContainer container = getListenerContainerFromContext(context); + container.pausePartition(topicPartition); + this.backOffHandler.onNextBackOff(container, topicPartition, backOffTime); + throw new KafkaBackoffException(String.format("Partition %s from topic %s is not ready for consumption, " + + "backing off for approx. %s millis.", topicPartition.partition(), + topicPartition.topic(), backOffTime), + topicPartition, context.getListenerId(), context.getDueTimestamp()); + } + + private MessageListenerContainer getListenerContainerFromContext(Context context) { + MessageListenerContainer container = this.listenerContainerRegistry.getListenerContainer(context.getListenerId()); // NOSONAR + if (container == null) { + container = this.listenerContainerRegistry.getUnregisteredListenerContainer(context.getListenerId()); + } + Assert.notNull(container, () -> "No container found with id: " + context.getListenerId()); + return container; + } + +} diff --git a/spring-kafka/src/main/java/org/springframework/kafka/listener/ContainerPartitionPausingBackOffManagerFactory.java b/spring-kafka/src/main/java/org/springframework/kafka/listener/ContainerPartitionPausingBackOffManagerFactory.java new file mode 100644 index 0000000000..a4b4df8f21 --- /dev/null +++ b/spring-kafka/src/main/java/org/springframework/kafka/listener/ContainerPartitionPausingBackOffManagerFactory.java @@ -0,0 +1,54 @@ +/* + * Copyright 2022 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.kafka.listener; + +import org.springframework.util.Assert; + +/** + * A factory for {@link ContainerPartitionPausingBackoffManager}. + * + * @author Gary Russell + * @since 2.9 + * + */ +public class ContainerPartitionPausingBackOffManagerFactory extends AbstractKafkaBackOffManagerFactory { + + private BackOffHandler backOffHandler; + + /** + * Construct an instance with the provided properties. + * @param listenerContainerRegistry the registry. + */ + public ContainerPartitionPausingBackOffManagerFactory(ListenerContainerRegistry listenerContainerRegistry) { + super(listenerContainerRegistry); + } + + @Override + protected KafkaConsumerBackoffManager doCreateManager(ListenerContainerRegistry registry) { + Assert.notNull(this.backOffHandler, "a BackOffHandler is required"); + return new ContainerPartitionPausingBackOffManager(getListenerContainerRegistry(), this.backOffHandler); + } + + /** + * Set the back off handler to use in the created handlers. + * @param backOffHandler the handler. + */ + public void setBackOffHandler(BackOffHandler backOffHandler) { + this.backOffHandler = backOffHandler; + } + +} diff --git a/spring-kafka/src/main/java/org/springframework/kafka/listener/ContainerPausingBackOffHandler.java b/spring-kafka/src/main/java/org/springframework/kafka/listener/ContainerPausingBackOffHandler.java index 2562f840ec..aff58d2a19 100644 --- a/spring-kafka/src/main/java/org/springframework/kafka/listener/ContainerPausingBackOffHandler.java +++ b/spring-kafka/src/main/java/org/springframework/kafka/listener/ContainerPausingBackOffHandler.java @@ -18,6 +18,8 @@ import java.time.Duration; +import org.apache.kafka.common.TopicPartition; + import org.springframework.lang.Nullable; /** @@ -51,4 +53,9 @@ public void onNextBackOff(@Nullable MessageListenerContainer container, Exceptio } } + @Override + public void onNextBackOff(MessageListenerContainer container, TopicPartition partition, long nextBackOff) { + this.pauser.pausePartition(container, partition, Duration.ofMillis(nextBackOff)); + } + } diff --git a/spring-kafka/src/main/java/org/springframework/kafka/listener/KafkaConsumerTimingAdjuster.java b/spring-kafka/src/main/java/org/springframework/kafka/listener/KafkaConsumerTimingAdjuster.java deleted file mode 100644 index 4dd5e0ab41..0000000000 --- a/spring-kafka/src/main/java/org/springframework/kafka/listener/KafkaConsumerTimingAdjuster.java +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Copyright 2018-2021 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.kafka.listener; - -import org.apache.kafka.clients.consumer.Consumer; -import org.apache.kafka.common.TopicPartition; - -/** - * - * Adjusts the consumption timing of the given consumer to try to have it consume the - * next message at a given time until due. Since the {@link org.apache.kafka.clients.consumer.KafkaConsumer} - * executes on a single thread, this is done in a best-effort basis. - * - * @author Tomaz Fernandes - * @since 2.7 - * @see KafkaConsumerBackoffManager - */ -public interface KafkaConsumerTimingAdjuster { - - /** - * Executes the timing adjustment. - * - * @param consumerToAdjust the consumer that will have consumption adjusted - * @param topicPartitionToAdjust the consumer's topic partition to be adjusted - * @param containerPollTimeout the consumer's container pollTimeout property - * @param timeUntilNextMessageIsDue the time when the next message should be consumed - * - * @return the applied adjustment amount - */ - long adjustTiming(Consumer consumerToAdjust, TopicPartition topicPartitionToAdjust, - long containerPollTimeout, long timeUntilNextMessageIsDue); -} diff --git a/spring-kafka/src/main/java/org/springframework/kafka/listener/KafkaMessageListenerContainer.java b/spring-kafka/src/main/java/org/springframework/kafka/listener/KafkaMessageListenerContainer.java index cd98cab805..dc904a7caf 100644 --- a/spring-kafka/src/main/java/org/springframework/kafka/listener/KafkaMessageListenerContainer.java +++ b/spring-kafka/src/main/java/org/springframework/kafka/listener/KafkaMessageListenerContainer.java @@ -319,6 +319,15 @@ public void resume() { } } + @Override + public void resumePartition(TopicPartition topicPartition) { + super.resumePartition(topicPartition); + KafkaMessageListenerContainer.ListenerConsumer consumer = this.listenerConsumer; + if (consumer != null) { + this.listenerConsumer.wakeIfNecessary(); + } + } + @Override public Map> metrics() { ListenerConsumer listenerConsumerForMetrics = this.listenerConsumer; diff --git a/spring-kafka/src/main/java/org/springframework/kafka/listener/ListenerContainerPauseService.java b/spring-kafka/src/main/java/org/springframework/kafka/listener/ListenerContainerPauseService.java index 6fe241da2e..5f8120b4e0 100644 --- a/spring-kafka/src/main/java/org/springframework/kafka/listener/ListenerContainerPauseService.java +++ b/spring-kafka/src/main/java/org/springframework/kafka/listener/ListenerContainerPauseService.java @@ -22,6 +22,7 @@ import java.util.Optional; import org.apache.commons.logging.LogFactory; +import org.apache.kafka.common.TopicPartition; import org.springframework.core.log.LogAccessor; import org.springframework.lang.Nullable; @@ -81,13 +82,37 @@ public void pause(MessageListenerContainer messageListenerContainer, Duration pa } else { Instant resumeAt = Instant.now().plusMillis(pauseDuration.toMillis()); - LOGGER.debug(() -> "Pausing container " + messageListenerContainer + "resume scheduled for " + LOGGER.debug(() -> "Pausing container " + messageListenerContainer + ", resume scheduled for " + resumeAt.atZone(ZoneId.systemDefault()).toLocalDateTime()); messageListenerContainer.pause(); - this.scheduler.schedule(() -> resume(messageListenerContainer), resumeAt); + this.scheduler.schedule(() -> { + LOGGER.debug(() -> "Pausing container " + messageListenerContainer); + resume(messageListenerContainer); + }, resumeAt); } } + /** + * Pause consumption from a given partition for the duration. + * @param messageListenerContainer the container. + * @param partition the partition. + * @param pauseDuration the duration. + */ + public void pausePartition(MessageListenerContainer messageListenerContainer, TopicPartition partition, + Duration pauseDuration) { + + Instant resumeAt = Instant.now().plusMillis(pauseDuration.toMillis()); + LOGGER.debug(() -> "Pausing container: " + messageListenerContainer + " partition: " + partition + + ", resume scheduled for " + + resumeAt.atZone(ZoneId.systemDefault()).toLocalDateTime()); + messageListenerContainer.pausePartition(partition); + this.scheduler.schedule(() -> { + LOGGER.debug(() -> "Resuming container: " + messageListenerContainer + " partition: " + partition); + messageListenerContainer.resumePartition(partition); + }, resumeAt); + + } + /** * Resume the listener container by given id. * @param listenerId the id of the listener diff --git a/spring-kafka/src/main/java/org/springframework/kafka/listener/ListenerContainerRegistry.java b/spring-kafka/src/main/java/org/springframework/kafka/listener/ListenerContainerRegistry.java index 6420de1de8..968d3a8705 100644 --- a/spring-kafka/src/main/java/org/springframework/kafka/listener/ListenerContainerRegistry.java +++ b/spring-kafka/src/main/java/org/springframework/kafka/listener/ListenerContainerRegistry.java @@ -1,5 +1,5 @@ /* - * Copyright 2021 the original author or authors. + * Copyright 2021-2022 the original author or authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -41,6 +41,17 @@ public interface ListenerContainerRegistry { @Nullable MessageListenerContainer getListenerContainer(String id); + /** + * Return the {@link MessageListenerContainer} with the specified id or {@code null} + * if no such container exists. Returns containers that are not registered with the + * registry, but exist in the application context. + * @param id the id of the container + * @return the container or {@code null} if no container with that id exists + * @see #getListenerContainerIds() + */ + @Nullable + MessageListenerContainer getUnregisteredListenerContainer(String id); + /** * Return the ids of the managed {@link MessageListenerContainer} instance(s). * @return the ids. diff --git a/spring-kafka/src/main/java/org/springframework/kafka/listener/PartitionPausingBackOffManagerFactory.java b/spring-kafka/src/main/java/org/springframework/kafka/listener/PartitionPausingBackOffManagerFactory.java deleted file mode 100644 index 63dc63f096..0000000000 --- a/spring-kafka/src/main/java/org/springframework/kafka/listener/PartitionPausingBackOffManagerFactory.java +++ /dev/null @@ -1,175 +0,0 @@ -/* - * Copyright 2018-2022 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.kafka.listener; - -import java.time.Clock; - -import org.springframework.core.task.TaskExecutor; -import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; -import org.springframework.util.Assert; - -/** - * - * Creates a {@link KafkaConsumerBackoffManager} instance - * with or without a {@link KafkaConsumerTimingAdjuster}. - * IMPORTANT: Since 2.9 this class doesn't create a {@link ThreadPoolTaskExecutor} - * by default. In order for the factory to create a {@link KafkaConsumerTimingAdjuster}, - * such thread executor must be provided. - * - * @author Tomaz Fernandes - * @since 2.7 - */ -public class PartitionPausingBackOffManagerFactory extends AbstractKafkaBackOffManagerFactory { - - private boolean timingAdjustmentEnabled = true; - - private KafkaConsumerTimingAdjuster timingAdjustmentManager; - - private TaskExecutor taskExecutor; - - private Clock clock; - - /** - * Construct a factory instance that will create the {@link KafkaConsumerBackoffManager} - * instances with the provided {@link KafkaConsumerTimingAdjuster}. - * - * @param timingAdjustmentManager the {@link KafkaConsumerTimingAdjuster} to be used. - */ - public PartitionPausingBackOffManagerFactory(KafkaConsumerTimingAdjuster timingAdjustmentManager) { - this.clock = getDefaultClock(); - doSetTimingAdjustmentManager(timingAdjustmentManager); - } - - /** - * Construct a factory instance that will create the {@link KafkaConsumerBackoffManager} - * instances with the provided {@link TaskExecutor} in its {@link KafkaConsumerTimingAdjuster}. - * - * @param timingAdjustmentManagerTaskExecutor the {@link TaskExecutor} to be used. - */ - public PartitionPausingBackOffManagerFactory(TaskExecutor timingAdjustmentManagerTaskExecutor) { - this.clock = getDefaultClock(); - doSetTaskExecutor(timingAdjustmentManagerTaskExecutor); - } - - /** - * Construct a factory instance specifying whether or not timing adjustment is enabled - * for this factories {@link KafkaConsumerBackoffManager}. - * - * @param timingAdjustmentEnabled the {@link KafkaConsumerTimingAdjuster} to be used. - */ - public PartitionPausingBackOffManagerFactory(boolean timingAdjustmentEnabled) { - this.clock = getDefaultClock(); - this.timingAdjustmentEnabled = timingAdjustmentEnabled; - } - - /** - * Construct a factory instance using the provided {@link ListenerContainerRegistry}. - * - * @param listenerContainerRegistry the {@link ListenerContainerRegistry} to be used. - */ - public PartitionPausingBackOffManagerFactory(ListenerContainerRegistry listenerContainerRegistry) { - super(listenerContainerRegistry); - this.clock = getDefaultClock(); - } - - /** - * Construct a factory instance with default dependencies. - */ - public PartitionPausingBackOffManagerFactory() { - this.clock = getDefaultClock(); - } - - /** - * Construct an factory instance that will create the {@link KafkaConsumerBackoffManager} - * with the provided {@link Clock}. - * @param clock the clock instance to be used. - */ - public PartitionPausingBackOffManagerFactory(Clock clock) { - this.clock = clock; - } - - /** - * Set this property to false if you don't want the resulting KafkaBackOffManager - * to adjust the precision of the topics' consumption timing. - * - * @param timingAdjustmentEnabled set to false to disable timing adjustment. - */ - public void setTimingAdjustmentEnabled(boolean timingAdjustmentEnabled) { - this.timingAdjustmentEnabled = timingAdjustmentEnabled; - } - - /** - * Set the {@link WakingKafkaConsumerTimingAdjuster} that will be used - * with the resulting {@link KafkaConsumerBackoffManager}. - * - * @param timingAdjustmentManager the adjustmentManager to be used. - */ - public void setTimingAdjustmentManager(KafkaConsumerTimingAdjuster timingAdjustmentManager) { - doSetTimingAdjustmentManager(timingAdjustmentManager); - } - - private void doSetTimingAdjustmentManager(KafkaConsumerTimingAdjuster timingAdjustmentManager) { - Assert.isTrue(this.timingAdjustmentEnabled, () -> "TimingAdjustment is disabled for this factory."); - this.timingAdjustmentManager = timingAdjustmentManager; - } - - /** - * Set the {@link TaskExecutor} that will be used in the {@link KafkaConsumerTimingAdjuster}. - * @param taskExecutor the taskExecutor to be used. - */ - public void setTaskExecutor(TaskExecutor taskExecutor) { - doSetTaskExecutor(taskExecutor); - } - - private void doSetTaskExecutor(TaskExecutor taskExecutor) { - Assert.isTrue(this.timingAdjustmentEnabled, () -> "TimingAdjustment is disabled for this factory."); - this.taskExecutor = taskExecutor; - } - - /** - * Set the {@link Clock} instance that will be used in the - * {@link KafkaConsumerBackoffManager}. - * @param clock the clock instance. - * @since 2.9 - */ - public void setClock(Clock clock) { - this.clock = clock; - } - - @Override - protected KafkaConsumerBackoffManager doCreateManager(ListenerContainerRegistry registry) { - return getKafkaConsumerBackoffManager(registry); - } - - protected final Clock getDefaultClock() { - return Clock.systemUTC(); - } - - private PartitionPausingBackoffManager getKafkaConsumerBackoffManager(ListenerContainerRegistry registry) { - return this.timingAdjustmentEnabled && this.taskExecutor != null - ? new PartitionPausingBackoffManager(registry, getOrCreateBackOffTimingAdjustmentManager(), this.clock) - : new PartitionPausingBackoffManager(registry, this.clock); - } - - private KafkaConsumerTimingAdjuster getOrCreateBackOffTimingAdjustmentManager() { - if (this.timingAdjustmentManager != null) { - return this.timingAdjustmentManager; - } - return new WakingKafkaConsumerTimingAdjuster(this.taskExecutor); - } - -} diff --git a/spring-kafka/src/main/java/org/springframework/kafka/listener/PartitionPausingBackoffManager.java b/spring-kafka/src/main/java/org/springframework/kafka/listener/PartitionPausingBackoffManager.java deleted file mode 100644 index 965a9359f9..0000000000 --- a/spring-kafka/src/main/java/org/springframework/kafka/listener/PartitionPausingBackoffManager.java +++ /dev/null @@ -1,221 +0,0 @@ -/* - * Copyright 2018-2022 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.kafka.listener; - -import java.time.Clock; -import java.time.Instant; -import java.util.HashMap; -import java.util.Map; - -import org.apache.commons.logging.LogFactory; -import org.apache.kafka.common.TopicPartition; - -import org.springframework.context.ApplicationListener; -import org.springframework.core.log.LogAccessor; -import org.springframework.kafka.event.ListenerContainerPartitionIdleEvent; -import org.springframework.lang.Nullable; - -/** - * - * A manager that backs off consumption for a given topic if the timestamp provided is not - * due. Use with {@link DefaultErrorHandler} to guarantee that the message is read - * again after partition consumption is resumed (or seek it manually by other means). - * It's also necessary to set a {@link ContainerProperties#setIdlePartitionEventInterval(Long)} - * so the Manager can resume the partition consumption. - * - * Note that when a record backs off the partition consumption gets paused for - * approximately that amount of time, so you must have a fixed backoff value per partition. - * - * @author Tomaz Fernandes - * @author Gary Russell - * @since 2.7 - * @see DefaultErrorHandler - */ -public class PartitionPausingBackoffManager implements KafkaConsumerBackoffManager, - ApplicationListener { - - private static final LogAccessor LOGGER = new LogAccessor(LogFactory.getLog(KafkaConsumerBackoffManager.class)); - - private final ListenerContainerRegistry listenerContainerRegistry; - - private final Map backOffContexts; - - private final Clock clock; - - private final KafkaConsumerTimingAdjuster kafkaConsumerTimingAdjuster; - - /** - * Constructs an instance with the provided {@link ListenerContainerRegistry} and - * {@link KafkaConsumerTimingAdjuster}. - * - * The ListenerContainerRegistry is used to fetch the {@link MessageListenerContainer} - * that will be backed off / resumed. - * - * The KafkaConsumerTimingAdjuster is used to make timing adjustments - * in the message consumption so that it processes the message closer - * to its due time rather than later. - * - * @param listenerContainerRegistry the listenerContainerRegistry to use. - * @param kafkaConsumerTimingAdjuster the kafkaConsumerTimingAdjuster to use. - */ - public PartitionPausingBackoffManager(ListenerContainerRegistry listenerContainerRegistry, - KafkaConsumerTimingAdjuster kafkaConsumerTimingAdjuster) { - - this(listenerContainerRegistry, kafkaConsumerTimingAdjuster, Clock.systemUTC()); - } - - /** - * Constructs an instance with the provided {@link ListenerContainerRegistry} - * and with no timing adjustment capabilities. - * - * The ListenerContainerRegistry is used to fetch the {@link MessageListenerContainer} - * that will be backed off / resumed. - * - * @param listenerContainerRegistry the listenerContainerRegistry to use. - */ - public PartitionPausingBackoffManager(ListenerContainerRegistry listenerContainerRegistry) { - this(listenerContainerRegistry, null, Clock.systemUTC()); - } - - /** - * Creates an instance with the provided {@link ListenerContainerRegistry}, - * {@link KafkaConsumerTimingAdjuster} and {@link Clock}. - * - * @param listenerContainerRegistry the listenerContainerRegistry to use. - * @param kafkaConsumerTimingAdjuster the kafkaConsumerTimingAdjuster to use. - * @param clock the clock to use. - */ - public PartitionPausingBackoffManager(ListenerContainerRegistry listenerContainerRegistry, - @Nullable KafkaConsumerTimingAdjuster kafkaConsumerTimingAdjuster, - Clock clock) { - - this.listenerContainerRegistry = listenerContainerRegistry; - this.clock = clock; - this.kafkaConsumerTimingAdjuster = kafkaConsumerTimingAdjuster; - this.backOffContexts = new HashMap<>(); - } - - /** - * Creates an instance with the provided {@link ListenerContainerRegistry} - * and {@link Clock}, with no timing adjustment capabilities. - * - * @param listenerContainerRegistry the listenerContainerRegistry to use. - * @param clock the clock to use. - */ - public PartitionPausingBackoffManager(ListenerContainerRegistry listenerContainerRegistry, Clock clock) { - this(listenerContainerRegistry, null, clock); - } - - /** - * Backs off if the current time is before the dueTimestamp provided - * in the {@link Context} object. - * @param context the back off context for this execution. - */ - @Override - public void backOffIfNecessary(Context context) { - long backoffTime = context.getDueTimestamp() - getCurrentMillisFromClock(); - LOGGER.debug(() -> "Back off time: " + backoffTime + " Context: " + context); - if (backoffTime > 0) { - pauseConsumptionAndThrow(context, backoffTime); - } - } - - private void pauseConsumptionAndThrow(Context context, Long backOffTime) throws KafkaBackoffException { - TopicPartition topicPartition = context.getTopicPartition(); - getListenerContainerFromContext(context).pausePartition(topicPartition); - addBackoff(context, topicPartition); - throw new KafkaBackoffException(String.format("Partition %s from topic %s is not ready for consumption, " + - "backing off for approx. %s millis.", context.getTopicPartition().partition(), - context.getTopicPartition().topic(), backOffTime), - topicPartition, context.getListenerId(), context.getDueTimestamp()); - } - - @Override - public void onApplicationEvent(ListenerContainerPartitionIdleEvent partitionIdleEvent) { - LOGGER.debug(() -> String.format("partitionIdleEvent received at %s. Partition: %s", - getCurrentMillisFromClock(), partitionIdleEvent.getTopicPartition())); - - Context backOffContext = getBackOffContext(partitionIdleEvent.getTopicPartition()); - maybeResumeConsumption(backOffContext); - } - - private long getCurrentMillisFromClock() { - return Instant.now(this.clock).toEpochMilli(); - } - - private void maybeResumeConsumption(@Nullable Context context) { - if (context == null) { - return; - } - long now = getCurrentMillisFromClock(); - long timeUntilDue = context.getDueTimestamp() - now; - long pollTimeout = getListenerContainerFromContext(context) - .getContainerProperties() - .getPollTimeout(); - boolean isDue = timeUntilDue <= pollTimeout; - - long adjustedAmount = applyTimingAdjustment(context, timeUntilDue, pollTimeout); - - if (adjustedAmount != 0L || isDue) { - resumePartition(context); - } - else { - LOGGER.debug(() -> String.format("TopicPartition %s not due. DueTimestamp: %s Now: %s ", - context.getTopicPartition(), context.getDueTimestamp(), now)); - } - } - - private long applyTimingAdjustment(Context context, long timeUntilDue, long pollTimeout) { - if (this.kafkaConsumerTimingAdjuster == null || context.getConsumerForTimingAdjustment() == null) { - LOGGER.debug(() -> String.format( - "Skipping timing adjustment for TopicPartition %s.", context.getTopicPartition())); - return 0L; - } - return this.kafkaConsumerTimingAdjuster.adjustTiming( - context.getConsumerForTimingAdjustment(), - context.getTopicPartition(), pollTimeout, timeUntilDue); - } - - private void resumePartition(Context context) { - MessageListenerContainer container = getListenerContainerFromContext(context); - LOGGER.debug(() -> "Resuming partition at " + getCurrentMillisFromClock()); - container.resumePartition(context.getTopicPartition()); - removeBackoff(context.getTopicPartition()); - } - - private MessageListenerContainer getListenerContainerFromContext(Context context) { - return this.listenerContainerRegistry.getListenerContainer(context.getListenerId()); // NOSONAR - } - - protected void addBackoff(Context context, TopicPartition topicPartition) { - synchronized (this.backOffContexts) { - this.backOffContexts.put(topicPartition, context); - } - } - - protected @Nullable Context getBackOffContext(TopicPartition topicPartition) { - synchronized (this.backOffContexts) { - return this.backOffContexts.get(topicPartition); - } - } - - protected void removeBackoff(TopicPartition topicPartition) { - synchronized (this.backOffContexts) { - this.backOffContexts.remove(topicPartition); - } - } -} diff --git a/spring-kafka/src/main/java/org/springframework/kafka/listener/WakingKafkaConsumerTimingAdjuster.java b/spring-kafka/src/main/java/org/springframework/kafka/listener/WakingKafkaConsumerTimingAdjuster.java deleted file mode 100644 index 4e57f06435..0000000000 --- a/spring-kafka/src/main/java/org/springframework/kafka/listener/WakingKafkaConsumerTimingAdjuster.java +++ /dev/null @@ -1,149 +0,0 @@ -/* - * Copyright 2018-2022 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.kafka.listener; - -import java.time.Duration; - -import org.apache.commons.logging.LogFactory; -import org.apache.kafka.clients.consumer.Consumer; -import org.apache.kafka.common.TopicPartition; - -import org.springframework.core.log.LogAccessor; -import org.springframework.core.task.TaskExecutor; -import org.springframework.retry.backoff.Sleeper; -import org.springframework.util.Assert; - - -/** - * - * Adjusts timing by creating a thread that will - * wakeup the consumer from polling, considering that, if consumption is paused, - * it will check for consumption resuming in increments of 'pollTimeout'. This works best - * if the consumer is handling a single partition. - * - * @author Tomaz Fernandes - * @since 2.7 - * @see KafkaConsumerBackoffManager - */ -public class WakingKafkaConsumerTimingAdjuster implements KafkaConsumerTimingAdjuster { - - private static final LogAccessor LOGGER = - new LogAccessor(LogFactory.getLog(WakingKafkaConsumerTimingAdjuster.class)); - - private static final long HUNDRED = 100L; - - private static final Duration DEFAULT_TIMING_ADJUSTMENT_THRESHOLD = Duration.ofMillis(HUNDRED); - - private static final int DEFAULT_POLL_TIMEOUTS_FOR_ADJUSTMENT_WINDOW = 2; - - private Duration timingAdjustmentThreshold = DEFAULT_TIMING_ADJUSTMENT_THRESHOLD; - - private int pollTimeoutsForAdjustmentWindow = DEFAULT_POLL_TIMEOUTS_FOR_ADJUSTMENT_WINDOW; - - private final TaskExecutor taskExecutor; - - private final Sleeper sleeper; - - /** - * Create an instance with the provided TaskExecutor and Sleeper. - * @param taskExecutor the task executor. - * @param sleeper the sleeper. - */ - public WakingKafkaConsumerTimingAdjuster(TaskExecutor taskExecutor, Sleeper sleeper) { - Assert.notNull(taskExecutor, "Task executor cannot be null."); - Assert.notNull(sleeper, "Sleeper cannot be null."); - this.taskExecutor = taskExecutor; - this.sleeper = sleeper; - } - - /** - * Create an instance with the provided {@link TaskExecutor} and a thread sleeper. - * @param taskExecutor the task executor. - */ - public WakingKafkaConsumerTimingAdjuster(TaskExecutor taskExecutor) { - this(taskExecutor, Thread::sleep); - } - - /** - * - * Set how many pollTimeouts prior to the dueTimeout the adjustment will take place. - * Default is 2. - * - * @param pollTimeoutsForAdjustmentWindow the amount of pollTimeouts in the adjustment window. - */ - public void setPollTimeoutsForAdjustmentWindow(int pollTimeoutsForAdjustmentWindow) { - this.pollTimeoutsForAdjustmentWindow = pollTimeoutsForAdjustmentWindow; - } - - /** - * - * Set the threshold for the timing adjustment to take place. If the time difference between - * the probable instant the message will be consumed and the instant it should is lower than - * this value, no adjustment will be applied. - * Default is 100ms. - * - * @param timingAdjustmentThreshold the threshold to be set. - */ - public void setTimingAdjustmentThreshold(Duration timingAdjustmentThreshold) { - this.timingAdjustmentThreshold = timingAdjustmentThreshold; - } - - /** - * Adjust the timing with the provided parameters. - * - * @param consumerToAdjust the {@link Consumer} that will be adjusted - * @param topicPartition the {@link TopicPartition} that will be adjusted - * @param pollTimeout the pollConfiguration for the consumer's container - * @param timeUntilDue the amount of time until the message is due for consumption - * @return the adjusted amount in milliseconds - */ - public long adjustTiming(Consumer consumerToAdjust, TopicPartition topicPartition, - long pollTimeout, long timeUntilDue) { - - boolean isInAdjustmentWindow = timeUntilDue > pollTimeout && timeUntilDue <= - pollTimeout * this.pollTimeoutsForAdjustmentWindow; - - long adjustmentAmount = timeUntilDue % pollTimeout; - if (isInAdjustmentWindow && adjustmentAmount > this.timingAdjustmentThreshold.toMillis()) { - this.taskExecutor.execute(() -> - doApplyTimingAdjustment(consumerToAdjust, topicPartition, adjustmentAmount)); - return adjustmentAmount; - } - return 0L; - } - - private void doApplyTimingAdjustment(Consumer consumerForTimingAdjustment, - TopicPartition topicPartition, long adjustmentAmount) { - try { - LOGGER.debug(() -> String.format("Applying timing adjustment of %s millis for TopicPartition %s", - adjustmentAmount, topicPartition)); - this.sleeper.sleep(adjustmentAmount); - LOGGER.debug(() -> "Waking up consumer for partition topic: " + topicPartition); - consumerForTimingAdjustment.wakeup(); - } - catch (InterruptedException e) { - Thread.currentThread().interrupt(); - throw new IllegalStateException("Interrupted waking up consumer while applying timing adjustment " + - "for TopicPartition " + topicPartition, e); - } - catch (Exception e) { // NOSONAR - LOGGER.error(e, () -> "Error waking up consumer while applying timing adjustment " + - "for TopicPartition " + topicPartition); - } - } - -} diff --git a/spring-kafka/src/main/java/org/springframework/kafka/retrytopic/ListenerContainerFactoryConfigurer.java b/spring-kafka/src/main/java/org/springframework/kafka/retrytopic/ListenerContainerFactoryConfigurer.java index 9230a9e4e3..e0b8202d63 100644 --- a/spring-kafka/src/main/java/org/springframework/kafka/retrytopic/ListenerContainerFactoryConfigurer.java +++ b/spring-kafka/src/main/java/org/springframework/kafka/retrytopic/ListenerContainerFactoryConfigurer.java @@ -18,16 +18,12 @@ import java.time.Clock; import java.util.Arrays; -import java.util.Comparator; -import java.util.HashSet; import java.util.List; -import java.util.Set; import java.util.function.Consumer; import java.util.regex.Pattern; import org.apache.commons.logging.LogFactory; -import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.core.log.LogAccessor; import org.springframework.kafka.KafkaException; import org.springframework.kafka.config.ConcurrentKafkaListenerContainerFactory; @@ -65,23 +61,9 @@ */ public class ListenerContainerFactoryConfigurer { - private static final Set> CONFIGURED_FACTORIES_CACHE; - private static final LogAccessor LOGGER = new LogAccessor( LogFactory.getLog(ListenerContainerFactoryConfigurer.class)); - static { - CONFIGURED_FACTORIES_CACHE = new HashSet<>(); - } - - private static final int MIN_POLL_TIMEOUT_VALUE = 100; - - private static final int MAX_POLL_TIMEOUT_VALUE = 5000; - - private static final int POLL_TIMEOUT_DIVISOR = 4; - - private static final long LOWEST_BACKOFF_THRESHOLD = 1500L; - private BackOff providedBlockingBackOff = null; private Class[] blockingExceptionTypes = null; @@ -89,7 +71,7 @@ public class ListenerContainerFactoryConfigurer { private Consumer> containerCustomizer = container -> { }; - private Consumer errorHandlerCustomizer = errorHandler -> { + private Consumer errorHandlerCustomizer = errorHandler -> { }; private final DeadLetterPublishingRecovererFactory deadLetterPublishingRecovererFactory; @@ -100,45 +82,12 @@ public class ListenerContainerFactoryConfigurer { public ListenerContainerFactoryConfigurer(KafkaConsumerBackoffManager kafkaConsumerBackoffManager, DeadLetterPublishingRecovererFactory deadLetterPublishingRecovererFactory, - @Qualifier("internalBackOffClock") Clock clock) { + Clock clock) { this.kafkaConsumerBackoffManager = kafkaConsumerBackoffManager; this.deadLetterPublishingRecovererFactory = deadLetterPublishingRecovererFactory; this.clock = clock; } - /** - * Configures the provided {@link ConcurrentKafkaListenerContainerFactory}. - * @param containerFactory the factory instance to be configured. - * @param configuration the configuration provided by the {@link RetryTopicConfiguration}. - * @return the configured factory instance. - * @deprecated in favor of - * {@link #decorateFactory(ConcurrentKafkaListenerContainerFactory, Configuration)}. - */ - @Deprecated - public ConcurrentKafkaListenerContainerFactory configure( - ConcurrentKafkaListenerContainerFactory containerFactory, Configuration configuration) { - return isCached(containerFactory) - ? containerFactory - : addToCache(doConfigure(containerFactory, configuration, true)); - } - - /** - * Configures the provided {@link ConcurrentKafkaListenerContainerFactory}. - * Meant to be used for the main endpoint, this method ignores the provided backOff values. - * @param containerFactory the factory instance to be configured. - * @param configuration the configuration provided by the {@link RetryTopicConfiguration}. - * @return the configured factory instance. - * @deprecated in favor of - * {@link #decorateFactoryWithoutSettingContainerProperties(ConcurrentKafkaListenerContainerFactory, Configuration)}. - */ - @Deprecated - public ConcurrentKafkaListenerContainerFactory configureWithoutBackOffValues( - ConcurrentKafkaListenerContainerFactory containerFactory, Configuration configuration) { - return isCached(containerFactory) - ? containerFactory - : doConfigure(containerFactory, configuration, false); - } - /** * Decorates the provided {@link ConcurrentKafkaListenerContainerFactory}. * @param factory the factory instance to be decorated. @@ -198,36 +147,12 @@ public final void setBlockingRetryableExceptions(Class... e this.blockingExceptionTypes = Arrays.copyOf(exceptionTypes, exceptionTypes.length); } - private ConcurrentKafkaListenerContainerFactory doConfigure( - ConcurrentKafkaListenerContainerFactory containerFactory, Configuration configuration, - boolean isSetContainerProperties) { - - containerFactory - .setContainerCustomizer(container -> setupBackoffAwareMessageListenerAdapter(container, configuration, isSetContainerProperties)); - containerFactory - .setCommonErrorHandler(createErrorHandler(this.deadLetterPublishingRecovererFactory.create(), configuration)); - return containerFactory; - } - - private boolean isCached(ConcurrentKafkaListenerContainerFactory containerFactory) { - synchronized (CONFIGURED_FACTORIES_CACHE) { - return CONFIGURED_FACTORIES_CACHE.contains(containerFactory); - } - } - - private ConcurrentKafkaListenerContainerFactory addToCache(ConcurrentKafkaListenerContainerFactory containerFactory) { - synchronized (CONFIGURED_FACTORIES_CACHE) { - CONFIGURED_FACTORIES_CACHE.add(containerFactory); - return containerFactory; - } - } - public void setContainerCustomizer(Consumer> containerCustomizer) { Assert.notNull(containerCustomizer, "'containerCustomizer' cannot be null"); this.containerCustomizer = containerCustomizer; } - public void setErrorHandlerCustomizer(Consumer errorHandlerCustomizer) { + public void setErrorHandlerCustomizer(Consumer errorHandlerCustomizer) { this.errorHandlerCustomizer = errorHandlerCustomizer; } @@ -255,60 +180,12 @@ protected void setupBackoffAwareMessageListenerAdapter(ConcurrentMessageListener MessageListener listener = checkAndCast(container.getContainerProperties() .getMessageListener(), MessageListener.class); - if (isSetContainerProperties && !configuration.backOffValues.isEmpty()) { - configurePollTimeoutAndIdlePartitionInterval(container, configuration); - } - container.setupMessageListener(new KafkaBackoffAwareMessageListenerAdapter<>(listener, this.kafkaConsumerBackoffManager, container.getListenerId(), this.clock)); // NOSONAR this.containerCustomizer.accept(container); } - protected void configurePollTimeoutAndIdlePartitionInterval(ConcurrentMessageListenerContainer container, - Configuration configuration) { - - ContainerProperties containerProperties = container.getContainerProperties(); - - long pollTimeoutValue = getPollTimeoutValue(containerProperties, configuration); - long idlePartitionEventInterval = getIdlePartitionInterval(containerProperties, pollTimeoutValue); - - LOGGER.debug(() -> "pollTimeout and idlePartitionEventInterval for back off values " - + configuration.backOffValues + " will be set to " + pollTimeoutValue - + " and " + idlePartitionEventInterval); - - containerProperties - .setIdlePartitionEventInterval(idlePartitionEventInterval); - containerProperties.setPollTimeout(pollTimeoutValue); - } - - protected long getIdlePartitionInterval(ContainerProperties containerProperties, long pollTimeoutValue) { - Long idlePartitionEventInterval = containerProperties.getIdlePartitionEventInterval(); - return idlePartitionEventInterval != null && idlePartitionEventInterval > 0 - ? idlePartitionEventInterval - : pollTimeoutValue; - } - - protected long getPollTimeoutValue(ContainerProperties containerProperties, Configuration configuration) { - if (containerProperties.getPollTimeout() != ContainerProperties.DEFAULT_POLL_TIMEOUT - || configuration.backOffValues.isEmpty()) { - return containerProperties.getPollTimeout(); - } - - Long lowestBackOff = configuration.backOffValues - .stream() - .min(Comparator.naturalOrder()) - .orElseThrow(() -> new IllegalArgumentException("No back off values found!")); - - return lowestBackOff > LOWEST_BACKOFF_THRESHOLD - ? applyLimits(lowestBackOff / POLL_TIMEOUT_DIVISOR) - : MIN_POLL_TIMEOUT_VALUE; - } - - private long applyLimits(long pollTimeoutValue) { - return Math.min(Math.max(pollTimeoutValue, MIN_POLL_TIMEOUT_VALUE), MAX_POLL_TIMEOUT_VALUE); - } - @SuppressWarnings("unchecked") private T checkAndCast(Object obj, Class clazz) { Assert.isAssignable(clazz, obj.getClass(), diff --git a/spring-kafka/src/main/java/org/springframework/kafka/retrytopic/RetryTopicBootstrapper.java b/spring-kafka/src/main/java/org/springframework/kafka/retrytopic/RetryTopicBootstrapper.java deleted file mode 100644 index 168da4ca52..0000000000 --- a/spring-kafka/src/main/java/org/springframework/kafka/retrytopic/RetryTopicBootstrapper.java +++ /dev/null @@ -1,161 +0,0 @@ -/* - * Copyright 2018-2022 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.kafka.retrytopic; - -import java.time.Clock; -import java.util.function.Supplier; - -import org.springframework.beans.factory.BeanFactory; -import org.springframework.beans.factory.NoSuchBeanDefinitionException; -import org.springframework.beans.factory.config.SingletonBeanRegistry; -import org.springframework.beans.factory.support.BeanDefinitionRegistry; -import org.springframework.beans.factory.support.RootBeanDefinition; -import org.springframework.context.ApplicationContext; -import org.springframework.context.ApplicationContextAware; -import org.springframework.context.ConfigurableApplicationContext; -import org.springframework.core.task.TaskExecutor; -import org.springframework.kafka.listener.KafkaBackOffManagerFactory; -import org.springframework.kafka.listener.KafkaConsumerBackoffManager; -import org.springframework.kafka.listener.KafkaConsumerTimingAdjuster; -import org.springframework.kafka.listener.PartitionPausingBackOffManagerFactory; -import org.springframework.kafka.listener.PartitionPausingBackoffManager; -import org.springframework.retry.backoff.ThreadWaitSleeper; - -/** - * - * Bootstraps the {@link RetryTopicConfigurer} context, registering the dependency - * beans and configuring the {@link org.springframework.context.ApplicationListener}s. - * - * Note that if a bean with the same name already exists in the context that one will - * be used instead. - * - * @author Tomaz Fernandes - * @since 2.7 - * @deprecated in favor of {@link org.springframework.kafka.retrytopic.RetryTopicConfigurationSupport} - * - */ -@Deprecated -public class RetryTopicBootstrapper { - - private final ApplicationContext applicationContext; - - private final BeanFactory beanFactory; - - public RetryTopicBootstrapper(ApplicationContext applicationContext, BeanFactory beanFactory) { - if (!ConfigurableApplicationContext.class.isAssignableFrom(applicationContext.getClass()) || - !BeanDefinitionRegistry.class.isAssignableFrom(applicationContext.getClass())) { - throw new IllegalStateException(String.format("ApplicationContext must be implement %s and %s interfaces. Provided: %s", - ConfigurableApplicationContext.class.getSimpleName(), - BeanDefinitionRegistry.class.getSimpleName(), - applicationContext.getClass().getSimpleName())); - } - if (!SingletonBeanRegistry.class.isAssignableFrom(beanFactory.getClass())) { - throw new IllegalStateException("BeanFactory must implement " + SingletonBeanRegistry.class + - " interface. Provided: " + beanFactory.getClass().getSimpleName()); - } - this.beanFactory = beanFactory; - this.applicationContext = applicationContext; - } - - public void bootstrapRetryTopic() { - registerBeans(); - registerSingletons(); - addApplicationListeners(); - } - - private void registerBeans() { - registerIfNotContains(RetryTopicInternalBeanNames.LISTENER_CONTAINER_FACTORY_RESOLVER_NAME, - ListenerContainerFactoryResolver.class); - registerIfNotContains(RetryTopicInternalBeanNames.DESTINATION_TOPIC_PROCESSOR_NAME, - DefaultDestinationTopicProcessor.class); - registerIfNotContains(RetryTopicInternalBeanNames.LISTENER_CONTAINER_FACTORY_CONFIGURER_NAME, - ListenerContainerFactoryConfigurer.class); - registerIfNotContains(RetryTopicInternalBeanNames.DEAD_LETTER_PUBLISHING_RECOVERER_FACTORY_BEAN_NAME, - DeadLetterPublishingRecovererFactory.class); - registerIfNotContains(RetryTopicInternalBeanNames.RETRY_TOPIC_CONFIGURER, RetryTopicConfigurer.class); - registerIfNotContains(RetryTopicInternalBeanNames.DESTINATION_TOPIC_CONTAINER_NAME, - DefaultDestinationTopicResolver.class); - registerIfNotContains(RetryTopicInternalBeanNames.BACKOFF_SLEEPER_BEAN_NAME, ThreadWaitSleeper.class); - registerIfNotContains(RetryTopicInternalBeanNames.INTERNAL_KAFKA_CONSUMER_BACKOFF_MANAGER_FACTORY, - PartitionPausingBackOffManagerFactory.class); - - // Register a RetryTopicNamesProviderFactory implementation only if none is already present in the context - try { - this.applicationContext.getBean(RetryTopicNamesProviderFactory.class); - } - catch (NoSuchBeanDefinitionException e) { - ((BeanDefinitionRegistry) this.applicationContext).registerBeanDefinition( - RetryTopicInternalBeanNames.RETRY_TOPIC_NAMES_PROVIDER_FACTORY, - new RootBeanDefinition(SuffixingRetryTopicNamesProviderFactory.class)); - } - } - - private void registerSingletons() { - registerSingletonIfNotContains(RetryTopicInternalBeanNames.INTERNAL_BACKOFF_CLOCK_BEAN_NAME, Clock::systemUTC); - registerSingletonIfNotContains(RetryTopicInternalBeanNames.KAFKA_CONSUMER_BACKOFF_MANAGER, - this::createKafkaConsumerBackoffManager); - } - - private void addApplicationListeners() { - ConfigurableApplicationContext context = (ConfigurableApplicationContext) this.applicationContext; - context.addApplicationListener(this.applicationContext.getBean( - RetryTopicInternalBeanNames.DESTINATION_TOPIC_CONTAINER_NAME, DefaultDestinationTopicResolver.class)); - context.addApplicationListener(this.applicationContext.getBean( - RetryTopicInternalBeanNames.KAFKA_CONSUMER_BACKOFF_MANAGER, PartitionPausingBackoffManager.class)); - } - - private KafkaConsumerBackoffManager createKafkaConsumerBackoffManager() { - KafkaBackOffManagerFactory factory = this.applicationContext - .getBean(RetryTopicInternalBeanNames.INTERNAL_KAFKA_CONSUMER_BACKOFF_MANAGER_FACTORY, - KafkaBackOffManagerFactory.class); - if (ApplicationContextAware.class.isAssignableFrom(factory.getClass())) { - ((ApplicationContextAware) factory).setApplicationContext(this.applicationContext); - } - if (PartitionPausingBackOffManagerFactory.class.isAssignableFrom(factory.getClass())) { - setupTimingAdjustingBackOffFactory((PartitionPausingBackOffManagerFactory) factory); - } - return factory.create(); - } - - private void setupTimingAdjustingBackOffFactory(PartitionPausingBackOffManagerFactory factory) { - if (this.applicationContext.containsBean(RetryTopicInternalBeanNames.BACKOFF_TASK_EXECUTOR)) { - factory.setTaskExecutor(this.applicationContext - .getBean(RetryTopicInternalBeanNames.BACKOFF_TASK_EXECUTOR, TaskExecutor.class)); - } - if (this.applicationContext.containsBean( - RetryTopicInternalBeanNames.INTERNAL_BACKOFF_TIMING_ADJUSTMENT_MANAGER)) { - factory.setTimingAdjustmentManager(this.applicationContext - .getBean(RetryTopicInternalBeanNames.INTERNAL_BACKOFF_TIMING_ADJUSTMENT_MANAGER, - KafkaConsumerTimingAdjuster.class)); - } - } - - private void registerIfNotContains(String beanName, Class beanClass) { - BeanDefinitionRegistry registry = (BeanDefinitionRegistry) this.applicationContext; - if (!registry.containsBeanDefinition(beanName)) { - registry.registerBeanDefinition(beanName, - new RootBeanDefinition(beanClass)); - } - } - - private void registerSingletonIfNotContains(String beanName, Supplier singletonSupplier) { - if (!this.applicationContext.containsBeanDefinition(beanName)) { - ((SingletonBeanRegistry) this.beanFactory).registerSingleton(beanName, singletonSupplier.get()); - } - } - -} diff --git a/spring-kafka/src/main/java/org/springframework/kafka/retrytopic/RetryTopicComponentFactory.java b/spring-kafka/src/main/java/org/springframework/kafka/retrytopic/RetryTopicComponentFactory.java index cfae581f93..e378e321ba 100644 --- a/spring-kafka/src/main/java/org/springframework/kafka/retrytopic/RetryTopicComponentFactory.java +++ b/spring-kafka/src/main/java/org/springframework/kafka/retrytopic/RetryTopicComponentFactory.java @@ -21,12 +21,12 @@ import org.springframework.beans.factory.BeanFactory; import org.springframework.kafka.config.KafkaListenerContainerFactory; import org.springframework.kafka.config.KafkaListenerEndpoint; +import org.springframework.kafka.listener.ContainerPartitionPausingBackOffManagerFactory; import org.springframework.kafka.listener.DeadLetterPublishingRecoverer; import org.springframework.kafka.listener.KafkaBackOffManagerFactory; import org.springframework.kafka.listener.KafkaConsumerBackoffManager; import org.springframework.kafka.listener.ListenerContainerRegistry; import org.springframework.kafka.listener.MessageListenerContainer; -import org.springframework.kafka.listener.PartitionPausingBackOffManagerFactory; import org.springframework.kafka.listener.adapter.KafkaBackoffAwareMessageListenerAdapter; /** @@ -153,7 +153,7 @@ public RetryTopicNamesProviderFactory retryTopicNamesProviderFactory() { * @return the instance. */ public KafkaBackOffManagerFactory kafkaBackOffManagerFactory(ListenerContainerRegistry registry) { - return new PartitionPausingBackOffManagerFactory(registry); + return new ContainerPartitionPausingBackOffManagerFactory(registry); } /** diff --git a/spring-kafka/src/main/java/org/springframework/kafka/retrytopic/RetryTopicConfigurationSupport.java b/spring-kafka/src/main/java/org/springframework/kafka/retrytopic/RetryTopicConfigurationSupport.java index 390caaf63a..d77ab1c4b7 100644 --- a/spring-kafka/src/main/java/org/springframework/kafka/retrytopic/RetryTopicConfigurationSupport.java +++ b/spring-kafka/src/main/java/org/springframework/kafka/retrytopic/RetryTopicConfigurationSupport.java @@ -16,10 +16,10 @@ package org.springframework.kafka.retrytopic; -import java.time.Clock; import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.Consumer; import java.util.stream.Collectors; @@ -27,28 +27,27 @@ import org.springframework.beans.factory.BeanFactory; import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.context.ApplicationContext; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; -import org.springframework.core.task.TaskExecutor; import org.springframework.kafka.annotation.EnableKafkaRetryTopic; import org.springframework.kafka.annotation.KafkaListenerAnnotationBeanPostProcessor; import org.springframework.kafka.config.KafkaListenerConfigUtils; import org.springframework.kafka.config.KafkaListenerEndpoint; import org.springframework.kafka.listener.CommonErrorHandler; import org.springframework.kafka.listener.ConcurrentMessageListenerContainer; +import org.springframework.kafka.listener.ContainerPartitionPausingBackOffManagerFactory; +import org.springframework.kafka.listener.ContainerPausingBackOffHandler; import org.springframework.kafka.listener.DeadLetterPublishingRecoverer; import org.springframework.kafka.listener.DefaultErrorHandler; import org.springframework.kafka.listener.ExceptionClassifier; import org.springframework.kafka.listener.KafkaBackOffManagerFactory; import org.springframework.kafka.listener.KafkaConsumerBackoffManager; -import org.springframework.kafka.listener.KafkaConsumerTimingAdjuster; +import org.springframework.kafka.listener.ListenerContainerPauseService; import org.springframework.kafka.listener.ListenerContainerRegistry; import org.springframework.kafka.listener.MessageListenerContainer; -import org.springframework.kafka.listener.PartitionPausingBackOffManagerFactory; -import org.springframework.kafka.listener.WakingKafkaConsumerTimingAdjuster; import org.springframework.kafka.support.JavaUtils; -import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; +import org.springframework.lang.Nullable; +import org.springframework.scheduling.TaskScheduler; import org.springframework.util.Assert; import org.springframework.util.backoff.BackOff; import org.springframework.util.backoff.FixedBackOff; @@ -64,13 +63,20 @@ * {@link EnableKafkaRetryTopic @EnableRetryTopic}. * * @author Tomaz Fernandes + * @author Gary Russell * @since 2.9 */ public class RetryTopicConfigurationSupport { + private static final String BACK_OFF_MANAGER_THREAD_EXECUTOR_BEAN_NAME = "backOffManagerThreadExecutor"; + + private static final AtomicBoolean ONLY_ONE_ALLOWED = new AtomicBoolean(true); + private final RetryTopicComponentFactory componentFactory = createComponentFactory(); - private static final String BACK_OFF_MANAGER_THREAD_EXECUTOR_BEAN_NAME = "backOffManagerThreadExecutor"; + protected RetryTopicConfigurationSupport() { + Assert.state(ONLY_ONE_ALLOWED.getAndSet(false), "Only one 'RetryTopicConfigurationSupport' is allowed"); + } /** * Return a global {@link RetryTopicConfigurer} for configuring retry topics @@ -265,69 +271,30 @@ protected Consumer configureDestinationTopicResolver() * and return a different {@link KafkaBackOffManagerFactory}. * @param registry the {@link ListenerContainerRegistry} to be used to fetch the * {@link MessageListenerContainer} at runtime to be backed off. - * @param taskExecutor the {@link TaskExecutor} to be used with the - * {@link KafkaConsumerTimingAdjuster}. + * @param wrapper a {@link RetryTopicSchedulerWrapper}. + * @param taskScheduler a {@link TaskScheduler}. * @return the instance. */ @Bean(name = KafkaListenerConfigUtils.KAFKA_CONSUMER_BACK_OFF_MANAGER_BEAN_NAME) public KafkaConsumerBackoffManager kafkaConsumerBackoffManager( @Qualifier(KafkaListenerConfigUtils.KAFKA_LISTENER_ENDPOINT_REGISTRY_BEAN_NAME) - ListenerContainerRegistry registry, - @Qualifier(BACK_OFF_MANAGER_THREAD_EXECUTOR_BEAN_NAME) TaskExecutor taskExecutor) { + ListenerContainerRegistry registry, @Nullable RetryTopicSchedulerWrapper wrapper, + @Nullable TaskScheduler taskScheduler) { KafkaBackOffManagerFactory backOffManagerFactory = this.componentFactory.kafkaBackOffManagerFactory(registry); - JavaUtils.INSTANCE.acceptIfInstanceOf(PartitionPausingBackOffManagerFactory.class, backOffManagerFactory, - factory -> configurePartitionPausingFactory(taskExecutor, factory)); + JavaUtils.INSTANCE.acceptIfInstanceOf(ContainerPartitionPausingBackOffManagerFactory.class, backOffManagerFactory, + factory -> configurePartitionPausingFactory(factory, registry, + wrapper != null ? wrapper.getScheduler() : taskScheduler)); return backOffManagerFactory.create(); } - /** - * Internal method for processing the {@link PartitionPausingBackOffManagerFactory}. - * @param taskExecutor the {@link TaskExecutor} instance to be used with - * {@link WakingKafkaConsumerTimingAdjuster}. Consider overriding the - * {@link #configureKafkaBackOffManager} method for furher customization. - * @param factory the factory instance. - */ - private void configurePartitionPausingFactory(TaskExecutor taskExecutor, - PartitionPausingBackOffManagerFactory factory) { - KafkaBackOffManagerConfigurer configurer = new KafkaBackOffManagerConfigurer(); - configureKafkaBackOffManager(configurer); - Assert.isTrue(!configurer.timingAdjustmentEnabled - || configurer.maxThreadPoolSize == null - || ThreadPoolTaskExecutor.class.isAssignableFrom(taskExecutor.getClass()), - "TaskExecutor must be an instance of ThreadPoolTaskExecutor to set maxThreadPoolSize"); - factory.setTimingAdjustmentEnabled(configurer.timingAdjustmentEnabled); - if (ThreadPoolTaskExecutor.class.isAssignableFrom(taskExecutor.getClass())) { - JavaUtils.INSTANCE - .acceptIfNotNull(configurer.maxThreadPoolSize, poolSize -> ((ThreadPoolTaskExecutor) taskExecutor) - .setMaxPoolSize(poolSize)); - } - JavaUtils.INSTANCE - .acceptIfCondition(configurer.timingAdjustmentEnabled, taskExecutor, factory::setTaskExecutor) - .acceptIfNotNull(configurer.clock, factory::setClock); - } - - /** - * Create the {@link TaskExecutor} instance that will be used with the - * {@link WakingKafkaConsumerTimingAdjuster}, if timing adjustment is enabled. - * @return the instance. - */ - @Bean(name = BACK_OFF_MANAGER_THREAD_EXECUTOR_BEAN_NAME) - public TaskExecutor backoffManagerTaskExecutor() { - KafkaBackOffManagerConfigurer configurer = new KafkaBackOffManagerConfigurer(); - configureKafkaBackOffManager(configurer); - return configurer.timingAdjustmentEnabled - ? new ThreadPoolTaskExecutor() - : task -> { - }; - } + private void configurePartitionPausingFactory(ContainerPartitionPausingBackOffManagerFactory factory, + ListenerContainerRegistry registry, @Nullable TaskScheduler scheduler) { - /** - * Override this method to configure the {@link KafkaConsumerBackoffManager}. - * @param backOffManagerConfigurer a {@link KafkaBackOffManagerConfigurer}. - */ - protected void configureKafkaBackOffManager(KafkaBackOffManagerConfigurer backOffManagerConfigurer) { + Assert.notNull(scheduler, "Either a RetryTopicSchedulerWrapper or TaskScheduler bean is required"); + factory.setBackOffHandler(new ContainerPausingBackOffHandler( + new ListenerContainerPauseService(registry, scheduler))); } /** @@ -339,12 +306,6 @@ protected RetryTopicComponentFactory createComponentFactory() { return new RetryTopicComponentFactory(); } - @Deprecated - @Bean(name = RetryTopicInternalBeanNames.RETRY_TOPIC_BOOTSTRAPPER) - RetryTopicBootstrapper retryTopicBootstrapper(ApplicationContext context) { - return new RetryTopicBootstrapper(context, context.getAutowireCapableBeanFactory()); - } - /** * Configure blocking retries to be used along non-blocking. */ @@ -383,62 +344,12 @@ public BlockingRetriesConfigurer backOff(BackOff backoff) { } } - /** - * Configure the {@link KafkaConsumerBackoffManager} instance. - */ - public static class KafkaBackOffManagerConfigurer { - - boolean timingAdjustmentEnabled = true; - - private Integer maxThreadPoolSize = null; - - private Clock clock; - - /** - * Disable timing adjustment for the delays. By choosing this option records - * won't be processed exactly at the proper time. It's guaranteed however that - * records won't be processed before their due time. - * @return the configurer. - * @see WakingKafkaConsumerTimingAdjuster - */ - public KafkaBackOffManagerConfigurer disableTimingAdjustment() { - this.timingAdjustmentEnabled = false; - return this; - } - - /** - * Set the maximum thread pool size to be used by the - * {@link WakingKafkaConsumerTimingAdjuster}. This - * {@link KafkaConsumerTimingAdjuster} implementation spawns threads that will - * sleep for a calculated time, and after that will - * {@link org.apache.kafka.clients.consumer.Consumer#wakeup()} the consumer, in - * order to improve delay precision. - * @param maxThreadPoolSize the maximum thread pool size. - * @return the configurer. - */ - public KafkaBackOffManagerConfigurer setMaxThreadPoolSize(int maxThreadPoolSize) { - this.maxThreadPoolSize = maxThreadPoolSize; - return this; - } - - /** - * Set the {@link Clock} instance to be used with the - * {@link KafkaConsumerBackoffManager}. - * @param clock the clock instance. - * @return the configurer. - */ - public KafkaBackOffManagerConfigurer setClock(Clock clock) { - this.clock = clock; - return this; - } - } - /** * Configure customizers for components instantiated by the retry topics feature. */ public static class CustomizersConfigurer { - private Consumer errorHandlerCustomizer; + private Consumer errorHandlerCustomizer; private Consumer> listenerContainerCustomizer; @@ -451,7 +362,7 @@ public static class CustomizersConfigurer { * @return the configurer. * @see DefaultErrorHandler */ - public CustomizersConfigurer customizeErrorHandler(Consumer errorHandlerCustomizer) { + public CustomizersConfigurer customizeErrorHandler(Consumer errorHandlerCustomizer) { this.errorHandlerCustomizer = errorHandlerCustomizer; return this; } diff --git a/spring-kafka/src/main/java/org/springframework/kafka/retrytopic/RetryTopicConfigurer.java b/spring-kafka/src/main/java/org/springframework/kafka/retrytopic/RetryTopicConfigurer.java index 292d6d3f6c..2473799332 100644 --- a/spring-kafka/src/main/java/org/springframework/kafka/retrytopic/RetryTopicConfigurer.java +++ b/spring-kafka/src/main/java/org/springframework/kafka/retrytopic/RetryTopicConfigurer.java @@ -222,8 +222,6 @@ public class RetryTopicConfigurer implements BeanFactoryAware { private final RetryTopicNamesProviderFactory retryTopicNamesProviderFactory; - private boolean useLegacyFactoryConfigurer = false; - /** * Create an instance with the provided properties. * @param destinationTopicProcessor the destination topic processor. @@ -385,35 +383,29 @@ private EndpointHandlerMethod getDltEndpointHandlerMethodOrDefault(EndpointHandl return dltEndpointHandlerMethod != null ? dltEndpointHandlerMethod : DEFAULT_DLT_HANDLER; } - @SuppressWarnings("deprecation") private KafkaListenerContainerFactory resolveAndConfigureFactoryForMainEndpoint( KafkaListenerContainerFactory providedFactory, String defaultFactoryBeanName, RetryTopicConfiguration configuration) { + ConcurrentKafkaListenerContainerFactory resolvedFactory = this.containerFactoryResolver .resolveFactoryForMainEndpoint(providedFactory, defaultFactoryBeanName, configuration.forContainerFactoryResolver()); - return this.useLegacyFactoryConfigurer - ? this.listenerContainerFactoryConfigurer - .configureWithoutBackOffValues(resolvedFactory, configuration.forContainerFactoryConfigurer()) - : this.listenerContainerFactoryConfigurer - .decorateFactoryWithoutSettingContainerProperties(resolvedFactory, - configuration.forContainerFactoryConfigurer()); + return this.listenerContainerFactoryConfigurer + .decorateFactoryWithoutSettingContainerProperties(resolvedFactory, + configuration.forContainerFactoryConfigurer()); } - @SuppressWarnings("deprecation") private KafkaListenerContainerFactory resolveAndConfigureFactoryForRetryEndpoint( KafkaListenerContainerFactory providedFactory, String defaultFactoryBeanName, RetryTopicConfiguration configuration) { + ConcurrentKafkaListenerContainerFactory resolvedFactory = this.containerFactoryResolver.resolveFactoryForRetryEndpoint(providedFactory, defaultFactoryBeanName, configuration.forContainerFactoryResolver()); - return this.useLegacyFactoryConfigurer - ? this.listenerContainerFactoryConfigurer.configure(resolvedFactory, - configuration.forContainerFactoryConfigurer()) - : this.listenerContainerFactoryConfigurer - .decorateFactory(resolvedFactory, configuration.forContainerFactoryConfigurer()); + return this.listenerContainerFactoryConfigurer + .decorateFactory(resolvedFactory, configuration.forContainerFactoryConfigurer()); } private void throwIfMultiMethodEndpoint(MethodKafkaListenerEndpoint mainEndpoint) { @@ -430,17 +422,6 @@ public static EndpointHandlerMethod createHandlerMethodWith(Object bean, Method return new EndpointHandlerMethod(bean, method); } - /** - * Set to true if you want the {@link ListenerContainerFactoryConfigurer} to - * behave as before 2.8.3. - * @param useLegacyFactoryConfigurer Whether to use the legacy factory configuration. - * @deprecated for removal after the deprecated legacy configuration methods are removed. - */ - @Deprecated - public void useLegacyFactoryConfigurer(boolean useLegacyFactoryConfigurer) { - this.useLegacyFactoryConfigurer = useLegacyFactoryConfigurer; - } - @Override public void setBeanFactory(BeanFactory beanFactory) throws BeansException { this.beanFactory = beanFactory; diff --git a/spring-kafka/src/main/java/org/springframework/kafka/retrytopic/RetryTopicInternalBeanNames.java b/spring-kafka/src/main/java/org/springframework/kafka/retrytopic/RetryTopicInternalBeanNames.java deleted file mode 100644 index 97ab0f7aca..0000000000 --- a/spring-kafka/src/main/java/org/springframework/kafka/retrytopic/RetryTopicInternalBeanNames.java +++ /dev/null @@ -1,117 +0,0 @@ -/* - * Copyright 2021-2022 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.kafka.retrytopic; - -/** - * - * Contains the internal bean names that will be used by the retryable topic configuration. - * - * If you provide a bean of your own with the same name, your instance will be used instead - * of the default one. - * - * @author Tomaz Fernandes - * @since 2.7 - * @deprecated in favor of {@link RetryTopicBeanNames} - * - */ -@Deprecated -public abstract class RetryTopicInternalBeanNames { - - /** - * {@link DestinationTopicProcessor} bean name. - */ - public static final String DESTINATION_TOPIC_PROCESSOR_NAME = "internalDestinationTopicProcessor"; - - /** - * {@link org.springframework.kafka.listener.KafkaConsumerBackoffManager} bean name. - */ - public static final String KAFKA_CONSUMER_BACKOFF_MANAGER = "internalKafkaConsumerBackoffManager"; - - /** - * {@link ListenerContainerFactoryResolver} bean name. - */ - public static final String LISTENER_CONTAINER_FACTORY_RESOLVER_NAME = "internalListenerContainerFactoryResolver"; - - /** - * {@link ListenerContainerFactoryConfigurer} bean name. - */ - public static final String LISTENER_CONTAINER_FACTORY_CONFIGURER_NAME = "internalListenerContainerFactoryConfigurer"; - - /** - * {@link DeadLetterPublishingRecovererFactory} bean name. - */ - public static final String DEAD_LETTER_PUBLISHING_RECOVERER_FACTORY_BEAN_NAME = - "internalDeadLetterPublishingRecovererProvider"; - - /** - * {@link DestinationTopicContainer} bean name. - */ - public static final String DESTINATION_TOPIC_CONTAINER_NAME = "internalDestinationTopicContainer"; - - /** - * The default {@link org.springframework.kafka.config.KafkaListenerContainerFactory} - * bean name that will be looked up if no other is provided. - */ - public static final String DEFAULT_LISTENER_FACTORY_BEAN_NAME = "internalRetryTopicListenerContainerFactory"; - - /** - * {@link org.springframework.retry.backoff.Sleeper} bean name. - */ - public static final String BACKOFF_SLEEPER_BEAN_NAME = "internalBackoffSleeper"; - - /** - * {@link org.springframework.core.task.TaskExecutor} bean name to be used. - * in the {@link org.springframework.kafka.listener.WakingKafkaConsumerTimingAdjuster} - */ - public static final String BACKOFF_TASK_EXECUTOR = "internalBackOffTaskExecutor"; - - /** - * {@link org.springframework.kafka.listener.KafkaConsumerTimingAdjuster} bean name. - */ - public static final String INTERNAL_BACKOFF_TIMING_ADJUSTMENT_MANAGER = "internalKafkaConsumerTimingAdjustmentManager"; - - /** - * {@link org.springframework.kafka.listener.KafkaBackOffManagerFactory} bean name. - */ - public static final String INTERNAL_KAFKA_CONSUMER_BACKOFF_MANAGER_FACTORY = "internalKafkaConsumerBackOffManagerFactory"; - - /** - * {@link RetryTopicNamesProviderFactory} bean name. - */ - public static final String RETRY_TOPIC_NAMES_PROVIDER_FACTORY = "internalRetryTopicNamesProviderFactory"; - - /** - * The {@link java.time.Clock} bean name that will be used for backing off partitions. - */ - public static final String INTERNAL_BACKOFF_CLOCK_BEAN_NAME = "internalBackOffClock"; - - /** - * Default {@link org.springframework.kafka.core.KafkaTemplate} bean name for publishing to retry topics. - */ - public static final String DEFAULT_KAFKA_TEMPLATE_BEAN_NAME = "retryTopicDefaultKafkaTemplate"; - - /** - * {@link RetryTopicBootstrapper} bean name. - */ - public static final String RETRY_TOPIC_BOOTSTRAPPER = "internalRetryTopicBootstrapper"; - - /** - * {@link RetryTopicConfigurer} bean name. - */ - public static final String RETRY_TOPIC_CONFIGURER = "internalRetryTopicConfigurer"; - -} diff --git a/spring-kafka/src/main/java/org/springframework/kafka/retrytopic/RetryTopicSchedulerWrapper.java b/spring-kafka/src/main/java/org/springframework/kafka/retrytopic/RetryTopicSchedulerWrapper.java new file mode 100644 index 0000000000..ec7cc13d90 --- /dev/null +++ b/spring-kafka/src/main/java/org/springframework/kafka/retrytopic/RetryTopicSchedulerWrapper.java @@ -0,0 +1,68 @@ +/* + * Copyright 2022 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.kafka.retrytopic; + +import org.springframework.beans.factory.DisposableBean; +import org.springframework.beans.factory.InitializingBean; +import org.springframework.scheduling.TaskScheduler; + +/** + * A wrapper class for a {@link TaskScheduler} to use for scheduling container resumption + * when a partition has been paused for a retry topic. Using this class prevents breaking + * Spring Boot's auto configuration for other frameworks. Use this if you are using Spring + * Boot and do not want to use that auto configured scheduler (if it is configured). This + * framework requires a scheduler bean and looks for one in this order: 1. A single + * instance of this class, 2. a single {@link TaskScheduler} bean, 3. when multiple + * {@link TaskScheduler}s are present, a bean with the name {@code taskScheduler}. + * If you use this class, you should provide a {@link TaskScheduler} that is not defined + * as a bean; this class will maintain the scheduler's lifecycle. + * + * @author Gary Russell + * @since 2.9 + */ +public class RetryTopicSchedulerWrapper implements InitializingBean, DisposableBean { + + private final TaskScheduler scheduler; + + /** + * Create a wrapper for the supplied scheduler. + * @param scheduler the scheduler + */ + public RetryTopicSchedulerWrapper(TaskScheduler scheduler) { + this.scheduler = scheduler; + } + + public TaskScheduler getScheduler() { + return this.scheduler; + } + + @Override + public void afterPropertiesSet() throws Exception { + if (this.scheduler instanceof InitializingBean) { + ((InitializingBean) this.scheduler).afterPropertiesSet(); + } + } + + @Override + public void destroy() throws Exception { + if (this.scheduler instanceof DisposableBean) { + ((DisposableBean) this.scheduler).destroy(); + } + + } + +} diff --git a/spring-kafka/src/test/java/org/springframework/kafka/annotation/EnableKafkaIntegrationTests.java b/spring-kafka/src/test/java/org/springframework/kafka/annotation/EnableKafkaIntegrationTests.java index 69504e843a..9da4731c29 100644 --- a/spring-kafka/src/test/java/org/springframework/kafka/annotation/EnableKafkaIntegrationTests.java +++ b/spring-kafka/src/test/java/org/springframework/kafka/annotation/EnableKafkaIntegrationTests.java @@ -42,8 +42,10 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutionException; +import java.util.concurrent.Semaphore; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; import java.util.stream.Collectors; @@ -2240,6 +2242,10 @@ public static class SeekToLastOnIdleListener extends AbstractConsumerSeekAware { final CountDownLatch latch4 = new CountDownLatch(2); + final AtomicInteger idleRewinds = new AtomicInteger(); + + final Semaphore semaphore = new Semaphore(0); + final Set consumerThreads = ConcurrentHashMap.newKeySet(); @KafkaListener(id = "seekOnIdle", topics = "seekOnIdle", autoStartup = "false", concurrency = "2", @@ -2249,13 +2255,20 @@ public void listen(@SuppressWarnings("unused") String in, Acknowledgment ack) { this.latch2.countDown(); this.latch1.countDown(); ack.acknowledge(); + this.semaphore.release(); } @Override public void onIdleContainer(Map assignments, ConsumerSeekCallback callback) { - if (this.latch1.getCount() > 0) { + if (this.semaphore.availablePermits() > 0 && this.idleRewinds.getAndIncrement() < 10) { + try { + this.semaphore.acquire(); + } + catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } assignments.keySet().forEach(tp -> callback.seekRelative(tp.topic(), tp.partition(), -1, true)); } } diff --git a/spring-kafka/src/test/java/org/springframework/kafka/listener/PartitionPausingBackoffManagerTests.java b/spring-kafka/src/test/java/org/springframework/kafka/listener/PartitionPausingBackoffManagerTests.java deleted file mode 100644 index 6094080eab..0000000000 --- a/spring-kafka/src/test/java/org/springframework/kafka/listener/PartitionPausingBackoffManagerTests.java +++ /dev/null @@ -1,211 +0,0 @@ -/* - * Copyright 2019-2021 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.kafka.listener; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.catchThrowableOfType; -import static org.mockito.BDDMockito.given; -import static org.mockito.BDDMockito.then; -import static org.mockito.Mockito.times; - -import java.time.Clock; -import java.time.Instant; - -import org.apache.kafka.clients.consumer.Consumer; -import org.apache.kafka.common.TopicPartition; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.mockito.Mock; -import org.mockito.junit.jupiter.MockitoExtension; - -import org.springframework.kafka.config.KafkaListenerEndpointRegistry; -import org.springframework.kafka.event.ListenerContainerPartitionIdleEvent; -import org.springframework.kafka.retrytopic.TestClockUtils; - -/** - * @author Tomaz Fernandes - * @since 2.7 - */ -@ExtendWith(MockitoExtension.class) -class PartitionPausingBackoffManagerTests { - - @Mock - private KafkaListenerEndpointRegistry registry; - - @Mock - private MessageListenerContainer listenerContainer; - - @Mock - private WakingKafkaConsumerTimingAdjuster timingAdjustmentManager; - - @Mock - private ListenerContainerPartitionIdleEvent partitionIdleEvent; - - @Mock - private Consumer consumer; - - @Mock - private ContainerProperties containerProperties; - - private static final String testListenerId = "testListenerId"; - - private static final Clock clock = TestClockUtils.CLOCK; - - private static final String testTopic = "testTopic"; - - private static final int testPartition = 0; - - private static final long pollTimeout = 500L; - - private static final TopicPartition topicPartition = new TopicPartition(testTopic, testPartition); - - private static final long originalTimestamp = Instant.now(clock).minusMillis(2500L).toEpochMilli(); - - @Test - void shouldBackOffGivenDueTimestampIsLater() { - - // given - given(this.registry.getListenerContainer(testListenerId)).willReturn(listenerContainer); - given(registry.getListenerContainer(testListenerId)).willReturn(listenerContainer); - PartitionPausingBackoffManager backoffManager = - new PartitionPausingBackoffManager(registry, timingAdjustmentManager, clock); - - long dueTimestamp = originalTimestamp + 5000; - KafkaConsumerBackoffManager.Context context = - backoffManager.createContext(dueTimestamp, testListenerId, topicPartition, consumer); - - // then - KafkaBackoffException backoffException = catchThrowableOfType(() -> backoffManager.backOffIfNecessary(context), - KafkaBackoffException.class); - - // when - assertThat(backoffException.getDueTimestamp()).isEqualTo(dueTimestamp); - assertThat(backoffException.getListenerId()).isEqualTo(testListenerId); - assertThat(backoffException.getTopicPartition()).isEqualTo(topicPartition); - assertThat(backoffManager.getBackOffContext(topicPartition)).isEqualTo(context); - then(listenerContainer).should(times(1)).pausePartition(topicPartition); - } - - @Test - void shouldNotBackoffGivenDueTimestampIsPast() { - - // given - PartitionPausingBackoffManager backoffManager = - new PartitionPausingBackoffManager(registry, timingAdjustmentManager, clock); - KafkaConsumerBackoffManager.Context context = - backoffManager.createContext(originalTimestamp - 5000, testListenerId, topicPartition, consumer); - - // then - backoffManager.backOffIfNecessary(context); - - // when - assertThat(backoffManager.getBackOffContext(topicPartition)).isNull(); - then(listenerContainer).should(times(0)).pausePartition(topicPartition); - } - - @Test - void shouldDoNothingIfIdleBeforeDueTimestamp() { - - // given - given(this.partitionIdleEvent.getTopicPartition()).willReturn(topicPartition); - given(registry.getListenerContainer(testListenerId)).willReturn(listenerContainer); - given(listenerContainer.getContainerProperties()).willReturn(containerProperties); - given(containerProperties.getPollTimeout()).willReturn(pollTimeout); - - PartitionPausingBackoffManager backoffManager = - new PartitionPausingBackoffManager(registry, timingAdjustmentManager, clock); - - long dueTimestamp = originalTimestamp + 5000; - KafkaConsumerBackoffManager.Context context = - backoffManager.createContext(dueTimestamp, testListenerId, topicPartition, consumer); - backoffManager.addBackoff(context, topicPartition); - - // then - backoffManager.onApplicationEvent(partitionIdleEvent); - - // when - assertThat(backoffManager.getBackOffContext(topicPartition)).isEqualTo(context); - then(timingAdjustmentManager).should(times(1)).adjustTiming( - consumer, topicPartition, pollTimeout, getTimeUntilDue(dueTimestamp)); - then(listenerContainer).should(times(0)).resumePartition(topicPartition); - } - - private long getTimeUntilDue(long dueTimestamp) { - return dueTimestamp - Instant.now(clock).toEpochMilli(); - } - - @Test - void shouldResumePartitionIfIdleAfterDueTimestamp() { - - // given - given(registry.getListenerContainer(testListenerId)).willReturn(listenerContainer); - given(listenerContainer.getContainerProperties()).willReturn(containerProperties); - given(containerProperties.getPollTimeout()).willReturn(500L); - - given(this.registry.getListenerContainer(testListenerId)).willReturn(listenerContainer); - given(this.partitionIdleEvent.getTopicPartition()).willReturn(topicPartition); - long dueTimestamp = originalTimestamp - 5000; - given(timingAdjustmentManager - .adjustTiming(consumer, topicPartition, pollTimeout, getTimeUntilDue(dueTimestamp))) - .willReturn(0L); - PartitionPausingBackoffManager backoffManager = - new PartitionPausingBackoffManager(registry, timingAdjustmentManager, clock); - KafkaConsumerBackoffManager.Context context = - backoffManager.createContext(dueTimestamp, testListenerId, topicPartition, consumer); - backoffManager.addBackoff(context, topicPartition); - - // when - backoffManager.onApplicationEvent(partitionIdleEvent); - - // then - then(timingAdjustmentManager).should(times(1)).adjustTiming( - consumer, topicPartition, pollTimeout, getTimeUntilDue(dueTimestamp)); - assertThat(backoffManager.getBackOffContext(topicPartition)).isNull(); - then(listenerContainer).should(times(1)).resumePartition(topicPartition); - } - - @Test - void shouldResumePartitionIfCorrectionIsApplied() { - - // given - given(registry.getListenerContainer(testListenerId)).willReturn(listenerContainer); - given(listenerContainer.getContainerProperties()).willReturn(containerProperties); - long pollTimeout = 500L; - given(containerProperties.getPollTimeout()).willReturn(pollTimeout); - - given(this.registry.getListenerContainer(testListenerId)).willReturn(listenerContainer); - given(this.partitionIdleEvent.getTopicPartition()).willReturn(topicPartition); - long dueTimestamp = originalTimestamp + 5000; - given(timingAdjustmentManager - .adjustTiming(consumer, topicPartition, pollTimeout, getTimeUntilDue(dueTimestamp))) - .willReturn(1000L); - PartitionPausingBackoffManager backoffManager = - new PartitionPausingBackoffManager(registry, timingAdjustmentManager, clock); - KafkaConsumerBackoffManager.Context context = - backoffManager.createContext(dueTimestamp, testListenerId, topicPartition, consumer); - backoffManager.addBackoff(context, topicPartition); - - // when - backoffManager.onApplicationEvent(partitionIdleEvent); - - // then - then(timingAdjustmentManager).should(times(1)).adjustTiming( - consumer, topicPartition, pollTimeout, getTimeUntilDue(dueTimestamp)); - then(listenerContainer).should(times(1)).resumePartition(topicPartition); - assertThat(backoffManager.getBackOffContext(topicPartition)).isNull(); - } -} diff --git a/spring-kafka/src/test/java/org/springframework/kafka/listener/WakingKafkaConsumerTimingAdjusterTests.java b/spring-kafka/src/test/java/org/springframework/kafka/listener/WakingKafkaConsumerTimingAdjusterTests.java deleted file mode 100644 index 837d59217d..0000000000 --- a/spring-kafka/src/test/java/org/springframework/kafka/listener/WakingKafkaConsumerTimingAdjusterTests.java +++ /dev/null @@ -1,82 +0,0 @@ -/* - * Copyright 2019-2021 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.kafka.listener; - -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.BDDMockito.then; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.never; -import static org.mockito.Mockito.times; - -import org.apache.kafka.clients.consumer.Consumer; -import org.apache.kafka.common.TopicPartition; -import org.junit.jupiter.api.Test; -import org.mockito.ArgumentCaptor; - -import org.springframework.core.task.TaskExecutor; -import org.springframework.retry.backoff.Sleeper; - -/** - * @author Tomaz Fernandes - * @since 2.7 - */ -class WakingKafkaConsumerTimingAdjusterTests { - - @Test - void testAppliesCorrectionIfInCorrectionWindow() throws InterruptedException { - Sleeper sleeper = mock(Sleeper.class); - TaskExecutor taskExecutor = mock(TaskExecutor.class); - Consumer consumer = mock(Consumer.class); - TopicPartition topicPartition = new TopicPartition("test-topic", 0); - long pollTimout = 500L; - long dueBackOffTime = 750L; - ArgumentCaptor correctionRunnableCaptor = ArgumentCaptor.forClass(Runnable.class); - WakingKafkaConsumerTimingAdjuster timingAdjuster = new WakingKafkaConsumerTimingAdjuster(taskExecutor, sleeper); - timingAdjuster.adjustTiming(consumer, topicPartition, pollTimout, dueBackOffTime); - then(taskExecutor).should(times(1)).execute(correctionRunnableCaptor.capture()); - Runnable correctionRunnable = correctionRunnableCaptor.getValue(); - correctionRunnable.run(); - then(sleeper).should(times(1)).sleep(dueBackOffTime - pollTimout); - then(consumer).should(times(1)).wakeup(); - } - - @Test - void testDoesNotApplyCorrectionIfTooLateForCorrectionWindow() { - Sleeper sleeper = mock(Sleeper.class); - TaskExecutor taskExecutor = mock(TaskExecutor.class); - Consumer consumer = mock(Consumer.class); - TopicPartition topicPartition = new TopicPartition("test-topic", 0); - long pollTimout = 500L; - long dueBackOffTime = 250L; - WakingKafkaConsumerTimingAdjuster timingAdjuster = new WakingKafkaConsumerTimingAdjuster(taskExecutor, sleeper); - timingAdjuster.adjustTiming(consumer, topicPartition, pollTimout, dueBackOffTime); - then(taskExecutor).should(never()).execute(any(Runnable.class)); - } - - @Test - void testDoesNotApplyCorrectionIfTooSoonForCorrectionWindow() { - Sleeper sleeper = mock(Sleeper.class); - TaskExecutor taskExecutor = mock(TaskExecutor.class); - Consumer consumer = mock(Consumer.class); - TopicPartition topicPartition = new TopicPartition("test-topic", 0); - long pollTimout = 500L; - long dueBackOffTime = 1250L; - WakingKafkaConsumerTimingAdjuster timingAdjuster = new WakingKafkaConsumerTimingAdjuster(taskExecutor, sleeper); - timingAdjuster.adjustTiming(consumer, topicPartition, pollTimout, dueBackOffTime); - then(taskExecutor).should(never()).execute(any(Runnable.class)); - } -} diff --git a/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/AbstractRetryTopicIntegrationTests.java b/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/AbstractRetryTopicIntegrationTests.java new file mode 100644 index 0000000000..b46cb7cb14 --- /dev/null +++ b/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/AbstractRetryTopicIntegrationTests.java @@ -0,0 +1,41 @@ +/* + * Copyright 2022 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.kafka.retrytopic; + +import java.lang.reflect.Field; +import java.util.concurrent.atomic.AtomicBoolean; + +import org.junit.jupiter.api.BeforeAll; + +/** + * @author Gary Russell + * @since 2.9 + * + */ +public class AbstractRetryTopicIntegrationTests { + + protected AbstractRetryTopicIntegrationTests() { + } + + @BeforeAll + static void reset() throws Exception { // NOSONAR + Field field = RetryTopicConfigurationSupport.class.getDeclaredField("ONLY_ONE_ALLOWED"); + field.setAccessible(true); + ((AtomicBoolean) field.get(null)).set(true); + } + +} diff --git a/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/CircularDltHandlerTests.java b/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/CircularDltHandlerTests.java index cd9ce8800f..3872491c3f 100644 --- a/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/CircularDltHandlerTests.java +++ b/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/CircularDltHandlerTests.java @@ -1,5 +1,5 @@ /* - * Copyright 2021 the original author or authors. + * Copyright 2021-2022 the original author or authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -28,13 +28,15 @@ import org.springframework.kafka.config.ConcurrentKafkaListenerContainerFactory; import org.springframework.kafka.core.ConsumerFactory; import org.springframework.kafka.core.KafkaTemplate; +import org.springframework.scheduling.TaskScheduler; +import org.springframework.scheduling.concurrent.ThreadPoolTaskScheduler; /** * @author Gary Russell * @since 2.8 * */ -public class CircularDltHandlerTests { +public class CircularDltHandlerTests extends AbstractRetryTopicIntegrationTests { @Test void contextLoads() { @@ -46,7 +48,7 @@ void contextLoads() { @Configuration @EnableKafka - public static class Config { + public static class Config extends RetryTopicConfigurationSupport { @SuppressWarnings("unchecked") @Bean @@ -71,6 +73,11 @@ Listener listener() { return new Listener(); } + @Bean + TaskScheduler sched() { + return new ThreadPoolTaskScheduler(); + } + } public static class Listener { diff --git a/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/DltStartupTests.java b/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/DltStartupTests.java index c90f814bec..7b22efc41d 100644 --- a/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/DltStartupTests.java +++ b/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/DltStartupTests.java @@ -1,5 +1,5 @@ /* - * Copyright 2021 the original author or authors. + * Copyright 2021-2022 the original author or authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -39,6 +39,8 @@ import org.springframework.kafka.test.EmbeddedKafkaBroker; import org.springframework.kafka.test.context.EmbeddedKafka; import org.springframework.kafka.test.utils.KafkaTestUtils; +import org.springframework.scheduling.TaskScheduler; +import org.springframework.scheduling.concurrent.ThreadPoolTaskScheduler; import org.springframework.test.context.junit.jupiter.SpringJUnitConfig; /** @@ -51,7 +53,7 @@ */ @SpringJUnitConfig @EmbeddedKafka -public class DltStartupTests { +public class DltStartupTests extends AbstractRetryTopicIntegrationTests { @Test void dltStartOverridesCorrect(@Autowired KafkaListenerEndpointRegistry registry) { @@ -82,7 +84,7 @@ void dltStartOverridesCorrect(@Autowired KafkaListenerEndpointRegistry registry) @Configuration @EnableKafka - public static class Config { + public static class Config extends RetryTopicConfigurationSupport { @KafkaListener(id = "shouldStartDlq1", topics = "DltStartupTests.1", containerFactory = "cf1") void shouldStartDlq1(String in) { @@ -191,6 +193,11 @@ KafkaOperations template() { return mock(KafkaOperations.class); } + @Bean + TaskScheduler sched() { + return new ThreadPoolTaskScheduler(); + } + } } diff --git a/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/ExistingRetryTopicIntegrationTests.java b/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/ExistingRetryTopicIntegrationTests.java index 22cc73870e..c46953f0cc 100644 --- a/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/ExistingRetryTopicIntegrationTests.java +++ b/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/ExistingRetryTopicIntegrationTests.java @@ -56,7 +56,9 @@ import org.springframework.kafka.test.context.EmbeddedKafka; import org.springframework.messaging.handler.annotation.Header; import org.springframework.retry.annotation.Backoff; +import org.springframework.scheduling.TaskScheduler; import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; +import org.springframework.scheduling.concurrent.ThreadPoolTaskScheduler; import org.springframework.test.annotation.DirtiesContext; import org.springframework.test.context.TestPropertySource; import org.springframework.test.context.junit.jupiter.SpringJUnitConfig; @@ -74,7 +76,7 @@ ExistingRetryTopicIntegrationTests.MAIN_TOPIC_WITH_PARTITION_INFO, ExistingRetryTopicIntegrationTests.RETRY_TOPIC_WITH_PARTITION_INFO}, partitions = 4) @TestPropertySource(properties = "two.attempts=2") -public class ExistingRetryTopicIntegrationTests { +public class ExistingRetryTopicIntegrationTests extends AbstractRetryTopicIntegrationTests { private static final Logger logger = LoggerFactory.getLogger(ExistingRetryTopicIntegrationTests.class); @@ -291,7 +293,7 @@ public KafkaTemplate kafkaTemplate() { @EnableKafka @Configuration - public static class KafkaConsumerConfig { + public static class KafkaConsumerConfig extends RetryTopicConfigurationSupport { @Autowired EmbeddedKafkaBroker broker; @@ -350,6 +352,12 @@ public ConcurrentKafkaListenerContainerFactory kafkaListenerCont factory.setConcurrency(1); return factory; } + + @Bean + TaskScheduler sched() { + return new ThreadPoolTaskScheduler(); + } + } } diff --git a/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/ListenerContainerFactoryConfigurerTests.java b/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/ListenerContainerFactoryConfigurerTests.java index 48f24c19a2..dd7cd0a9a3 100644 --- a/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/ListenerContainerFactoryConfigurerTests.java +++ b/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/ListenerContainerFactoryConfigurerTests.java @@ -25,7 +25,6 @@ import static org.mockito.BDDMockito.then; import static org.mockito.BDDMockito.willReturn; import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.never; import static org.mockito.Mockito.times; import java.math.BigInteger; @@ -88,7 +87,7 @@ class ListenerContainerFactoryConfigurerTests { private ContainerProperties containerProperties; @Captor - private ArgumentCaptor errorHandlerCaptor; + private ArgumentCaptor errorHandlerCaptor; private final ConsumerRecord record = new ConsumerRecord<>("test-topic", 1, 1234L, new Object(), new Object()); @@ -105,7 +104,7 @@ class ListenerContainerFactoryConfigurerTests { private OffsetCommitCallback offsetCommitCallback; @Mock - private java.util.function.Consumer errorHandlerCustomizer; + private java.util.function.Consumer errorHandlerCustomizer; @SuppressWarnings("rawtypes") @Captor @@ -159,24 +158,25 @@ void shouldSetupErrorHandling() { given(deadLetterPublishingRecovererFactory.create()).willReturn(recoverer); given(containerProperties.getAckMode()).willReturn(ContainerProperties.AckMode.MANUAL_IMMEDIATE); given(containerProperties.getCommitCallback()).willReturn(offsetCommitCallback); + given(containerProperties.getMessageListener()).willReturn(listener); given(configuration.forContainerFactoryConfigurer()).willReturn(lcfcConfiguration); + willReturn(container).given(containerFactory).createListenerContainer(endpoint); // when ListenerContainerFactoryConfigurer configurer = new ListenerContainerFactoryConfigurer(kafkaConsumerBackoffManager, deadLetterPublishingRecovererFactory, clock); configurer.setErrorHandlerCustomizer(errorHandlerCustomizer); - configurer - .configure(containerFactory, configuration.forContainerFactoryConfigurer()); + KafkaListenerContainerFactory factory = configurer.decorateFactory(containerFactory, + configuration.forContainerFactoryConfigurer()); + factory.createListenerContainer(endpoint); // then - then(containerFactory).should(times(1)).setCommonErrorHandler(errorHandlerCaptor.capture()); - CommonErrorHandler errorHandler = errorHandlerCaptor.getValue(); - assertThat(DefaultErrorHandler.class.isAssignableFrom(errorHandler.getClass())).isTrue(); - DefaultErrorHandler seekToCurrent = (DefaultErrorHandler) errorHandler; + then(container).should(times(1)).setCommonErrorHandler(errorHandlerCaptor.capture()); + DefaultErrorHandler errorHandler = errorHandlerCaptor.getValue(); RuntimeException ex = new RuntimeException(); - seekToCurrent.handleRemaining(ex, records, consumer, container); + errorHandler.handleRemaining(ex, records, consumer, container); then(recoverer).should(times(1)).accept(record, consumer, ex); then(consumer).should(times(1)).commitAsync(any(Map.class), eq(offsetCommitCallback)); @@ -184,157 +184,6 @@ void shouldSetupErrorHandling() { } - @SuppressWarnings("deprecation") - @Test - void shouldSetPartitionEventIntervalAndPollTimout() { - - // given - given(container.getContainerProperties()).willReturn(containerProperties); - given(deadLetterPublishingRecovererFactory.create()).willReturn(recoverer); - given(containerProperties.getMessageListener()).willReturn(listener); - given(configuration.forContainerFactoryConfigurer()).willReturn(lcfcConfiguration); - given(containerProperties.getPollTimeout()).willReturn(ContainerProperties.DEFAULT_POLL_TIMEOUT); - - // when - ListenerContainerFactoryConfigurer configurer = - new ListenerContainerFactoryConfigurer(kafkaConsumerBackoffManager, - deadLetterPublishingRecovererFactory, clock); - configurer - .configure(containerFactory, configuration.forContainerFactoryConfigurer()); - - // then - then(containerFactory).should(times(1)) - .setContainerCustomizer(containerCustomizerCaptor.capture()); - ContainerCustomizer containerCustomizer = containerCustomizerCaptor.getValue(); - containerCustomizer.configure(container); - - then(containerProperties).should(times(1)) - .setIdlePartitionEventInterval(backOffValue / 4); - then(containerProperties).should(times(1)) - .setPollTimeout(backOffValue / 4); - } - - @SuppressWarnings("deprecation") - @Test - void shouldNotOverridePollTimeoutIfNotDefault() { - - // given - given(container.getContainerProperties()).willReturn(containerProperties); - given(deadLetterPublishingRecovererFactory.create()).willReturn(recoverer); - given(containerProperties.getMessageListener()).willReturn(listener); - given(configuration.forContainerFactoryConfigurer()).willReturn(lcfcConfiguration); - long previousPollTimoutValue = 3000L; - given(containerProperties.getPollTimeout()).willReturn(previousPollTimoutValue); - - // when - ListenerContainerFactoryConfigurer configurer = - new ListenerContainerFactoryConfigurer(kafkaConsumerBackoffManager, - deadLetterPublishingRecovererFactory, clock); - configurer - .configure(containerFactory, configuration.forContainerFactoryConfigurer()); - - // then - then(containerFactory).should(times(1)) - .setContainerCustomizer(containerCustomizerCaptor.capture()); - ContainerCustomizer containerCustomizer = containerCustomizerCaptor.getValue(); - containerCustomizer.configure(container); - - then(containerProperties).should(times(1)) - .setIdlePartitionEventInterval(previousPollTimoutValue); - then(containerProperties).should(times(1)) - .setPollTimeout(previousPollTimoutValue); - } - - @SuppressWarnings("deprecation") - @Test - void shouldApplyMinimumPollTimeoutLimit() { - - // given - given(container.getContainerProperties()).willReturn(containerProperties); - given(deadLetterPublishingRecovererFactory.create()).willReturn(recoverer); - given(containerProperties.getMessageListener()).willReturn(listener); - given(configuration.forContainerFactoryConfigurer()) - .willReturn(new ListenerContainerFactoryConfigurer.Configuration(Collections.singletonList(500L))); - given(containerProperties.getPollTimeout()).willReturn(ContainerProperties.DEFAULT_POLL_TIMEOUT); - - // when - ListenerContainerFactoryConfigurer configurer = - new ListenerContainerFactoryConfigurer(kafkaConsumerBackoffManager, - deadLetterPublishingRecovererFactory, clock); - configurer - .configure(containerFactory, configuration.forContainerFactoryConfigurer()); - - // then - then(containerFactory).should(times(1)) - .setContainerCustomizer(containerCustomizerCaptor.capture()); - ContainerCustomizer containerCustomizer = containerCustomizerCaptor.getValue(); - containerCustomizer.configure(container); - - then(containerProperties).should(times(1)) - .setIdlePartitionEventInterval(100L); - then(containerProperties).should(times(1)) - .setPollTimeout(100L); - } - - @Test - void shouldApplyMaximumPollTimeoutLimit() { - - // given - given(container.getContainerProperties()).willReturn(containerProperties); - given(deadLetterPublishingRecovererFactory.create()).willReturn(recoverer); - given(containerProperties.getMessageListener()).willReturn(listener); - given(configuration.forContainerFactoryConfigurer()) - .willReturn(new ListenerContainerFactoryConfigurer.Configuration(Collections.singletonList(30000L))); - given(containerProperties.getPollTimeout()).willReturn(ContainerProperties.DEFAULT_POLL_TIMEOUT); - - // when - ListenerContainerFactoryConfigurer configurer = - new ListenerContainerFactoryConfigurer(kafkaConsumerBackoffManager, - deadLetterPublishingRecovererFactory, clock); - configurer - .configure(containerFactory, configuration.forContainerFactoryConfigurer()); - - // then - then(containerFactory).should(times(1)) - .setContainerCustomizer(containerCustomizerCaptor.capture()); - ContainerCustomizer containerCustomizer = containerCustomizerCaptor.getValue(); - containerCustomizer.configure(container); - - then(containerProperties).should(times(1)) - .setIdlePartitionEventInterval(5000L); - then(containerProperties).should(times(1)) - .setPollTimeout(5000L); - } - - @Test - void shouldNotSetPolltimoutAndPartitionIdleIfNoBackOffProvided() { - - // given - given(container.getContainerProperties()).willReturn(containerProperties); - given(deadLetterPublishingRecovererFactory.create()).willReturn(recoverer); - given(containerProperties.getMessageListener()).willReturn(listener); - given(configuration.forContainerFactoryConfigurer()) - .willReturn(lcfcConfiguration); - - // when - ListenerContainerFactoryConfigurer configurer = - new ListenerContainerFactoryConfigurer(kafkaConsumerBackoffManager, - deadLetterPublishingRecovererFactory, clock); - configurer - .configureWithoutBackOffValues(containerFactory, configuration.forContainerFactoryConfigurer()); - - // then - then(containerFactory).should(times(1)) - .setContainerCustomizer(containerCustomizerCaptor.capture()); - ContainerCustomizer containerCustomizer = containerCustomizerCaptor.getValue(); - containerCustomizer.configure(container); - - then(containerProperties).should(never()) - .setIdlePartitionEventInterval(anyLong()); - then(containerProperties).should(never()) - .setPollTimeout(anyLong()); - } - @Test void shouldSetupMessageListenerAdapter() { @@ -348,22 +197,18 @@ void shouldSetupMessageListenerAdapter() { String testListenerId = "testListenerId"; given(container.getListenerId()).willReturn(testListenerId); given(configuration.forContainerFactoryConfigurer()).willReturn(lcfcConfiguration); + willReturn(container).given(containerFactory).createListenerContainer(endpoint); // when ListenerContainerFactoryConfigurer configurer = new ListenerContainerFactoryConfigurer(kafkaConsumerBackoffManager, deadLetterPublishingRecovererFactory, clock); configurer.setContainerCustomizer(configurerContainerCustomizer); - ConcurrentKafkaListenerContainerFactory factory = configurer - .configure(containerFactory, configuration.forContainerFactoryConfigurer()); + KafkaListenerContainerFactory factory = configurer + .decorateFactory(containerFactory, configuration.forContainerFactoryConfigurer()); + factory.createListenerContainer(endpoint); // then - then(containerFactory) - .should(times(1)) - .setContainerCustomizer(containerCustomizerCaptor.capture()); - ContainerCustomizer containerCustomizer = containerCustomizerCaptor.getValue(); - containerCustomizer.configure(container); - then(container).should(times(1)).setupMessageListener(listenerAdapterCaptor.capture()); KafkaBackoffAwareMessageListenerAdapter listenerAdapter = (KafkaBackoffAwareMessageListenerAdapter) listenerAdapterCaptor.getValue(); @@ -467,26 +312,4 @@ void shouldThrowIfBackOffOrRetryablesAlreadySet() { .isInstanceOf(IllegalStateException.class); } - - @Test - void shouldCacheFactoryInstances() { - - // given - given(deadLetterPublishingRecovererFactory.create()).willReturn(recoverer); - given(configuration.forContainerFactoryConfigurer()).willReturn(lcfcConfiguration); - - // when - ListenerContainerFactoryConfigurer configurer = - new ListenerContainerFactoryConfigurer(kafkaConsumerBackoffManager, - deadLetterPublishingRecovererFactory, clock); - ConcurrentKafkaListenerContainerFactory factory = configurer - .configure(containerFactory, configuration.forContainerFactoryConfigurer()); - ConcurrentKafkaListenerContainerFactory secondFactory = configurer - .configure(containerFactory, configuration.forContainerFactoryConfigurer()); - - // then - assertThat(secondFactory).isEqualTo(factory); - then(containerFactory).should(times(1)).setContainerCustomizer(any()); - then(containerFactory).should(times(1)).setCommonErrorHandler(any()); - } } diff --git a/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/RetryTopicBootstrapperTests.java b/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/RetryTopicBootstrapperTests.java deleted file mode 100644 index f96c73ac15..0000000000 --- a/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/RetryTopicBootstrapperTests.java +++ /dev/null @@ -1,250 +0,0 @@ -/* - * Copyright 2018-2022 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.kafka.retrytopic; - -import static org.assertj.core.api.Assertions.assertThatIllegalStateException; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.BDDMockito.given; -import static org.mockito.BDDMockito.then; -import static org.mockito.Mockito.never; -import static org.mockito.Mockito.times; - -import java.time.Clock; - -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.mockito.Mock; -import org.mockito.junit.jupiter.MockitoExtension; - -import org.springframework.beans.factory.BeanFactory; -import org.springframework.beans.factory.NoSuchBeanDefinitionException; -import org.springframework.beans.factory.config.SingletonBeanRegistry; -import org.springframework.beans.factory.support.DefaultListableBeanFactory; -import org.springframework.beans.factory.support.RootBeanDefinition; -import org.springframework.context.ApplicationContext; -import org.springframework.context.support.GenericApplicationContext; -import org.springframework.kafka.listener.KafkaBackOffManagerFactory; -import org.springframework.kafka.listener.KafkaConsumerBackoffManager; -import org.springframework.kafka.listener.PartitionPausingBackOffManagerFactory; -import org.springframework.retry.backoff.ThreadWaitSleeper; - -/** - * @author Tomaz Fernandes - * @since 2.7 - */ -@SuppressWarnings("deprecation") -@ExtendWith(MockitoExtension.class) -class RetryTopicBootstrapperTests { - - @Mock - private ApplicationContext wrongApplicationContext; - - @Mock - private GenericApplicationContext applicationContext; - - @Mock - private DefaultListableBeanFactory beanFactory; - - @Mock - private BeanFactory wrongBeanFactory; - - @Mock - private DefaultDestinationTopicResolver defaultDestinationTopicResolver; - - @Mock - private PartitionPausingBackOffManagerFactory kafkaBackOffManagerFactory; - - @Mock - private KafkaConsumerBackoffManager kafkaConsumerBackOffManager; - - @Mock - private RetryTopicNamesProviderFactory retryTopicNamesProviderFactory; - - @Test - void shouldThrowIfACDoesntImplementInterfaces() { - assertThatIllegalStateException() - .isThrownBy(() -> new RetryTopicBootstrapper(wrongApplicationContext, beanFactory)); - } - - @Test - void shouldThrowIfBFDoesntImplementInterfaces() { - assertThatIllegalStateException() - .isThrownBy(() -> new RetryTopicBootstrapper(applicationContext, wrongBeanFactory)); - } - - @Test - void shouldRegisterBeansIfNotRegistered() { - - // given - given(applicationContext.containsBeanDefinition(any(String.class))).willReturn(false); - given(this.applicationContext.getBean( - RetryTopicInternalBeanNames.DESTINATION_TOPIC_CONTAINER_NAME, DefaultDestinationTopicResolver.class)) - .willReturn(defaultDestinationTopicResolver); - given(this.applicationContext.getBean( - RetryTopicInternalBeanNames.INTERNAL_KAFKA_CONSUMER_BACKOFF_MANAGER_FACTORY, - KafkaBackOffManagerFactory.class)) - .willReturn(kafkaBackOffManagerFactory); - given(this.applicationContext.getBean( - RetryTopicNamesProviderFactory.class)) - .willThrow(NoSuchBeanDefinitionException.class); - // when - RetryTopicBootstrapper bootstrapper = new RetryTopicBootstrapper(applicationContext, beanFactory); - bootstrapper.bootstrapRetryTopic(); - - // then - then(this.applicationContext).should(times(1)) - .registerBeanDefinition(RetryTopicInternalBeanNames.LISTENER_CONTAINER_FACTORY_RESOLVER_NAME, - new RootBeanDefinition(ListenerContainerFactoryResolver.class)); - then(this.applicationContext).should(times(1)) - .registerBeanDefinition(RetryTopicInternalBeanNames.DESTINATION_TOPIC_PROCESSOR_NAME, - new RootBeanDefinition(DefaultDestinationTopicProcessor.class)); - then(this.applicationContext).should(times(1)) - .registerBeanDefinition(RetryTopicInternalBeanNames.LISTENER_CONTAINER_FACTORY_CONFIGURER_NAME, - new RootBeanDefinition(ListenerContainerFactoryConfigurer.class)); - then(this.applicationContext).should(times(1)) - .registerBeanDefinition(RetryTopicInternalBeanNames.DEAD_LETTER_PUBLISHING_RECOVERER_FACTORY_BEAN_NAME, - new RootBeanDefinition(DeadLetterPublishingRecovererFactory.class)); - then(this.applicationContext).should(times(1)) - .registerBeanDefinition(RetryTopicInternalBeanNames.RETRY_TOPIC_CONFIGURER, - new RootBeanDefinition(RetryTopicConfigurer.class)); - then(this.applicationContext).should(times(1)) - .registerBeanDefinition(RetryTopicInternalBeanNames.DESTINATION_TOPIC_CONTAINER_NAME, - new RootBeanDefinition(DefaultDestinationTopicResolver.class)); - then(this.applicationContext).should(times(1)) - .registerBeanDefinition(RetryTopicInternalBeanNames.BACKOFF_SLEEPER_BEAN_NAME, - new RootBeanDefinition(ThreadWaitSleeper.class)); - then(this.applicationContext).should(times(1)) - .registerBeanDefinition(RetryTopicInternalBeanNames.RETRY_TOPIC_NAMES_PROVIDER_FACTORY, - new RootBeanDefinition(SuffixingRetryTopicNamesProviderFactory.class)); - } - - @Test - void shouldNotRegisterBeansIfRegistered() { - - // given - given(applicationContext.containsBeanDefinition(any(String.class))).willReturn(true); - given(this.applicationContext.getBean( - RetryTopicInternalBeanNames.DESTINATION_TOPIC_CONTAINER_NAME, DefaultDestinationTopicResolver.class)) - .willReturn(defaultDestinationTopicResolver); - given(this.applicationContext.getBean( - RetryTopicNamesProviderFactory.class)) - .willReturn(this.retryTopicNamesProviderFactory); - - // when - RetryTopicBootstrapper bootstrapper = new RetryTopicBootstrapper(applicationContext, beanFactory); - bootstrapper.bootstrapRetryTopic(); - - // then - then(this.applicationContext).should(times(0)) - .registerBeanDefinition(RetryTopicInternalBeanNames.DESTINATION_TOPIC_PROCESSOR_NAME, - new RootBeanDefinition(DefaultDestinationTopicProcessor.class)); - then(this.applicationContext).should(times(0)) - .registerBeanDefinition(RetryTopicInternalBeanNames.LISTENER_CONTAINER_FACTORY_CONFIGURER_NAME, - new RootBeanDefinition(ListenerContainerFactoryConfigurer.class)); - then(this.applicationContext).should(times(0)) - .registerBeanDefinition(RetryTopicInternalBeanNames.LISTENER_CONTAINER_FACTORY_RESOLVER_NAME, - new RootBeanDefinition(ListenerContainerFactoryResolver.class)); - then(this.applicationContext).should(times(0)) - .registerBeanDefinition(RetryTopicInternalBeanNames.DEAD_LETTER_PUBLISHING_RECOVERER_FACTORY_BEAN_NAME, - new RootBeanDefinition(DeadLetterPublishingRecovererFactory.class)); - then(this.applicationContext).should(times(0)) - .registerBeanDefinition(RetryTopicInternalBeanNames.RETRY_TOPIC_CONFIGURER, - new RootBeanDefinition(RetryTopicConfigurer.class)); - then(this.applicationContext).should(times(0)) - .registerBeanDefinition(RetryTopicInternalBeanNames.DESTINATION_TOPIC_CONTAINER_NAME, - new RootBeanDefinition(DefaultDestinationTopicResolver.class)); - then(this.applicationContext).should(times(0)) - .registerBeanDefinition(RetryTopicInternalBeanNames.BACKOFF_SLEEPER_BEAN_NAME, - new RootBeanDefinition(ThreadWaitSleeper.class)); - then(this.applicationContext).should(times(0)) - .registerBeanDefinition(RetryTopicInternalBeanNames.RETRY_TOPIC_NAMES_PROVIDER_FACTORY, - new RootBeanDefinition(SuffixingRetryTopicNamesProviderFactory.class)); - } - - @Test - void shouldRegisterSingletonsIfNotExists() { - - // given - given(applicationContext.containsBeanDefinition(any(String.class))) - .willReturn(false); - given(this.applicationContext - .getBean(RetryTopicInternalBeanNames.INTERNAL_KAFKA_CONSUMER_BACKOFF_MANAGER_FACTORY, - KafkaBackOffManagerFactory.class)).willReturn(kafkaBackOffManagerFactory); - given(this.applicationContext - .getBean(RetryTopicNamesProviderFactory.class)) - .willThrow(NoSuchBeanDefinitionException.class); - - given(kafkaBackOffManagerFactory.create()).willReturn(kafkaConsumerBackOffManager); - - // when - RetryTopicBootstrapper bootstrapper = new RetryTopicBootstrapper(applicationContext, beanFactory); - bootstrapper.bootstrapRetryTopic(); - - // then - then((SingletonBeanRegistry) this.beanFactory).should(times(1)).registerSingleton( - RetryTopicInternalBeanNames.INTERNAL_BACKOFF_CLOCK_BEAN_NAME, Clock.systemUTC()); - then((SingletonBeanRegistry) this.beanFactory).should(times(1)).registerSingleton( - RetryTopicInternalBeanNames.KAFKA_CONSUMER_BACKOFF_MANAGER, kafkaConsumerBackOffManager); - } - - @Test - void shouldNotRegisterSingletonsIfExists() { - - // given - given(applicationContext.containsBeanDefinition(any(String.class))) - .willReturn(false); - given(applicationContext.containsBeanDefinition(RetryTopicInternalBeanNames.INTERNAL_BACKOFF_CLOCK_BEAN_NAME)) - .willReturn(true); - given(applicationContext.containsBeanDefinition(RetryTopicInternalBeanNames.KAFKA_CONSUMER_BACKOFF_MANAGER)) - .willReturn(true); - - // when - RetryTopicBootstrapper bootstrapper = new RetryTopicBootstrapper(applicationContext, beanFactory); - bootstrapper.bootstrapRetryTopic(); - - // then - then((SingletonBeanRegistry) this.beanFactory).should(never()).registerSingleton( - RetryTopicInternalBeanNames.INTERNAL_BACKOFF_CLOCK_BEAN_NAME, Clock.systemUTC()); - then((SingletonBeanRegistry) this.beanFactory).should(never()).registerSingleton( - RetryTopicInternalBeanNames.KAFKA_CONSUMER_BACKOFF_MANAGER, kafkaConsumerBackOffManager); - } - - @Test - void shouldAddApplicationListeners() { - - // given - given(applicationContext.containsBeanDefinition(any(String.class))) - .willReturn(false); - given(this.applicationContext.getBean( - RetryTopicInternalBeanNames.DESTINATION_TOPIC_CONTAINER_NAME, DefaultDestinationTopicResolver.class)) - .willReturn(defaultDestinationTopicResolver); - given(this.applicationContext.getBean( - RetryTopicInternalBeanNames.INTERNAL_KAFKA_CONSUMER_BACKOFF_MANAGER_FACTORY, - KafkaBackOffManagerFactory.class)) - .willReturn(kafkaBackOffManagerFactory); - given(this.applicationContext.getBean( - RetryTopicNamesProviderFactory.class)) - .willThrow(NoSuchBeanDefinitionException.class); - // when - RetryTopicBootstrapper bootstrapper = new RetryTopicBootstrapper(applicationContext, beanFactory); - bootstrapper.bootstrapRetryTopic(); - - // then - then(this.applicationContext).should(times(1)) - .addApplicationListener(defaultDestinationTopicResolver); - } -} diff --git a/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/RetryTopicConfigurationIntegrationTests.java b/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/RetryTopicConfigurationIntegrationTests.java index c2ac98668b..341adb30da 100644 --- a/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/RetryTopicConfigurationIntegrationTests.java +++ b/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/RetryTopicConfigurationIntegrationTests.java @@ -44,6 +44,8 @@ import org.springframework.kafka.test.EmbeddedKafkaBroker; import org.springframework.kafka.test.context.EmbeddedKafka; import org.springframework.kafka.test.utils.KafkaTestUtils; +import org.springframework.scheduling.TaskScheduler; +import org.springframework.scheduling.concurrent.ThreadPoolTaskScheduler; import org.springframework.test.annotation.DirtiesContext; import org.springframework.test.context.junit.jupiter.SpringJUnitConfig; @@ -55,7 +57,7 @@ @SpringJUnitConfig @DirtiesContext @EmbeddedKafka(topics = RetryTopicConfigurationIntegrationTests.TOPIC1, partitions = 1) -class RetryTopicConfigurationIntegrationTests { +class RetryTopicConfigurationIntegrationTests extends AbstractRetryTopicIntegrationTests { public static final String TOPIC1 = "RetryTopicConfigurationIntegrationTests.1"; @@ -74,7 +76,7 @@ void includeTopic(@Autowired EmbeddedKafkaBroker broker, @Autowired ConsumerFact @Configuration(proxyBeanMethods = false) @EnableKafka - static class Config { + static class Config extends RetryTopicConfigurationSupport { private final CountDownLatch latch = new CountDownLatch(1); @@ -128,6 +130,11 @@ RetryTopicConfiguration retryTopicConfiguration1(KafkaTemplate .create(template); } + @Bean + TaskScheduler sched() { + return new ThreadPoolTaskScheduler(); + } + } } diff --git a/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/RetryTopicConfigurationManualAssignmentIntegrationTests.java b/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/RetryTopicConfigurationManualAssignmentIntegrationTests.java index ccd6df6827..2cc011ba0d 100644 --- a/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/RetryTopicConfigurationManualAssignmentIntegrationTests.java +++ b/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/RetryTopicConfigurationManualAssignmentIntegrationTests.java @@ -46,6 +46,8 @@ import org.springframework.kafka.test.EmbeddedKafkaBroker; import org.springframework.kafka.test.context.EmbeddedKafka; import org.springframework.kafka.test.utils.KafkaTestUtils; +import org.springframework.scheduling.TaskScheduler; +import org.springframework.scheduling.concurrent.ThreadPoolTaskScheduler; import org.springframework.test.annotation.DirtiesContext; import org.springframework.test.context.junit.jupiter.SpringJUnitConfig; @@ -58,7 +60,7 @@ @DirtiesContext @EmbeddedKafka(topics = { RetryTopicConfigurationManualAssignmentIntegrationTests.TOPIC1, RetryTopicConfigurationManualAssignmentIntegrationTests.TOPIC2 }, partitions = 1) -class RetryTopicConfigurationManualAssignmentIntegrationTests { +class RetryTopicConfigurationManualAssignmentIntegrationTests extends AbstractRetryTopicIntegrationTests { public static final String TOPIC1 = "RetryTopicConfigurationManualAssignmentIntegrationTests.1"; @@ -84,7 +86,7 @@ void includeTopic(@Autowired EmbeddedKafkaBroker broker, @Autowired ConsumerFact @Configuration(proxyBeanMethods = false) @EnableKafka - static class Config { + static class Config extends RetryTopicConfigurationSupport { private final CountDownLatch latch = new CountDownLatch(1); @@ -143,6 +145,11 @@ RetryTopicConfiguration retryTopicConfiguration1(KafkaTemplate .create(template); } + @Bean + TaskScheduler sched() { + return new ThreadPoolTaskScheduler(); + } + } } diff --git a/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/RetryTopicConfigurationSupportTests.java b/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/RetryTopicConfigurationSupportTests.java index cf6dd844f9..0386b5fd4f 100644 --- a/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/RetryTopicConfigurationSupportTests.java +++ b/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/RetryTopicConfigurationSupportTests.java @@ -22,39 +22,43 @@ import static org.mockito.BDDMockito.given; import static org.mockito.BDDMockito.then; import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.never; -import java.lang.reflect.Method; +import java.lang.reflect.Field; import java.time.Clock; import java.util.List; import java.util.Map; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.Consumer; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.mockito.ArgumentCaptor; import org.springframework.beans.factory.BeanFactory; -import org.springframework.beans.factory.support.DefaultListableBeanFactory; -import org.springframework.context.support.GenericApplicationContext; -import org.springframework.core.task.TaskExecutor; -import org.springframework.kafka.listener.CommonErrorHandler; import org.springframework.kafka.listener.ConcurrentMessageListenerContainer; +import org.springframework.kafka.listener.ContainerPartitionPausingBackOffManagerFactory; import org.springframework.kafka.listener.DeadLetterPublishingRecoverer; +import org.springframework.kafka.listener.DefaultErrorHandler; import org.springframework.kafka.listener.KafkaConsumerBackoffManager; -import org.springframework.kafka.listener.KafkaConsumerTimingAdjuster; import org.springframework.kafka.listener.ListenerContainerRegistry; -import org.springframework.kafka.listener.PartitionPausingBackOffManagerFactory; import org.springframework.kafka.support.converter.ConversionException; -import org.springframework.kafka.test.utils.KafkaTestUtils; -import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; +import org.springframework.scheduling.TaskScheduler; import org.springframework.util.backoff.BackOff; /** * @author Tomaz Fernandes + * @author Gary Russell * @since 2.9 */ class RetryTopicConfigurationSupportTests { + @BeforeEach + void reset() throws Exception { // NOSONAR + Field field = RetryTopicConfigurationSupport.class.getDeclaredField("ONLY_ONE_ALLOWED"); + field.setAccessible(true); + ((AtomicBoolean) field.get(null)).set(true); + } + @SuppressWarnings("unchecked") @Test void testCreateConfigurer() { @@ -83,7 +87,7 @@ void testCreateConfigurer() { Consumer dlprfCustomizer = mock(Consumer.class); Consumer rtconfigurer = mock(Consumer.class); Consumer lcfcConsumer = mock(Consumer.class); - Consumer errorHandlerCustomizer = mock(Consumer.class); + Consumer errorHandlerCustomizer = mock(Consumer.class); BackOff backoff = mock(BackOff.class); RetryTopicConfigurationSupport support = new RetryTopicConfigurationSupport() { @@ -158,129 +162,34 @@ void testRetryTopicConfigurerNoConfiguration() { void testCreateBackOffManager() { ListenerContainerRegistry registry = mock(ListenerContainerRegistry.class); RetryTopicComponentFactory componentFactory = mock(RetryTopicComponentFactory.class); - PartitionPausingBackOffManagerFactory factory = mock(PartitionPausingBackOffManagerFactory.class); + ContainerPartitionPausingBackOffManagerFactory factory = mock( + ContainerPartitionPausingBackOffManagerFactory.class); KafkaConsumerBackoffManager backoffManagerMock = mock(KafkaConsumerBackoffManager.class); - ThreadPoolTaskExecutor taskExecutorMock = mock(ThreadPoolTaskExecutor.class); + TaskScheduler taskSchedulerMock = mock(TaskScheduler.class); Clock clock = mock(Clock.class); given(componentFactory.kafkaBackOffManagerFactory(registry)).willReturn(factory); given(factory.create()).willReturn(backoffManagerMock); RetryTopicConfigurationSupport support = new RetryTopicConfigurationSupport() { - @Override - protected void configureKafkaBackOffManager(KafkaBackOffManagerConfigurer configurer) { - configurer - .setMaxThreadPoolSize(10) - .setClock(clock); - } @Override protected RetryTopicComponentFactory createComponentFactory() { return componentFactory; } - }; - KafkaConsumerBackoffManager backoffManager = support.kafkaConsumerBackoffManager(registry, taskExecutorMock); - assertThat(backoffManager).isEqualTo(backoffManagerMock); - then(componentFactory).should().kafkaBackOffManagerFactory(registry); - then(factory).should().create(); - then(factory).should().setTaskExecutor(taskExecutorMock); - then(factory).should().setClock(clock); - then(taskExecutorMock).should().setMaxPoolSize(10); - } - - @Test - void testCreateBackOffManagerWithDisableTimingAdjustment() { - ListenerContainerRegistry registry = mock(ListenerContainerRegistry.class); - RetryTopicComponentFactory componentFactory = mock(RetryTopicComponentFactory.class); - PartitionPausingBackOffManagerFactory factory = mock(PartitionPausingBackOffManagerFactory.class); - KafkaConsumerBackoffManager backoffManagerMock = mock(KafkaConsumerBackoffManager.class); - ThreadPoolTaskExecutor taskExecutorMock = mock(ThreadPoolTaskExecutor.class); - given(componentFactory.kafkaBackOffManagerFactory(registry)).willReturn(factory); - given(factory.create()).willReturn(backoffManagerMock); - RetryTopicConfigurationSupport support = new RetryTopicConfigurationSupport() { - - @Override - protected void configureKafkaBackOffManager(KafkaBackOffManagerConfigurer configurer) { - configurer - .disableTimingAdjustment(); - } - @Override - protected RetryTopicComponentFactory createComponentFactory() { - return componentFactory; - } }; - KafkaConsumerBackoffManager backoffManager = support.kafkaConsumerBackoffManager(registry, taskExecutorMock); + KafkaConsumerBackoffManager backoffManager = support.kafkaConsumerBackoffManager(registry, null, + taskSchedulerMock); assertThat(backoffManager).isEqualTo(backoffManagerMock); then(componentFactory).should().kafkaBackOffManagerFactory(registry); then(factory).should().create(); - then(factory).should().setTimingAdjustmentEnabled(false); - then(factory).should(never()).setTaskExecutor(taskExecutorMock); - } - - @Test - void backOffManagerFactoryCoverage() throws Exception { - Method create = PartitionPausingBackOffManagerFactory.class.getDeclaredMethod("doCreateManager", - ListenerContainerRegistry.class); - create.setAccessible(true); - TaskExecutor te = mock(TaskExecutor.class); - KafkaConsumerTimingAdjuster mock = mock(KafkaConsumerTimingAdjuster.class); - PartitionPausingBackOffManagerFactory factory = new PartitionPausingBackOffManagerFactory(mock); - assertThat(KafkaTestUtils.getPropertyValue(factory, "clock")).isEqualTo(Clock.systemUTC()); - assertThat(KafkaTestUtils.getPropertyValue(factory, "timingAdjustmentManager")).isEqualTo(mock); - create.invoke(factory, mock(ListenerContainerRegistry.class)); - factory.setTaskExecutor(te); - assertThat(KafkaTestUtils.getPropertyValue(factory, "taskExecutor")).isEqualTo(te); - factory = new PartitionPausingBackOffManagerFactory(te); - assertThat(KafkaTestUtils.getPropertyValue(factory, "clock")).isEqualTo(Clock.systemUTC()); - assertThat(KafkaTestUtils.getPropertyValue(factory, "taskExecutor")).isEqualTo(te); - create.invoke(factory, mock(ListenerContainerRegistry.class)); - factory.setTimingAdjustmentManager(mock); - assertThat(KafkaTestUtils.getPropertyValue(factory, "timingAdjustmentManager")).isEqualTo(mock); - create.invoke(factory, mock(ListenerContainerRegistry.class)); - factory = new PartitionPausingBackOffManagerFactory(false); - assertThat(KafkaTestUtils.getPropertyValue(factory, "clock")).isEqualTo(Clock.systemUTC()); - assertThat(KafkaTestUtils.getPropertyValue(factory, "timingAdjustmentEnabled")).isEqualTo(Boolean.FALSE); - factory.setClock(Clock.systemDefaultZone()); - assertThat(KafkaTestUtils.getPropertyValue(factory, "clock")).isEqualTo(Clock.systemDefaultZone()); - factory = new PartitionPausingBackOffManagerFactory(Clock.systemDefaultZone()); - assertThat(KafkaTestUtils.getPropertyValue(factory, "clock")).isEqualTo(Clock.systemDefaultZone()); - assertThat(KafkaTestUtils.getPropertyValue(factory, "timingAdjustmentEnabled")).isEqualTo(Boolean.TRUE); - factory.setTimingAdjustmentEnabled(false); - assertThat(KafkaTestUtils.getPropertyValue(factory, "timingAdjustmentEnabled")).isEqualTo(Boolean.FALSE); - } - - @Test - void testCreatesTaskExecutor() { - RetryTopicConfigurationSupport support = new RetryTopicConfigurationSupport(); - TaskExecutor taskExecutor = support.backoffManagerTaskExecutor(); - assertThat(taskExecutor).isInstanceOf(ThreadPoolTaskExecutor.class); - } - - @Test - void testDoesNotCreateTaskExecutorIfTimingAdjustmentDisabled() { - RetryTopicComponentFactory componentFactory = mock(RetryTopicComponentFactory.class); - RetryTopicConfigurationSupport support = new RetryTopicConfigurationSupport() { - @Override - protected void configureKafkaBackOffManager(KafkaBackOffManagerConfigurer configurer) { - configurer - .disableTimingAdjustment(); - } - - @Override - protected RetryTopicComponentFactory createComponentFactory() { - return componentFactory; - } - }; - TaskExecutor taskExecutor = support.backoffManagerTaskExecutor(); - then(componentFactory).shouldHaveNoInteractions(); - assertThat(taskExecutor).isNotInstanceOf(ThreadPoolTaskExecutor.class); } @Test void testCreateBackOffManagerNoConfiguration() { ListenerContainerRegistry registry = mock(ListenerContainerRegistry.class); - TaskExecutor taskExecutor = mock(TaskExecutor.class); + TaskScheduler scheduler = mock(TaskScheduler.class); RetryTopicConfigurationSupport support = new RetryTopicConfigurationSupport(); - KafkaConsumerBackoffManager backoffManager = support.kafkaConsumerBackoffManager(registry, taskExecutor); + KafkaConsumerBackoffManager backoffManager = support.kafkaConsumerBackoffManager(registry, null, scheduler); assertThat(backoffManager).isNotNull(); } @@ -328,13 +237,4 @@ void testCreatesComponentFactory() { assertThat(configurationSupport).hasFieldOrProperty("componentFactory").isNotNull(); } - @Deprecated - @Test - void testCreatesBootstrapper() { - GenericApplicationContext context = mock(GenericApplicationContext.class); - given(context.getAutowireCapableBeanFactory()).willReturn(mock(DefaultListableBeanFactory.class)); - RetryTopicConfigurationSupport configurationSupport = new RetryTopicConfigurationSupport(); - assertThat(configurationSupport.retryTopicBootstrapper(context)).isNotNull(); - } - } diff --git a/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/RetryTopicExceptionRoutingIntegrationTests.java b/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/RetryTopicExceptionRoutingIntegrationTests.java index af6bd48d44..e8f5282a34 100644 --- a/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/RetryTopicExceptionRoutingIntegrationTests.java +++ b/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/RetryTopicExceptionRoutingIntegrationTests.java @@ -56,6 +56,8 @@ import org.springframework.kafka.test.context.EmbeddedKafka; import org.springframework.messaging.handler.annotation.Header; import org.springframework.retry.annotation.Backoff; +import org.springframework.scheduling.TaskScheduler; +import org.springframework.scheduling.concurrent.ThreadPoolTaskScheduler; import org.springframework.test.annotation.DirtiesContext; import org.springframework.test.context.junit.jupiter.SpringJUnitConfig; import org.springframework.util.backoff.FixedBackOff; @@ -68,7 +70,7 @@ @SpringJUnitConfig @DirtiesContext @EmbeddedKafka -public class RetryTopicExceptionRoutingIntegrationTests { +public class RetryTopicExceptionRoutingIntegrationTests extends AbstractRetryTopicIntegrationTests { private static final Logger logger = LoggerFactory.getLogger(RetryTopicExceptionRoutingIntegrationTests.class); @@ -379,6 +381,11 @@ DltProcessorWithError dltProcessorWithError() { return new DltProcessorWithError(); } + @Bean + TaskScheduler sched() { + return new ThreadPoolTaskScheduler(); + } + } @Configuration diff --git a/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/RetryTopicIntegrationTests.java b/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/RetryTopicIntegrationTests.java index 57ce304031..fb2702f326 100644 --- a/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/RetryTopicIntegrationTests.java +++ b/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/RetryTopicIntegrationTests.java @@ -41,7 +41,7 @@ import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.kafka.annotation.DltHandler; -import org.springframework.kafka.annotation.EnableKafkaRetryTopic; +import org.springframework.kafka.annotation.EnableKafka; import org.springframework.kafka.annotation.KafkaListener; import org.springframework.kafka.annotation.PartitionOffset; import org.springframework.kafka.annotation.RetryableTopic; @@ -64,6 +64,8 @@ import org.springframework.messaging.converter.SmartMessageConverter; import org.springframework.messaging.handler.annotation.Header; import org.springframework.retry.annotation.Backoff; +import org.springframework.scheduling.TaskScheduler; +import org.springframework.scheduling.concurrent.ThreadPoolTaskScheduler; import org.springframework.stereotype.Component; import org.springframework.test.annotation.DirtiesContext; import org.springframework.test.context.TestPropertySource; @@ -83,7 +85,7 @@ RetryTopicIntegrationTests.FOURTH_TOPIC, RetryTopicIntegrationTests.FIFTH_TOPIC }) @TestPropertySource(properties = "five.attempts=5") -public class RetryTopicIntegrationTests { +public class RetryTopicIntegrationTests extends AbstractRetryTopicIntegrationTests { private static final Logger logger = LoggerFactory.getLogger(RetryTopicIntegrationTests.class); @@ -388,7 +390,7 @@ public MyDontRetryException(String msg) { } @Configuration - static class RetryTopicConfigurations { + static class RetryTopicConfigurations extends RetryTopicConfigurationSupport { private static final String DLT_METHOD_NAME = "processDltMessage"; @@ -511,7 +513,7 @@ public KafkaTemplate kafkaTemplate() { } } - @EnableKafkaRetryTopic + @EnableKafka @Configuration public static class KafkaConsumerConfig { @@ -572,6 +574,12 @@ public ConcurrentKafkaListenerContainerFactory kafkaListenerCont factory.setConcurrency(1); return factory; } + + @Bean + TaskScheduler sched() { + return new ThreadPoolTaskScheduler(); + } + } } diff --git a/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/RetryTopicInternalBeanNamesTests.java b/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/RetryTopicInternalBeanNamesTests.java deleted file mode 100644 index 29d9137d6b..0000000000 --- a/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/RetryTopicInternalBeanNamesTests.java +++ /dev/null @@ -1,70 +0,0 @@ -/* - * Copyright 2018-2022 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.kafka.retrytopic; - -import static org.assertj.core.api.Assertions.assertThat; - -import org.junit.jupiter.api.Test; - -/** - * @author Tomaz Fernandes - * @author Gary Russell - * @since 2.7 - */ -class RetryTopicInternalBeanNamesTests { - - static final String DESTINATION_TOPIC_PROCESSOR_NAME = "internalDestinationTopicProcessor"; - - static final String KAFKA_CONSUMER_BACKOFF_MANAGER = "internalKafkaConsumerBackoffManager"; - - static final String RETRY_TOPIC_CONFIGURER = "internalRetryTopicConfigurer"; - - static final String LISTENER_CONTAINER_FACTORY_RESOLVER_NAME = "internalListenerContainerFactoryResolver"; - - static final String LISTENER_CONTAINER_FACTORY_CONFIGURER_NAME = "internalListenerContainerFactoryConfigurer"; - - static final String DEAD_LETTER_PUBLISHING_RECOVERER_PROVIDER_NAME = "internalDeadLetterPublishingRecovererProvider"; - - static final String DESTINATION_TOPIC_CONTAINER_NAME = "internalDestinationTopicContainer"; - - static final String DEFAULT_LISTENER_FACTORY_BEAN_NAME = "internalRetryTopicListenerContainerFactory"; - - static final String DEFAULT_KAFKA_TEMPLATE_BEAN_NAME = "retryTopicDefaultKafkaTemplate"; - - @SuppressWarnings("deprecation") - @Test - public void assertRetryTopicInternalBeanNamesConstants() { - new RetryTopicInternalBeanNames() { }; // for coverage - assertThat(RetryTopicInternalBeanNames.DESTINATION_TOPIC_PROCESSOR_NAME) - .isEqualTo(DESTINATION_TOPIC_PROCESSOR_NAME); - assertThat(RetryTopicInternalBeanNames.KAFKA_CONSUMER_BACKOFF_MANAGER) - .isEqualTo(KAFKA_CONSUMER_BACKOFF_MANAGER); - assertThat(RetryTopicInternalBeanNames.RETRY_TOPIC_CONFIGURER).isEqualTo(RETRY_TOPIC_CONFIGURER); - assertThat(RetryTopicInternalBeanNames.LISTENER_CONTAINER_FACTORY_RESOLVER_NAME) - .isEqualTo(LISTENER_CONTAINER_FACTORY_RESOLVER_NAME); - assertThat(RetryTopicInternalBeanNames.LISTENER_CONTAINER_FACTORY_CONFIGURER_NAME) - .isEqualTo(LISTENER_CONTAINER_FACTORY_CONFIGURER_NAME); - assertThat(RetryTopicInternalBeanNames.DEAD_LETTER_PUBLISHING_RECOVERER_FACTORY_BEAN_NAME) - .isEqualTo(DEAD_LETTER_PUBLISHING_RECOVERER_PROVIDER_NAME); - assertThat(RetryTopicInternalBeanNames.DESTINATION_TOPIC_CONTAINER_NAME) - .isEqualTo(DESTINATION_TOPIC_CONTAINER_NAME); - assertThat(RetryTopicInternalBeanNames.DEFAULT_LISTENER_FACTORY_BEAN_NAME) - .isEqualTo(DEFAULT_LISTENER_FACTORY_BEAN_NAME); - assertThat(RetryTopicInternalBeanNames.DEFAULT_KAFKA_TEMPLATE_BEAN_NAME) - .isEqualTo(DEFAULT_KAFKA_TEMPLATE_BEAN_NAME); - } -} diff --git a/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/RetryTopicLegacyFactoryConfigurerIntegrationTests.java b/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/RetryTopicLegacyFactoryConfigurerIntegrationTests.java deleted file mode 100644 index b80b5d48ae..0000000000 --- a/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/RetryTopicLegacyFactoryConfigurerIntegrationTests.java +++ /dev/null @@ -1,217 +0,0 @@ -/* - * Copyright 2021-2022 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.kafka.retrytopic; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.fail; - -import java.util.HashMap; -import java.util.Map; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.TimeUnit; - -import org.apache.kafka.clients.consumer.ConsumerConfig; -import org.apache.kafka.clients.producer.ProducerConfig; -import org.apache.kafka.common.serialization.StringDeserializer; -import org.apache.kafka.common.serialization.StringSerializer; -import org.junit.jupiter.api.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import org.springframework.beans.factory.BeanFactory; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.kafka.annotation.DltHandler; -import org.springframework.kafka.annotation.EnableKafka; -import org.springframework.kafka.annotation.KafkaListener; -import org.springframework.kafka.annotation.RetryableTopic; -import org.springframework.kafka.config.ConcurrentKafkaListenerContainerFactory; -import org.springframework.kafka.core.ConsumerFactory; -import org.springframework.kafka.core.DefaultKafkaConsumerFactory; -import org.springframework.kafka.core.DefaultKafkaProducerFactory; -import org.springframework.kafka.core.KafkaTemplate; -import org.springframework.kafka.core.ProducerFactory; -import org.springframework.kafka.listener.ContainerProperties; -import org.springframework.kafka.support.KafkaHeaders; -import org.springframework.kafka.test.EmbeddedKafkaBroker; -import org.springframework.kafka.test.context.EmbeddedKafka; -import org.springframework.messaging.handler.annotation.Header; -import org.springframework.retry.annotation.Backoff; -import org.springframework.stereotype.Component; -import org.springframework.test.annotation.DirtiesContext; -import org.springframework.test.context.junit.jupiter.SpringJUnitConfig; - -/** - * @author Tomaz Fernandes - * @since 2.8.3 - */ -@SpringJUnitConfig -@DirtiesContext -@EmbeddedKafka(topics = RetryTopicLegacyFactoryConfigurerIntegrationTests.FIRST_TOPIC, partitions = 1) -public class RetryTopicLegacyFactoryConfigurerIntegrationTests { - - private static final Logger logger = LoggerFactory.getLogger(RetryTopicLegacyFactoryConfigurerIntegrationTests.class); - - public final static String FIRST_TOPIC = "myRetryTopic1"; - - @Autowired - private KafkaTemplate sendKafkaTemplate; - - @Autowired - private CountDownLatchContainer latchContainer; - - @Test - void shouldRetryFirstAndSecondTopics() { - logger.debug("Sending message to topic " + FIRST_TOPIC); - sendKafkaTemplate.send(FIRST_TOPIC, "Testing topic 1"); - assertThat(awaitLatch(latchContainer.countDownLatch1)).isTrue(); - assertThat(awaitLatch(latchContainer.countDownLatchDltOne)).isTrue(); - } - - private boolean awaitLatch(CountDownLatch latch) { - try { - return latch.await(150, TimeUnit.SECONDS); - } - catch (Exception e) { - fail(e.getMessage()); - throw new RuntimeException(e); - } - } - - @Component - static class RetryableKafkaListener { - - @Autowired - CountDownLatchContainer countDownLatchContainer; - - @RetryableTopic( - attempts = "4", - backoff = @Backoff(delay = 1000, multiplier = 2.0), - autoCreateTopics = "false", - topicSuffixingStrategy = TopicSuffixingStrategy.SUFFIX_WITH_INDEX_VALUE) - @KafkaListener(topics = RetryTopicLegacyFactoryConfigurerIntegrationTests.FIRST_TOPIC) - public void listen(String in, @Header(KafkaHeaders.RECEIVED_TOPIC) String topic) { - countDownLatchContainer.countDownLatch1.countDown(); - logger.warn(in + " from " + topic); - throw new RuntimeException("test"); - } - - @DltHandler - public void dlt(String in, @Header(KafkaHeaders.RECEIVED_TOPIC) String topic) { - countDownLatchContainer.countDownLatchDltOne.countDown(); - logger.warn(in + " from " + topic); - } - } - - @Component - static class CountDownLatchContainer { - - CountDownLatch countDownLatch1 = new CountDownLatch(4); - CountDownLatch countDownLatchDltOne = new CountDownLatch(1); - CountDownLatch customizerLatch = new CountDownLatch(6); - } - - @EnableKafka - @Configuration - static class Config { - - @Autowired - EmbeddedKafkaBroker broker; - - @Bean - CountDownLatchContainer latchContainer() { - return new CountDownLatchContainer(); - } - - @Bean - RetryableKafkaListener retryableKafkaListener() { - return new RetryableKafkaListener(); - } - - @Bean - public ConcurrentKafkaListenerContainerFactory kafkaListenerContainerFactory( - ConsumerFactory consumerFactory, CountDownLatchContainer latchContainer) { - - ConcurrentKafkaListenerContainerFactory factory = new ConcurrentKafkaListenerContainerFactory<>(); - factory.setConsumerFactory(consumerFactory); - ContainerProperties props = factory.getContainerProperties(); - props.setIdleEventInterval(100L); - props.setPollTimeout(50L); - props.setIdlePartitionEventInterval(100L); - factory.setConsumerFactory(consumerFactory); - factory.setConcurrency(1); - factory.setContainerCustomizer( - container -> latchContainer.customizerLatch.countDown()); - return factory; - } - - @Bean - public ProducerFactory producerFactory() { - Map configProps = new HashMap<>(); - configProps.put( - ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, - this.broker.getBrokersAsString()); - configProps.put( - ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, - StringSerializer.class); - configProps.put( - ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, - StringSerializer.class); - return new DefaultKafkaProducerFactory<>(configProps); - } - - @Bean - public KafkaTemplate kafkaTemplate() { - return new KafkaTemplate<>(producerFactory()); - } - - @SuppressWarnings("deprecation") - @Bean(name = RetryTopicInternalBeanNames.RETRY_TOPIC_CONFIGURER) - public RetryTopicConfigurer retryTopicConfigurer(DestinationTopicProcessor destinationTopicProcessor, - ListenerContainerFactoryResolver containerFactoryResolver, - ListenerContainerFactoryConfigurer listenerContainerFactoryConfigurer, - BeanFactory beanFactory, - RetryTopicNamesProviderFactory retryTopicNamesProviderFactory) { - RetryTopicConfigurer retryTopicConfigurer = new RetryTopicConfigurer(destinationTopicProcessor, containerFactoryResolver, listenerContainerFactoryConfigurer, beanFactory, retryTopicNamesProviderFactory); - retryTopicConfigurer.useLegacyFactoryConfigurer(true); - return retryTopicConfigurer; - } - - @Bean - public ConsumerFactory consumerFactory() { - Map props = new HashMap<>(); - props.put( - ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, - this.broker.getBrokersAsString()); - props.put( - ConsumerConfig.GROUP_ID_CONFIG, - "groupId"); - props.put( - ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, - StringDeserializer.class); - props.put( - ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, - StringDeserializer.class); - props.put( - ConsumerConfig.ALLOW_AUTO_CREATE_TOPICS_CONFIG, false); - props.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest"); - - return new DefaultKafkaConsumerFactory<>(props); - } - } -} diff --git a/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/RetryTopicSameContainerFactoryIntegrationTests.java b/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/RetryTopicSameContainerFactoryIntegrationTests.java index 454cd42b8e..4f7385e5e2 100644 --- a/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/RetryTopicSameContainerFactoryIntegrationTests.java +++ b/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/RetryTopicSameContainerFactoryIntegrationTests.java @@ -52,6 +52,8 @@ import org.springframework.kafka.test.context.EmbeddedKafka; import org.springframework.messaging.handler.annotation.Header; import org.springframework.retry.annotation.Backoff; +import org.springframework.scheduling.TaskScheduler; +import org.springframework.scheduling.concurrent.ThreadPoolTaskScheduler; import org.springframework.stereotype.Component; import org.springframework.test.annotation.DirtiesContext; import org.springframework.test.context.junit.jupiter.SpringJUnitConfig; @@ -65,7 +67,7 @@ @DirtiesContext @EmbeddedKafka(topics = { RetryTopicSameContainerFactoryIntegrationTests.FIRST_TOPIC, RetryTopicSameContainerFactoryIntegrationTests.SECOND_TOPIC}, partitions = 1) -public class RetryTopicSameContainerFactoryIntegrationTests { +public class RetryTopicSameContainerFactoryIntegrationTests extends AbstractRetryTopicIntegrationTests { private static final Logger logger = LoggerFactory.getLogger(RetryTopicSameContainerFactoryIntegrationTests.class); @@ -150,7 +152,7 @@ static class CountDownLatchContainer { @EnableKafka @Configuration - static class Config { + static class Config extends RetryTopicConfigurationSupport { @Autowired EmbeddedKafkaBroker broker; @@ -232,5 +234,12 @@ public ConsumerFactory consumerFactory() { return new DefaultKafkaConsumerFactory<>(props); } + + @Bean + TaskScheduler sched() { + return new ThreadPoolTaskScheduler(); + } + } + } diff --git a/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/RetryableTopicAnnotationProcessorTests.java b/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/RetryableTopicAnnotationProcessorTests.java index d19f052565..5daa04fcd2 100644 --- a/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/RetryableTopicAnnotationProcessorTests.java +++ b/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/RetryableTopicAnnotationProcessorTests.java @@ -18,6 +18,7 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatExceptionOfType; +import static org.assertj.core.api.Assertions.assertThatIllegalStateException; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.BDDMockito.given; import static org.mockito.BDDMockito.willAnswer; @@ -33,6 +34,7 @@ import org.springframework.beans.factory.BeanInitializationException; import org.springframework.beans.factory.NoSuchBeanDefinitionException; +import org.springframework.beans.factory.ObjectProvider; import org.springframework.beans.factory.config.ConfigurableBeanFactory; import org.springframework.core.annotation.AnnotationUtils; import org.springframework.kafka.annotation.DltHandler; @@ -109,7 +111,7 @@ private Object createBean() { void shouldGetDltHandlerMethod() { // setup - given(beanFactory.getBean(RetryTopicInternalBeanNames.DEFAULT_KAFKA_TEMPLATE_BEAN_NAME, KafkaOperations.class)) + given(beanFactory.getBean(RetryTopicBeanNames.DEFAULT_KAFKA_TEMPLATE_BEAN_NAME, KafkaOperations.class)) .willReturn(kafkaOperationsFromDefaultName); RetryableTopicAnnotationProcessor processor = new RetryableTopicAnnotationProcessor(beanFactory); @@ -163,19 +165,19 @@ void shouldThrowIfProvidedKafkaTemplateNotFound() { void shouldThrowIfNoKafkaTemplateFound() { // setup - given(this.beanFactory.getBean(RetryTopicInternalBeanNames.DEFAULT_KAFKA_TEMPLATE_BEAN_NAME, KafkaOperations.class)) - .willThrow(NoSuchBeanDefinitionException.class); - given(this.beanFactory.getBean(RetryTopicBeanNames.DEFAULT_KAFKA_TEMPLATE_BEAN_NAME, KafkaOperations.class)) .willThrow(NoSuchBeanDefinitionException.class); - given(this.beanFactory.getBean("kafkaTemplate", KafkaOperations.class)) - .willThrow(NoSuchBeanDefinitionException.class); + @SuppressWarnings({ "unchecked", "rawtypes" }) + ObjectProvider templateProvider = mock(ObjectProvider.class); + given(templateProvider.getIfUnique()).willReturn(null); + given(this.beanFactory.getBeanProvider(KafkaOperations.class)) + .willReturn(templateProvider); RetryableTopicAnnotationProcessor processor = new RetryableTopicAnnotationProcessor(beanFactory); // given - then - assertThatExceptionOfType(BeanInitializationException.class).isThrownBy(() -> + assertThatIllegalStateException().isThrownBy(() -> processor.processAnnotation(topics, listenWithRetryAndDlt, annotationWithDlt, beanWithDlt)); } @@ -183,12 +185,13 @@ void shouldThrowIfNoKafkaTemplateFound() { void shouldTrySpringBootDefaultKafkaTemplate() { // setup - given(this.beanFactory.getBean(RetryTopicInternalBeanNames.DEFAULT_KAFKA_TEMPLATE_BEAN_NAME, KafkaOperations.class)) - .willThrow(NoSuchBeanDefinitionException.class); given(this.beanFactory.getBean(RetryTopicBeanNames.DEFAULT_KAFKA_TEMPLATE_BEAN_NAME, KafkaOperations.class)) .willThrow(NoSuchBeanDefinitionException.class); - given(this.beanFactory.getBean("kafkaTemplate", KafkaOperations.class)) - .willReturn(kafkaOperationsFromDefaultName); + @SuppressWarnings({ "unchecked", "rawtypes" }) + ObjectProvider templateProvider = mock(ObjectProvider.class); + given(templateProvider.getIfUnique()).willReturn(kafkaOperationsFromDefaultName); + given(this.beanFactory.getBeanProvider(KafkaOperations.class)) + .willReturn(templateProvider); RetryableTopicAnnotationProcessor processor = new RetryableTopicAnnotationProcessor(beanFactory); // given - then @@ -219,7 +222,7 @@ void shouldGetKafkaTemplateFromBeanName() { void shouldGetKafkaTemplateFromDefaultBeanName() { // setup - given(this.beanFactory.getBean(RetryTopicInternalBeanNames.DEFAULT_KAFKA_TEMPLATE_BEAN_NAME, KafkaOperations.class)) + given(this.beanFactory.getBean(RetryTopicBeanNames.DEFAULT_KAFKA_TEMPLATE_BEAN_NAME, KafkaOperations.class)) .willReturn(kafkaOperationsFromDefaultName); RetryableTopicAnnotationProcessor processor = new RetryableTopicAnnotationProcessor(beanFactory); @@ -237,7 +240,7 @@ void shouldGetKafkaTemplateFromDefaultBeanName() { void shouldCreateExponentialBackoff() { // setup - given(this.beanFactory.getBean(RetryTopicInternalBeanNames.DEFAULT_KAFKA_TEMPLATE_BEAN_NAME, KafkaOperations.class)) + given(this.beanFactory.getBean(RetryTopicBeanNames.DEFAULT_KAFKA_TEMPLATE_BEAN_NAME, KafkaOperations.class)) .willReturn(kafkaOperationsFromDefaultName); RetryableTopicAnnotationProcessor processor = new RetryableTopicAnnotationProcessor(beanFactory); @@ -264,7 +267,7 @@ void shouldCreateExponentialBackoff() { void shouldSetAbort() { // setup - given(this.beanFactory.getBean(RetryTopicInternalBeanNames.DEFAULT_KAFKA_TEMPLATE_BEAN_NAME, KafkaOperations.class)) + given(this.beanFactory.getBean(RetryTopicBeanNames.DEFAULT_KAFKA_TEMPLATE_BEAN_NAME, KafkaOperations.class)) .willReturn(kafkaOperationsFromDefaultName); RetryableTopicAnnotationProcessor processor = new RetryableTopicAnnotationProcessor(beanFactory);