-
Notifications
You must be signed in to change notification settings - Fork 1.5k
/
RetryTopicConfigurationSupport.java
392 lines (349 loc) · 17.3 KB
/
RetryTopicConfigurationSupport.java
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
/*
* Copyright 2022 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.kafka.retrytopic;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.function.Consumer;
import java.util.stream.Collectors;
import org.apache.kafka.clients.consumer.ConsumerRecord;
import org.springframework.beans.factory.BeanFactory;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.context.ApplicationContext;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.kafka.annotation.EnableKafkaRetryTopic;
import org.springframework.kafka.annotation.KafkaListenerAnnotationBeanPostProcessor;
import org.springframework.kafka.config.KafkaListenerConfigUtils;
import org.springframework.kafka.config.KafkaListenerEndpoint;
import org.springframework.kafka.listener.CommonErrorHandler;
import org.springframework.kafka.listener.ConcurrentMessageListenerContainer;
import org.springframework.kafka.listener.ContainerPartitionPausingBackOffManagerFactory;
import org.springframework.kafka.listener.ContainerPausingBackOffHandler;
import org.springframework.kafka.listener.DeadLetterPublishingRecoverer;
import org.springframework.kafka.listener.DefaultErrorHandler;
import org.springframework.kafka.listener.ExceptionClassifier;
import org.springframework.kafka.listener.KafkaBackOffManagerFactory;
import org.springframework.kafka.listener.KafkaConsumerBackoffManager;
import org.springframework.kafka.listener.ListenerContainerPauseService;
import org.springframework.kafka.listener.ListenerContainerRegistry;
import org.springframework.kafka.listener.MessageListenerContainer;
import org.springframework.kafka.support.JavaUtils;
import org.springframework.lang.Nullable;
import org.springframework.scheduling.TaskScheduler;
import org.springframework.util.Assert;
import org.springframework.util.backoff.BackOff;
import org.springframework.util.backoff.FixedBackOff;
/**
* This is the main class providing the configuration behind the non-blocking,
* topic-based delayed retries feature. It is typically imported by adding
* {@link EnableKafkaRetryTopic @EnableKafkaRetryTopic} to an application
* {@link Configuration @Configuration} class. An alternative more advanced option
* is to extend directly from this class and override methods as necessary, remembering
* to add {@link Configuration @Configuration} to the subclass and {@link Bean @Bean}
* to overridden {@link Bean @Bean} methods. For more details see the javadoc of
* {@link EnableKafkaRetryTopic @EnableRetryTopic}.
*
* @author Tomaz Fernandes
* @author Gary Russell
* @since 2.9
*/
public class RetryTopicConfigurationSupport {
private static final AtomicBoolean ONLY_ONE_ALLOWED = new AtomicBoolean(true);
private final RetryTopicComponentFactory componentFactory = createComponentFactory();
public RetryTopicConfigurationSupport() {
Assert.state(ONLY_ONE_ALLOWED.getAndSet(false), "Only one 'RetryTopicConfigurationSupport' is allowed");
}
/**
* Return a global {@link RetryTopicConfigurer} for configuring retry topics
* for {@link KafkaListenerEndpoint} instances with a corresponding
* {@link org.springframework.kafka.retrytopic.RetryTopicConfiguration}.
* To configure it, consider overriding the {@link #configureRetryTopicConfigurer()}.
* @param kafkaConsumerBackoffManager the global {@link KafkaConsumerBackoffManager}.
* @param destinationTopicResolver the global {@link DestinationTopicResolver}.
* @param beanFactory the {@link BeanFactory}.
* @return the instance.
* @see KafkaListenerAnnotationBeanPostProcessor
*/
@Bean(name = RetryTopicBeanNames.RETRY_TOPIC_CONFIGURER_BEAN_NAME)
public RetryTopicConfigurer retryTopicConfigurer(@Qualifier(KafkaListenerConfigUtils.KAFKA_CONSUMER_BACK_OFF_MANAGER_BEAN_NAME)
KafkaConsumerBackoffManager kafkaConsumerBackoffManager,
@Qualifier(RetryTopicBeanNames.DESTINATION_TOPIC_RESOLVER_BEAN_NAME)
DestinationTopicResolver destinationTopicResolver,
BeanFactory beanFactory) {
DestinationTopicProcessor destinationTopicProcessor = this.componentFactory
.destinationTopicProcessor(destinationTopicResolver);
DeadLetterPublishingRecovererFactory dlprf = this.componentFactory
.deadLetterPublishingRecovererFactory(destinationTopicResolver);
ListenerContainerFactoryConfigurer lcfc = this.componentFactory
.listenerContainerFactoryConfigurer(kafkaConsumerBackoffManager,
dlprf, this.componentFactory.internalRetryTopicClock());
ListenerContainerFactoryResolver factoryResolver = this.componentFactory
.listenerContainerFactoryResolver(beanFactory);
RetryTopicNamesProviderFactory retryTopicNamesProviderFactory =
this.componentFactory.retryTopicNamesProviderFactory();
processDeadLetterPublishingContainerFactory(dlprf);
processListenerContainerFactoryConfigurer(lcfc);
RetryTopicConfigurer retryTopicConfigurer = this.componentFactory
.retryTopicConfigurer(destinationTopicProcessor, lcfc,
factoryResolver, retryTopicNamesProviderFactory);
Consumer<RetryTopicConfigurer> configurerConsumer = configureRetryTopicConfigurer();
Assert.notNull(configurerConsumer, "configureRetryTopicConfigurer cannot return null.");
configurerConsumer.accept(retryTopicConfigurer);
return retryTopicConfigurer;
}
/**
* Override this method if you need to configure the {@link RetryTopicConfigurer}.
* @return a {@link RetryTopicConfigurer} consumer.
*/
protected Consumer<RetryTopicConfigurer> configureRetryTopicConfigurer() {
return retryTopicConfigurer -> {
};
}
/**
* Internal method for processing the {@link DeadLetterPublishingRecovererFactory}.
* Consider overriding the {@link #configureDeadLetterPublishingContainerFactory()}
* method if further customization is required.
* @param deadLetterPublishingRecovererFactory the instance.
*/
private void processDeadLetterPublishingContainerFactory(
DeadLetterPublishingRecovererFactory deadLetterPublishingRecovererFactory) {
CustomizersConfigurer customizersConfigurer = new CustomizersConfigurer();
configureCustomizers(customizersConfigurer);
JavaUtils.INSTANCE
.acceptIfNotNull(customizersConfigurer.deadLetterPublishingRecovererCustomizer,
deadLetterPublishingRecovererFactory::setDeadLetterPublishingRecovererCustomizer);
Consumer<DeadLetterPublishingRecovererFactory> dlprfConsumer = configureDeadLetterPublishingContainerFactory();
Assert.notNull(dlprfConsumer, "configureDeadLetterPublishingContainerFactory must not return null");
dlprfConsumer.accept(deadLetterPublishingRecovererFactory);
}
/**
* Override this method to further configure the {@link DeadLetterPublishingRecovererFactory}.
* @return a {@link DeadLetterPublishingRecovererFactory} consumer.
*/
protected Consumer<DeadLetterPublishingRecovererFactory> configureDeadLetterPublishingContainerFactory() {
return dlprf -> {
};
}
/**
* Internal method for processing the {@link ListenerContainerFactoryConfigurer}.
* Consider overriding {@link #configureListenerContainerFactoryConfigurer()}
* if further customization is required.
* @param listenerContainerFactoryConfigurer the {@link ListenerContainerFactoryConfigurer} instance.
*/
private void processListenerContainerFactoryConfigurer(ListenerContainerFactoryConfigurer listenerContainerFactoryConfigurer) {
CustomizersConfigurer customizersConfigurer = new CustomizersConfigurer();
configureCustomizers(customizersConfigurer);
BlockingRetriesConfigurer blockingRetriesConfigurer = new BlockingRetriesConfigurer();
configureBlockingRetries(blockingRetriesConfigurer);
JavaUtils.INSTANCE
.acceptIfNotNull(blockingRetriesConfigurer.backOff,
listenerContainerFactoryConfigurer::setBlockingRetriesBackOff)
.acceptIfNotNull(blockingRetriesConfigurer.retryableExceptions,
listenerContainerFactoryConfigurer::setBlockingRetryableExceptions)
.acceptIfNotNull(customizersConfigurer.errorHandlerCustomizer,
listenerContainerFactoryConfigurer::setErrorHandlerCustomizer)
.acceptIfNotNull(customizersConfigurer.listenerContainerCustomizer,
listenerContainerFactoryConfigurer::setContainerCustomizer);
Consumer<ListenerContainerFactoryConfigurer> lcfcConfigurer = configureListenerContainerFactoryConfigurer();
Assert.notNull(lcfcConfigurer, "configureListenerContainerFactoryConfigurer must not return null.");
lcfcConfigurer.accept(listenerContainerFactoryConfigurer);
}
/**
* Override this method to further configure the {@link ListenerContainerFactoryConfigurer}.
* @return a {@link ListenerContainerFactoryConfigurer} consumer.
*/
protected Consumer<ListenerContainerFactoryConfigurer> configureListenerContainerFactoryConfigurer() {
return lcfc -> {
};
}
/**
* Override this method to configure blocking retries parameters
* such as exceptions to be retried and the {@link BackOff} to be used.
* @param blockingRetries a {@link BlockingRetriesConfigurer}.
*/
protected void configureBlockingRetries(BlockingRetriesConfigurer blockingRetries) {
}
/**
* Override this method to manage non-blocking retries fatal exceptions.
* Records which processing throws an exception present in this list will be
* forwarded directly to the DLT, if one is configured, or stop being processed
* otherwise.
* @param nonBlockingRetriesExceptions a {@link List} of fatal exceptions
* containing the framework defaults.
*/
protected void manageNonBlockingFatalExceptions(List<Class<? extends Throwable>> nonBlockingRetriesExceptions) {
}
/**
* Override this method to configure customizers for components created
* by non-blocking retries' configuration, such as {@link MessageListenerContainer},
* {@link DeadLetterPublishingRecoverer} and {@link DefaultErrorHandler}.
* @param customizersConfigurer a {@link CustomizersConfigurer}.
*/
protected void configureCustomizers(CustomizersConfigurer customizersConfigurer) {
}
/**
* Return a global {@link DestinationTopicResolver} for resolving
* the {@link DestinationTopic} to which a given {@link ConsumerRecord}
* should be sent for retry.
*
* To configure it, consider overriding one of these other more
* fine-grained methods:
* <ul>
* <li>{@link #manageNonBlockingFatalExceptions} to configure non-blocking retries.
* <li>{@link #configureDestinationTopicResolver} to further customize the component.
* <li>{@link #createComponentFactory} to provide a subclass instance.
* </ul>
* @return the instance.
*/
@Bean(name = RetryTopicBeanNames.DESTINATION_TOPIC_RESOLVER_BEAN_NAME)
public DestinationTopicResolver destinationTopicResolver() {
DestinationTopicResolver destinationTopicResolver = this.componentFactory.destinationTopicResolver();
JavaUtils.INSTANCE.acceptIfInstanceOf(DefaultDestinationTopicResolver.class, destinationTopicResolver,
this::configureNonBlockingFatalExceptions);
Consumer<DestinationTopicResolver> resolverConsumer = configureDestinationTopicResolver();
Assert.notNull(resolverConsumer, "customizeDestinationTopicResolver must not return null");
resolverConsumer.accept(destinationTopicResolver);
return destinationTopicResolver;
}
private void configureNonBlockingFatalExceptions(DefaultDestinationTopicResolver destinationTopicResolver) {
List<Class<? extends Throwable>> fatalExceptions =
new ArrayList<>(ExceptionClassifier.defaultFatalExceptionsList());
manageNonBlockingFatalExceptions(fatalExceptions);
destinationTopicResolver.setClassifications(fatalExceptions.stream()
.collect(Collectors.toMap(ex -> ex, ex -> false)), true);
}
/**
* Override this method to configure the {@link DestinationTopicResolver}.
* @return a {@link DestinationTopicResolver} consumer.
*/
protected Consumer<DestinationTopicResolver> configureDestinationTopicResolver() {
return dtr -> {
};
}
/**
* Create the {@link KafkaConsumerBackoffManager} instance that will be used to
* back off partitions.
* To provide a custom implementation, either override this method, or
* override the {@link RetryTopicComponentFactory#kafkaBackOffManagerFactory} method
* and return a different {@link KafkaBackOffManagerFactory}.
* @param applicationContext the application context.
* @param registry the {@link ListenerContainerRegistry} to be used to fetch the
* {@link MessageListenerContainer} at runtime to be backed off.
* @param wrapper a {@link RetryTopicSchedulerWrapper}.
* @param taskScheduler a {@link TaskScheduler}.
* @return the instance.
*/
@Bean(name = KafkaListenerConfigUtils.KAFKA_CONSUMER_BACK_OFF_MANAGER_BEAN_NAME)
public KafkaConsumerBackoffManager kafkaConsumerBackoffManager(ApplicationContext applicationContext,
@Qualifier(KafkaListenerConfigUtils.KAFKA_LISTENER_ENDPOINT_REGISTRY_BEAN_NAME)
ListenerContainerRegistry registry, @Nullable RetryTopicSchedulerWrapper wrapper,
@Nullable TaskScheduler taskScheduler) {
KafkaBackOffManagerFactory backOffManagerFactory =
this.componentFactory.kafkaBackOffManagerFactory(registry, applicationContext);
JavaUtils.INSTANCE.acceptIfInstanceOf(ContainerPartitionPausingBackOffManagerFactory.class, backOffManagerFactory,
factory -> configurePartitionPausingFactory(factory, registry,
wrapper != null ? wrapper.getScheduler() : taskScheduler));
return backOffManagerFactory.create();
}
private void configurePartitionPausingFactory(ContainerPartitionPausingBackOffManagerFactory factory,
ListenerContainerRegistry registry, @Nullable TaskScheduler scheduler) {
Assert.notNull(scheduler, "Either a RetryTopicSchedulerWrapper or TaskScheduler bean is required");
factory.setBackOffHandler(new ContainerPausingBackOffHandler(
new ListenerContainerPauseService(registry, scheduler)));
}
/**
* Override this method to provide a subclass of {@link RetryTopicComponentFactory}
* with different component implementations or subclasses.
* @return the instance.
*/
protected RetryTopicComponentFactory createComponentFactory() {
return new RetryTopicComponentFactory();
}
/**
* Configure blocking retries to be used along non-blocking.
*/
public static class BlockingRetriesConfigurer {
private BackOff backOff;
private Class<? extends Exception>[] retryableExceptions;
/**
* Set the exceptions that should be retried by the blocking retry mechanism.
* @param exceptions the exceptions.
* @return the configurer.
* @see DefaultErrorHandler
*/
@SuppressWarnings("varargs")
@SafeVarargs
public final BlockingRetriesConfigurer retryOn(Class<? extends Exception>... exceptions) {
this.retryableExceptions = Arrays.copyOf(exceptions, exceptions.length);
return this;
}
/**
* Set the {@link BackOff} that should be used with the blocking retry mechanism.
* By default, a {@link FixedBackOff} with 0 delay and 9 retry attempts
* is configured. Note that this only has any effect for exceptions specified
* with the {@link #retryOn} method - by default blocking retries are disabled
* when using the non-blocking retries feature.
* @param backoff the {@link BackOff} instance.
* @return the configurer.
* @see DefaultErrorHandler
*/
public BlockingRetriesConfigurer backOff(BackOff backoff) {
this.backOff = backoff;
return this;
}
}
/**
* Configure customizers for components instantiated by the retry topics feature.
*/
public static class CustomizersConfigurer {
private Consumer<DefaultErrorHandler> errorHandlerCustomizer;
private Consumer<ConcurrentMessageListenerContainer<?, ?>> listenerContainerCustomizer;
private Consumer<DeadLetterPublishingRecoverer> deadLetterPublishingRecovererCustomizer;
/**
* Customize the {@link CommonErrorHandler} instances that will be used for the
* feature.
* @param errorHandlerCustomizer the customizer.
* @return the configurer.
* @see DefaultErrorHandler
*/
public CustomizersConfigurer customizeErrorHandler(Consumer<DefaultErrorHandler> errorHandlerCustomizer) {
this.errorHandlerCustomizer = errorHandlerCustomizer;
return this;
}
/**
* Customize the {@link ConcurrentMessageListenerContainer} instances created
* for the retry and DLT consumers.
* @param listenerContainerCustomizer the customizer.
* @return the configurer.
*/
public CustomizersConfigurer customizeListenerContainer(Consumer<ConcurrentMessageListenerContainer<?, ?>> listenerContainerCustomizer) {
this.listenerContainerCustomizer = listenerContainerCustomizer;
return this;
}
/**
* Customize the {@link DeadLetterPublishingRecoverer} that will be used to
* forward the records to the retry topics and DLT.
* @param dlprCustomizer the customizer.
* @return the configurer.
*/
public CustomizersConfigurer customizeDeadLetterPublishingRecoverer(Consumer<DeadLetterPublishingRecoverer> dlprCustomizer) {
this.deadLetterPublishingRecovererCustomizer = dlprCustomizer;
return this;
}
}
}