From d95cceb8a41b5050f6949889d043927f70ca7b09 Mon Sep 17 00:00:00 2001 From: Philipp Schirmer Date: Thu, 26 Feb 2026 10:31:58 +0100 Subject: [PATCH 01/15] Clean up Consumer/ConsumerProducer --- .../SimpleKafkaConsumerApplication.java | 4 +- .../KafkaConsumerProducerApplication.java | 7 +-- ...impleKafkaConsumerProducerApplication.java | 4 +- ...fkaConsumerProducerApplicationCliTest.java | 4 +- .../consumerproducer/apps/CloseFlagApp.java | 12 ++-- .../kafka/consumerproducer/apps/Mirror.java | 12 ++-- .../kafka/admin/ConsumerGroupsClient.java | 39 ++++++++++++ .../kafka/admin/KafkaAdminException.java | 6 +- .../kafka/consumer/ConfiguredConsumerApp.java | 20 ++++--- .../bakdata/kafka/consumer/ConsumerApp.java | 6 +- .../ConsumerApplicationException.java | 4 +- .../kafka/consumer/ConsumerBuilder.java | 20 +++++-- .../kafka/consumer/ConsumerCleanUpRunner.java | 59 ++----------------- .../consumer/ConsumerExecutionOptions.java | 3 +- .../kafka/consumer/ConsumerRunner.java | 9 +-- .../consumer/DefaultConsumerRunnable.java | 4 +- .../kafka/consumer/ExecutableConsumerApp.java | 9 ++- .../ConfiguredConsumerProducerApp.java | 19 +++--- .../consumerproducer/ConsumerProducerApp.java | 7 ++- .../ConsumerProducerAppConfiguration.java | 4 +- .../ConsumerProducerBuilder.java | 15 +++-- .../ConsumerProducerCleanUpRunner.java | 6 +- .../ConsumerProducerRunner.java | 9 +-- .../ConsumerProducerTopicConfig.java | 4 +- .../DefaultConsumerProducerRunnable.java | 4 +- .../ExecutableConsumerProducerApp.java | 12 ++-- .../SerializerDeserializerConfig.java | 21 +++---- .../kafka/producer/ConfiguredProducerApp.java | 3 +- .../consumer/ConsumerCleanUpRunnerTest.java | 44 +++++++------- .../kafka/consumer/ConsumerRunnerTest.java | 7 ++- .../consumer/DefaultConsumerRunnableTest.java | 39 ++++++------ .../bakdata/kafka/consumer/TestHelper.java | 5 +- .../ConsumerProducerCleanUpRunnerTest.java | 34 +++++------ .../ConsumerProducerRunnerTest.java | 7 ++- .../MirrorKeyWithAvroConsumerProducer.java | 12 ++-- .../MirrorValueWithAvroConsumerProducer.java | 12 ++-- .../apps/StringConsumerProducer.java | 12 ++-- .../apps/StringPatternConsumerProducer.java | 12 ++-- .../apps/SimpleConsumerProducerApp.java | 12 ++-- 39 files changed, 282 insertions(+), 240 deletions(-) diff --git a/streams-bootstrap-cli/src/main/java/com/bakdata/kafka/consumer/SimpleKafkaConsumerApplication.java b/streams-bootstrap-cli/src/main/java/com/bakdata/kafka/consumer/SimpleKafkaConsumerApplication.java index c89e0ac8d..58d5b34ce 100644 --- a/streams-bootstrap-cli/src/main/java/com/bakdata/kafka/consumer/SimpleKafkaConsumerApplication.java +++ b/streams-bootstrap-cli/src/main/java/com/bakdata/kafka/consumer/SimpleKafkaConsumerApplication.java @@ -1,7 +1,7 @@ /* * MIT License * - * Copyright (c) 2025 bakdata + * Copyright (c) 2026 bakdata * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -29,7 +29,7 @@ import lombok.RequiredArgsConstructor; /** - * {@code KafkaConsumerApplication} without any additional configuration options. + * {@link KafkaConsumerApplication} without any additional configuration options. * * @param type of {@link ConsumerApp} created by this application */ diff --git a/streams-bootstrap-cli/src/main/java/com/bakdata/kafka/consumerproducer/KafkaConsumerProducerApplication.java b/streams-bootstrap-cli/src/main/java/com/bakdata/kafka/consumerproducer/KafkaConsumerProducerApplication.java index 510c7c200..7267a60e5 100644 --- a/streams-bootstrap-cli/src/main/java/com/bakdata/kafka/consumerproducer/KafkaConsumerProducerApplication.java +++ b/streams-bootstrap-cli/src/main/java/com/bakdata/kafka/consumerproducer/KafkaConsumerProducerApplication.java @@ -1,7 +1,7 @@ /* * MIT License * - * Copyright (c) 2025 bakdata + * Copyright (c) 2026 bakdata * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -94,10 +94,9 @@ public void clean() { } /** - * Clear all state stores and consumer group offsets associated with the Kafka ConsumerProducer application. + * Reset consumer group offsets associated with the Kafka ConsumerProducer application. */ - @Command(description = "Clear all state stores, consumer group offsets, and internal topics associated with the " - + "Kafka ConsumerProducer application.") + @Command(description = "Reset consumer group offsets associated with the Kafka ConsumerProducer application.") public void reset() { this.prepareClean(); try (final CleanableApp app = this.createCleanableApp()) { diff --git a/streams-bootstrap-cli/src/main/java/com/bakdata/kafka/consumerproducer/SimpleKafkaConsumerProducerApplication.java b/streams-bootstrap-cli/src/main/java/com/bakdata/kafka/consumerproducer/SimpleKafkaConsumerProducerApplication.java index b11ae695b..6618ab9fc 100644 --- a/streams-bootstrap-cli/src/main/java/com/bakdata/kafka/consumerproducer/SimpleKafkaConsumerProducerApplication.java +++ b/streams-bootstrap-cli/src/main/java/com/bakdata/kafka/consumerproducer/SimpleKafkaConsumerProducerApplication.java @@ -1,7 +1,7 @@ /* * MIT License * - * Copyright (c) 2024 bakdata + * Copyright (c) 2026 bakdata * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -29,7 +29,7 @@ import lombok.RequiredArgsConstructor; /** - * {@code KafkaConsumerProducerApplication} without any additional configuration options. + * {@link KafkaConsumerProducerApplication} without any additional configuration options. * * @param type of {@link ConsumerProducerApp} created by this application */ diff --git a/streams-bootstrap-cli/src/test/java/com/bakdata/kafka/consumerproducer/KafkaConsumerProducerApplicationCliTest.java b/streams-bootstrap-cli/src/test/java/com/bakdata/kafka/consumerproducer/KafkaConsumerProducerApplicationCliTest.java index b0555eaf7..050cb04e3 100644 --- a/streams-bootstrap-cli/src/test/java/com/bakdata/kafka/consumerproducer/KafkaConsumerProducerApplicationCliTest.java +++ b/streams-bootstrap-cli/src/test/java/com/bakdata/kafka/consumerproducer/KafkaConsumerProducerApplicationCliTest.java @@ -1,7 +1,7 @@ /* * MIT License * - * Copyright (c) 2025 bakdata + * Copyright (c) 2026 bakdata * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -265,7 +265,7 @@ void shouldExitWithSuccessCodeOnShutdown() { @Override public ConsumerProducerRunnable buildRunnable(final ConsumerProducerBuilder builder) { return (consumerConfig, producerConfig) -> { - try (final Producer producer = builder.producerBuilder() + try (final Producer producer = builder.getProducerBuilder() .createProducer()) { final ProducerRecord producerRecord = new ProducerRecord<>(output, "foo", "bar"); diff --git a/streams-bootstrap-cli/src/test/java/com/bakdata/kafka/consumerproducer/apps/CloseFlagApp.java b/streams-bootstrap-cli/src/test/java/com/bakdata/kafka/consumerproducer/apps/CloseFlagApp.java index 182172524..946e8a489 100644 --- a/streams-bootstrap-cli/src/test/java/com/bakdata/kafka/consumerproducer/apps/CloseFlagApp.java +++ b/streams-bootstrap-cli/src/test/java/com/bakdata/kafka/consumerproducer/apps/CloseFlagApp.java @@ -1,7 +1,7 @@ /* * MIT License * - * Copyright (c) 2025 bakdata + * Copyright (c) 2026 bakdata * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -60,13 +60,13 @@ public ConsumerProducerApp createApp() { @Override public ConsumerProducerRunnable buildRunnable(final ConsumerProducerBuilder builder) { - final Producer producer = builder.producerBuilder().createProducer(); - final Consumer consumer = builder.consumerBuilder().createConsumer(); - builder.consumerBuilder().subscribeToAllTopics(consumer); + final Producer producer = builder.getProducerBuilder().createProducer(); + final Consumer consumer = builder.getConsumerBuilder().createConsumer(); + builder.getConsumerBuilder().subscribeToAllTopics(consumer); final ConsumerRunnable consumerRunnable = - builder.consumerBuilder().createDefaultConsumerRunnable(consumer, records -> + builder.getConsumerBuilder().createDefaultConsumerRunnable(consumer, records -> records.forEach(consumerRecord -> - producer.send(new ProducerRecord<>(builder.topics().getOutputTopic(), + producer.send(new ProducerRecord<>(builder.getTopics().getOutputTopic(), consumerRecord.key(), consumerRecord.value())))); return new DefaultConsumerProducerRunnable<>(producer, consumerRunnable); } diff --git a/streams-bootstrap-cli/src/test/java/com/bakdata/kafka/consumerproducer/apps/Mirror.java b/streams-bootstrap-cli/src/test/java/com/bakdata/kafka/consumerproducer/apps/Mirror.java index 2e2df3738..fa8df8269 100644 --- a/streams-bootstrap-cli/src/test/java/com/bakdata/kafka/consumerproducer/apps/Mirror.java +++ b/streams-bootstrap-cli/src/test/java/com/bakdata/kafka/consumerproducer/apps/Mirror.java @@ -1,7 +1,7 @@ /* * MIT License * - * Copyright (c) 2025 bakdata + * Copyright (c) 2026 bakdata * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -43,13 +43,13 @@ public class Mirror implements ConsumerProducerApp { @Override public ConsumerProducerRunnable buildRunnable(final ConsumerProducerBuilder builder) { - final Producer producer = builder.producerBuilder().createProducer(); - final Consumer consumer = builder.consumerBuilder().createConsumer(); - builder.consumerBuilder().subscribeToAllTopics(consumer); + final Producer producer = builder.getProducerBuilder().createProducer(); + final Consumer consumer = builder.getConsumerBuilder().createConsumer(); + builder.getConsumerBuilder().subscribeToAllTopics(consumer); final ConsumerRunnable - consumerRunnable = builder.consumerBuilder().createDefaultConsumerRunnable(consumer, records -> + consumerRunnable = builder.getConsumerBuilder().createDefaultConsumerRunnable(consumer, records -> records.forEach(consumerRecord -> - producer.send(new ProducerRecord<>(builder.topics().getOutputTopic(), + producer.send(new ProducerRecord<>(builder.getTopics().getOutputTopic(), consumerRecord.key(), consumerRecord.value())))); return new DefaultConsumerProducerRunnable<>(producer, consumerRunnable); } diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/admin/ConsumerGroupsClient.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/admin/ConsumerGroupsClient.java index 0096663c9..3a9a1132c 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/admin/ConsumerGroupsClient.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/admin/ConsumerGroupsClient.java @@ -28,6 +28,7 @@ import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.stream.Collectors; import lombok.AccessLevel; import lombok.NonNull; import lombok.RequiredArgsConstructor; @@ -39,7 +40,11 @@ import org.apache.kafka.clients.admin.GroupListing; import org.apache.kafka.clients.admin.ListConsumerGroupOffsetsResult; import org.apache.kafka.clients.admin.ListGroupsResult; +import org.apache.kafka.clients.admin.ListOffsetsResult; +import org.apache.kafka.clients.admin.ListOffsetsResult.ListOffsetsResultInfo; +import org.apache.kafka.clients.admin.OffsetSpec; import org.apache.kafka.clients.consumer.OffsetAndMetadata; +import org.apache.kafka.common.GroupState; import org.apache.kafka.common.KafkaFuture; import org.apache.kafka.common.TopicPartition; import org.apache.kafka.common.config.ConfigResource; @@ -160,6 +165,40 @@ public void deleteIfExists() { } } + /** + * Reset consumer group offset. + * + * @param offsetSpec specification where offsets should be reset to + */ + public void reset(final OffsetSpec offsetSpec) { + final Optional groupDescription = this.describe(); + if (groupDescription.isEmpty()) { + return; + } + if (groupDescription.get().groupState() != GroupState.EMPTY) { + throw new KafkaAdminException( + "Error resetting consumer group %s, consumer group is not empty".formatted(this.groupName)); + } + + final Map groupOffsets = this.listOffsets(); + + final Map request = groupOffsets.keySet().stream() + .collect(Collectors.toMap(tp -> tp, tp -> offsetSpec)); + final KafkaFuture> offsetsFuture = + ConsumerGroupsClient.this.adminClient.listOffsets(request).all(); + final Map offsets = + ConsumerGroupsClient.this.timeout.get(offsetsFuture, + () -> "Error resetting consumer group %s, could not find offsets for spec %s".formatted( + this.groupName, offsetSpec)); + + final Map resetOffsets = offsets.entrySet().stream() + .collect(Collectors.toMap(Map.Entry::getKey, e -> new OffsetAndMetadata(e.getValue().offset()))); + final KafkaFuture alterOffsetResult = + ConsumerGroupsClient.this.adminClient.alterConsumerGroupOffsets(this.groupName, resetOffsets).all(); + ConsumerGroupsClient.this.timeout.get(alterOffsetResult, + () -> "Error resetting consumer group %s, could not alter offsets".formatted(this.groupName)); + } + /** * Create a client for the configuration of this consumer group. * diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/admin/KafkaAdminException.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/admin/KafkaAdminException.java index 116c87743..4dbfe356a 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/admin/KafkaAdminException.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/admin/KafkaAdminException.java @@ -1,7 +1,7 @@ /* * MIT License * - * Copyright (c) 2025 bakdata + * Copyright (c) 2026 bakdata * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -32,4 +32,8 @@ public class KafkaAdminException extends RuntimeException { KafkaAdminException(final String message, final Throwable cause) { super(message, cause); } + + KafkaAdminException(final String message) { + super(message); + } } diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumer/ConfiguredConsumerApp.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumer/ConfiguredConsumerApp.java index 31aa062b4..fd69b3eae 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumer/ConfiguredConsumerApp.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumer/ConfiguredConsumerApp.java @@ -31,7 +31,9 @@ import java.util.HashMap; import java.util.Map; import java.util.Objects; +import lombok.Getter; import lombok.NonNull; +import lombok.RequiredArgsConstructor; import org.apache.kafka.clients.consumer.ConsumerConfig; import org.apache.kafka.clients.consumer.internals.AutoOffsetResetStrategy; import org.apache.kafka.common.IsolationLevel; @@ -41,9 +43,13 @@ * * @param type of {@link ConsumerApp} */ -public record ConfiguredConsumerApp(@NonNull T app, - @NonNull ConsumerAppConfiguration configuration) - implements ConfiguredApp> { +@RequiredArgsConstructor +public class ConfiguredConsumerApp implements ConfiguredApp> { + @Getter + private final @NonNull T app; + private final @NonNull ConsumerAppConfiguration configuration; + + //TODO javadoc public static Map createBaseConfig() { final Map kafkaConfig = new HashMap<>(); @@ -96,8 +102,8 @@ public Map getKafkaProperties(final RuntimeConfiguration runtime * Get unique group identifier of {@link ConsumerApp} * * @return unique group identifier - * @throws IllegalArgumentException if unique group identifier of {@link ConsumerApp} is different from - * provided group identifier in {@link ConsumerAppConfiguration} + * @throws IllegalArgumentException if unique group identifier of {@link ConsumerApp} is different from provided + * group identifier in {@link ConsumerAppConfiguration} * @see ConsumerApp#getUniqueGroupId(ConsumerAppConfiguration) */ public String getUniqueGroupId() { @@ -111,9 +117,9 @@ public String getUniqueGroupId() { } /** - * Create an {@code ExecutableConsumerApp} using the provided {@link RuntimeConfiguration} + * Create an {@link ExecutableConsumerApp} using the provided {@link RuntimeConfiguration} * - * @return {@code ExecutableConsumerApp} + * @return {@link ExecutableConsumerApp} */ @Override public ExecutableConsumerApp withRuntimeConfiguration(final RuntimeConfiguration runtimeConfiguration) { diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumer/ConsumerApp.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumer/ConsumerApp.java index 72fdd51b2..ffcaabcd9 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumer/ConsumerApp.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumer/ConsumerApp.java @@ -1,7 +1,7 @@ /* * MIT License * - * Copyright (c) 2025 bakdata + * Copyright (c) 2026 bakdata * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -37,12 +37,12 @@ public interface ConsumerApp extends App kafkaProperties, - @NonNull ConsumerExecutionOptions executionOptions) { +@RequiredArgsConstructor +@Value +public class ConsumerBuilder { + + @NonNull + ConsumerTopicConfig topics; + @NonNull + Map kafkaProperties; + @NonNull + ConsumerExecutionOptions executionOptions; /** - * Create a new {@code Consumer} using {@link #kafkaProperties} + * Create a new {@link Consumer} using {@link #kafkaProperties} * * @param type of keys * @param type of values - * @return {@code Consumer} + * @return {@link Consumer} * @see KafkaConsumer#KafkaConsumer(Map) */ public Consumer createConsumer() { diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumer/ConsumerCleanUpRunner.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumer/ConsumerCleanUpRunner.java index 57fbe220b..5e1b2ed11 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumer/ConsumerCleanUpRunner.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumer/ConsumerCleanUpRunner.java @@ -1,7 +1,7 @@ /* * MIT License * - * Copyright (c) 2025 bakdata + * Copyright (c) 2026 bakdata * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -24,25 +24,15 @@ package com.bakdata.kafka.consumer; -import com.bakdata.kafka.CleanUpException; import com.bakdata.kafka.CleanUpRunner; import com.bakdata.kafka.admin.AdminClientX; import com.bakdata.kafka.admin.ConsumerGroupsClient.ConsumerGroupClient; import java.util.Map; -import java.util.Optional; -import java.util.concurrent.ExecutionException; -import java.util.stream.Collectors; import lombok.AccessLevel; import lombok.NonNull; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; -import org.apache.kafka.clients.admin.ConsumerGroupDescription; -import org.apache.kafka.clients.admin.ListOffsetsResult; import org.apache.kafka.clients.admin.OffsetSpec; -import org.apache.kafka.clients.consumer.OffsetAndMetadata; -import org.apache.kafka.common.GroupState; -import org.apache.kafka.common.KafkaFuture; -import org.apache.kafka.common.TopicPartition; /** * Clean up all topics specified by a {@link ConsumerTopicConfig} @@ -50,7 +40,6 @@ @Slf4j @RequiredArgsConstructor(access = AccessLevel.PRIVATE) public final class ConsumerCleanUpRunner implements CleanUpRunner { - private final @NonNull ConsumerTopicConfig topics; private final @NonNull Map kafkaProperties; private final @NonNull String groupId; private final @NonNull ConsumerCleanUpConfiguration cleanHooks; @@ -58,31 +47,27 @@ public final class ConsumerCleanUpRunner implements CleanUpRunner { /** * Create a new {@code ConsumerCleanUpRunner} with default {@link ConsumerCleanUpConfiguration} * - * @param topics topic configuration * @param kafkaProperties configuration to connect to Kafka admin tools * @param groupId consumer group id to clean up * @return {@code ConsumerCleanUpRunner} */ - public static ConsumerCleanUpRunner create(@NonNull final ConsumerTopicConfig topics, - @NonNull final Map kafkaProperties, + public static ConsumerCleanUpRunner create(@NonNull final Map kafkaProperties, @NonNull final String groupId) { - return create(topics, kafkaProperties, groupId, new ConsumerCleanUpConfiguration()); + return create(kafkaProperties, groupId, new ConsumerCleanUpConfiguration()); } /** * Create a new {@code ConsumerCleanUpRunner} * - * @param topics topic configuration * @param kafkaProperties configuration to connect to Kafka admin tools * @param groupId consumer group id to clean up * @param configuration configuration for hooks that are called when running {@link #clean()} * @return {@code ConsumerCleanUpRunner} */ - public static ConsumerCleanUpRunner create(@NonNull final ConsumerTopicConfig topics, - @NonNull final Map kafkaProperties, + public static ConsumerCleanUpRunner create(@NonNull final Map kafkaProperties, @NonNull final String groupId, @NonNull final ConsumerCleanUpConfiguration configuration) { - return new ConsumerCleanUpRunner(topics, kafkaProperties, groupId, configuration); + return new ConsumerCleanUpRunner(kafkaProperties, groupId, configuration); } @Override @@ -120,43 +105,11 @@ private class Task { private void reset() { final ConsumerGroupClient groupClient = this.adminClient.consumerGroups() .group(ConsumerCleanUpRunner.this.groupId); - final Optional groupDescription = groupClient.describe(); - if (groupDescription.isEmpty()) { - return; - } - if (groupDescription.get().groupState() != GroupState.EMPTY) { - throw new CleanUpException("Error resetting application, consumer group is not empty"); - } - - final Map groupOffsets = groupClient.listOffsets(); - - final Map request = groupOffsets.keySet().stream() - .collect(Collectors.toMap(tp -> tp, tp -> OffsetSpec.earliest())); - final Map earliestOffsets = - runAdminFuture(this.adminClient.admin().listOffsets(request).all(), - "Error resetting application, beginning consumer group offset could not be found"); - - final Map resetOffsets = earliestOffsets.entrySet().stream() - .collect(Collectors.toMap(Map.Entry::getKey, - e -> new OffsetAndMetadata(e.getValue().offset()))); - runAdminFuture( - this.adminClient.admin().alterConsumerGroupOffsets(ConsumerCleanUpRunner.this.groupId, resetOffsets) - .all(), "Error resetting application, could not alter consumer group offsets"); + groupClient.reset(OffsetSpec.earliest()); ConsumerCleanUpRunner.this.cleanHooks.runResetHooks(); } - private static T runAdminFuture(final KafkaFuture action, final String errorMessage) { - try { - return action.get(); - } catch (final InterruptedException e) { - Thread.currentThread().interrupt(); - throw new CleanUpException(errorMessage, e); - } catch (final ExecutionException e) { - throw new CleanUpException(errorMessage, e); - } - } - private void clean() { this.deleteConsumerGroup(); ConsumerCleanUpRunner.this.cleanHooks.runCleanHooks(); diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumer/ConsumerExecutionOptions.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumer/ConsumerExecutionOptions.java index af8408aef..73765ac1b 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumer/ConsumerExecutionOptions.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumer/ConsumerExecutionOptions.java @@ -1,7 +1,7 @@ /* * MIT License * - * Copyright (c) 2025 bakdata + * Copyright (c) 2026 bakdata * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -68,6 +68,7 @@ public class ConsumerExecutionOptions { @Getter private final Duration pollTimeout = Duration.ofMillis(Long.MAX_VALUE); + //TODO reuse close options in StreamsExecutionOptions private static boolean isStaticMembershipDisabled(final Map originals) { return originals.get(ConsumerConfig.GROUP_INSTANCE_ID_CONFIG) == null; } diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumer/ConsumerRunner.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumer/ConsumerRunner.java index 3623eba9f..42d956f16 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumer/ConsumerRunner.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumer/ConsumerRunner.java @@ -1,7 +1,7 @@ /* * MIT License * - * Copyright (c) 2025 bakdata + * Copyright (c) 2026 bakdata * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -49,18 +49,19 @@ public void close() { @Override public void run() { - log.info("Starting consumer"); this.runConsumer(); + //TODO await? } private void runConsumer() { - log.info("Starting Kafka Consumer and calling start hook"); + log.info("Starting Kafka Consumer"); final RunningConsumer runningConsumer = RunningConsumer.builder() .consumerRunnable(this.runnable) .config(this.config) .build(); + log.debug("Calling start hook"); this.executionOptions.onStart(runningConsumer); // Run Kafka consumer until it shuts down - this.runnable.run(this.config); + this.runnable.run(this.config); //TODO call before onStart } } diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumer/DefaultConsumerRunnable.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumer/DefaultConsumerRunnable.java index 1a892e648..ce19b5373 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumer/DefaultConsumerRunnable.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumer/DefaultConsumerRunnable.java @@ -1,7 +1,7 @@ /* * MIT License * - * Copyright (c) 2025 bakdata + * Copyright (c) 2026 bakdata * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -46,6 +46,7 @@ */ @RequiredArgsConstructor(access = AccessLevel.PROTECTED) @Slf4j +//TODO check public class DefaultConsumerRunnable implements ConsumerRunnable { @Getter @@ -84,6 +85,7 @@ private void pollLoop(final ConsumerConfig consumerConfig) { } catch (final WakeupException exception) { log.info("Consumer poll loop waking up for shutdown", exception); } catch (final RuntimeException exception) { + //TODO why catch? log.error("RuntimeException while running consumer loop", exception); } finally { log.info("Closing consumer"); diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumer/ExecutableConsumerApp.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumer/ExecutableConsumerApp.java index 4474e7d47..12ac8e0f8 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumer/ExecutableConsumerApp.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumer/ExecutableConsumerApp.java @@ -1,7 +1,7 @@ /* * MIT License * - * Copyright (c) 2025 bakdata + * Copyright (c) 2026 bakdata * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -31,7 +31,6 @@ import lombok.Builder; import lombok.Getter; import lombok.NonNull; -import lombok.RequiredArgsConstructor; import org.apache.kafka.clients.consumer.ConsumerConfig; /** @@ -49,15 +48,15 @@ public class ExecutableConsumerApp private final @NonNull T app; /** - * Create {@code ConsumerCleanUpRunner} in order to clean application + * Create {@link ConsumerCleanUpRunner} in order to clean application * - * @return {@code ConsumerCleanUpRunner} + * @return {@link ConsumerCleanUpRunner} */ @Override public ConsumerCleanUpRunner createCleanUpRunner() { final AppConfiguration configuration = this.createConfiguration(); final ConsumerCleanUpConfiguration configurer = this.app.setupCleanUp(configuration); - return ConsumerCleanUpRunner.create(this.topics, this.kafkaProperties, this.groupId, configurer); + return ConsumerCleanUpRunner.create(this.kafkaProperties, this.groupId, configurer); } /** diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumerproducer/ConfiguredConsumerProducerApp.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumerproducer/ConfiguredConsumerProducerApp.java index 497f1ebb9..8c0b99147 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumerproducer/ConfiguredConsumerProducerApp.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumerproducer/ConfiguredConsumerProducerApp.java @@ -1,7 +1,7 @@ /* * MIT License * - * Copyright (c) 2025 bakdata + * Copyright (c) 2026 bakdata * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -32,7 +32,9 @@ import com.bakdata.kafka.producer.ConfiguredProducerApp; import java.util.Map; import java.util.Objects; +import lombok.Getter; import lombok.NonNull; +import lombok.RequiredArgsConstructor; import org.apache.kafka.clients.CommonClientConfigs; import org.apache.kafka.clients.consumer.ConsumerConfig; import org.apache.kafka.clients.producer.ProducerConfig; @@ -42,9 +44,12 @@ * * @param type of {@link ConsumerProducerApp} */ -public record ConfiguredConsumerProducerApp( - @NonNull T app, @NonNull ConsumerProducerAppConfiguration configuration) +@RequiredArgsConstructor +public class ConfiguredConsumerProducerApp implements ConfiguredApp> { + @Getter + private final @NonNull T app; + private final @NonNull ConsumerProducerAppConfiguration configuration; /** *

This method creates the configuration to run a {@link ConsumerProducerApp}.

@@ -97,8 +102,8 @@ public Map getKafkaProperties(final RuntimeConfiguration runtime * Get unique group identifier of {@link ConsumerProducerApp} * * @return unique group identifier - * @throws IllegalArgumentException if unique group identifier of {@link ConsumerProducerApp} is different - * from provided group identifier in {@link ConsumerProducerAppConfiguration} + * @throws IllegalArgumentException if unique group identifier of {@link ConsumerProducerApp} is different from + * provided group identifier in {@link ConsumerProducerAppConfiguration} * @see ConsumerProducerApp#getUniqueAppId(ConsumerProducerAppConfiguration) */ public String getUniqueAppId() { @@ -112,9 +117,9 @@ public String getUniqueAppId() { } /** - * Create an {@code ExecutableConsumerProducerApp} using the provided {@link RuntimeConfiguration} + * Create an {@link ExecutableConsumerProducerApp} using the provided {@link RuntimeConfiguration} * - * @return {@code ExecutableConsumerProducerApp} + * @return {@link ExecutableConsumerProducerApp} */ @Override public ExecutableConsumerProducerApp withRuntimeConfiguration(final RuntimeConfiguration runtimeConfiguration) { diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumerproducer/ConsumerProducerApp.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumerproducer/ConsumerProducerApp.java index b331f42ea..a3961c275 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumerproducer/ConsumerProducerApp.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumerproducer/ConsumerProducerApp.java @@ -1,7 +1,7 @@ /* * MIT License * - * Copyright (c) 2025 bakdata + * Copyright (c) 2026 bakdata * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -38,7 +38,7 @@ public interface ConsumerProducerApp extends App new IllegalArgumentException("Please provide a group ID")); } /** - * @return {@code StreamsCleanUpConfiguration} + * @return {@link StreamsCleanUpConfiguration} * @see StreamsCleanUpRunner */ @Override diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumerproducer/ConsumerProducerAppConfiguration.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumerproducer/ConsumerProducerAppConfiguration.java index 834695cf6..32f2ae90b 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumerproducer/ConsumerProducerAppConfiguration.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumerproducer/ConsumerProducerAppConfiguration.java @@ -1,7 +1,7 @@ /* * MIT License * - * Copyright (c) 2025 bakdata + * Copyright (c) 2026 bakdata * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -40,7 +40,7 @@ public class ConsumerProducerAppConfiguration { private final String uniqueAppId; /** - * Create a new {@code ConsumerAppConfiguration} with no provided {@link #uniqueAppId} + * Create a new {@code ConsumerProducerAppConfiguration} with no provided {@link #uniqueAppId} * * @param topics topics to use for app */ diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumerproducer/ConsumerProducerBuilder.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumerproducer/ConsumerProducerBuilder.java index 6b1371f43..f954f4e23 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumerproducer/ConsumerProducerBuilder.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumerproducer/ConsumerProducerBuilder.java @@ -1,7 +1,7 @@ /* * MIT License * - * Copyright (c) 2024 bakdata + * Copyright (c) 2026 bakdata * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -27,13 +27,20 @@ import com.bakdata.kafka.consumer.ConsumerBuilder; import com.bakdata.kafka.producer.ProducerBuilder; import lombok.NonNull; +import lombok.Value; /** * Provides all runtime configurations when running a {@link ConsumerProducerApp} * * @see ConsumerProducerApp#buildRunnable(ConsumerProducerBuilder) */ -public record ConsumerProducerBuilder(@NonNull ConsumerProducerTopicConfig topics, - @NonNull ConsumerBuilder consumerBuilder, - @NonNull ProducerBuilder producerBuilder) { +@Value +public class ConsumerProducerBuilder { + + @NonNull + ConsumerProducerTopicConfig topics; + @NonNull + ConsumerBuilder consumerBuilder; + @NonNull + ProducerBuilder producerBuilder; } diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumerproducer/ConsumerProducerCleanUpRunner.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumerproducer/ConsumerProducerCleanUpRunner.java index d8a813c84..f919f4b66 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumerproducer/ConsumerProducerCleanUpRunner.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumerproducer/ConsumerProducerCleanUpRunner.java @@ -1,7 +1,7 @@ /* * MIT License * - * Copyright (c) 2025 bakdata + * Copyright (c) 2026 bakdata * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -68,7 +68,7 @@ public static ConsumerProducerCleanUpRunner create(@NonNull final ConsumerProduc * @param kafkaProperties configuration to connect to Kafka admin tools * @param groupId group id of the consumer * @param configuration configuration for hooks that are called when running {@link #clean()} - * @return {@code ConsumerCleanUpRunner} + * @return {@code ConsumerProducerCleanUpRunner} */ public static ConsumerProducerCleanUpRunner create(@NonNull final ConsumerProducerTopicConfig topics, @NonNull final Map kafkaProperties, @@ -79,7 +79,7 @@ public static ConsumerProducerCleanUpRunner create(@NonNull final ConsumerProduc final ConsumerCleanUpConfiguration consumerConfig = configuration.toConsumerCleanUpConfiguration(); final ProducerCleanUpConfiguration producerConfig = configuration.toProducerCleanUpConfiguration(); final ConsumerCleanUpRunner consumerCleanUpRunner = - ConsumerCleanUpRunner.create(consumerTopicConfig, kafkaProperties, groupId, consumerConfig); + ConsumerCleanUpRunner.create(kafkaProperties, groupId, consumerConfig); final ProducerCleanUpRunner producerCleanUpRunner = ProducerCleanUpRunner.create(producerTopicConfig, kafkaProperties, producerConfig); return new ConsumerProducerCleanUpRunner(consumerCleanUpRunner, producerCleanUpRunner); diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumerproducer/ConsumerProducerRunner.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumerproducer/ConsumerProducerRunner.java index 23c311469..a0a92b350 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumerproducer/ConsumerProducerRunner.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumerproducer/ConsumerProducerRunner.java @@ -1,7 +1,7 @@ /* * MIT License * - * Copyright (c) 2025 bakdata + * Copyright (c) 2026 bakdata * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -51,19 +51,20 @@ public void close() { @Override public void run() { - log.info("Starting consumer and producer"); this.runConsumerProducer(); + //TODO await? } private void runConsumerProducer() { - log.info("Starting Kafka ConsumerProducer and calling start hook"); + log.info("Starting Kafka ConsumerProducer"); final RunningConsumerProducer runningConsumer = RunningConsumerProducer.builder() .consumerProducerRunnable(this.runnable) .consumerConfig(this.consumerConfig) .producerConfig(this.producerConfig) .build(); + log.debug("Calling start hook"); this.executionOptions.onStart(runningConsumer); // Run Kafka application until it shuts down - this.runnable.run(this.consumerConfig, this.producerConfig); + this.runnable.run(this.consumerConfig, this.producerConfig); //TODO call before onStart } } diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumerproducer/ConsumerProducerTopicConfig.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumerproducer/ConsumerProducerTopicConfig.java index af3f40d1a..f54f71df8 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumerproducer/ConsumerProducerTopicConfig.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumerproducer/ConsumerProducerTopicConfig.java @@ -1,7 +1,7 @@ /* * MIT License * - * Copyright (c) 2025 bakdata + * Copyright (c) 2026 bakdata * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -70,6 +70,7 @@ public class ConsumerProducerTopicConfig { Map labeledOutputTopics = emptyMap(); String errorTopic; + //TODO javadoc public ConsumerTopicConfig toConsumerTopicConfig() { return ConsumerTopicConfig.builder() .inputTopics(this.getInputTopics()) @@ -79,6 +80,7 @@ public ConsumerTopicConfig toConsumerTopicConfig() { .build(); } + //TODO javadoc public ProducerTopicConfig toProducerTopicConfig() { return ProducerTopicConfig.builder() .outputTopic(this.getOutputTopic()) diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumerproducer/DefaultConsumerProducerRunnable.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumerproducer/DefaultConsumerProducerRunnable.java index 1a2a3a318..3664790e0 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumerproducer/DefaultConsumerProducerRunnable.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumerproducer/DefaultConsumerProducerRunnable.java @@ -1,7 +1,7 @@ /* * MIT License * - * Copyright (c) 2025 bakdata + * Copyright (c) 2026 bakdata * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -51,6 +51,7 @@ public void close() { log.debug("Closing consumer runnable"); this.consumerRunnable.close(); } catch (final RuntimeException e) { + //TODO why catch? log.warn("Error closing consumer runnable", e); } @@ -58,6 +59,7 @@ public void close() { log.debug("Closing producer"); this.producer.close(); } catch (final RuntimeException e) { + //TODO why catch? log.warn("Error closing producer", e); } diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumerproducer/ExecutableConsumerProducerApp.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumerproducer/ExecutableConsumerProducerApp.java index 8c4c04a4a..59ef2259d 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumerproducer/ExecutableConsumerProducerApp.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumerproducer/ExecutableConsumerProducerApp.java @@ -1,7 +1,7 @@ /* * MIT License * - * Copyright (c) 2025 bakdata + * Copyright (c) 2026 bakdata * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -27,9 +27,7 @@ import com.bakdata.kafka.AppConfiguration; import com.bakdata.kafka.ExecutableApp; import com.bakdata.kafka.consumer.ConsumerBuilder; -import com.bakdata.kafka.consumer.ConsumerTopicConfig; import com.bakdata.kafka.producer.ProducerBuilder; -import com.bakdata.kafka.producer.ProducerTopicConfig; import com.bakdata.kafka.streams.StreamsCleanUpConfiguration; import java.util.Map; import lombok.AccessLevel; @@ -56,9 +54,9 @@ public class ExecutableConsumerProducerApp private final @NonNull String groupId; /** - * Create {@code ConsumerProducerCleanUpRunner} in order to clean application + * Create {@link ConsumerProducerCleanUpRunner} in order to clean application * - * @return {@code ConsumerProducerCleanUpRunner} + * @return {@link ConsumerProducerCleanUpRunner} */ @Override public ConsumerProducerCleanUpRunner createCleanUpRunner() { @@ -69,9 +67,9 @@ public ConsumerProducerCleanUpRunner createCleanUpRunner() { } /** - * Create {@code ConsumerProducerRunner} in order to run application + * Create {@link ConsumerProducerRunner} in order to run application * - * @return {@code ConsumerProducerRunner} + * @return {@link ConsumerProducerRunner} */ @Override public ConsumerProducerRunner createRunner() { diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumerproducer/SerializerDeserializerConfig.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumerproducer/SerializerDeserializerConfig.java index 2855ad47b..360273808 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumerproducer/SerializerDeserializerConfig.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumerproducer/SerializerDeserializerConfig.java @@ -1,7 +1,7 @@ /* * MIT License * - * Copyright (c) 2024 bakdata + * Copyright (c) 2026 bakdata * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -46,6 +46,15 @@ public class SerializerDeserializerConfig implements SerializationConfig { private final @NonNull SerializerConfig serializerConfig; private final @NonNull DeserializerConfig deserializerConfig; + //TODO javadoc + public SerializerDeserializerConfig(final @NonNull Class keySerializer, + final @NonNull Class valueSerializer, + final @NonNull Class keyDeserializer, + final @NonNull Class valueDeserializer) { + this.serializerConfig = new SerializerConfig(keySerializer, valueSerializer); + this.deserializerConfig = new DeserializerConfig(keyDeserializer, valueDeserializer); + } + @Override public Map createProperties() { return Stream.concat(Stream.of(this.serializerConfig.createProperties()), @@ -54,15 +63,7 @@ public Map createProperties() { .collect(Collectors.toMap( Map.Entry::getKey, Map.Entry::getValue, - (v1, v2) -> v2 + (v1, v2) -> v2 // v1 and v2 should always be different )); } - - public SerializerDeserializerConfig(final @NonNull Class keySerializer, - final @NonNull Class valueSerializer, - final @NonNull Class keyDeserializer, - final @NonNull Class valueDeserializer) { - this.serializerConfig = new SerializerConfig(keySerializer, valueSerializer); - this.deserializerConfig = new DeserializerConfig(keyDeserializer, valueDeserializer); - } } diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/producer/ConfiguredProducerApp.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/producer/ConfiguredProducerApp.java index 747d8bd16..65d5aba8d 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/producer/ConfiguredProducerApp.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/producer/ConfiguredProducerApp.java @@ -1,7 +1,7 @@ /* * MIT License * - * Copyright (c) 2025 bakdata + * Copyright (c) 2026 bakdata * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -48,6 +48,7 @@ public class ConfiguredProducerApp implements ConfiguredA private final @NonNull T app; private final @NonNull ProducerAppConfiguration configuration; + //TODO javadoc public static Map createBaseConfig() { final Map kafkaConfig = new HashMap<>(); diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/consumer/ConsumerCleanUpRunnerTest.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/consumer/ConsumerCleanUpRunnerTest.java index 8f2dafcf8..6bcb90bfc 100644 --- a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/consumer/ConsumerCleanUpRunnerTest.java +++ b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/consumer/ConsumerCleanUpRunnerTest.java @@ -1,7 +1,7 @@ /* * MIT License * - * Copyright (c) 2025 bakdata + * Copyright (c) 2026 bakdata * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -60,14 +60,14 @@ class ConsumerCleanUpRunnerTest extends KafkaTest { @InjectSoftAssertions private SoftAssertions softly; - static ConfiguredConsumerApp createStringApplication() { + static ConfiguredConsumerApp createStringApplication() { final ConsumerTopicConfig topics = ConsumerTopicConfig.builder() .inputTopics(List.of("input")) .build(); return new ConfiguredConsumerApp<>(new StringConsumer(), new ConsumerAppConfiguration(topics)); } - static ConfiguredConsumerApp createStringPatternApplication() { + static ConfiguredConsumerApp createStringPatternApplication() { final ConsumerTopicConfig topics = ConsumerTopicConfig.builder() .inputPattern(Pattern.compile(".*_topic")) .build(); @@ -82,8 +82,8 @@ private void assertSize(final Collection> records @Test void shouldDeleteConsumerGroup() { - try (final ConfiguredConsumerApp app = createStringApplication(); - final ExecutableConsumerApp executableApp = createExecutableApp(app, + try (final ConfiguredConsumerApp app = createStringApplication(); + final ExecutableConsumerApp executableApp = createExecutableApp(app, this.createConfig())) { final KafkaTestClient testClient = this.newTestClient(); testClient.createTopic(app.getTopics().getInputTopics().get(0)); @@ -102,7 +102,7 @@ void shouldDeleteConsumerGroup() { new KeyValue<>("blub", "blub") ); - final StringConsumer stringConsumer = (StringConsumer) app.app(); + final StringConsumer stringConsumer = app.getApp(); run(executableApp); assertContent(this.softly, stringConsumer.getConsumedRecords(), expectedValues, @@ -131,8 +131,8 @@ void shouldDeleteConsumerGroup() { @Test void shouldNotThrowAnErrorIfConsumerGroupDoesNotExist() { - try (final ConfiguredConsumerApp app = createStringApplication(); - final ExecutableConsumerApp executableApp = createExecutableApp(app, + try (final ConfiguredConsumerApp app = createStringApplication(); + final ExecutableConsumerApp executableApp = createExecutableApp(app, this.createConfig())) { final KafkaTestClient testClient = this.newTestClient(); testClient.createTopic(app.getTopics().getInputTopics().get(0)); @@ -151,7 +151,7 @@ void shouldNotThrowAnErrorIfConsumerGroupDoesNotExist() { new KeyValue<>("blub", "blub") ); - final StringConsumer stringConsumer = (StringConsumer) app.app(); + final StringConsumer stringConsumer = app.getApp(); run(executableApp); assertContent(this.softly, stringConsumer.getConsumedRecords(), expectedValues, @@ -179,8 +179,8 @@ void shouldNotThrowAnErrorIfConsumerGroupDoesNotExist() { @Test void shouldReprocessAlreadySeenRecords() { - try (final ConfiguredConsumerApp app = createStringApplication(); - final ExecutableConsumerApp executableApp = createExecutableApp(app, + try (final ConfiguredConsumerApp app = createStringApplication(); + final ExecutableConsumerApp executableApp = createExecutableApp(app, this.createConfig())) { final KafkaTestClient testClient = this.newTestClient(); testClient.createTopic(app.getTopics().getInputTopics().get(0)); @@ -193,7 +193,7 @@ void shouldReprocessAlreadySeenRecords() { new SimpleProducerRecord<>("blub", "blub") )); - final StringConsumer stringConsumer = (StringConsumer) app.app(); + final StringConsumer stringConsumer = app.getApp(); run(executableApp); this.assertSize(stringConsumer.getConsumedRecords(), 3); @@ -212,8 +212,8 @@ void shouldReprocessAlreadySeenRecords() { @Test void shouldNotThrowExceptionOnMissingInputTopic() { - try (final ConfiguredConsumerApp app = createStringApplication(); - final ExecutableConsumerApp executableApp = createExecutableApp(app, + try (final ConfiguredConsumerApp app = createStringApplication(); + final ExecutableConsumerApp executableApp = createExecutableApp(app, this.createConfig())) { this.softly.assertThatCode(() -> clean(executableApp)).doesNotThrowAnyException(); } @@ -221,8 +221,9 @@ void shouldNotThrowExceptionOnMissingInputTopic() { @Test void shouldThrowExceptionOnResetterError() { - try (final ConfiguredConsumerApp app = createStringApplication(); - final ExecutableConsumerApp executableApp = createExecutableApp(app, this.createConfig()); + try (final ConfiguredConsumerApp app = createStringApplication(); + final ExecutableConsumerApp executableApp = createExecutableApp(app, + this.createConfig()); final ConsumerRunner runner = executableApp.createRunner()) { final KafkaTestClient testClient = this.newTestClient(); testClient.createTopic(app.getTopics().getInputTopics().get(0)); @@ -238,8 +239,8 @@ void shouldThrowExceptionOnResetterError() { @Test void shouldReprocessAlreadySeenRecordsWithPattern() { - try (final ConfiguredConsumerApp app = createStringPatternApplication(); - final ExecutableConsumerApp executableApp = createExecutableApp(app, + try (final ConfiguredConsumerApp app = createStringPatternApplication(); + final ExecutableConsumerApp executableApp = createExecutableApp(app, this.createConfig())) { final String topic = "input_topic"; final KafkaTestClient testClient = this.newTestClient(); @@ -253,7 +254,7 @@ void shouldReprocessAlreadySeenRecordsWithPattern() { new SimpleProducerRecord<>("blub", "blub") )); - final StringPatternConsumer stringConsumer = (StringPatternConsumer) app.app(); + final StringPatternConsumer stringConsumer = app.getApp(); run(executableApp); this.assertSize(stringConsumer.getConsumedRecords(), 3); @@ -272,8 +273,9 @@ void shouldReprocessAlreadySeenRecordsWithPattern() { @Test void shouldNotThrowExceptionOnResetIfConsumerGroupNotExists() { - try (final ConfiguredConsumerApp app = createStringApplication(); - final ExecutableConsumerApp executableApp = createExecutableApp(app, this.createConfig())) { + try (final ConfiguredConsumerApp app = createStringApplication(); + final ExecutableConsumerApp executableApp = createExecutableApp(app, + this.createConfig())) { // The app is not run so the consumer group is never created this.softly.assertThatCode(() -> reset(executableApp)).doesNotThrowAnyException(); } diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/consumer/ConsumerRunnerTest.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/consumer/ConsumerRunnerTest.java index 881010f60..d5ef19980 100644 --- a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/consumer/ConsumerRunnerTest.java +++ b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/consumer/ConsumerRunnerTest.java @@ -50,8 +50,9 @@ class ConsumerRunnerTest extends KafkaTest { @Test void shouldRunApp() { - try (final ConfiguredConsumerApp app = createStringApplication(); - final ExecutableConsumerApp executableApp = createExecutableApp(app, this.createConfig()); + try (final ConfiguredConsumerApp app = createStringApplication(); + final ExecutableConsumerApp executableApp = createExecutableApp(app, + this.createConfig()); final ConsumerRunner runner = executableApp.createRunner()) { runAsync(runner); @@ -60,7 +61,7 @@ void shouldRunApp() { final SimpleProducerRecord simpleProducerRecord = new SimpleProducerRecord<>("foo", "bar"); this.writeInputTopic(app.getTopics().getInputTopics().get(0), simpleProducerRecord); - final StringConsumer stringConsumer = (StringConsumer) app.app(); + final StringConsumer stringConsumer = app.getApp(); awaitProcessing(executableApp); diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/consumer/DefaultConsumerRunnableTest.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/consumer/DefaultConsumerRunnableTest.java index 64092977e..1e9cdcb5e 100644 --- a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/consumer/DefaultConsumerRunnableTest.java +++ b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/consumer/DefaultConsumerRunnableTest.java @@ -1,7 +1,7 @@ /* * MIT License * - * Copyright (c) 2025 bakdata + * Copyright (c) 2026 bakdata * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -57,14 +57,14 @@ class DefaultConsumerRunnableTest extends KafkaTest { @InjectSoftAssertions private SoftAssertions softly; - static ConfiguredConsumerApp createStringApplication() { + static ConfiguredConsumerApp createStringApplication() { final ConsumerTopicConfig topics = ConsumerTopicConfig.builder() .inputTopics(List.of("input")) .build(); return createStringApplication(new ConsumerAppConfiguration(topics)); } - static ConfiguredConsumerApp createStringApplication(final ConsumerAppConfiguration configuration) { + static ConfiguredConsumerApp createStringApplication(final ConsumerAppConfiguration configuration) { return new ConfiguredConsumerApp<>(new StringConsumer(), configuration); } @@ -79,8 +79,9 @@ static ConfiguredConsumerApp createCustomProcessorConsumer( @Test void shouldRunProcessAndShutdownGracefully() { - try (final ConfiguredConsumerApp app = createStringApplication(); - final ExecutableConsumerApp executableApp = createExecutableApp(app, this.createConfig()); + try (final ConfiguredConsumerApp app = createStringApplication(); + final ExecutableConsumerApp executableApp = createExecutableApp(app, + this.createConfig()); final ConsumerRunner runner = executableApp.createRunner()) { final KafkaTestClient testClient = this.newTestClient(); testClient.createTopic(app.getTopics().getInputTopics().get(0)); @@ -98,7 +99,7 @@ void shouldRunProcessAndShutdownGracefully() { new KeyValue<>("blub", "blub") ); - final StringConsumer stringConsumer = (StringConsumer) app.app(); + final StringConsumer stringConsumer = app.getApp(); runAsync(runner); awaitActive(executableApp); @@ -109,8 +110,9 @@ void shouldRunProcessAndShutdownGracefully() { @Test void shouldCommitOffsets() { - try (final ConfiguredConsumerApp app = createStringApplication(); - final ExecutableConsumerApp executableApp = createExecutableApp(app, this.createConfig()); + try (final ConfiguredConsumerApp app = createStringApplication(); + final ExecutableConsumerApp executableApp = createExecutableApp(app, + this.createConfig()); final ConsumerRunner runner = executableApp.createRunner()) { final KafkaTestClient testClient = this.newTestClient(); testClient.createTopic(app.getTopics().getInputTopics().get(0)); @@ -203,9 +205,10 @@ void shouldSubscribeToInputPattern() { final ConsumerTopicConfig topics = ConsumerTopicConfig.builder() .inputPattern(Pattern.compile("inp.*")) .build(); - try (final ConfiguredConsumerApp app = createStringApplication( + try (final ConfiguredConsumerApp app = createStringApplication( new ConsumerAppConfiguration(topics)); - final ExecutableConsumerApp executableApp = createExecutableApp(app, this.createConfig()); + final ExecutableConsumerApp executableApp = createExecutableApp(app, + this.createConfig()); final ConsumerRunner runner = executableApp.createRunner()) { final KafkaTestClient testClient = this.newTestClient(); testClient.createTopic("input"); @@ -217,7 +220,7 @@ void shouldSubscribeToInputPattern() { )); final List> expectedValues = List.of(new KeyValue<>("blub", "blub")); - final StringConsumer stringConsumer = (StringConsumer) app.app(); + final StringConsumer stringConsumer = app.getApp(); runAsync(runner); awaitActive(executableApp); @@ -231,9 +234,10 @@ void shouldSubscribeToLabeledInputPattern() { final ConsumerTopicConfig topics = ConsumerTopicConfig.builder() .labeledInputPatterns(Map.of("LABEL", Pattern.compile("inp.*"))) .build(); - try (final ConfiguredConsumerApp app = createStringApplication( + try (final ConfiguredConsumerApp app = createStringApplication( new ConsumerAppConfiguration(topics)); - final ExecutableConsumerApp executableApp = createExecutableApp(app, this.createConfig()); + final ExecutableConsumerApp executableApp = createExecutableApp(app, + this.createConfig()); final ConsumerRunner runner = executableApp.createRunner()) { final KafkaTestClient testClient = this.newTestClient(); testClient.createTopic("input"); @@ -245,7 +249,7 @@ void shouldSubscribeToLabeledInputPattern() { )); final List> expectedValues = List.of(new KeyValue<>("blub", "blub")); - final StringConsumer stringConsumer = (StringConsumer) app.app(); + final StringConsumer stringConsumer = app.getApp(); runAsync(runner); awaitActive(executableApp); @@ -259,9 +263,10 @@ void shouldSubscribeToLabeledInputTopics() { final ConsumerTopicConfig topics = ConsumerTopicConfig.builder() .labeledInputTopics(Map.of("LABEL", List.of("input"))) .build(); - try (final ConfiguredConsumerApp app = createStringApplication( + try (final ConfiguredConsumerApp app = createStringApplication( new ConsumerAppConfiguration(topics)); - final ExecutableConsumerApp executableApp = createExecutableApp(app, this.createConfig()); + final ExecutableConsumerApp executableApp = createExecutableApp(app, + this.createConfig()); final ConsumerRunner runner = executableApp.createRunner()) { final KafkaTestClient testClient = this.newTestClient(); testClient.createTopic("input"); @@ -273,7 +278,7 @@ void shouldSubscribeToLabeledInputTopics() { )); final List> expectedValues = List.of(new KeyValue<>("blub", "blub")); - final StringConsumer stringConsumer = (StringConsumer) app.app(); + final StringConsumer stringConsumer = app.getApp(); runAsync(runner); awaitActive(executableApp); diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/consumer/TestHelper.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/consumer/TestHelper.java index fe8fca84f..d7aefeffd 100644 --- a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/consumer/TestHelper.java +++ b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/consumer/TestHelper.java @@ -1,7 +1,7 @@ /* * MIT License * - * Copyright (c) 2025 bakdata + * Copyright (c) 2026 bakdata * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -59,7 +59,8 @@ public static void assertContent(final SoftAssertions softly, }); } - public static ExecutableConsumerApp createExecutableApp(final ConfiguredConsumerApp app, + public static ExecutableConsumerApp createExecutableApp( + final ConfiguredConsumerApp app, final RuntimeConfiguration runtimeConfiguration) { return app.withRuntimeConfiguration(runtimeConfiguration); } diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/consumerproducer/ConsumerProducerCleanUpRunnerTest.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/consumerproducer/ConsumerProducerCleanUpRunnerTest.java index abd99a731..af4d486b7 100644 --- a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/consumerproducer/ConsumerProducerCleanUpRunnerTest.java +++ b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/consumerproducer/ConsumerProducerCleanUpRunnerTest.java @@ -1,7 +1,7 @@ /* * MIT License * - * Copyright (c) 2025 bakdata + * Copyright (c) 2026 bakdata * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -97,7 +97,7 @@ private static void clean(final ExecutableApp app } } - static ConfiguredConsumerProducerApp createStringConsumerProducer() { + static ConfiguredConsumerProducerApp createStringConsumerProducer() { return new ConfiguredConsumerProducerApp<>(new StringConsumerProducer(), new ConsumerProducerAppConfiguration(TOPIC_CONFIG)); } @@ -132,16 +132,16 @@ public StreamsCleanUpConfiguration setupCleanUp( }, new ConsumerProducerAppConfiguration(TOPIC_CONFIG)); } - static ExecutableConsumerProducerApp createExecutableApp( - final ConfiguredConsumerProducerApp app, + static ExecutableConsumerProducerApp createExecutableApp( + final ConfiguredConsumerProducerApp app, final RuntimeConfiguration runtimeConfiguration) { return app.withRuntimeConfiguration(runtimeConfiguration); } @Test void shouldDeleteTopic() { - try (final ConfiguredConsumerProducerApp app = createStringConsumerProducer(); - final ExecutableConsumerProducerApp executableApp = createExecutableApp(app, + try (final ConfiguredConsumerProducerApp app = createStringConsumerProducer(); + final ExecutableConsumerProducerApp executableApp = createExecutableApp(app, this.createConfig())) { final KafkaTestClient testClient = this.newTestClient(); testClient.createTopic(app.getTopics().getInputTopics().get(0)); @@ -179,8 +179,8 @@ void shouldDeleteTopic() { @Test void shouldDeleteConsumerGroup() { - try (final ConfiguredConsumerProducerApp app = createStringConsumerProducer(); - final ExecutableConsumerProducerApp executableApp = createExecutableApp(app, + try (final ConfiguredConsumerProducerApp app = createStringConsumerProducer(); + final ExecutableConsumerProducerApp executableApp = createExecutableApp(app, this.createConfig())) { final KafkaTestClient testClient = this.newTestClient(); testClient.createTopic(app.getTopics().getInputTopics().get(0)); @@ -199,7 +199,7 @@ void shouldDeleteConsumerGroup() { new KeyValue<>("blub", "blub") ); - final StringConsumerProducer stringConsumer = (StringConsumerProducer) app.app(); + final StringConsumerProducer stringConsumer = app.getApp(); run(executableApp); this.assertContent(app.getTopics().getOutputTopic(), expectedValues, @@ -229,8 +229,8 @@ void shouldDeleteConsumerGroup() { @Test void shouldNotThrowAnErrorIfConsumerGroupDoesNotExist() { - try (final ConfiguredConsumerProducerApp app = createStringConsumerProducer(); - final ExecutableConsumerProducerApp executableApp = createExecutableApp(app, + try (final ConfiguredConsumerProducerApp app = createStringConsumerProducer(); + final ExecutableConsumerProducerApp executableApp = createExecutableApp(app, this.createConfig())) { final KafkaTestClient testClient = this.newTestClient(); testClient.createTopic(app.getTopics().getOutputTopic()); @@ -381,8 +381,8 @@ void shouldCallCleanUpHookForAllTopics() { @Test void shouldNotThrowExceptionOnMissingInputTopic() { - try (final ConfiguredConsumerProducerApp app = createStringConsumerProducer(); - final ExecutableConsumerProducerApp executableApp = createExecutableApp(app, + try (final ConfiguredConsumerProducerApp app = createStringConsumerProducer(); + final ExecutableConsumerProducerApp executableApp = createExecutableApp(app, this.createConfig())) { this.softly.assertThatCode(() -> clean(executableApp)).doesNotThrowAnyException(); } @@ -390,8 +390,8 @@ void shouldNotThrowExceptionOnMissingInputTopic() { @Test void shouldThrowExceptionOnResetterError() { - try (final ConfiguredConsumerProducerApp app = createStringConsumerProducer(); - final ExecutableConsumerProducerApp executableApp = createExecutableApp(app, + try (final ConfiguredConsumerProducerApp app = createStringConsumerProducer(); + final ExecutableConsumerProducerApp executableApp = createExecutableApp(app, this.createConfig()); final ConsumerProducerRunner runner = executableApp.createRunner()) { final KafkaTestClient testClient = this.newTestClient(); @@ -408,8 +408,8 @@ void shouldThrowExceptionOnResetterError() { @Test void shouldReprocessAlreadySeenRecords() { - try (final ConfiguredConsumerProducerApp app = createStringConsumerProducer(); - final ExecutableConsumerProducerApp executableApp = createExecutableApp(app, + try (final ConfiguredConsumerProducerApp app = createStringConsumerProducer(); + final ExecutableConsumerProducerApp executableApp = createExecutableApp(app, this.createConfig())) { final KafkaTestClient testClient = this.newTestClient(); testClient.createTopic(app.getTopics().getOutputTopic()); diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/consumerproducer/ConsumerProducerRunnerTest.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/consumerproducer/ConsumerProducerRunnerTest.java index 6503d581b..a96dd7fc7 100644 --- a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/consumerproducer/ConsumerProducerRunnerTest.java +++ b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/consumerproducer/ConsumerProducerRunnerTest.java @@ -1,7 +1,7 @@ /* * MIT License * - * Copyright (c) 2025 bakdata + * Copyright (c) 2026 bakdata * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -31,6 +31,7 @@ import com.bakdata.kafka.KafkaTest; import com.bakdata.kafka.KafkaTestClient; import com.bakdata.kafka.SenderBuilder.SimpleProducerRecord; +import com.bakdata.kafka.consumerproducer.apps.StringConsumerProducer; import java.util.List; import org.apache.kafka.clients.consumer.ConsumerConfig; import org.apache.kafka.clients.producer.ProducerConfig; @@ -49,8 +50,8 @@ class ConsumerProducerRunnerTest extends KafkaTest { @Test void shouldRunApp() { - try (final ConfiguredConsumerProducerApp app = createStringConsumerProducer(); - final ExecutableConsumerProducerApp executableApp = createExecutableApp(app, + try (final ConfiguredConsumerProducerApp app = createStringConsumerProducer(); + final ExecutableConsumerProducerApp executableApp = createExecutableApp(app, this.createConfig()); final ConsumerProducerRunner runner = executableApp.createRunner()) { final KafkaTestClient testClient = this.newTestClient(); diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/consumerproducer/apps/MirrorKeyWithAvroConsumerProducer.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/consumerproducer/apps/MirrorKeyWithAvroConsumerProducer.java index 798b7fd30..c25ae3df9 100644 --- a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/consumerproducer/apps/MirrorKeyWithAvroConsumerProducer.java +++ b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/consumerproducer/apps/MirrorKeyWithAvroConsumerProducer.java @@ -1,7 +1,7 @@ /* * MIT License * - * Copyright (c) 2025 bakdata + * Copyright (c) 2026 bakdata * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -54,13 +54,13 @@ public SerializerDeserializerConfig defaultSerializationConfig() { @Override public ConsumerProducerRunnable buildRunnable(final ConsumerProducerBuilder builder) { - final Consumer consumer = builder.consumerBuilder().createConsumer(); - builder.consumerBuilder().subscribeToAllTopics(consumer); - final Producer producer = builder.producerBuilder().createProducer(); - final ConsumerRunnable consumerRunnable = builder.consumerBuilder().createDefaultConsumerRunnable(consumer, + final Consumer consumer = builder.getConsumerBuilder().createConsumer(); + builder.getConsumerBuilder().subscribeToAllTopics(consumer); + final Producer producer = builder.getProducerBuilder().createProducer(); + final ConsumerRunnable consumerRunnable = builder.getConsumerBuilder().createDefaultConsumerRunnable(consumer, records -> records.forEach( consumerRecord -> - producer.send(new ProducerRecord<>(builder.topics().getOutputTopic(), + producer.send(new ProducerRecord<>(builder.getTopics().getOutputTopic(), consumerRecord.key(), consumerRecord.value())))); return new DefaultConsumerProducerRunnable<>(producer, consumerRunnable); } diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/consumerproducer/apps/MirrorValueWithAvroConsumerProducer.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/consumerproducer/apps/MirrorValueWithAvroConsumerProducer.java index 7f42f13f7..eab4a83d0 100644 --- a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/consumerproducer/apps/MirrorValueWithAvroConsumerProducer.java +++ b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/consumerproducer/apps/MirrorValueWithAvroConsumerProducer.java @@ -1,7 +1,7 @@ /* * MIT License * - * Copyright (c) 2025 bakdata + * Copyright (c) 2026 bakdata * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -54,13 +54,13 @@ public SerializerDeserializerConfig defaultSerializationConfig() { @Override public ConsumerProducerRunnable buildRunnable(final ConsumerProducerBuilder builder) { - final Consumer consumer = builder.consumerBuilder().createConsumer(); - builder.consumerBuilder().subscribeToAllTopics(consumer); - final Producer producer = builder.producerBuilder().createProducer(); - final ConsumerRunnable consumerRunnable = builder.consumerBuilder().createDefaultConsumerRunnable(consumer, + final Consumer consumer = builder.getConsumerBuilder().createConsumer(); + builder.getConsumerBuilder().subscribeToAllTopics(consumer); + final Producer producer = builder.getProducerBuilder().createProducer(); + final ConsumerRunnable consumerRunnable = builder.getConsumerBuilder().createDefaultConsumerRunnable(consumer, records -> records.forEach( consumerRecord -> - producer.send(new ProducerRecord<>(builder.topics().getOutputTopic(), + producer.send(new ProducerRecord<>(builder.getTopics().getOutputTopic(), consumerRecord.key(), consumerRecord.value())))); return new DefaultConsumerProducerRunnable<>(producer, consumerRunnable); } diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/consumerproducer/apps/StringConsumerProducer.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/consumerproducer/apps/StringConsumerProducer.java index c9a754c23..91a63e77b 100644 --- a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/consumerproducer/apps/StringConsumerProducer.java +++ b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/consumerproducer/apps/StringConsumerProducer.java @@ -1,7 +1,7 @@ /* * MIT License * - * Copyright (c) 2025 bakdata + * Copyright (c) 2026 bakdata * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -51,13 +51,13 @@ public SerializerDeserializerConfig defaultSerializationConfig() { @Override public ConsumerProducerRunnable buildRunnable(final ConsumerProducerBuilder builder) { - final Producer producer = builder.producerBuilder().createProducer(); - final Consumer consumer = builder.consumerBuilder().createConsumer(); - builder.consumerBuilder().subscribeToAllTopics(consumer); + final Producer producer = builder.getProducerBuilder().createProducer(); + final Consumer consumer = builder.getConsumerBuilder().createConsumer(); + builder.getConsumerBuilder().subscribeToAllTopics(consumer); final ConsumerRunnable - consumerRunnable = builder.consumerBuilder().createDefaultConsumerRunnable(consumer, records -> + consumerRunnable = builder.getConsumerBuilder().createDefaultConsumerRunnable(consumer, records -> records.forEach(consumerRecord -> - producer.send(new ProducerRecord<>(builder.topics().getOutputTopic(), + producer.send(new ProducerRecord<>(builder.getTopics().getOutputTopic(), consumerRecord.key(), consumerRecord.value())))); return new DefaultConsumerProducerRunnable<>(producer, consumerRunnable); } diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/consumerproducer/apps/StringPatternConsumerProducer.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/consumerproducer/apps/StringPatternConsumerProducer.java index a29d0334e..688aeaba5 100644 --- a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/consumerproducer/apps/StringPatternConsumerProducer.java +++ b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/consumerproducer/apps/StringPatternConsumerProducer.java @@ -1,7 +1,7 @@ /* * MIT License * - * Copyright (c) 2025 bakdata + * Copyright (c) 2026 bakdata * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -51,13 +51,13 @@ public SerializerDeserializerConfig defaultSerializationConfig() { @Override public ConsumerProducerRunnable buildRunnable(final ConsumerProducerBuilder builder) { - final Consumer consumer = builder.consumerBuilder().createConsumer(); - builder.consumerBuilder().subscribeToAllTopics(consumer); - final Producer producer = builder.producerBuilder().createProducer(); - final ConsumerRunnable consumerRunnable = builder.consumerBuilder().createDefaultConsumerRunnable(consumer, + final Consumer consumer = builder.getConsumerBuilder().createConsumer(); + builder.getConsumerBuilder().subscribeToAllTopics(consumer); + final Producer producer = builder.getProducerBuilder().createProducer(); + final ConsumerRunnable consumerRunnable = builder.getConsumerBuilder().createDefaultConsumerRunnable(consumer, records -> records.forEach( consumerRecord -> - producer.send(new ProducerRecord<>(builder.topics().getOutputTopic(), + producer.send(new ProducerRecord<>(builder.getTopics().getOutputTopic(), consumerRecord.key(), consumerRecord.value())))); return new DefaultConsumerProducerRunnable<>(producer, consumerRunnable); } diff --git a/streams-bootstrap-test/src/testFixtures/java/com/bakdata/kafka/streams/apps/SimpleConsumerProducerApp.java b/streams-bootstrap-test/src/testFixtures/java/com/bakdata/kafka/streams/apps/SimpleConsumerProducerApp.java index f8067f163..4db773ab4 100644 --- a/streams-bootstrap-test/src/testFixtures/java/com/bakdata/kafka/streams/apps/SimpleConsumerProducerApp.java +++ b/streams-bootstrap-test/src/testFixtures/java/com/bakdata/kafka/streams/apps/SimpleConsumerProducerApp.java @@ -1,7 +1,7 @@ /* * MIT License * - * Copyright (c) 2025 bakdata + * Copyright (c) 2026 bakdata * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -47,13 +47,13 @@ public SerializerDeserializerConfig defaultSerializationConfig() { @Override public ConsumerProducerRunnable buildRunnable(final ConsumerProducerBuilder builder) { - final Producer producer = builder.producerBuilder().createProducer(); - final Consumer consumer = builder.consumerBuilder().createConsumer(); - builder.consumerBuilder().subscribeToAllTopics(consumer); + final Producer producer = builder.getProducerBuilder().createProducer(); + final Consumer consumer = builder.getConsumerBuilder().createConsumer(); + builder.getConsumerBuilder().subscribeToAllTopics(consumer); final ConsumerRunnable - consumerRunnable = builder.consumerBuilder().createDefaultConsumerRunnable(consumer, records -> + consumerRunnable = builder.getConsumerBuilder().createDefaultConsumerRunnable(consumer, records -> records.forEach(consumerRecord -> - producer.send(new ProducerRecord<>(builder.topics().getOutputTopic(), + producer.send(new ProducerRecord<>(builder.getTopics().getOutputTopic(), consumerRecord.key(), consumerRecord.value())))); return new DefaultConsumerProducerRunnable<>(producer, consumerRunnable); } From e037196006e3cd264bfb8f519967bc0298400729 Mon Sep 17 00:00:00 2001 From: Philipp Schirmer Date: Thu, 26 Feb 2026 10:58:31 +0100 Subject: [PATCH 02/15] Clean up Consumer/ConsumerProducer --- .../consumerproducer/DefaultConsumerProducerRunnable.java | 1 + .../com/bakdata/kafka/consumer/ConsumerCleanUpRunnerTest.java | 3 ++- .../consumerproducer/ConsumerProducerCleanUpRunnerTest.java | 3 ++- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumerproducer/DefaultConsumerProducerRunnable.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumerproducer/DefaultConsumerProducerRunnable.java index 3664790e0..ed0664f56 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumerproducer/DefaultConsumerProducerRunnable.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumerproducer/DefaultConsumerProducerRunnable.java @@ -34,6 +34,7 @@ @AllArgsConstructor @Slf4j +//TODO check public class DefaultConsumerProducerRunnable implements ConsumerProducerRunnable { private final Producer producer; diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/consumer/ConsumerCleanUpRunnerTest.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/consumer/ConsumerCleanUpRunnerTest.java index 6bcb90bfc..47ef616a9 100644 --- a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/consumer/ConsumerCleanUpRunnerTest.java +++ b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/consumer/ConsumerCleanUpRunnerTest.java @@ -233,7 +233,8 @@ void shouldThrowExceptionOnResetterError() { // should throw exception because consumer group is still active this.softly.assertThatThrownBy(() -> reset(executableApp)) .isInstanceOf(CleanUpException.class) - .hasMessageContaining("Error resetting application, consumer group is not empty"); + .hasMessageContaining("Error resetting consumer group %s, consumer group is not empty", + app.getUniqueGroupId()); } } diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/consumerproducer/ConsumerProducerCleanUpRunnerTest.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/consumerproducer/ConsumerProducerCleanUpRunnerTest.java index af4d486b7..7dc6edeb9 100644 --- a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/consumerproducer/ConsumerProducerCleanUpRunnerTest.java +++ b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/consumerproducer/ConsumerProducerCleanUpRunnerTest.java @@ -402,7 +402,8 @@ void shouldThrowExceptionOnResetterError() { // should throw exception because consumer group is still active this.softly.assertThatThrownBy(() -> reset(executableApp)) .isInstanceOf(CleanUpException.class) - .hasMessageContaining("Error resetting application, consumer group is not empty"); + .hasMessageContaining("Error resetting consumer group %s, consumer group is not empty", + app.getUniqueAppId()); } } From 25fdb26ef98a1cf9f0afc9322be90a03f71a1ddf Mon Sep 17 00:00:00 2001 From: Philipp Schirmer Date: Thu, 26 Feb 2026 11:23:07 +0100 Subject: [PATCH 03/15] Clean up Consumer/ConsumerProducer --- .../com/bakdata/kafka/consumer/ConsumerCleanUpRunnerTest.java | 4 ++-- .../consumerproducer/ConsumerProducerCleanUpRunnerTest.java | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/consumer/ConsumerCleanUpRunnerTest.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/consumer/ConsumerCleanUpRunnerTest.java index 47ef616a9..eb4c10391 100644 --- a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/consumer/ConsumerCleanUpRunnerTest.java +++ b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/consumer/ConsumerCleanUpRunnerTest.java @@ -31,13 +31,13 @@ import static com.bakdata.kafka.consumer.TestHelper.run; import static java.util.concurrent.CompletableFuture.runAsync; -import com.bakdata.kafka.CleanUpException; import com.bakdata.kafka.KafkaTest; import com.bakdata.kafka.KafkaTestClient; import com.bakdata.kafka.SenderBuilder.SimpleProducerRecord; import com.bakdata.kafka.admin.AdminClientX; import com.bakdata.kafka.admin.ConsumerGroupsClient; import com.bakdata.kafka.admin.ConsumerGroupsClient.ConsumerGroupClient; +import com.bakdata.kafka.admin.KafkaAdminException; import com.bakdata.kafka.consumer.apps.StringConsumer; import com.bakdata.kafka.consumer.apps.StringPatternConsumer; import java.time.Duration; @@ -232,7 +232,7 @@ void shouldThrowExceptionOnResetterError() { awaitActive(executableApp); // should throw exception because consumer group is still active this.softly.assertThatThrownBy(() -> reset(executableApp)) - .isInstanceOf(CleanUpException.class) + .isInstanceOf(KafkaAdminException.class) .hasMessageContaining("Error resetting consumer group %s, consumer group is not empty", app.getUniqueGroupId()); } diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/consumerproducer/ConsumerProducerCleanUpRunnerTest.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/consumerproducer/ConsumerProducerCleanUpRunnerTest.java index 7dc6edeb9..a3b8b5dca 100644 --- a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/consumerproducer/ConsumerProducerCleanUpRunnerTest.java +++ b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/consumerproducer/ConsumerProducerCleanUpRunnerTest.java @@ -31,7 +31,6 @@ import static org.mockito.Mockito.verifyNoMoreInteractions; import com.bakdata.kafka.AppConfiguration; -import com.bakdata.kafka.CleanUpException; import com.bakdata.kafka.CleanUpRunner; import com.bakdata.kafka.ExecutableApp; import com.bakdata.kafka.HasTopicHooks.TopicHook; @@ -44,6 +43,7 @@ import com.bakdata.kafka.admin.AdminClientX; import com.bakdata.kafka.admin.ConsumerGroupsClient; import com.bakdata.kafka.admin.ConsumerGroupsClient.ConsumerGroupClient; +import com.bakdata.kafka.admin.KafkaAdminException; import com.bakdata.kafka.admin.TopicsClient; import com.bakdata.kafka.consumerproducer.apps.MirrorKeyWithAvroConsumerProducer; import com.bakdata.kafka.consumerproducer.apps.MirrorValueWithAvroConsumerProducer; @@ -401,7 +401,7 @@ void shouldThrowExceptionOnResetterError() { awaitActive(executableApp); // should throw exception because consumer group is still active this.softly.assertThatThrownBy(() -> reset(executableApp)) - .isInstanceOf(CleanUpException.class) + .isInstanceOf(KafkaAdminException.class) .hasMessageContaining("Error resetting consumer group %s, consumer group is not empty", app.getUniqueAppId()); } From be3f445d0091c0debea8461bbde041b9c6786a95 Mon Sep 17 00:00:00 2001 From: Philipp Schirmer Date: Thu, 26 Feb 2026 12:41:51 +0100 Subject: [PATCH 04/15] Clean up Consumer/ConsumerProducer --- .../kafka/consumerproducer/ConsumerProducerCleanUpRunner.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumerproducer/ConsumerProducerCleanUpRunner.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumerproducer/ConsumerProducerCleanUpRunner.java index f919f4b66..3244c4972 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumerproducer/ConsumerProducerCleanUpRunner.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumerproducer/ConsumerProducerCleanUpRunner.java @@ -27,7 +27,6 @@ import com.bakdata.kafka.CleanUpRunner; import com.bakdata.kafka.consumer.ConsumerCleanUpConfiguration; import com.bakdata.kafka.consumer.ConsumerCleanUpRunner; -import com.bakdata.kafka.consumer.ConsumerTopicConfig; import com.bakdata.kafka.producer.ProducerCleanUpConfiguration; import com.bakdata.kafka.producer.ProducerCleanUpRunner; import com.bakdata.kafka.producer.ProducerTopicConfig; @@ -74,7 +73,6 @@ public static ConsumerProducerCleanUpRunner create(@NonNull final ConsumerProduc @NonNull final Map kafkaProperties, @NonNull final String groupId, @NonNull final StreamsCleanUpConfiguration configuration) { - final ConsumerTopicConfig consumerTopicConfig = topics.toConsumerTopicConfig(); final ProducerTopicConfig producerTopicConfig = topics.toProducerTopicConfig(); final ConsumerCleanUpConfiguration consumerConfig = configuration.toConsumerCleanUpConfiguration(); final ProducerCleanUpConfiguration producerConfig = configuration.toProducerCleanUpConfiguration(); From 75f6b2e751d4ab0371f511c9dbdd99edc30e8170 Mon Sep 17 00:00:00 2001 From: Philipp Schirmer Date: Thu, 26 Feb 2026 12:56:17 +0100 Subject: [PATCH 05/15] Clean up Consumer/ConsumerProducer --- .../com/bakdata/kafka/admin/ConsumerGroupsClient.java | 8 +++++--- .../consumerproducer/SerializerDeserializerConfig.java | 2 +- .../bakdata/kafka/consumer/ConsumerCleanUpRunnerTest.java | 2 +- .../ConsumerProducerCleanUpRunnerTest.java | 2 +- 4 files changed, 8 insertions(+), 6 deletions(-) diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/admin/ConsumerGroupsClient.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/admin/ConsumerGroupsClient.java index 3a9a1132c..effdaeeeb 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/admin/ConsumerGroupsClient.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/admin/ConsumerGroupsClient.java @@ -177,7 +177,8 @@ public void reset(final OffsetSpec offsetSpec) { } if (groupDescription.get().groupState() != GroupState.EMPTY) { throw new KafkaAdminException( - "Error resetting consumer group %s, consumer group is not empty".formatted(this.groupName)); + "Failed to reset offsets for consumer group %s: consumer group is not empty".formatted( + this.groupName)); } final Map groupOffsets = this.listOffsets(); @@ -188,7 +189,7 @@ public void reset(final OffsetSpec offsetSpec) { ConsumerGroupsClient.this.adminClient.listOffsets(request).all(); final Map offsets = ConsumerGroupsClient.this.timeout.get(offsetsFuture, - () -> "Error resetting consumer group %s, could not find offsets for spec %s".formatted( + () -> "Failed to reset offsets for consumer group %s: could not find offsets for spec %s".formatted( this.groupName, offsetSpec)); final Map resetOffsets = offsets.entrySet().stream() @@ -196,7 +197,8 @@ public void reset(final OffsetSpec offsetSpec) { final KafkaFuture alterOffsetResult = ConsumerGroupsClient.this.adminClient.alterConsumerGroupOffsets(this.groupName, resetOffsets).all(); ConsumerGroupsClient.this.timeout.get(alterOffsetResult, - () -> "Error resetting consumer group %s, could not alter offsets".formatted(this.groupName)); + () -> "Failed to reset offsets for consumer group %s: could not alter offsets".formatted( + this.groupName)); } /** diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumerproducer/SerializerDeserializerConfig.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumerproducer/SerializerDeserializerConfig.java index 360273808..48c768c36 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumerproducer/SerializerDeserializerConfig.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumerproducer/SerializerDeserializerConfig.java @@ -63,7 +63,7 @@ public Map createProperties() { .collect(Collectors.toMap( Map.Entry::getKey, Map.Entry::getValue, - (v1, v2) -> v2 // v1 and v2 should always be different + (v1, v2) -> v2 // v1 and v2 are always different )); } } diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/consumer/ConsumerCleanUpRunnerTest.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/consumer/ConsumerCleanUpRunnerTest.java index eb4c10391..963de5815 100644 --- a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/consumer/ConsumerCleanUpRunnerTest.java +++ b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/consumer/ConsumerCleanUpRunnerTest.java @@ -233,7 +233,7 @@ void shouldThrowExceptionOnResetterError() { // should throw exception because consumer group is still active this.softly.assertThatThrownBy(() -> reset(executableApp)) .isInstanceOf(KafkaAdminException.class) - .hasMessageContaining("Error resetting consumer group %s, consumer group is not empty", + .hasMessageContaining("Failed to reset offsets for consumer group %s: consumer group is not empty", app.getUniqueGroupId()); } } diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/consumerproducer/ConsumerProducerCleanUpRunnerTest.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/consumerproducer/ConsumerProducerCleanUpRunnerTest.java index a3b8b5dca..a06fc2f07 100644 --- a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/consumerproducer/ConsumerProducerCleanUpRunnerTest.java +++ b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/consumerproducer/ConsumerProducerCleanUpRunnerTest.java @@ -402,7 +402,7 @@ void shouldThrowExceptionOnResetterError() { // should throw exception because consumer group is still active this.softly.assertThatThrownBy(() -> reset(executableApp)) .isInstanceOf(KafkaAdminException.class) - .hasMessageContaining("Error resetting consumer group %s, consumer group is not empty", + .hasMessageContaining("Failed to reset offsets for consumer group %s: consumer group is not empty", app.getUniqueAppId()); } } From 56795a06b62e6bee83e3f734c99dcf56ebad6c09 Mon Sep 17 00:00:00 2001 From: Philipp Schirmer Date: Thu, 26 Feb 2026 13:07:44 +0100 Subject: [PATCH 06/15] Remove start hooks --- .../consumer/KafkaConsumerApplication.java | 13 +---- ...fkaConsumerProducerApplicationCliTest.java | 6 +-- .../consumer/ConsumerExecutionOptions.java | 11 ---- .../kafka/consumer/ConsumerRunner.java | 9 +--- .../kafka/consumer/ExecutableConsumerApp.java | 2 +- .../kafka/consumer/RunningConsumer.java | 49 ------------------ .../ConsumerProducerExecutionOptions.java | 13 +---- .../ConsumerProducerRunnable.java | 6 +-- .../ConsumerProducerRunner.java | 12 +---- .../DefaultConsumerProducerRunnable.java | 3 +- .../ExecutableConsumerProducerApp.java | 4 +- .../RunningConsumerProducer.java | 51 ------------------- 12 files changed, 12 insertions(+), 167 deletions(-) delete mode 100644 streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumer/RunningConsumer.java delete mode 100644 streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumerproducer/RunningConsumerProducer.java diff --git a/streams-bootstrap-cli/src/main/java/com/bakdata/kafka/consumer/KafkaConsumerApplication.java b/streams-bootstrap-cli/src/main/java/com/bakdata/kafka/consumer/KafkaConsumerApplication.java index 822301015..057ed55b4 100644 --- a/streams-bootstrap-cli/src/main/java/com/bakdata/kafka/consumer/KafkaConsumerApplication.java +++ b/streams-bootstrap-cli/src/main/java/com/bakdata/kafka/consumer/KafkaConsumerApplication.java @@ -1,7 +1,7 @@ /* * MIT License * - * Copyright (c) 2025 bakdata + * Copyright (c) 2026 bakdata * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -35,7 +35,6 @@ import lombok.ToString; import lombok.experimental.Delegate; import lombok.extern.slf4j.Slf4j; -import org.apache.kafka.clients.consumer.ConsumerConfig; import picocli.CommandLine.Command; import picocli.CommandLine.Mixin; import picocli.CommandLine.Option; @@ -102,7 +101,6 @@ public void reset() { public final Optional createExecutionOptions() { final ConsumerExecutionOptions executionOptions = ConsumerExecutionOptions.builder() .volatileGroupInstanceId(this.isVolatileGroupInstanceId()) - .onStart(this::onConsumerStart) .pollTimeout(this.getPollTimeout()) .build(); return Optional.of(executionOptions); @@ -128,13 +126,4 @@ public final ConfiguredConsumerApp createConfiguredApp(final T app, public ConsumerAppConfiguration createConfiguration(final ConsumerTopicConfig topics) { return new ConsumerAppConfiguration(topics, this.getGroupId()); } - - /** - * Called after starting Kafka Consumer - * - * @param runningConsumer running {@link ConsumerRunnable} instance along with its {@link ConsumerConfig} - */ - protected void onConsumerStart(final RunningConsumer runningConsumer) { - // do nothing by default - } } diff --git a/streams-bootstrap-cli/src/test/java/com/bakdata/kafka/consumerproducer/KafkaConsumerProducerApplicationCliTest.java b/streams-bootstrap-cli/src/test/java/com/bakdata/kafka/consumerproducer/KafkaConsumerProducerApplicationCliTest.java index 050cb04e3..942df6fce 100644 --- a/streams-bootstrap-cli/src/test/java/com/bakdata/kafka/consumerproducer/KafkaConsumerProducerApplicationCliTest.java +++ b/streams-bootstrap-cli/src/test/java/com/bakdata/kafka/consumerproducer/KafkaConsumerProducerApplicationCliTest.java @@ -191,7 +191,7 @@ public ConsumerProducerApp createApp() { return new ConsumerProducerApp() { @Override public ConsumerProducerRunnable buildRunnable(final ConsumerProducerBuilder builder) { - return (consumerConfig, producerConfig) -> {}; + return (consumerConfig) -> {}; } @Override @@ -223,7 +223,7 @@ void shouldExitWithErrorInBuildRunnable() { () -> new ConsumerProducerApp() { @Override public ConsumerProducerRunnable buildRunnable(final ConsumerProducerBuilder builder) { - return (consumerConfig, producerConfig) -> { + return (consumerConfig) -> { throw new RuntimeException("Error building runnable"); }; } @@ -264,7 +264,7 @@ void shouldExitWithSuccessCodeOnShutdown() { () -> new ConsumerProducerApp() { @Override public ConsumerProducerRunnable buildRunnable(final ConsumerProducerBuilder builder) { - return (consumerConfig, producerConfig) -> { + return (consumerConfig) -> { try (final Producer producer = builder.getProducerBuilder() .createProducer()) { final ProducerRecord producerRecord = diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumer/ConsumerExecutionOptions.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumer/ConsumerExecutionOptions.java index 73765ac1b..a36399b6d 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumer/ConsumerExecutionOptions.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumer/ConsumerExecutionOptions.java @@ -28,7 +28,6 @@ import java.util.Map; import lombok.Builder; import lombok.Getter; -import lombok.NonNull; import org.apache.kafka.clients.consumer.CloseOptions; import org.apache.kafka.clients.consumer.CloseOptions.GroupMembershipOperation; import org.apache.kafka.clients.consumer.Consumer; @@ -40,12 +39,6 @@ @Builder public class ConsumerExecutionOptions { - /** - * Hook that is called after the {@link ConsumerRunnable} is started - */ - @Builder.Default - private final @NonNull java.util.function.Consumer onStart = runningConsumer -> {}; - /** * Defines if {@link ConsumerConfig#GROUP_INSTANCE_ID_CONFIG} is volatile. If it is configured and non-volatile, * {@link Consumer#close(CloseOptions)} is called with @@ -84,8 +77,4 @@ boolean shouldLeaveGroup(final Map originals) { final boolean staticMembershipDisabled = isStaticMembershipDisabled(originals); return staticMembershipDisabled || this.volatileGroupInstanceId; } - - void onStart(final RunningConsumer runningConsumer) { - this.onStart.accept(runningConsumer); - } } diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumer/ConsumerRunner.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumer/ConsumerRunner.java index 42d956f16..2361ff38c 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumer/ConsumerRunner.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumer/ConsumerRunner.java @@ -39,7 +39,6 @@ public class ConsumerRunner implements Runner { private final @NonNull ConsumerRunnable runnable; private final @NonNull ConsumerConfig config; - private final @NonNull ConsumerExecutionOptions executionOptions; @Override public void close() { @@ -55,13 +54,7 @@ public void run() { private void runConsumer() { log.info("Starting Kafka Consumer"); - final RunningConsumer runningConsumer = RunningConsumer.builder() - .consumerRunnable(this.runnable) - .config(this.config) - .build(); - log.debug("Calling start hook"); - this.executionOptions.onStart(runningConsumer); // Run Kafka consumer until it shuts down - this.runnable.run(this.config); //TODO call before onStart + this.runnable.run(this.config); } } diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumer/ExecutableConsumerApp.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumer/ExecutableConsumerApp.java index 12ac8e0f8..bf94a796f 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumer/ExecutableConsumerApp.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumer/ExecutableConsumerApp.java @@ -74,7 +74,7 @@ public ConsumerRunner createRunner(final ConsumerExecutionOptions options) { final ConsumerBuilder consumerBuilder = new ConsumerBuilder(this.topics, this.kafkaProperties, options); final AppConfiguration configuration = this.createConfiguration(); this.app.setup(configuration); - return new ConsumerRunner(this.app.buildRunnable(consumerBuilder), this.getConfig(), options); + return new ConsumerRunner(this.app.buildRunnable(consumerBuilder), this.getConfig()); } @Override diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumer/RunningConsumer.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumer/RunningConsumer.java deleted file mode 100644 index c0f141b62..000000000 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumer/RunningConsumer.java +++ /dev/null @@ -1,49 +0,0 @@ -/* - * MIT License - * - * Copyright (c) 2025 bakdata - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package com.bakdata.kafka.consumer; - -import com.bakdata.kafka.consumer.ConsumerExecutionOptions.ConsumerExecutionOptionsBuilder; -import java.util.function.Consumer; -import lombok.Builder; -import lombok.NonNull; -import lombok.Value; -import org.apache.kafka.clients.consumer.ConsumerConfig; -import org.apache.kafka.clients.consumer.KafkaConsumer; - -/** - * A running {@link KafkaConsumer} instance along with its {@link ConsumerConfig} and - * {@link ConsumerRunnable} - * - * @see ConsumerExecutionOptionsBuilder#onStart(Consumer) - */ -@Builder -@Value -public class RunningConsumer { - - @NonNull - ConsumerConfig config; - @NonNull - ConsumerRunnable consumerRunnable; -} diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumerproducer/ConsumerProducerExecutionOptions.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumerproducer/ConsumerProducerExecutionOptions.java index 280c28549..6a0203c1e 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumerproducer/ConsumerProducerExecutionOptions.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumerproducer/ConsumerProducerExecutionOptions.java @@ -1,7 +1,7 @@ /* * MIT License * - * Copyright (c) 2025 bakdata + * Copyright (c) 2026 bakdata * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -28,7 +28,6 @@ import java.time.Duration; import lombok.Builder; import lombok.Getter; -import lombok.NonNull; import org.apache.kafka.clients.consumer.CloseOptions; import org.apache.kafka.clients.consumer.CloseOptions.GroupMembershipOperation; import org.apache.kafka.clients.consumer.Consumer; @@ -40,12 +39,6 @@ @Builder public final class ConsumerProducerExecutionOptions { - /** - * Hook that is called after the {@link ConsumerProducerRunnable} is started - */ - @Builder.Default - private final @NonNull java.util.function.Consumer onStart = runningConsumerProducer -> {}; - /** * Defines if {@link ConsumerConfig#GROUP_INSTANCE_ID_CONFIG} is volatile. If it is configured and non-volatile, * {@link Consumer#close(CloseOptions)} is called with @@ -68,10 +61,6 @@ public final class ConsumerProducerExecutionOptions { @Getter private final Duration pollTimeout = Duration.ofMillis(Long.MAX_VALUE); - void onStart(final RunningConsumerProducer runningConsumerProducer) { - this.onStart.accept(runningConsumerProducer); - } - ConsumerExecutionOptions toConsumerExecutionOptions() { return ConsumerExecutionOptions.builder() .volatileGroupInstanceId(this.volatileGroupInstanceId) diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumerproducer/ConsumerProducerRunnable.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumerproducer/ConsumerProducerRunnable.java index a8b4bfad7..78fba25a9 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumerproducer/ConsumerProducerRunnable.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumerproducer/ConsumerProducerRunnable.java @@ -1,7 +1,7 @@ /* * MIT License * - * Copyright (c) 2025 bakdata + * Copyright (c) 2026 bakdata * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -25,7 +25,6 @@ package com.bakdata.kafka.consumerproducer; import org.apache.kafka.clients.consumer.ConsumerConfig; -import org.apache.kafka.clients.producer.ProducerConfig; /** * Produce or consume messages to or from Kafka @@ -36,9 +35,8 @@ public interface ConsumerProducerRunnable extends AutoCloseable { * Produce or Consume messages from Kafka * * @param consumerConfig configuration for the consumer - * @param producerConfig configuration for the producer */ - void run(ConsumerConfig consumerConfig, ProducerConfig producerConfig); + void run(ConsumerConfig consumerConfig); @Override default void close() { diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumerproducer/ConsumerProducerRunner.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumerproducer/ConsumerProducerRunner.java index a0a92b350..e86d46e8b 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumerproducer/ConsumerProducerRunner.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumerproducer/ConsumerProducerRunner.java @@ -29,7 +29,6 @@ import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.apache.kafka.clients.consumer.ConsumerConfig; -import org.apache.kafka.clients.producer.ProducerConfig; /** * Runs a Kafka Consumer and Producer application @@ -40,8 +39,6 @@ public class ConsumerProducerRunner implements Runner { private final @NonNull ConsumerProducerRunnable runnable; private final @NonNull ConsumerConfig consumerConfig; - private final @NonNull ProducerConfig producerConfig; - private final @NonNull ConsumerProducerExecutionOptions executionOptions; @Override public void close() { @@ -57,14 +54,7 @@ public void run() { private void runConsumerProducer() { log.info("Starting Kafka ConsumerProducer"); - final RunningConsumerProducer runningConsumer = RunningConsumerProducer.builder() - .consumerProducerRunnable(this.runnable) - .consumerConfig(this.consumerConfig) - .producerConfig(this.producerConfig) - .build(); - log.debug("Calling start hook"); - this.executionOptions.onStart(runningConsumer); // Run Kafka application until it shuts down - this.runnable.run(this.consumerConfig, this.producerConfig); //TODO call before onStart + this.runnable.run(this.consumerConfig); } } diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumerproducer/DefaultConsumerProducerRunnable.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumerproducer/DefaultConsumerProducerRunnable.java index ed0664f56..3145a0707 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumerproducer/DefaultConsumerProducerRunnable.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumerproducer/DefaultConsumerProducerRunnable.java @@ -30,7 +30,6 @@ import lombok.extern.slf4j.Slf4j; import org.apache.kafka.clients.consumer.ConsumerConfig; import org.apache.kafka.clients.producer.Producer; -import org.apache.kafka.clients.producer.ProducerConfig; @AllArgsConstructor @Slf4j @@ -42,7 +41,7 @@ public class DefaultConsumerProducerRunnable implements ConsumerProd private final ConsumerRunnable consumerRunnable; @Override - public void run(final ConsumerConfig consumerConfig, final ProducerConfig producerConfig) { + public void run(final ConsumerConfig consumerConfig) { this.consumerRunnable.run(consumerConfig); } diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumerproducer/ExecutableConsumerProducerApp.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumerproducer/ExecutableConsumerProducerApp.java index 59ef2259d..37ee1af03 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumerproducer/ExecutableConsumerProducerApp.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumerproducer/ExecutableConsumerProducerApp.java @@ -88,9 +88,7 @@ public ConsumerProducerRunner createRunner(final ConsumerProducerExecutionOption final AppConfiguration configuration = this.createConfiguration(); this.app.setup(configuration); return new ConsumerProducerRunner(this.app.buildRunnable(consumerProducerBuilder), - this.getConsumerConfig(), - this.getProducerConfig(), - options); + this.getConsumerConfig()); } @Override diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumerproducer/RunningConsumerProducer.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumerproducer/RunningConsumerProducer.java deleted file mode 100644 index 5981d8521..000000000 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumerproducer/RunningConsumerProducer.java +++ /dev/null @@ -1,51 +0,0 @@ -/* - * MIT License - * - * Copyright (c) 2025 bakdata - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package com.bakdata.kafka.consumerproducer; - -import com.bakdata.kafka.consumer.ConsumerRunnable; -import lombok.Builder; -import lombok.NonNull; -import lombok.Value; -import org.apache.kafka.clients.consumer.ConsumerConfig; -import org.apache.kafka.clients.consumer.KafkaConsumer; -import org.apache.kafka.clients.producer.ProducerConfig; - -/** - * A running {@link KafkaConsumer} instance along with its {@link ConsumerConfig} and - * {@link ConsumerRunnable} - * - * @see ConsumerProducerExecutionOptions#onStart(RunningConsumerProducer) - */ -@Builder -@Value -public class RunningConsumerProducer { - - @NonNull - ConsumerConfig consumerConfig; - @NonNull - ProducerConfig producerConfig; - @NonNull - ConsumerProducerRunnable consumerProducerRunnable; -} From 9ee6c2f0f0ef9b8070f6dd2b2b62c1d8421590ed Mon Sep 17 00:00:00 2001 From: Philipp Schirmer Date: Thu, 26 Feb 2026 21:05:05 +0100 Subject: [PATCH 07/15] More clean up --- .../consumer/KafkaConsumerApplication.java | 11 +-- .../KafkaConsumerProducerApplication.java | 9 ++- .../bakdata/kafka/mixin/ConsumerOptions.java | 5 ++ .../streams/KafkaStreamsApplication.java | 7 +- ...fkaConsumerProducerApplicationCliTest.java | 18 ++--- .../consumerproducer/apps/CloseFlagApp.java | 2 +- .../kafka/consumerproducer/apps/Mirror.java | 2 +- .../bakdata/kafka/CloseExecutionOptions.java | 78 +++++++++++++++++++ .../consumer/ConsumerExecutionOptions.java | 38 ++------- .../kafka/consumer/ConsumerRunner.java | 5 -- .../consumer/DefaultConsumerRunnable.java | 8 +- .../ConfiguredConsumerProducerApp.java | 19 ++--- .../consumerproducer/ConsumerProducerApp.java | 7 +- .../ConsumerProducerAppConfiguration.java | 4 +- .../ConsumerProducerExecutionOptions.java | 23 +----- .../ConsumerProducerRunner.java | 5 -- .../DefaultConsumerProducerRunnable.java | 8 +- .../streams/StreamsExecutionOptions.java | 25 ++---- ...st.java => CloseExecutionOptionsTest.java} | 12 +-- .../ConsumerExecutionOptionsTest.java | 62 --------------- .../ConfiguredConsumerProducerAppTest.java | 4 +- .../ConsumerProducerCleanUpRunnerTest.java | 12 +-- .../MirrorKeyWithAvroConsumerProducer.java | 2 +- .../MirrorValueWithAvroConsumerProducer.java | 2 +- .../apps/StringConsumerProducer.java | 2 +- .../apps/StringPatternConsumerProducer.java | 2 +- .../apps/SimpleConsumerProducerApp.java | 2 +- 27 files changed, 167 insertions(+), 207 deletions(-) create mode 100644 streams-bootstrap-core/src/main/java/com/bakdata/kafka/CloseExecutionOptions.java rename streams-bootstrap-core/src/test/java/com/bakdata/kafka/{streams/StreamsExecutionOptionsTest.java => CloseExecutionOptionsTest.java} (85%) delete mode 100644 streams-bootstrap-core/src/test/java/com/bakdata/kafka/consumer/ConsumerExecutionOptionsTest.java diff --git a/streams-bootstrap-cli/src/main/java/com/bakdata/kafka/consumer/KafkaConsumerApplication.java b/streams-bootstrap-cli/src/main/java/com/bakdata/kafka/consumer/KafkaConsumerApplication.java index 057ed55b4..780afa8ee 100644 --- a/streams-bootstrap-cli/src/main/java/com/bakdata/kafka/consumer/KafkaConsumerApplication.java +++ b/streams-bootstrap-cli/src/main/java/com/bakdata/kafka/consumer/KafkaConsumerApplication.java @@ -24,10 +24,10 @@ package com.bakdata.kafka.consumer; +import com.bakdata.kafka.CloseExecutionOptions; import com.bakdata.kafka.KafkaApplication; import com.bakdata.kafka.mixin.ConsumerOptions; import com.bakdata.kafka.mixin.InputOptions; -import java.time.Duration; import java.util.Optional; import lombok.Getter; import lombok.RequiredArgsConstructor; @@ -37,7 +37,6 @@ import lombok.extern.slf4j.Slf4j; import picocli.CommandLine.Command; import picocli.CommandLine.Mixin; -import picocli.CommandLine.Option; /** @@ -71,10 +70,6 @@ public abstract class KafkaConsumerApplication extends @Mixin @Delegate private ConsumerOptions consumerOptions = new ConsumerOptions(); - @Option(names = {"--poll-timeout"}, - description = "The maximum time to block in the consumer poll loop. Examples: 'PT0.1S', 'PT2S', 'PT1M'.", - defaultValue = "PT0.1S") - private Duration pollTimeout = Duration.ofMillis(100); /** * Reset the Kafka Consumer application. Additionally, delete the consumer group. @@ -100,7 +95,9 @@ public void reset() { @Override public final Optional createExecutionOptions() { final ConsumerExecutionOptions executionOptions = ConsumerExecutionOptions.builder() - .volatileGroupInstanceId(this.isVolatileGroupInstanceId()) + .closeExecutionOptions(CloseExecutionOptions.builder() + .volatileGroupInstanceId(this.isVolatileGroupInstanceId()) + .build()) .pollTimeout(this.getPollTimeout()) .build(); return Optional.of(executionOptions); diff --git a/streams-bootstrap-cli/src/main/java/com/bakdata/kafka/consumerproducer/KafkaConsumerProducerApplication.java b/streams-bootstrap-cli/src/main/java/com/bakdata/kafka/consumerproducer/KafkaConsumerProducerApplication.java index 7267a60e5..d763a7505 100644 --- a/streams-bootstrap-cli/src/main/java/com/bakdata/kafka/consumerproducer/KafkaConsumerProducerApplication.java +++ b/streams-bootstrap-cli/src/main/java/com/bakdata/kafka/consumerproducer/KafkaConsumerProducerApplication.java @@ -24,6 +24,7 @@ package com.bakdata.kafka.consumerproducer; +import com.bakdata.kafka.CloseExecutionOptions; import com.bakdata.kafka.KafkaApplication; import com.bakdata.kafka.mixin.ConsumerOptions; import com.bakdata.kafka.mixin.ErrorOptions; @@ -107,7 +108,13 @@ public void reset() { @Override public final Optional createExecutionOptions() { - return Optional.empty(); + final ConsumerProducerExecutionOptions executionOptions = ConsumerProducerExecutionOptions.builder() + .closeExecutionOptions(CloseExecutionOptions.builder() + .volatileGroupInstanceId(this.isVolatileGroupInstanceId()) + .build()) + .pollTimeout(this.getPollTimeout()) + .build(); + return Optional.of(executionOptions); } @Override diff --git a/streams-bootstrap-cli/src/main/java/com/bakdata/kafka/mixin/ConsumerOptions.java b/streams-bootstrap-cli/src/main/java/com/bakdata/kafka/mixin/ConsumerOptions.java index db4448f3f..ae4bd2f89 100644 --- a/streams-bootstrap-cli/src/main/java/com/bakdata/kafka/mixin/ConsumerOptions.java +++ b/streams-bootstrap-cli/src/main/java/com/bakdata/kafka/mixin/ConsumerOptions.java @@ -24,8 +24,10 @@ package com.bakdata.kafka.mixin; +import java.time.Duration; import lombok.Data; import picocli.CommandLine; +import picocli.CommandLine.Option; /** * Shared CLI options to configure Kafka Consumer applications. @@ -38,4 +40,7 @@ public class ConsumerOptions { @CommandLine.Option(names = "--group-id", description = "Unique identifier for the Kafka Consumer applications, used as 'group.id'.") private String groupId; + @Option(names = {"--poll-timeout"}, + description = "The maximum time to block in the consumer poll loop. Examples: 'PT0.1S', 'PT2S', 'PT1M'.") + private Duration pollTimeout = Duration.ofMillis(100); } diff --git a/streams-bootstrap-cli/src/main/java/com/bakdata/kafka/streams/KafkaStreamsApplication.java b/streams-bootstrap-cli/src/main/java/com/bakdata/kafka/streams/KafkaStreamsApplication.java index 68a6ef033..b589bf0f1 100644 --- a/streams-bootstrap-cli/src/main/java/com/bakdata/kafka/streams/KafkaStreamsApplication.java +++ b/streams-bootstrap-cli/src/main/java/com/bakdata/kafka/streams/KafkaStreamsApplication.java @@ -1,7 +1,7 @@ /* * MIT License * - * Copyright (c) 2025 bakdata + * Copyright (c) 2026 bakdata * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -24,6 +24,7 @@ package com.bakdata.kafka.streams; +import com.bakdata.kafka.CloseExecutionOptions; import com.bakdata.kafka.KafkaApplication; import com.bakdata.kafka.mixin.ErrorOptions; import com.bakdata.kafka.mixin.InputOptions; @@ -116,7 +117,9 @@ public void reset() { @Override public final Optional createExecutionOptions() { final StreamsExecutionOptions options = StreamsExecutionOptions.builder() - .volatileGroupInstanceId(this.isVolatileGroupInstanceId()) + .closeExecutionOptions(CloseExecutionOptions.builder() + .volatileGroupInstanceId(this.isVolatileGroupInstanceId()) + .build()) .uncaughtExceptionHandler(this::createUncaughtExceptionHandler) .stateListener(this::createStateListener) .onStart(this::onStreamsStart) diff --git a/streams-bootstrap-cli/src/test/java/com/bakdata/kafka/consumerproducer/KafkaConsumerProducerApplicationCliTest.java b/streams-bootstrap-cli/src/test/java/com/bakdata/kafka/consumerproducer/KafkaConsumerProducerApplicationCliTest.java index 942df6fce..569fc9813 100644 --- a/streams-bootstrap-cli/src/test/java/com/bakdata/kafka/consumerproducer/KafkaConsumerProducerApplicationCliTest.java +++ b/streams-bootstrap-cli/src/test/java/com/bakdata/kafka/consumerproducer/KafkaConsumerProducerApplicationCliTest.java @@ -65,7 +65,7 @@ public ConsumerProducerRunnable buildRunnable(final ConsumerProducerBuilder buil } @Override - public String getUniqueAppId(final ConsumerProducerAppConfiguration configuration) { + public String getUniqueGroupId(final ConsumerProducerAppConfiguration configuration) { throw new UnsupportedOperationException(); } @@ -97,7 +97,7 @@ public ConsumerProducerRunnable buildRunnable(final ConsumerProducerBuilder buil } @Override - public String getUniqueAppId(final ConsumerProducerAppConfiguration configuration) { + public String getUniqueGroupId(final ConsumerProducerAppConfiguration configuration) { throw new UnsupportedOperationException(); } @@ -125,7 +125,7 @@ public ConsumerProducerRunnable buildRunnable(final ConsumerProducerBuilder buil } @Override - public String getUniqueAppId(final ConsumerProducerAppConfiguration configuration) { + public String getUniqueGroupId(final ConsumerProducerAppConfiguration configuration) { throw new UnsupportedOperationException(); } @@ -161,7 +161,7 @@ public ConsumerProducerRunnable buildRunnable(final ConsumerProducerBuilder buil } @Override - public String getUniqueAppId(final ConsumerProducerAppConfiguration configuration) { + public String getUniqueGroupId(final ConsumerProducerAppConfiguration configuration) { throw new UnsupportedOperationException(); } @@ -195,7 +195,7 @@ public ConsumerProducerRunnable buildRunnable(final ConsumerProducerBuilder buil } @Override - public String getUniqueAppId(final ConsumerProducerAppConfiguration configuration) { + public String getUniqueGroupId(final ConsumerProducerAppConfiguration configuration) { return "my-id"; } @@ -229,7 +229,7 @@ public ConsumerProducerRunnable buildRunnable(final ConsumerProducerBuilder buil } @Override - public String getUniqueAppId(final ConsumerProducerAppConfiguration configuration) { + public String getUniqueGroupId(final ConsumerProducerAppConfiguration configuration) { return "app"; } @@ -275,7 +275,7 @@ public ConsumerProducerRunnable buildRunnable(final ConsumerProducerBuilder buil } @Override - public String getUniqueAppId(final ConsumerProducerAppConfiguration configuration) { + public String getUniqueGroupId(final ConsumerProducerAppConfiguration configuration) { return "app"; } @@ -321,7 +321,7 @@ public ConsumerProducerRunnable buildRunnable(final ConsumerProducerBuilder buil } @Override - public String getUniqueAppId(final ConsumerProducerAppConfiguration configuration) { + public String getUniqueGroupId(final ConsumerProducerAppConfiguration configuration) { throw new UnsupportedOperationException(); } @@ -351,7 +351,7 @@ public ConsumerProducerRunnable buildRunnable(final ConsumerProducerBuilder buil } @Override - public String getUniqueAppId(final ConsumerProducerAppConfiguration configuration) { + public String getUniqueGroupId(final ConsumerProducerAppConfiguration configuration) { throw new UnsupportedOperationException(); } diff --git a/streams-bootstrap-cli/src/test/java/com/bakdata/kafka/consumerproducer/apps/CloseFlagApp.java b/streams-bootstrap-cli/src/test/java/com/bakdata/kafka/consumerproducer/apps/CloseFlagApp.java index 946e8a489..b433ed67b 100644 --- a/streams-bootstrap-cli/src/test/java/com/bakdata/kafka/consumerproducer/apps/CloseFlagApp.java +++ b/streams-bootstrap-cli/src/test/java/com/bakdata/kafka/consumerproducer/apps/CloseFlagApp.java @@ -72,7 +72,7 @@ public ConsumerProducerRunnable buildRunnable(final ConsumerProducerBuilder buil } @Override - public String getUniqueAppId(final ConsumerProducerAppConfiguration configuration) { + public String getUniqueGroupId(final ConsumerProducerAppConfiguration configuration) { return CloseFlagApp.this.getClass().getSimpleName() + "-" + configuration.getTopics().getOutputTopic(); } diff --git a/streams-bootstrap-cli/src/test/java/com/bakdata/kafka/consumerproducer/apps/Mirror.java b/streams-bootstrap-cli/src/test/java/com/bakdata/kafka/consumerproducer/apps/Mirror.java index fa8df8269..3a46644dc 100644 --- a/streams-bootstrap-cli/src/test/java/com/bakdata/kafka/consumerproducer/apps/Mirror.java +++ b/streams-bootstrap-cli/src/test/java/com/bakdata/kafka/consumerproducer/apps/Mirror.java @@ -55,7 +55,7 @@ public ConsumerProducerRunnable buildRunnable(final ConsumerProducerBuilder buil } @Override - public String getUniqueAppId(final ConsumerProducerAppConfiguration configuration) { + public String getUniqueGroupId(final ConsumerProducerAppConfiguration configuration) { return this.getClass().getSimpleName() + "-" + configuration.getTopics().getOutputTopic(); } diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/CloseExecutionOptions.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/CloseExecutionOptions.java new file mode 100644 index 000000000..94195ee31 --- /dev/null +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/CloseExecutionOptions.java @@ -0,0 +1,78 @@ +/* + * MIT License + * + * Copyright (c) 2026 bakdata + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package com.bakdata.kafka; + +import com.bakdata.kafka.streams.StreamsRunner; +import java.time.Duration; +import java.util.Map; +import lombok.Builder; +import org.apache.kafka.clients.consumer.CloseOptions.GroupMembershipOperation; +import org.apache.kafka.clients.consumer.ConsumerConfig; +import org.apache.kafka.streams.KafkaStreams; +import org.apache.kafka.streams.KafkaStreams.CloseOptions; +import org.apache.kafka.streams.StreamsConfig; + +//FIXME javadoc + +/** + * Options to run a Kafka Streams app using {@link StreamsRunner} + */ +@Builder +public class CloseExecutionOptions { + /** + * Defines if {@link ConsumerConfig#GROUP_INSTANCE_ID_CONFIG} is volatile. If it is configured and non-volatile, + * {@link KafkaStreams#close(CloseOptions)} is called with {@link CloseOptions#leaveGroup(boolean)} disabled + */ + @Builder.Default + private final boolean volatileGroupInstanceId = true; + /** + * Defines {@link CloseOptions#timeout(Duration)} when calling {@link KafkaStreams#close(CloseOptions)} + */ + @Builder.Default + private final Duration closeTimeout = Duration.ofMillis(Long.MAX_VALUE); + + private static boolean isStaticMembershipDisabled(final Map originals) { + return originals.get(ConsumerConfig.GROUP_INSTANCE_ID_CONFIG) == null; + } + + //TODO javadoc + public CloseOptions createCloseOptions(final StreamsConfig config) { + final boolean leaveGroup = this.shouldLeaveGroup(config.originals()); + return new CloseOptions().leaveGroup(leaveGroup).timeout(this.closeTimeout); + } + + public org.apache.kafka.clients.consumer.CloseOptions createCloseOptions(final ConsumerConfig config) { + final boolean leaveGroup = this.shouldLeaveGroup(config.originals()); + final GroupMembershipOperation operation = + leaveGroup ? GroupMembershipOperation.LEAVE_GROUP : GroupMembershipOperation.DEFAULT; + return org.apache.kafka.clients.consumer.CloseOptions.groupMembershipOperation(operation) + .withTimeout(this.closeTimeout); + } + + boolean shouldLeaveGroup(final Map originals) { + final boolean staticMembershipDisabled = isStaticMembershipDisabled(originals); + return staticMembershipDisabled || this.volatileGroupInstanceId; + } +} diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumer/ConsumerExecutionOptions.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumer/ConsumerExecutionOptions.java index a36399b6d..e3fa9c0e0 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumer/ConsumerExecutionOptions.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumer/ConsumerExecutionOptions.java @@ -24,12 +24,11 @@ package com.bakdata.kafka.consumer; +import com.bakdata.kafka.CloseExecutionOptions; import java.time.Duration; -import java.util.Map; import lombok.Builder; import lombok.Getter; import org.apache.kafka.clients.consumer.CloseOptions; -import org.apache.kafka.clients.consumer.CloseOptions.GroupMembershipOperation; import org.apache.kafka.clients.consumer.Consumer; import org.apache.kafka.clients.consumer.ConsumerConfig; @@ -39,42 +38,19 @@ @Builder public class ConsumerExecutionOptions { - /** - * Defines if {@link ConsumerConfig#GROUP_INSTANCE_ID_CONFIG} is volatile. If it is configured and non-volatile, - * {@link Consumer#close(CloseOptions)} is called with - * {@link CloseOptions#groupMembershipOperation(GroupMembershipOperation)} set to - * {@link GroupMembershipOperation#REMAIN_IN_GROUP} - */ - @Builder.Default - private final boolean volatileGroupInstanceId = true; - - /** - * Defines {@link CloseOptions#timeout(Duration)} when calling {@link Consumer#close(CloseOptions)} - */ + //TODO javadoc @Builder.Default - private final Duration closeTimeout = Duration.ofMillis(Long.MAX_VALUE); + private final CloseExecutionOptions closeExecutionOptions = CloseExecutionOptions.builder().build(); /** * Defines the timeout duration for the {@link Consumer#poll(Duration)} call */ @Builder.Default @Getter - private final Duration pollTimeout = Duration.ofMillis(Long.MAX_VALUE); - - //TODO reuse close options in StreamsExecutionOptions - private static boolean isStaticMembershipDisabled(final Map originals) { - return originals.get(ConsumerConfig.GROUP_INSTANCE_ID_CONFIG) == null; - } - - CloseOptions createCloseOptions(final ConsumerConfig config) { - final boolean leaveGroup = this.shouldLeaveGroup(config.originals()); - final GroupMembershipOperation operation = - leaveGroup ? GroupMembershipOperation.LEAVE_GROUP : GroupMembershipOperation.DEFAULT; - return CloseOptions.groupMembershipOperation(operation).withTimeout(this.closeTimeout); - } + private final Duration pollTimeout = Duration.ofMillis(100L); - boolean shouldLeaveGroup(final Map originals) { - final boolean staticMembershipDisabled = isStaticMembershipDisabled(originals); - return staticMembershipDisabled || this.volatileGroupInstanceId; + //TODO javadoc + public CloseOptions createCloseOptions(final ConsumerConfig config) { + return this.closeExecutionOptions.createCloseOptions(config); } } diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumer/ConsumerRunner.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumer/ConsumerRunner.java index 2361ff38c..8a20b7224 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumer/ConsumerRunner.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumer/ConsumerRunner.java @@ -48,11 +48,6 @@ public void close() { @Override public void run() { - this.runConsumer(); - //TODO await? - } - - private void runConsumer() { log.info("Starting Kafka Consumer"); // Run Kafka consumer until it shuts down this.runnable.run(this.config); diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumer/DefaultConsumerRunnable.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumer/DefaultConsumerRunnable.java index ce19b5373..1ba16784f 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumer/DefaultConsumerRunnable.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumer/DefaultConsumerRunnable.java @@ -26,8 +26,6 @@ import java.util.concurrent.CountDownLatch; import java.util.concurrent.atomic.AtomicBoolean; -import lombok.AccessLevel; -import lombok.Getter; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.apache.kafka.clients.consumer.CloseOptions; @@ -44,12 +42,10 @@ * @param type of keys * @param type of values */ -@RequiredArgsConstructor(access = AccessLevel.PROTECTED) +@RequiredArgsConstructor @Slf4j -//TODO check public class DefaultConsumerRunnable implements ConsumerRunnable { - @Getter private final Consumer consumer; private final ConsumerExecutionOptions executionOptions; /** @@ -85,7 +81,7 @@ private void pollLoop(final ConsumerConfig consumerConfig) { } catch (final WakeupException exception) { log.info("Consumer poll loop waking up for shutdown", exception); } catch (final RuntimeException exception) { - //TODO why catch? + // ensure latch count down for proper close log.error("RuntimeException while running consumer loop", exception); } finally { log.info("Closing consumer"); diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumerproducer/ConfiguredConsumerProducerApp.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumerproducer/ConfiguredConsumerProducerApp.java index 8c0b99147..48e0e9194 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumerproducer/ConfiguredConsumerProducerApp.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumerproducer/ConfiguredConsumerProducerApp.java @@ -94,7 +94,7 @@ public Map getKafkaProperties(final RuntimeConfiguration runtime config.putAll(ConfiguredConsumerApp.createBaseConfig()); final KafkaPropertiesFactory propertiesFactory = this.createPropertiesFactory(runtimeConfiguration, config); return propertiesFactory.createKafkaProperties(Map.of( - CommonClientConfigs.GROUP_ID_CONFIG, this.getUniqueAppId() + CommonClientConfigs.GROUP_ID_CONFIG, this.getUniqueGroupId() )); } @@ -104,16 +104,17 @@ public Map getKafkaProperties(final RuntimeConfiguration runtime * @return unique group identifier * @throws IllegalArgumentException if unique group identifier of {@link ConsumerProducerApp} is different from * provided group identifier in {@link ConsumerProducerAppConfiguration} - * @see ConsumerProducerApp#getUniqueAppId(ConsumerProducerAppConfiguration) + * @see ConsumerProducerApp#getUniqueGroupId(ConsumerProducerAppConfiguration) */ - public String getUniqueAppId() { - final String uniqueAppId = - Objects.requireNonNull(this.app.getUniqueAppId(this.configuration), "Group ID cannot be null"); - if (this.configuration.getUniqueAppId().map(configuredId -> !uniqueAppId.equals(configuredId)).orElse(false)) { + public String getUniqueGroupId() { + final String uniqueGroupId = + Objects.requireNonNull(this.app.getUniqueGroupId(this.configuration), "Group ID cannot be null"); + if (this.configuration.getUniqueGroupId().map(configuredId -> !uniqueGroupId.equals(configuredId)) + .orElse(false)) { throw new IllegalArgumentException( - "Provided group ID does not match ConsumerProducerApp#getUniqueAppId()"); + "Provided group ID does not match ConsumerProducerApp#getUniqueGroupId()"); } - return uniqueAppId; + return uniqueGroupId; } /** @@ -129,7 +130,7 @@ public ExecutableConsumerProducerApp withRuntimeConfiguration(final RuntimeCo .producerProperties(properties) .app(this.app) .topics(this.getTopics()) - .groupId(this.getUniqueAppId()) + .groupId(this.getUniqueGroupId()) .build(); } diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumerproducer/ConsumerProducerApp.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumerproducer/ConsumerProducerApp.java index a3961c275..c6acdd6dc 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumerproducer/ConsumerProducerApp.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumerproducer/ConsumerProducerApp.java @@ -46,15 +46,14 @@ public interface ConsumerProducerApp extends App - * User may provide a unique group identifier via {@link ConsumerProducerAppConfiguration#getUniqueAppId()}. + * User may provide a unique group identifier via {@link ConsumerProducerAppConfiguration#getUniqueGroupId()}. * If that is the case, the returned group ID should match the provided one. * * @param configuration provides runtime configuration * @return unique group identifier */ - //TODO rename to group Id? It is inconsistent with the CLI param - default String getUniqueAppId(final ConsumerProducerAppConfiguration configuration) { - return configuration.getUniqueAppId() + default String getUniqueGroupId(final ConsumerProducerAppConfiguration configuration) { + return configuration.getUniqueGroupId() .orElseThrow(() -> new IllegalArgumentException("Please provide a group ID")); } diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumerproducer/ConsumerProducerAppConfiguration.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumerproducer/ConsumerProducerAppConfiguration.java index 32f2ae90b..e3a379b95 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumerproducer/ConsumerProducerAppConfiguration.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumerproducer/ConsumerProducerAppConfiguration.java @@ -52,9 +52,9 @@ public ConsumerProducerAppConfiguration(final ConsumerProducerTopicConfig topics * Get the provided unique group ID. If user did not provide a unique group ID, this will return empty. * * @return provided unique group ID - * @see ConsumerProducerApp#getUniqueAppId(ConsumerProducerAppConfiguration) + * @see ConsumerProducerApp#getUniqueGroupId(ConsumerProducerAppConfiguration) */ - public Optional getUniqueAppId() { + public Optional getUniqueGroupId() { return Optional.ofNullable(this.uniqueAppId); } } diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumerproducer/ConsumerProducerExecutionOptions.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumerproducer/ConsumerProducerExecutionOptions.java index 6a0203c1e..7a2433327 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumerproducer/ConsumerProducerExecutionOptions.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumerproducer/ConsumerProducerExecutionOptions.java @@ -24,14 +24,12 @@ package com.bakdata.kafka.consumerproducer; +import com.bakdata.kafka.CloseExecutionOptions; import com.bakdata.kafka.consumer.ConsumerExecutionOptions; import java.time.Duration; import lombok.Builder; import lombok.Getter; -import org.apache.kafka.clients.consumer.CloseOptions; -import org.apache.kafka.clients.consumer.CloseOptions.GroupMembershipOperation; import org.apache.kafka.clients.consumer.Consumer; -import org.apache.kafka.clients.consumer.ConsumerConfig; /** * Options to run a Kafka ConsumerProducer app @@ -39,33 +37,20 @@ @Builder public final class ConsumerProducerExecutionOptions { - /** - * Defines if {@link ConsumerConfig#GROUP_INSTANCE_ID_CONFIG} is volatile. If it is configured and non-volatile, - * {@link Consumer#close(CloseOptions)} is called with - * {@link CloseOptions#groupMembershipOperation(GroupMembershipOperation)} set to - * {@link GroupMembershipOperation#REMAIN_IN_GROUP} - */ - @Builder.Default - private final boolean volatileGroupInstanceId = true; - - /** - * Defines {@link CloseOptions#timeout(Duration)} when calling {@link Consumer#close(CloseOptions)} - */ @Builder.Default - private final Duration closeTimeout = Duration.ofMillis(Long.MAX_VALUE); + private final CloseExecutionOptions closeExecutionOptions = CloseExecutionOptions.builder().build(); /** * Defines the timeout duration for the {@link Consumer#poll(Duration)} call */ @Builder.Default @Getter - private final Duration pollTimeout = Duration.ofMillis(Long.MAX_VALUE); + private final Duration pollTimeout = Duration.ofMillis(100L); ConsumerExecutionOptions toConsumerExecutionOptions() { return ConsumerExecutionOptions.builder() - .volatileGroupInstanceId(this.volatileGroupInstanceId) + .closeExecutionOptions(this.closeExecutionOptions) .pollTimeout(this.pollTimeout) - .closeTimeout(this.closeTimeout) .build(); } } diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumerproducer/ConsumerProducerRunner.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumerproducer/ConsumerProducerRunner.java index e86d46e8b..ddae257bb 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumerproducer/ConsumerProducerRunner.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumerproducer/ConsumerProducerRunner.java @@ -48,11 +48,6 @@ public void close() { @Override public void run() { - this.runConsumerProducer(); - //TODO await? - } - - private void runConsumerProducer() { log.info("Starting Kafka ConsumerProducer"); // Run Kafka application until it shuts down this.runnable.run(this.consumerConfig); diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumerproducer/DefaultConsumerProducerRunnable.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumerproducer/DefaultConsumerProducerRunnable.java index 3145a0707..e4e12dce8 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumerproducer/DefaultConsumerProducerRunnable.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumerproducer/DefaultConsumerProducerRunnable.java @@ -25,19 +25,17 @@ package com.bakdata.kafka.consumerproducer; import com.bakdata.kafka.consumer.ConsumerRunnable; -import lombok.AllArgsConstructor; -import lombok.Getter; +import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.apache.kafka.clients.consumer.ConsumerConfig; import org.apache.kafka.clients.producer.Producer; -@AllArgsConstructor +@RequiredArgsConstructor @Slf4j -//TODO check +//TODO javadoc public class DefaultConsumerProducerRunnable implements ConsumerProducerRunnable { private final Producer producer; - @Getter private final ConsumerRunnable consumerRunnable; @Override diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/streams/StreamsExecutionOptions.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/streams/StreamsExecutionOptions.java index 44104d7df..85be05133 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/streams/StreamsExecutionOptions.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/streams/StreamsExecutionOptions.java @@ -1,7 +1,7 @@ /* * MIT License * - * Copyright (c) 2025 bakdata + * Copyright (c) 2026 bakdata * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -24,13 +24,12 @@ package com.bakdata.kafka.streams; +import com.bakdata.kafka.CloseExecutionOptions; import java.time.Duration; -import java.util.Map; import java.util.function.Consumer; import java.util.function.Supplier; import lombok.Builder; import lombok.NonNull; -import org.apache.kafka.clients.consumer.ConsumerConfig; import org.apache.kafka.streams.KafkaStreams; import org.apache.kafka.streams.KafkaStreams.CloseOptions; import org.apache.kafka.streams.KafkaStreams.StateListener; @@ -58,30 +57,18 @@ public class StreamsExecutionOptions { @Builder.Default private final @NonNull Supplier uncaughtExceptionHandler = DefaultStreamsUncaughtExceptionHandler::new; - /** - * Defines if {@link ConsumerConfig#GROUP_INSTANCE_ID_CONFIG} is volatile. If it is configured and non-volatile, - * {@link KafkaStreams#close(CloseOptions)} is called with {@link CloseOptions#leaveGroup(boolean)} disabled - */ + //TODO javadoc @Builder.Default - private final boolean volatileGroupInstanceId = true; + private final CloseExecutionOptions closeExecutionOptions = CloseExecutionOptions.builder() + .build(); /** * Defines {@link CloseOptions#timeout(Duration)} when calling {@link KafkaStreams#close(CloseOptions)} */ @Builder.Default private final Duration closeTimeout = Duration.ofMillis(Long.MAX_VALUE); - private static boolean isStaticMembershipDisabled(final Map originals) { - return originals.get(ConsumerConfig.GROUP_INSTANCE_ID_CONFIG) == null; - } - CloseOptions createCloseOptions(final StreamsConfig config) { - final boolean leaveGroup = this.shouldLeaveGroup(config.originals()); - return new CloseOptions().leaveGroup(leaveGroup).timeout(this.closeTimeout); - } - - boolean shouldLeaveGroup(final Map originals) { - final boolean staticMembershipDisabled = isStaticMembershipDisabled(originals); - return staticMembershipDisabled || this.volatileGroupInstanceId; + return this.closeExecutionOptions.createCloseOptions(config); } void onStart(final RunningStreams runningStreams) { diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/streams/StreamsExecutionOptionsTest.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/CloseExecutionOptionsTest.java similarity index 85% rename from streams-bootstrap-core/src/test/java/com/bakdata/kafka/streams/StreamsExecutionOptionsTest.java rename to streams-bootstrap-core/src/test/java/com/bakdata/kafka/CloseExecutionOptionsTest.java index a3837a0bc..e694fda20 100644 --- a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/streams/StreamsExecutionOptionsTest.java +++ b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/CloseExecutionOptionsTest.java @@ -1,7 +1,7 @@ /* * MIT License * - * Copyright (c) 2025 bakdata + * Copyright (c) 2026 bakdata * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -22,7 +22,7 @@ * SOFTWARE. */ -package com.bakdata.kafka.streams; +package com.bakdata.kafka; import static java.util.Collections.emptyMap; import static org.assertj.core.api.Assertions.assertThat; @@ -31,18 +31,18 @@ import org.apache.kafka.clients.consumer.ConsumerConfig; import org.junit.jupiter.api.Test; -class StreamsExecutionOptionsTest { +class CloseExecutionOptionsTest { @Test void shouldLeaveGroup() { - final StreamsExecutionOptions options = StreamsExecutionOptions.builder() + final CloseExecutionOptions options = CloseExecutionOptions.builder() .build(); assertThat(options.shouldLeaveGroup(emptyMap())).isTrue(); } @Test void shouldNotLeaveGroup() { - final StreamsExecutionOptions options = StreamsExecutionOptions.builder() + final CloseExecutionOptions options = CloseExecutionOptions.builder() .volatileGroupInstanceId(false) .build(); assertThat(options.shouldLeaveGroup(Map.of( @@ -52,7 +52,7 @@ void shouldNotLeaveGroup() { @Test void shouldLeaveGroupWithVolatileGroupId() { - final StreamsExecutionOptions options = StreamsExecutionOptions.builder() + final CloseExecutionOptions options = CloseExecutionOptions.builder() .volatileGroupInstanceId(true) .build(); assertThat(options.shouldLeaveGroup(Map.of( diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/consumer/ConsumerExecutionOptionsTest.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/consumer/ConsumerExecutionOptionsTest.java deleted file mode 100644 index 0147cf526..000000000 --- a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/consumer/ConsumerExecutionOptionsTest.java +++ /dev/null @@ -1,62 +0,0 @@ -/* - * MIT License - * - * Copyright (c) 2025 bakdata - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package com.bakdata.kafka.consumer; - -import static java.util.Collections.emptyMap; -import static org.assertj.core.api.Assertions.assertThat; - -import java.util.Map; -import org.apache.kafka.clients.consumer.ConsumerConfig; -import org.junit.jupiter.api.Test; - -class ConsumerExecutionOptionsTest { - - @Test - void shouldLeaveGroup() { - final ConsumerExecutionOptions options = ConsumerExecutionOptions.builder() - .build(); - assertThat(options.shouldLeaveGroup(emptyMap())).isTrue(); - } - - @Test - void shouldNotLeaveGroup() { - final ConsumerExecutionOptions options = ConsumerExecutionOptions.builder() - .volatileGroupInstanceId(false) - .build(); - assertThat(options.shouldLeaveGroup(Map.of( - ConsumerConfig.GROUP_INSTANCE_ID_CONFIG, "foo" - ))).isFalse(); - } - - @Test - void shouldLeaveGroupWithVolatileGroupId() { - final ConsumerExecutionOptions options = ConsumerExecutionOptions.builder() - .volatileGroupInstanceId(true) - .build(); - assertThat(options.shouldLeaveGroup(Map.of( - ConsumerConfig.GROUP_INSTANCE_ID_CONFIG, "foo" - ))).isTrue(); - } -} diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/consumerproducer/ConfiguredConsumerProducerAppTest.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/consumerproducer/ConfiguredConsumerProducerAppTest.java index 4dbc46197..af2b2cf7e 100644 --- a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/consumerproducer/ConfiguredConsumerProducerAppTest.java +++ b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/consumerproducer/ConfiguredConsumerProducerAppTest.java @@ -1,7 +1,7 @@ /* * MIT License * - * Copyright (c) 2025 bakdata + * Copyright (c) 2026 bakdata * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -183,7 +183,7 @@ public ConsumerProducerRunnable buildRunnable(final ConsumerProducerBuilder buil } @Override - public String getUniqueAppId(final ConsumerProducerAppConfiguration topics) { + public String getUniqueGroupId(final ConsumerProducerAppConfiguration topics) { return "app-id"; } diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/consumerproducer/ConsumerProducerCleanUpRunnerTest.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/consumerproducer/ConsumerProducerCleanUpRunnerTest.java index a06fc2f07..686083d18 100644 --- a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/consumerproducer/ConsumerProducerCleanUpRunnerTest.java +++ b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/consumerproducer/ConsumerProducerCleanUpRunnerTest.java @@ -207,7 +207,7 @@ void shouldDeleteConsumerGroup() { try (final AdminClientX adminClient = testClient.admin()) { final ConsumerGroupClient consumerGroupClient = - adminClient.consumerGroups().group(app.getUniqueAppId()); + adminClient.consumerGroups().group(app.getUniqueGroupId()); this.softly.assertThat(consumerGroupClient.exists()) .as("Consumer group exists") .isTrue(); @@ -219,7 +219,7 @@ void shouldDeleteConsumerGroup() { try (final AdminClientX adminClient = testClient.admin()) { final ConsumerGroupClient consumerGroupClient = - adminClient.consumerGroups().group(app.getUniqueAppId()); + adminClient.consumerGroups().group(app.getUniqueGroupId()); this.softly.assertThat(consumerGroupClient.exists()) .as("Consumer group is deleted") .isFalse(); @@ -255,7 +255,7 @@ void shouldNotThrowAnErrorIfConsumerGroupDoesNotExist() { try (final AdminClientX adminClient = testClient.admin()) { final ConsumerGroupsClient groups = adminClient.consumerGroups(); - this.softly.assertThat(groups.group(app.getUniqueAppId()).exists()) + this.softly.assertThat(groups.group(app.getUniqueGroupId()).exists()) .as("Consumer group exists") .isTrue(); } @@ -264,8 +264,8 @@ void shouldNotThrowAnErrorIfConsumerGroupDoesNotExist() { try (final AdminClientX adminClient = testClient.admin()) { final ConsumerGroupsClient groups = adminClient.consumerGroups(); - groups.group(app.getUniqueAppId()).delete(); - this.softly.assertThat(groups.group(app.getUniqueAppId()).exists()) + groups.group(app.getUniqueGroupId()).delete(); + this.softly.assertThat(groups.group(app.getUniqueGroupId()).exists()) .as("Consumer group is deleted") .isFalse(); } @@ -403,7 +403,7 @@ void shouldThrowExceptionOnResetterError() { this.softly.assertThatThrownBy(() -> reset(executableApp)) .isInstanceOf(KafkaAdminException.class) .hasMessageContaining("Failed to reset offsets for consumer group %s: consumer group is not empty", - app.getUniqueAppId()); + app.getUniqueGroupId()); } } diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/consumerproducer/apps/MirrorKeyWithAvroConsumerProducer.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/consumerproducer/apps/MirrorKeyWithAvroConsumerProducer.java index c25ae3df9..90742db18 100644 --- a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/consumerproducer/apps/MirrorKeyWithAvroConsumerProducer.java +++ b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/consumerproducer/apps/MirrorKeyWithAvroConsumerProducer.java @@ -66,7 +66,7 @@ public ConsumerProducerRunnable buildRunnable(final ConsumerProducerBuilder buil } @Override - public String getUniqueAppId(final ConsumerProducerAppConfiguration topics) { + public String getUniqueGroupId(final ConsumerProducerAppConfiguration topics) { return "app-id"; } } diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/consumerproducer/apps/MirrorValueWithAvroConsumerProducer.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/consumerproducer/apps/MirrorValueWithAvroConsumerProducer.java index eab4a83d0..97586d4f6 100644 --- a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/consumerproducer/apps/MirrorValueWithAvroConsumerProducer.java +++ b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/consumerproducer/apps/MirrorValueWithAvroConsumerProducer.java @@ -66,7 +66,7 @@ public ConsumerProducerRunnable buildRunnable(final ConsumerProducerBuilder buil } @Override - public String getUniqueAppId(final ConsumerProducerAppConfiguration topics) { + public String getUniqueGroupId(final ConsumerProducerAppConfiguration topics) { return "app-id"; } } diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/consumerproducer/apps/StringConsumerProducer.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/consumerproducer/apps/StringConsumerProducer.java index 91a63e77b..08c467bf0 100644 --- a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/consumerproducer/apps/StringConsumerProducer.java +++ b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/consumerproducer/apps/StringConsumerProducer.java @@ -63,7 +63,7 @@ public ConsumerProducerRunnable buildRunnable(final ConsumerProducerBuilder buil } @Override - public String getUniqueAppId(final ConsumerProducerAppConfiguration topics) { + public String getUniqueGroupId(final ConsumerProducerAppConfiguration topics) { return "app-id"; } } diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/consumerproducer/apps/StringPatternConsumerProducer.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/consumerproducer/apps/StringPatternConsumerProducer.java index 688aeaba5..f20183b2b 100644 --- a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/consumerproducer/apps/StringPatternConsumerProducer.java +++ b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/consumerproducer/apps/StringPatternConsumerProducer.java @@ -63,7 +63,7 @@ public ConsumerProducerRunnable buildRunnable(final ConsumerProducerBuilder buil } @Override - public String getUniqueAppId(final ConsumerProducerAppConfiguration topics) { + public String getUniqueGroupId(final ConsumerProducerAppConfiguration topics) { return "app-id"; } } diff --git a/streams-bootstrap-test/src/testFixtures/java/com/bakdata/kafka/streams/apps/SimpleConsumerProducerApp.java b/streams-bootstrap-test/src/testFixtures/java/com/bakdata/kafka/streams/apps/SimpleConsumerProducerApp.java index 4db773ab4..c35090d3c 100644 --- a/streams-bootstrap-test/src/testFixtures/java/com/bakdata/kafka/streams/apps/SimpleConsumerProducerApp.java +++ b/streams-bootstrap-test/src/testFixtures/java/com/bakdata/kafka/streams/apps/SimpleConsumerProducerApp.java @@ -59,7 +59,7 @@ public ConsumerProducerRunnable buildRunnable(final ConsumerProducerBuilder buil } @Override - public String getUniqueAppId(final ConsumerProducerAppConfiguration topics) { + public String getUniqueGroupId(final ConsumerProducerAppConfiguration topics) { return "app-id"; } } From 74a40f2ad47364286a6c23c0ab56b21a6587e46d Mon Sep 17 00:00:00 2001 From: Philipp Schirmer Date: Thu, 26 Feb 2026 21:09:28 +0100 Subject: [PATCH 08/15] Do not catch RuntimeException --- .../com/bakdata/kafka/consumer/DefaultConsumerRunnable.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumer/DefaultConsumerRunnable.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumer/DefaultConsumerRunnable.java index 1ba16784f..1eb0a44cd 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumer/DefaultConsumerRunnable.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumer/DefaultConsumerRunnable.java @@ -80,9 +80,6 @@ private void pollLoop(final ConsumerConfig consumerConfig) { } } catch (final WakeupException exception) { log.info("Consumer poll loop waking up for shutdown", exception); - } catch (final RuntimeException exception) { - // ensure latch count down for proper close - log.error("RuntimeException while running consumer loop", exception); } finally { log.info("Closing consumer"); final CloseOptions closeOptions = this.executionOptions.createCloseOptions(consumerConfig); From 31bbbc779b4ba55bb97dc3a5b2d5ac0c289d14e3 Mon Sep 17 00:00:00 2001 From: Philipp Schirmer Date: Thu, 26 Feb 2026 21:12:24 +0100 Subject: [PATCH 09/15] Do not catch RuntimeException --- .../DefaultConsumerProducerRunnable.java | 18 ++++-------------- 1 file changed, 4 insertions(+), 14 deletions(-) diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumerproducer/DefaultConsumerProducerRunnable.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumerproducer/DefaultConsumerProducerRunnable.java index e4e12dce8..f5c103df8 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumerproducer/DefaultConsumerProducerRunnable.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumerproducer/DefaultConsumerProducerRunnable.java @@ -45,21 +45,11 @@ public void run(final ConsumerConfig consumerConfig) { @Override public void close() { - try { - log.debug("Closing consumer runnable"); - this.consumerRunnable.close(); - } catch (final RuntimeException e) { - //TODO why catch? - log.warn("Error closing consumer runnable", e); - } + log.debug("Closing consumer runnable"); + this.consumerRunnable.close(); - try { - log.debug("Closing producer"); - this.producer.close(); - } catch (final RuntimeException e) { - //TODO why catch? - log.warn("Error closing producer", e); - } + log.debug("Closing producer"); + this.producer.close(); log.info("ConsumerProducer was shut down gracefully"); } From 835b15ebbc7516892075fb1d9a2dd264e7e29a18 Mon Sep 17 00:00:00 2001 From: Philipp Schirmer Date: Thu, 26 Feb 2026 22:01:31 +0100 Subject: [PATCH 10/15] Clean up --- .../main/java/com/bakdata/kafka/mixin/ConsumerOptions.java | 2 +- .../KafkaConsumerProducerApplicationCliTest.java | 6 +++--- .../consumerproducer/SerializerDeserializerConfig.java | 3 ++- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/streams-bootstrap-cli/src/main/java/com/bakdata/kafka/mixin/ConsumerOptions.java b/streams-bootstrap-cli/src/main/java/com/bakdata/kafka/mixin/ConsumerOptions.java index ae4bd2f89..4d2480c20 100644 --- a/streams-bootstrap-cli/src/main/java/com/bakdata/kafka/mixin/ConsumerOptions.java +++ b/streams-bootstrap-cli/src/main/java/com/bakdata/kafka/mixin/ConsumerOptions.java @@ -42,5 +42,5 @@ public class ConsumerOptions { private String groupId; @Option(names = {"--poll-timeout"}, description = "The maximum time to block in the consumer poll loop. Examples: 'PT0.1S', 'PT2S', 'PT1M'.") - private Duration pollTimeout = Duration.ofMillis(100); + private Duration pollTimeout = Duration.ofMillis(100L); } diff --git a/streams-bootstrap-cli/src/test/java/com/bakdata/kafka/consumerproducer/KafkaConsumerProducerApplicationCliTest.java b/streams-bootstrap-cli/src/test/java/com/bakdata/kafka/consumerproducer/KafkaConsumerProducerApplicationCliTest.java index 569fc9813..570b33a5d 100644 --- a/streams-bootstrap-cli/src/test/java/com/bakdata/kafka/consumerproducer/KafkaConsumerProducerApplicationCliTest.java +++ b/streams-bootstrap-cli/src/test/java/com/bakdata/kafka/consumerproducer/KafkaConsumerProducerApplicationCliTest.java @@ -191,7 +191,7 @@ public ConsumerProducerApp createApp() { return new ConsumerProducerApp() { @Override public ConsumerProducerRunnable buildRunnable(final ConsumerProducerBuilder builder) { - return (consumerConfig) -> {}; + return consumerConfig -> {}; } @Override @@ -223,7 +223,7 @@ void shouldExitWithErrorInBuildRunnable() { () -> new ConsumerProducerApp() { @Override public ConsumerProducerRunnable buildRunnable(final ConsumerProducerBuilder builder) { - return (consumerConfig) -> { + return consumerConfig -> { throw new RuntimeException("Error building runnable"); }; } @@ -264,7 +264,7 @@ void shouldExitWithSuccessCodeOnShutdown() { () -> new ConsumerProducerApp() { @Override public ConsumerProducerRunnable buildRunnable(final ConsumerProducerBuilder builder) { - return (consumerConfig) -> { + return consumerConfig -> { try (final Producer producer = builder.getProducerBuilder() .createProducer()) { final ProducerRecord producerRecord = diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumerproducer/SerializerDeserializerConfig.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumerproducer/SerializerDeserializerConfig.java index 48c768c36..efc7ec53c 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumerproducer/SerializerDeserializerConfig.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumerproducer/SerializerDeserializerConfig.java @@ -63,7 +63,8 @@ public Map createProperties() { .collect(Collectors.toMap( Map.Entry::getKey, Map.Entry::getValue, - (v1, v2) -> v2 // v1 and v2 are always different + // v1 and v2 are always different + (v1, v2) -> v2 )); } } From 05781095616ddc5d75fc778437e5329fb98f830f Mon Sep 17 00:00:00 2001 From: Philipp Schirmer Date: Thu, 26 Feb 2026 22:04:46 +0100 Subject: [PATCH 11/15] Clean up --- .../KafkaConsumerProducerApplication.java | 12 +++++++--- .../producer/KafkaProducerApplication.java | 6 +++-- .../ConsumerProducerExecutionOptions.java | 22 +++++-------------- .../ExecutableConsumerProducerApp.java | 2 +- 4 files changed, 20 insertions(+), 22 deletions(-) diff --git a/streams-bootstrap-cli/src/main/java/com/bakdata/kafka/consumerproducer/KafkaConsumerProducerApplication.java b/streams-bootstrap-cli/src/main/java/com/bakdata/kafka/consumerproducer/KafkaConsumerProducerApplication.java index d763a7505..02c69bfed 100644 --- a/streams-bootstrap-cli/src/main/java/com/bakdata/kafka/consumerproducer/KafkaConsumerProducerApplication.java +++ b/streams-bootstrap-cli/src/main/java/com/bakdata/kafka/consumerproducer/KafkaConsumerProducerApplication.java @@ -26,10 +26,12 @@ import com.bakdata.kafka.CloseExecutionOptions; import com.bakdata.kafka.KafkaApplication; +import com.bakdata.kafka.consumer.ConsumerExecutionOptions; import com.bakdata.kafka.mixin.ConsumerOptions; import com.bakdata.kafka.mixin.ErrorOptions; import com.bakdata.kafka.mixin.InputOptions; import com.bakdata.kafka.mixin.OutputOptions; +import com.bakdata.kafka.producer.ProducerExecutionOptions; import java.util.Optional; import lombok.Getter; import lombok.RequiredArgsConstructor; @@ -109,10 +111,14 @@ public void reset() { @Override public final Optional createExecutionOptions() { final ConsumerProducerExecutionOptions executionOptions = ConsumerProducerExecutionOptions.builder() - .closeExecutionOptions(CloseExecutionOptions.builder() - .volatileGroupInstanceId(this.isVolatileGroupInstanceId()) + .consumerExecutionOptions(ConsumerExecutionOptions.builder() + .closeExecutionOptions(CloseExecutionOptions.builder() + .volatileGroupInstanceId(this.isVolatileGroupInstanceId()) + .build()) + .pollTimeout(this.getPollTimeout()) + .build()) + .producerExecutionOptions(ProducerExecutionOptions.builder() .build()) - .pollTimeout(this.getPollTimeout()) .build(); return Optional.of(executionOptions); } diff --git a/streams-bootstrap-cli/src/main/java/com/bakdata/kafka/producer/KafkaProducerApplication.java b/streams-bootstrap-cli/src/main/java/com/bakdata/kafka/producer/KafkaProducerApplication.java index 2ed6bb485..9578429e8 100644 --- a/streams-bootstrap-cli/src/main/java/com/bakdata/kafka/producer/KafkaProducerApplication.java +++ b/streams-bootstrap-cli/src/main/java/com/bakdata/kafka/producer/KafkaProducerApplication.java @@ -1,7 +1,7 @@ /* * MIT License * - * Copyright (c) 2025 bakdata + * Copyright (c) 2026 bakdata * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -73,7 +73,9 @@ public void clean() { @Override public final Optional createExecutionOptions() { - return Optional.empty(); + final ProducerExecutionOptions executionOptions = ProducerExecutionOptions.builder() + .build(); + return Optional.of(executionOptions); } @Override diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumerproducer/ConsumerProducerExecutionOptions.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumerproducer/ConsumerProducerExecutionOptions.java index 7a2433327..559ef34fd 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumerproducer/ConsumerProducerExecutionOptions.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumerproducer/ConsumerProducerExecutionOptions.java @@ -24,33 +24,23 @@ package com.bakdata.kafka.consumerproducer; -import com.bakdata.kafka.CloseExecutionOptions; import com.bakdata.kafka.consumer.ConsumerExecutionOptions; -import java.time.Duration; +import com.bakdata.kafka.producer.ProducerExecutionOptions; import lombok.Builder; import lombok.Getter; -import org.apache.kafka.clients.consumer.Consumer; /** * Options to run a Kafka ConsumerProducer app */ @Builder +@Getter public final class ConsumerProducerExecutionOptions { @Builder.Default - private final CloseExecutionOptions closeExecutionOptions = CloseExecutionOptions.builder().build(); + private final ConsumerExecutionOptions consumerExecutionOptions = ConsumerExecutionOptions.builder() + .build(); - /** - * Defines the timeout duration for the {@link Consumer#poll(Duration)} call - */ @Builder.Default - @Getter - private final Duration pollTimeout = Duration.ofMillis(100L); - - ConsumerExecutionOptions toConsumerExecutionOptions() { - return ConsumerExecutionOptions.builder() - .closeExecutionOptions(this.closeExecutionOptions) - .pollTimeout(this.pollTimeout) - .build(); - } + private final ProducerExecutionOptions producerExecutionOptions = ProducerExecutionOptions.builder() + .build(); } diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumerproducer/ExecutableConsumerProducerApp.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumerproducer/ExecutableConsumerProducerApp.java index 37ee1af03..bb325138a 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumerproducer/ExecutableConsumerProducerApp.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumerproducer/ExecutableConsumerProducerApp.java @@ -80,7 +80,7 @@ public ConsumerProducerRunner createRunner() { @Override public ConsumerProducerRunner createRunner(final ConsumerProducerExecutionOptions options) { final ConsumerBuilder consumerBuilder = new ConsumerBuilder(this.topics.toConsumerTopicConfig(), - this.consumerProperties, options.toConsumerExecutionOptions()); + this.consumerProperties, options.getConsumerExecutionOptions()); final ProducerBuilder producerBuilder = new ProducerBuilder(this.topics.toProducerTopicConfig(), this.producerProperties); final ConsumerProducerBuilder From 976944757e8d65d92e8af6ad2b8fe1b56e47c4a1 Mon Sep 17 00:00:00 2001 From: Philipp Schirmer Date: Thu, 26 Feb 2026 22:05:30 +0100 Subject: [PATCH 12/15] Clean up --- .../com/bakdata/kafka/consumer/DefaultConsumerRunnable.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumer/DefaultConsumerRunnable.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumer/DefaultConsumerRunnable.java index 1eb0a44cd..856c1edc3 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumer/DefaultConsumerRunnable.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumer/DefaultConsumerRunnable.java @@ -26,6 +26,7 @@ import java.util.concurrent.CountDownLatch; import java.util.concurrent.atomic.AtomicBoolean; +import lombok.AccessLevel; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.apache.kafka.clients.consumer.CloseOptions; @@ -42,7 +43,7 @@ * @param type of keys * @param type of values */ -@RequiredArgsConstructor +@RequiredArgsConstructor(access = AccessLevel.PACKAGE) @Slf4j public class DefaultConsumerRunnable implements ConsumerRunnable { From aa919fd6ec1d5f412dbd651efb6bcb4c61ed0e36 Mon Sep 17 00:00:00 2001 From: Philipp Schirmer Date: Fri, 27 Feb 2026 08:56:49 +0100 Subject: [PATCH 13/15] Clean up --- .../bakdata/kafka/CloseExecutionOptions.java | 29 ++++++++++++++----- .../kafka/consumer/ConfiguredConsumerApp.java | 13 +++++++-- .../consumer/ConsumerExecutionOptions.java | 7 +++-- .../ConsumerProducerExecutionOptions.java | 6 ++++ .../ConsumerProducerTopicConfig.java | 11 +++++-- .../DefaultConsumerProducerRunnable.java | 8 ++++- .../SerializerDeserializerConfig.java | 11 +++++-- .../kafka/producer/ConfiguredProducerApp.java | 11 ++++++- .../streams/StreamsExecutionOptions.java | 4 ++- 9 files changed, 80 insertions(+), 20 deletions(-) diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/CloseExecutionOptions.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/CloseExecutionOptions.java index 94195ee31..90afda5d0 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/CloseExecutionOptions.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/CloseExecutionOptions.java @@ -24,31 +24,30 @@ package com.bakdata.kafka; -import com.bakdata.kafka.streams.StreamsRunner; import java.time.Duration; import java.util.Map; import lombok.Builder; import org.apache.kafka.clients.consumer.CloseOptions.GroupMembershipOperation; import org.apache.kafka.clients.consumer.ConsumerConfig; -import org.apache.kafka.streams.KafkaStreams; import org.apache.kafka.streams.KafkaStreams.CloseOptions; import org.apache.kafka.streams.StreamsConfig; -//FIXME javadoc - /** - * Options to run a Kafka Streams app using {@link StreamsRunner} + * Options to configure closing behavior of Kafka apps */ @Builder public class CloseExecutionOptions { /** * Defines if {@link ConsumerConfig#GROUP_INSTANCE_ID_CONFIG} is volatile. If it is configured and non-volatile, - * {@link KafkaStreams#close(CloseOptions)} is called with {@link CloseOptions#leaveGroup(boolean)} disabled + * {@link CloseOptions#leaveGroup(boolean)} is disabled and + * {@link org.apache.kafka.clients.consumer.CloseOptions#withGroupMembershipOperation(GroupMembershipOperation)} is + * set to {@link GroupMembershipOperation#DEFAULT}. */ @Builder.Default private final boolean volatileGroupInstanceId = true; /** - * Defines {@link CloseOptions#timeout(Duration)} when calling {@link KafkaStreams#close(CloseOptions)} + * Defines {@link CloseOptions#timeout(Duration)} and + * {@link org.apache.kafka.clients.consumer.CloseOptions#withTimeout(Duration)} */ @Builder.Default private final Duration closeTimeout = Duration.ofMillis(Long.MAX_VALUE); @@ -57,12 +56,26 @@ private static boolean isStaticMembershipDisabled(final Map orig return originals.get(ConsumerConfig.GROUP_INSTANCE_ID_CONFIG) == null; } - //TODO javadoc + /** + * Create {@link CloseOptions} for {@link org.apache.kafka.streams.KafkaStreams} + * + * @param config streams config + * @return {@link CloseOptions} + * @see org.apache.kafka.streams.KafkaStreams#close(CloseOptions) + */ public CloseOptions createCloseOptions(final StreamsConfig config) { final boolean leaveGroup = this.shouldLeaveGroup(config.originals()); return new CloseOptions().leaveGroup(leaveGroup).timeout(this.closeTimeout); } + /** + * Create {@link org.apache.kafka.clients.consumer.CloseOptions} for + * {@link org.apache.kafka.clients.consumer.Consumer} + * + * @param config consumer config + * @return {@link org.apache.kafka.clients.consumer.CloseOptions} + * @see org.apache.kafka.clients.consumer.Consumer#close(org.apache.kafka.clients.consumer.CloseOptions) + */ public org.apache.kafka.clients.consumer.CloseOptions createCloseOptions(final ConsumerConfig config) { final boolean leaveGroup = this.shouldLeaveGroup(config.originals()); final GroupMembershipOperation operation = diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumer/ConfiguredConsumerApp.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumer/ConfiguredConsumerApp.java index fd69b3eae..09b6272b3 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumer/ConfiguredConsumerApp.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumer/ConfiguredConsumerApp.java @@ -49,7 +49,16 @@ public class ConfiguredConsumerApp implements ConfiguredA private final @NonNull T app; private final @NonNull ConsumerAppConfiguration configuration; - //TODO javadoc + /** + * Base configuration for all consumer apps which includes + * 
+     * auto.offset.reset=earliest
+     * enable.auto.commit=false
+     * isolation.level=read_committed
+     *
+ * + * @return base configuration + */ public static Map createBaseConfig() { final Map kafkaConfig = new HashMap<>(); @@ -65,10 +74,10 @@ public static Map createBaseConfig() { * Configuration is created in the following order *
    *
  • - * Offset management: * 
          * auto.offset.reset=earliest
      * enable.auto.commit=false
+     * isolation.level=read_committed
      *
    *
    • *
  • diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumer/ConsumerExecutionOptions.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumer/ConsumerExecutionOptions.java index e3fa9c0e0..8b9270767 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumer/ConsumerExecutionOptions.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumer/ConsumerExecutionOptions.java @@ -38,7 +38,9 @@ @Builder public class ConsumerExecutionOptions { - //TODO javadoc + /** + * Defines the behavior when closing a consumer + */ @Builder.Default private final CloseExecutionOptions closeExecutionOptions = CloseExecutionOptions.builder().build(); @@ -49,8 +51,7 @@ public class ConsumerExecutionOptions { @Getter private final Duration pollTimeout = Duration.ofMillis(100L); - //TODO javadoc - public CloseOptions createCloseOptions(final ConsumerConfig config) { + CloseOptions createCloseOptions(final ConsumerConfig config) { return this.closeExecutionOptions.createCloseOptions(config); } } diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumerproducer/ConsumerProducerExecutionOptions.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumerproducer/ConsumerProducerExecutionOptions.java index 559ef34fd..beaf1ec5e 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumerproducer/ConsumerProducerExecutionOptions.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumerproducer/ConsumerProducerExecutionOptions.java @@ -36,10 +36,16 @@ @Getter public final class ConsumerProducerExecutionOptions { + /** + * Defines the execution options for the consumer part of the app. + */ @Builder.Default private final ConsumerExecutionOptions consumerExecutionOptions = ConsumerExecutionOptions.builder() .build(); + /** + * Defines the execution options for the producer part of the app. + */ @Builder.Default private final ProducerExecutionOptions producerExecutionOptions = ProducerExecutionOptions.builder() .build(); diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumerproducer/ConsumerProducerTopicConfig.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumerproducer/ConsumerProducerTopicConfig.java index f54f71df8..0f1a4e3ed 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumerproducer/ConsumerProducerTopicConfig.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumerproducer/ConsumerProducerTopicConfig.java @@ -70,7 +70,11 @@ public class ConsumerProducerTopicConfig { Map labeledOutputTopics = emptyMap(); String errorTopic; - //TODO javadoc + /** + * Convert this config to a {@link ConsumerTopicConfig} for the consumer part of the app. + * + * @return {@link ConsumerTopicConfig} + */ public ConsumerTopicConfig toConsumerTopicConfig() { return ConsumerTopicConfig.builder() .inputTopics(this.getInputTopics()) @@ -80,7 +84,10 @@ public ConsumerTopicConfig toConsumerTopicConfig() { .build(); } - //TODO javadoc + /** + * Convert this config to a {@link ProducerTopicConfig} for the producer part of the app. + * @return {@link ProducerTopicConfig} + */ public ProducerTopicConfig toProducerTopicConfig() { return ProducerTopicConfig.builder() .outputTopic(this.getOutputTopic()) diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumerproducer/DefaultConsumerProducerRunnable.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumerproducer/DefaultConsumerProducerRunnable.java index f5c103df8..8a050e053 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumerproducer/DefaultConsumerProducerRunnable.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumerproducer/DefaultConsumerProducerRunnable.java @@ -30,9 +30,15 @@ import org.apache.kafka.clients.consumer.ConsumerConfig; import org.apache.kafka.clients.producer.Producer; + +/** + * Creates a {@link ConsumerProducerRunnable} using the provided {@link ConsumerRunnable} and {@link Producer}. + * + * @param type of keys produced by this runnable + * @param type of values produced by this runnable + */ @RequiredArgsConstructor @Slf4j -//TODO javadoc public class DefaultConsumerProducerRunnable implements ConsumerProducerRunnable { private final Producer producer; diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumerproducer/SerializerDeserializerConfig.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumerproducer/SerializerDeserializerConfig.java index efc7ec53c..2c866e729 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumerproducer/SerializerDeserializerConfig.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumerproducer/SerializerDeserializerConfig.java @@ -37,7 +37,7 @@ import org.apache.kafka.common.serialization.Serializer; /** - * Defines how to (de)serialize the data in a Kafka consumer or producer + * Defines how to (de-)serialize the data in a Kafka consumer or producer */ @RequiredArgsConstructor @With @@ -46,7 +46,14 @@ public class SerializerDeserializerConfig implements SerializationConfig { private final @NonNull SerializerConfig serializerConfig; private final @NonNull DeserializerConfig deserializerConfig; - //TODO javadoc + /** + * Create a new {@code SerializerDeserializerConfig} + * + * @param keySerializer serializer for keys + * @param valueSerializer serializer for values + * @param keyDeserializer deserializer for keys + * @param valueDeserializer deserializer for values + */ public SerializerDeserializerConfig(final @NonNull Class keySerializer, final @NonNull Class valueSerializer, final @NonNull Class keyDeserializer, diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/producer/ConfiguredProducerApp.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/producer/ConfiguredProducerApp.java index 65d5aba8d..4d5498df3 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/producer/ConfiguredProducerApp.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/producer/ConfiguredProducerApp.java @@ -48,7 +48,16 @@ public class ConfiguredProducerApp implements ConfiguredA private final @NonNull T app; private final @NonNull ProducerAppConfiguration configuration; - //TODO javadoc + /** + * Base configuration for all producer apps which includes + * 
    +     * max.in.flight.requests.per.connection=1
+     * acks=all
+     * compression.type=gzip
+     *
    + * + * @return base configuration + */ public static Map createBaseConfig() { final Map kafkaConfig = new HashMap<>(); diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/streams/StreamsExecutionOptions.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/streams/StreamsExecutionOptions.java index 85be05133..795eb62c9 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/streams/StreamsExecutionOptions.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/streams/StreamsExecutionOptions.java @@ -57,7 +57,9 @@ public class StreamsExecutionOptions { @Builder.Default private final @NonNull Supplier uncaughtExceptionHandler = DefaultStreamsUncaughtExceptionHandler::new; - //TODO javadoc + /** + * Defines the behavior when closing a Kafka Streams app using {@link KafkaStreams#close(CloseOptions)}. + */ @Builder.Default private final CloseExecutionOptions closeExecutionOptions = CloseExecutionOptions.builder() .build(); From bee502777936686adb7e67d8de8d50eda324d069 Mon Sep 17 00:00:00 2001 From: Philipp Schirmer Date: Fri, 27 Feb 2026 08:57:59 +0100 Subject: [PATCH 14/15] Clean up --- .../src/main/java/com/bakdata/kafka/mixin/ConsumerOptions.java | 2 +- .../com/bakdata/kafka/consumer/ConsumerExecutionOptions.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/streams-bootstrap-cli/src/main/java/com/bakdata/kafka/mixin/ConsumerOptions.java b/streams-bootstrap-cli/src/main/java/com/bakdata/kafka/mixin/ConsumerOptions.java index 4d2480c20..bc055d0b7 100644 --- a/streams-bootstrap-cli/src/main/java/com/bakdata/kafka/mixin/ConsumerOptions.java +++ b/streams-bootstrap-cli/src/main/java/com/bakdata/kafka/mixin/ConsumerOptions.java @@ -42,5 +42,5 @@ public class ConsumerOptions { private String groupId; @Option(names = {"--poll-timeout"}, description = "The maximum time to block in the consumer poll loop. Examples: 'PT0.1S', 'PT2S', 'PT1M'.") - private Duration pollTimeout = Duration.ofMillis(100L); + private Duration pollTimeout = Duration.ofSeconds(10L); } diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumer/ConsumerExecutionOptions.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumer/ConsumerExecutionOptions.java index 8b9270767..674f3996c 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumer/ConsumerExecutionOptions.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/consumer/ConsumerExecutionOptions.java @@ -49,7 +49,7 @@ public class ConsumerExecutionOptions { */ @Builder.Default @Getter - private final Duration pollTimeout = Duration.ofMillis(100L); + private final Duration pollTimeout = Duration.ofSeconds(10L); CloseOptions createCloseOptions(final ConsumerConfig config) { return this.closeExecutionOptions.createCloseOptions(config); From 0dcaf8e664addbcc3b4c72dbb43a6d53d6133ba0 Mon Sep 17 00:00:00 2001 From: Philipp Schirmer Date: Fri, 27 Feb 2026 12:19:55 +0100 Subject: [PATCH 15/15] Clean up --- .../java/com/bakdata/kafka/admin/ConsumerGroupsClient.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/admin/ConsumerGroupsClient.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/admin/ConsumerGroupsClient.java index effdaeeeb..9b6f9c2be 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/admin/ConsumerGroupsClient.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/admin/ConsumerGroupsClient.java @@ -189,8 +189,8 @@ public void reset(final OffsetSpec offsetSpec) { ConsumerGroupsClient.this.adminClient.listOffsets(request).all(); final Map offsets = ConsumerGroupsClient.this.timeout.get(offsetsFuture, - () -> "Failed to reset offsets for consumer group %s: could not find offsets for spec %s".formatted( - this.groupName, offsetSpec)); + () -> "Failed to reset offsets for consumer group %s: could not find offsets for spec %s" + .formatted(this.groupName, offsetSpec)); final Map resetOffsets = offsets.entrySet().stream() .collect(Collectors.toMap(Map.Entry::getKey, e -> new OffsetAndMetadata(e.getValue().offset())));