diff --git a/ai-analytic-center-biz/src/main/java/com/volvo/ai/analytic/center/config/KafkaConfig.java b/ai-analytic-center-biz/src/main/java/com/volvo/ai/analytic/center/config/KafkaConfig.java index a0585ea..01beff9 100644 --- a/ai-analytic-center-biz/src/main/java/com/volvo/ai/analytic/center/config/KafkaConfig.java +++ b/ai-analytic-center-biz/src/main/java/com/volvo/ai/analytic/center/config/KafkaConfig.java @@ -68,7 +68,7 @@ public class KafkaConfig { props.put(ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG, 45000); // 会话1分钟 props.put(ConsumerConfig.MAX_POLL_INTERVAL_MS_CONFIG, 600000); // 可选:允许更长的消费间隔 props.put(ConsumerConfig.HEARTBEAT_INTERVAL_MS_CONFIG, 15000); - props.put(ConsumerConfig.MAX_POLL_RECORDS_CONFIG, 20); // 单次最多拉取的消息数 + /** props.put(ConsumerConfig.MAX_POLL_RECORDS_CONFIG, 20); // 单次最多拉取的消息数 **/ ConcurrentKafkaListenerContainerFactory factory = new ConcurrentKafkaListenerContainerFactory<>(); factory.setConsumerFactory(new DefaultKafkaConsumerFactory<>(props)); diff --git a/ai-analytic-center-biz/src/main/java/com/volvo/ai/analytic/center/mq/NameplateKafkaConsumer.java b/ai-analytic-center-biz/src/main/java/com/volvo/ai/analytic/center/mq/NameplateKafkaConsumer.java index f667a19..924e7b6 100644 --- a/ai-analytic-center-biz/src/main/java/com/volvo/ai/analytic/center/mq/NameplateKafkaConsumer.java +++ b/ai-analytic-center-biz/src/main/java/com/volvo/ai/analytic/center/mq/NameplateKafkaConsumer.java @@ -19,6 +19,7 @@ import org.springframework.stereotype.Component; import org.springframework.web.bind.annotation.RestController; import javax.annotation.Resource; +import java.util.concurrent.*; /** * @ClassName NameplateKafkaConsumer @@ -45,40 +46,50 @@ public class NameplateKafkaConsumer { @Resource private RocketMQTemplate rocketMqTemplate; - + private int messageCount = 0; @KafkaListener(topics = "${analyticCenterKafka.consumer.topic}", groupId = "${analyticCenterKafka.consumer.group}" , - containerFactory = "analyticCenterConsumerFactory", - concurrency = "3") + containerFactory = "analyticCenterConsumerFactory") public void listen(String recordMessages, Acknowledgment ack, @Header(KafkaHeaders.RECEIVED_PARTITION_ID) Integer partitionId, - @Header(KafkaHeaders.OFFSET) Long offset) { + @Header(KafkaHeaders.OFFSET) Long offset) throws InterruptedException { long startTime = System.currentTimeMillis(); - log.info("nameplateKafkaConsumer 当前线程: {}, 线程ID: {}", Thread.currentThread().getName(), Thread.currentThread().getId()); + messageCount++; + log.info("nameplateKafkaConsumer 当前线程: {}, 线程ID: {},计数:{}", Thread.currentThread().getName(), Thread.currentThread().getId(),messageCount); log.info("nameplateKafkaConsumerMessage: {}", recordMessages); if(StringUtils.isNotEmpty(recordMessages)){ - try { - NameplateTableKafkaDTO tmNameplateCorpus = JSON.parseObject(recordMessages, NameplateTableKafkaDTO.class); - log.info("nameplateKafkaConsumerParseType: {}", tmNameplateCorpus.getType()); - if(tmNameplateCorpus.getType().equals("INSERT")){ - tmNameplateCorpus.getData().forEach(nameplate -> { - log.info("nameplateKafkaConsumerCustomerFlowId: {}",nameplate.getCustomerFlowId()); - tmNameplateCorpusService.processItem(nameplate); - }); - } - } catch (Exception e) { - log.error("nameplateKafkaConsumer铭牌解析处理出错 {}:{},ex:{}", - partitionId, offset, e.getMessage()); - }finally { - log.info("nameplateKafkaConsumerack.acknowledge"); + + int optimalThreadPoolSize = Runtime.getRuntime().availableProcessors() + 1; + log.info("获取的线程数:{}",optimalThreadPoolSize); + // 创建线程池 + ExecutorService executor = Executors.newFixedThreadPool(optimalThreadPoolSize); // 根据需求调整线程池大小 + NameplateTableKafkaDTO tmNameplateCorpus = JSON.parseObject(recordMessages, NameplateTableKafkaDTO.class); + log.info("nameplateKafkaConsumerParseType: {},size:{}", tmNameplateCorpus.getType(), tmNameplateCorpus.getData().size()); + if(tmNameplateCorpus.getType().equals("INSERT")){ + CountDownLatch latch = new CountDownLatch(tmNameplateCorpus.getData().size()); + tmNameplateCorpus.getData().forEach(nameplate -> { + CompletableFuture.runAsync(() -> { + try { + log.info("nameplateKafkaConsumerCustomerFlowId: {}",nameplate.getCustomerFlowId()); + tmNameplateCorpusService.processItem(nameplate); + } catch (Exception e) { + log.error("nameplateKafkaConsumer铭牌解析处理出错 {}:{},ex:{}", + partitionId, offset, e.getMessage()); + }finally { + log.info("nameplateKafkaConsumerack.acknowledge"); + latch.countDown(); + } + }, executor); + }); + boolean allDone = latch.await(2, TimeUnit.MINUTES); + log.info("nameplateKafkaConsumerack.acknowledge:{}",allDone); try { ack.acknowledge(); // 确保无论成功与否都提交 offset } catch (IllegalStateException e) { log.warn("Offset 已提交,跳过重复提交"); } } - }else { ack.acknowledge(); // 空消息直接跳过 } @@ -86,7 +97,5 @@ public class NameplateKafkaConsumer { log.info("nameplateKafkaConsumer消息处理完成,耗时:{}", System.currentTimeMillis() - startTime); } - - }