admin/vlv-ai-analytic-center
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

添加日志，修改并发逻辑

Browse Source
This commit is contained in:
ZLI263
2025-09-19 13:06:39 +08:00
parent 1f6ce44c51
commit 34f8f108b3
1 changed files with 65 additions and 58 deletions
Download Patch File Download Diff File Expand all files Collapse all files

61
ai-analytic-center-biz/src/main/java/com/volvo/ai/analytic/center/mq/NameplateKafkaConsumer.java
View File

@@ -19,7 +19,10 @@ import org.springframework.web.bind.annotation.RestController;
import javax.annotation.Resource; import javax.annotation.Resource;
import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletableFuture;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.Semaphore; import java.util.concurrent.Semaphore;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
/** /**
* @ClassName NameplateKafkaConsumer * @ClassName NameplateKafkaConsumer
@@ -41,19 +44,16 @@ public class NameplateKafkaConsumer {
@Autowired @Autowired
@Resource(name = "threadPoolTaskExecutor") @Resource(name = "threadPoolTaskExecutor")
private ThreadPoolTaskExecutor executor; private ThreadPoolTaskExecutor executor;
private Semaphore semaphore = new Semaphore(10); // 限制并发数 private Semaphore semaphore = new Semaphore(2); // 限制并发数
@KafkaListener(topics = "${analyticCenterKafka.consumer.topic}", // = smart_assistant_nameplate_topic @KafkaListener(topics = "${analyticCenterKafka.consumer.topic}", // = smart_assistant_nameplate_topic
groupId = "${analyticCenterKafka.consumer.group}" , //smart_assistant_nameplate_topic_group groupId = "${analyticCenterKafka.consumer.group}", //smart_assistant_nameplate_topic_group
containerFactory = "analyticCenterConsumerFactory", containerFactory = "analyticCenterConsumerFactory",
concurrency = "3") concurrency = "3")
public void listen(String recordMessages, Acknowledgment ack, public void listen(String recordMessages, Acknowledgment ack,
@Header(KafkaHeaders.RECEIVED_PARTITION_ID) Integer partitionId, @Header(KafkaHeaders.RECEIVED_PARTITION_ID) Integer partitionId,
@Header(KafkaHeaders.OFFSET) Long offset) { @Header(KafkaHeaders.OFFSET) Long offset) {
long startTime = System.currentTimeMillis();
log.info("nameplateKafkaConsumer 当前线程: {}, 线程ID: {},计数：{}", Thread.currentThread().getName(), Thread.currentThread().getId());
log.info("nameplateKafkaConsumerMessage，消息：{}", recordMessages);
// 初始化绑定 Consumer
if (StringUtils.isEmpty(recordMessages)) { if (StringUtils.isEmpty(recordMessages)) {
ack.acknowledge(); ack.acknowledge();
return; return;
@@ -63,39 +63,46 @@ public class NameplateKafkaConsumer {
NameplateTableKafkaDTO tmNameplateCorpus = JSON.parseObject(recordMessages, NameplateTableKafkaDTO.class); NameplateTableKafkaDTO tmNameplateCorpus = JSON.parseObject(recordMessages, NameplateTableKafkaDTO.class);
if (!"INSERT".equals(tmNameplateCorpus.getType()) || CollectionUtils.isEmpty(tmNameplateCorpus.getData())) { if (!"INSERT".equals(tmNameplateCorpus.getType()) || CollectionUtils.isEmpty(tmNameplateCorpus.getData())) {
ack.acknowledge(); ack.acknowledge();
return;
} }
// 使用CountDownLatch等待所有异步任务完成
CountDownLatch latch = new CountDownLatch(tmNameplateCorpus.getData().size());
AtomicBoolean hasError = new AtomicBoolean(false);
tmNameplateCorpus.getData().forEach(nameplate -> tmNameplateCorpus.getData().forEach(nameplate ->
CompletableFuture.runAsync(() -> { CompletableFuture.runAsync(() -> {
try { try {
log.info(String.format("尝试获取许可，当前可用许可数: %d", if (!semaphore.tryAcquire(5, TimeUnit.SECONDS)) {
semaphore.availablePermits())); log.warn("获取许可超时，跳过处理");
if (semaphore.availablePermits() == 0) hasError.set(true);
log.info("获取许可 失败，❌ 任务被中断"); return;
semaphore.acquire(); // 获取许可 }
log.info(String.format("获取许可✅ 成功后，当前可用许可数:: %d",
semaphore.availablePermits()));
tmNameplateCorpusService.processItem(nameplate); tmNameplateCorpusService.processItem(nameplate);
} catch (Exception e) { } catch (Exception e) {
log.error("corpusPortrait画像铭牌异步任务执行失败", e); log.error("处理铭牌数据失败", e);
hasError.set(true);
} finally { } finally {
log.info("corpusPortrait 释放锁，availablePermits {}", semaphore.availablePermits()); semaphore.release();
try { latch.countDown();
ack.acknowledge(); // 提交 offset
} catch (IllegalStateException e) {
log.warn("Offset 已提交，跳过重复提交");
}
semaphore.release(); // 释放许可
log.error("释放许可，当前可用许可数:{}",semaphore.availablePermits());
} }
}, executor)); }, executor));
log.info("nameplateKafkaConsumeracknowledge：{}",partitionId, offset); // 等待所有任务完成
// 手动提交 offset if (latch.await(30, TimeUnit.SECONDS)) {
} catch (Exception e) { if (!hasError.get()) {
log.info("nameplateKafkaConsumerFailed to process message: {}", e); ack.acknowledge(); // 只有所有任务成功才提交offset
} else {
log.error("部分任务失败，不提交offset");
} }
log.info("nameplateKafkaConsumer消息处理开始，耗时：{}", System.currentTimeMillis() - startTime); } else {
log.error("任务执行超时，不提交offset");
}
} catch (Exception e) {
log.error("处理消息失败", e);
}
} }
} }