@@ -76,12 +76,12 @@ public class CorpusProcessKafkaProducer {
|
||||
@KafkaListener(topics = "${spring.kafka.topic}", groupId = "${spring.kafka.group}")
|
||||
public void listen(List<ConsumerRecord<String, Object>> recordMessages) {
|
||||
long startTime = System.currentTimeMillis();
|
||||
try {
|
||||
log.info("CorpusProcessKafkaProducer Received message: {}", recordMessages);
|
||||
// 获取消息列表
|
||||
int optimalThreadPoolSize = Runtime.getRuntime().availableProcessors() + 2;
|
||||
log.info("获取的线程数:{}", optimalThreadPoolSize);
|
||||
ExecutorService executor = Executors.newFixedThreadPool(optimalThreadPoolSize);
|
||||
try {
|
||||
if(CollectionUtils.isNotEmpty(recordMessages)){
|
||||
log.info("CorpusProcessKafkaProducer List size: {}", recordMessages.size());
|
||||
for (ConsumerRecord<String, Object> record : recordMessages) {
|
||||
@@ -120,6 +120,8 @@ public class CorpusProcessKafkaProducer {
|
||||
}
|
||||
} catch (Exception e) {
|
||||
log.error("CorpusProcessKafkaProducer 电话语料 解析JSON出错: {}", e.getMessage());
|
||||
}finally {
|
||||
executor.shutdown();
|
||||
}
|
||||
});
|
||||
|
||||
@@ -131,6 +133,8 @@ public class CorpusProcessKafkaProducer {
|
||||
// 在这里可以添加对解析后的对象的进一步处理逻辑
|
||||
} catch (Exception e) {
|
||||
log.error("CorpusProcessKafkaProducer 电话语料 解析JSON出错: {}" , e.getMessage());
|
||||
}finally {
|
||||
executor.shutdown();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -125,6 +125,7 @@ public class TmOdsVdqwMessagearchivingServiceImpl extends ServiceImpl<TmOdsVdqwM
|
||||
// 创建线程池
|
||||
ExecutorService executor = Executors.newFixedThreadPool(optimalThreadPoolSize); // 根据需求调整线程池大小
|
||||
|
||||
try {
|
||||
for (int i = 1; i <= totalPages; i++) {
|
||||
int offset = (i - 1) * pageSize;
|
||||
List<OdsVdqwMessageOTD> messageList = tmOdsVdqwMessagearchivingMapper.queryOdsVdqwMessageByData(statTime, endTime, offset, pageSize, retry);
|
||||
@@ -145,6 +146,12 @@ public class TmOdsVdqwMessagearchivingServiceImpl extends ServiceImpl<TmOdsVdqwM
|
||||
}
|
||||
// 关闭线程池
|
||||
executor.shutdown();
|
||||
} catch (Exception e) {
|
||||
log.error("企微数据跑批异常",e);
|
||||
} finally {
|
||||
// 关闭线程池
|
||||
executor.shutdown();
|
||||
}
|
||||
log.info("企微数据跑批结束 耗时:{}",System.currentTimeMillis()-startTime);
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user