admin/vlv-ai-analytic-center
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

增加语料判断

Browse Source
This commit is contained in:
zren25
2025-03-12 15:02:19 +08:00
parent 2f1fdac8c9
commit 41b858faee
17 changed files with 65 additions and 59 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
ai-analytic-center-biz/src/main/java/com/volvo/ai/analytic/center/controller/AiRunDataController.java
View File

@@ -1,22 +1,23 @@
package com.volvo.ai.analytic.center.controller;
import com.alibaba.fastjson.JSONObject;
import com.obs.services.model.ObsObject;
import com.volvo.ai.analytic.center.feign.DiFyFeign;
import com.volvo.ai.analytic.center.utils.ObsUtil;
import lombok.extern.slf4j.Slf4j;
import org.apache.poi.ss.usermodel.Cell;
import org.apache.poi.ss.usermodel.Row;
import org.apache.poi.ss.usermodel.Sheet;
import org.apache.poi.ss.usermodel.Workbook;
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.*;
import org.springframework.web.multipart.MultipartFile;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.HashMap;
import java.util.Map;
import org.apache.poi.ss.usermodel.*;
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
@Slf4j
@RestController

2
ai-analytic-center-biz/src/main/java/com/volvo/ai/analytic/center/mapper/TmCorpusReportMapper.java
View File

@@ -2,9 +2,7 @@ package com.volvo.ai.analytic.center.mapper;
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
import com.volvo.ai.analytic.center.entity.TmCorpusReport;
import com.volvo.ai.analytic.center.entity.TmTelephoneCorpus;
import org.apache.ibatis.annotations.Mapper;
import org.springframework.stereotype.Repository;
/**
* @description 电话语料表-同步表

5
ai-analytic-center-biz/src/main/java/com/volvo/ai/analytic/center/mapper/TmOdsVdqwExternalcontactMapper.java
View File

@@ -1,13 +1,8 @@
package com.volvo.ai.analytic.center.mapper;
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
import com.volvo.ai.analytic.center.dto.corpus.OdsVdqwMessageOTD;
import com.volvo.ai.analytic.center.entity.TmOdsVdqwExternalcontact;
import com.volvo.ai.analytic.center.entity.TmOdsVdqwMessagearchiving;
import org.apache.ibatis.annotations.Mapper;
import org.apache.ibatis.annotations.Param;
import java.util.List;
/**
* @description 会话存档消息记录表-湖仓同步表

1
ai-analytic-center-biz/src/main/java/com/volvo/ai/analytic/center/mapper/TmOdsVdqwMessagearchivingMapper.java
View File

@@ -5,7 +5,6 @@ import com.volvo.ai.analytic.center.dto.corpus.OdsVdqwMessageOTD;
import com.volvo.ai.analytic.center.entity.TmOdsVdqwMessagearchiving;
import org.apache.ibatis.annotations.Mapper;
import org.apache.ibatis.annotations.Param;
import org.springframework.stereotype.Repository;
import java.util.List;

5
ai-analytic-center-biz/src/main/java/com/volvo/ai/analytic/center/mapper/TmOdsVdqwWorkuserinfoMapper.java
View File

@@ -1,13 +1,8 @@
package com.volvo.ai.analytic.center.mapper;
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
import com.volvo.ai.analytic.center.dto.corpus.OdsVdqwMessageOTD;
import com.volvo.ai.analytic.center.entity.TmOdsVdqwMessagearchiving;
import com.volvo.ai.analytic.center.entity.TmOdsVdqwWorkuserinfo;
import org.apache.ibatis.annotations.Mapper;
import org.apache.ibatis.annotations.Param;
import java.util.List;
/**
* @description 会话存档消息记录表-湖仓同步表

14
ai-analytic-center-biz/src/main/java/com/volvo/ai/analytic/center/mq/CorpusProcessKafkaProducer.java
View File

@@ -1,6 +1,7 @@
package com.volvo.ai.analytic.center.mq;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.volvo.ai.analytic.center.constant.Constant;
import com.volvo.ai.analytic.center.dto.corpus.AicorpusTelephoneDTO;
import com.volvo.ai.analytic.center.dto.corpus.DisplayDTO;
import com.volvo.ai.analytic.center.entity.TmTelephoneCorpus;
@@ -36,7 +37,7 @@ public class CorpusProcessKafkaProducer {
@KafkaListener(topics = "${spring.kafka.topic}", groupId = "${spring.kafka.group}")
public void listen(String message) {
try {
log.info("CorpusProcessKafkaConsumer Received message: {}" , message);
log.info("CorpusProcessKafkaProducer Received message: {}" , message);
AicorpusTelephoneDTO aicorpusTelephone = objectMapper.readValue(message, AicorpusTelephoneDTO.class);
log.info("aicorpusTelephone categoryCode:{}, display: {}" ,aicorpusTelephone.getCategoryCode(), aicorpusTelephone.getDisplay());
@@ -47,11 +48,14 @@ public class CorpusProcessKafkaProducer {
tmTelephoneCorpus.setCreateBy("kafka");
tmTelephoneCorpus.setCreateTime(LocalDateTime.now());
// TODO 只处理dcc的
// 条件： 只处理dcc的 10s通话时间以上
log.info("CorpusProcessKafkaProducer getCategoryCode: {}" , tmTelephoneCorpus.getCategoryCode());
if (Constant.CHANNEL_DCC.equals(tmTelephoneCorpus.getCategoryCode())){
log.info(" dcc 语料开始处理: {}");
tmTelephoneCorpusService.saveTelephoneCorpus(tmTelephoneCorpus);
tmTelephoneCorpusService.runTelephoneCorpusDify(aicorpusTelephone);
}
tmTelephoneCorpusService.saveTelephoneCorpus(tmTelephoneCorpus);
tmTelephoneCorpusService.runTelephoneCorpusDify(aicorpusTelephone);
// 在这里可以添加对解析后的对象的进一步处理逻辑
} catch (Exception e) {

2
ai-analytic-center-biz/src/main/java/com/volvo/ai/analytic/center/mq/CorpushIsLikeConsumer.java
View File

@@ -9,7 +9,6 @@ import org.apache.rocketmq.spring.annotation.RocketMQMessageListener;
import org.apache.rocketmq.spring.core.RocketMQListener;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import org.springframework.web.bind.annotation.PostMapping;
import java.util.Date;
@@ -23,7 +22,6 @@ public class CorpushIsLikeConsumer implements RocketMQListener<MessageExt>{
@Autowired
private TmCorpusReportService tmCorpusReportService;
@Override
@PostMapping
public void onMessage(MessageExt messageExt) {
log.info("CorpushIsLikeConsumer message: " + messageExt);
String message = new String(messageExt.getBody());

1
ai-analytic-center-biz/src/main/java/com/volvo/ai/analytic/center/service/DiFyService.java
View File

@@ -2,7 +2,6 @@ package com.volvo.ai.analytic.center.service;
import com.alibaba.fastjson.JSONObject;
import com.volvo.ai.analytic.center.dto.req.DiFyReq;
import com.volvo.ai.analytic.center.enums.BusinessTypeEnum;
public interface DiFyService {

2
ai-analytic-center-biz/src/main/java/com/volvo/ai/analytic/center/service/TmOdsVdqwMessagearchivingService.java
View File

@@ -1,9 +1,7 @@
package com.volvo.ai.analytic.center.service;
import com.baomidou.mybatisplus.extension.service.IService;
import com.volvo.ai.analytic.center.dto.corpus.OdsVdqwMessageOTD;
import com.volvo.ai.analytic.center.entity.TmOdsVdqwMessagearchiving;
import java.util.*;
/**
* @description 电话语料表-同步表
* @author BEJSON

2
ai-analytic-center-biz/src/main/java/com/volvo/ai/analytic/center/service/TmTelephoneCorpusService.java
View File

@@ -4,8 +4,6 @@ import com.baomidou.mybatisplus.extension.service.IService;
import com.volvo.ai.analytic.center.dto.corpus.AicorpusTelephoneDTO;
import com.volvo.ai.analytic.center.entity.TmTelephoneCorpus;
import java.util.Map;
/**
* @description 电话语料表-同步表
* @author BEJSON

1
ai-analytic-center-biz/src/main/java/com/volvo/ai/analytic/center/service/impl/AiAnalysisErrorsServiceImpl.java
View File

@@ -1,7 +1,6 @@
package com.volvo.ai.analytic.center.service.impl;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
import com.volvo.ai.analytic.center.entity.AiAnalysisErrors;
import com.volvo.ai.analytic.center.mapper.AiAnalysisErrorsMapper;

1
ai-analytic-center-biz/src/main/java/com/volvo/ai/analytic/center/service/impl/AiAnalysisRequestLogsServiceImpl.java
View File

@@ -2,7 +2,6 @@ package com.volvo.ai.analytic.center.service.impl;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
import com.volvo.ai.analytic.center.entity.AiAnalysisErrors;
import com.volvo.ai.analytic.center.entity.AiAnalysisRequestLogs;
import com.volvo.ai.analytic.center.mapper.AiAnalysisRequestLogsMapper;
import com.volvo.ai.analytic.center.service.AiAnalysisRequestLogsService;

1
ai-analytic-center-biz/src/main/java/com/volvo/ai/analytic/center/service/impl/DiFyServiceImpl.java
View File

@@ -14,7 +14,6 @@ import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import java.util.Collections;
import java.util.HashMap;
import java.util.Map;

1
ai-analytic-center-biz/src/main/java/com/volvo/ai/analytic/center/service/impl/MqMessageRecordServiceImpl.java
View File

@@ -19,7 +19,6 @@ import com.volvo.ai.analytic.center.service.DiffdefeatApproveService;
import com.volvo.ai.analytic.center.service.MqMessageRecordService;
import com.volvo.ai.analytic.center.utils.AiAnalysisUtils;
import lombok.extern.slf4j.Slf4j;
//import org.springframework.amqp.core.AmqpTemplate;
import org.apache.rocketmq.spring.core.RocketMQTemplate;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;

39
ai-analytic-center-biz/src/main/java/com/volvo/ai/analytic/center/service/impl/TmOdsVdqwMessagearchivingServiceImpl.java
View File

@@ -38,6 +38,7 @@ import java.util.*;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.atomic.AtomicInteger;
/**
@@ -142,14 +143,15 @@ public class TmOdsVdqwMessagearchivingServiceImpl extends ServiceImpl<TmOdsVdqwM
log.info("企微语料内容：FromUserId：{}, AcceptUserId：{}", item.getFromUserId(), item.getAcceptUserId());
// 1，vdqw_workuserinfo 这个表对应是 B端认证中心userId
// 2，vdqw_externalcontact 这个表对应是 企微客户unionId
String unionId = getUnionId(Arrays.asList(item.getFromUserId(), item.getAcceptUserId()));
String userId = getUserId(Arrays.asList(item.getFromUserId(), item.getAcceptUserId()));
log.info("企微查询信息：unionId：{}, userId：{}", unionId, userId);
if (StringUtils.isEmpty(unionId) || StringUtils.isEmpty(userId)) {
TmOdsVdqwExternalcontact tmOdsVdqwExternalcontact = getUnionId(Arrays.asList(item.getFromUserId(), item.getAcceptUserId()));
TmOdsVdqwWorkuserinfo tmOdsVdqwWorkuserinfo = getUserId(Arrays.asList(item.getFromUserId(), item.getAcceptUserId()));
if (null == tmOdsVdqwExternalcontact || null == tmOdsVdqwWorkuserinfo) {
log.info("企微查询信息为空 ");
return;
}
String unionId = tmOdsVdqwExternalcontact.getUnionId();
String userId = tmOdsVdqwWorkuserinfo.getMiddleUserId().toString();
log.info("企微查询信息：unionId：{}, userId：{}", unionId, userId);
if (StringUtils.isNotBlank(item.getFromUserId()) && StringUtils.isNotBlank(item.getAcceptUserId())) {
List<OdsVdqwMessageOTD> contetnList = tmOdsVdqwMessagearchivingMapper.queryOdsVdqwMessageByFromUserIdAndAcceptUserId(statTime, endTime, Arrays.asList(item.getFromUserId(), item.getAcceptUserId()) );
OdsVdqwMessageOTD maxMsgTimeItem = contetnList.stream()
@@ -163,16 +165,27 @@ public class TmOdsVdqwMessagearchivingServiceImpl extends ServiceImpl<TmOdsVdqwM
RunMaskingRuleInput runMaskingRuleInput = new RunMaskingRuleInput();
runMaskingRuleInput.setDataMaskingRules(maskingRuleItems);
StringBuffer chatList = new StringBuffer();
AtomicInteger externalcontactCount = new AtomicInteger();
contetnList.forEach(contentItem -> {
String title = "";
if(tmOdsVdqwExternalcontact.getUnionId().equals(contentItem.getFromUserId())){
title="客户:";
externalcontactCount.addAndGet(1);
}else{
title="客服:";
}
JSONObject contentJson = JSONObject.parseObject(contentItem.getContent());
String content = contentJson.getString("content");
// 拼接 role 和 text
String chat = contentItem.getFromUserId().concat(":").concat(content);
String chat = title.concat(content);
runMaskingRuleInput.setOldStr(chat);
String corpusChat = dataMaskingRuleService.runMaskingRule(runMaskingRuleInput);
chatList.append(corpusChat).append("\n");
});
if(externalcontactCount.get()<1){
log.info("没有客户回复的语料，无需解析");
return;
}
inputMap.put("chat", chatList.toString());
// inputMap.put("model", tmTelephoneCorpusService.getCarModelList());
@@ -233,25 +246,25 @@ public class TmOdsVdqwMessagearchivingServiceImpl extends ServiceImpl<TmOdsVdqwM
}
}
private String getUnionId(List<String> userIds){
private TmOdsVdqwExternalcontact getUnionId(List<String> userIds){
LambdaQueryWrapper<TmOdsVdqwExternalcontact> queryWrapper = new LambdaQueryWrapper<>();
queryWrapper.in(TmOdsVdqwExternalcontact::getExternalUserId, userIds);
queryWrapper.eq(TmOdsVdqwExternalcontact::getIsDeleted, 0);
List<TmOdsVdqwExternalcontact> oldAiAnalysisRequestLogs= tmOdsVdqwExternalcontactMapper.selectList(queryWrapper);
if(CollectionUtils.isNotEmpty(oldAiAnalysisRequestLogs)){
return oldAiAnalysisRequestLogs.get(0).getUnionId();
return oldAiAnalysisRequestLogs.get(0);
}
return "";
return null;
}
private String getUserId(List<String> userIds){
private TmOdsVdqwWorkuserinfo getUserId(List<String> userIds){
LambdaQueryWrapper<TmOdsVdqwWorkuserinfo> queryWrapper = new LambdaQueryWrapper<>();
queryWrapper.in(TmOdsVdqwWorkuserinfo::getUserId, userIds);
queryWrapper.eq(TmOdsVdqwWorkuserinfo::getIsDeleted, 0);
List<TmOdsVdqwWorkuserinfo> tmOdsVdqwWorkuserinfoList = tmOdsVdqwWorkuserinfoMapper.selectList(queryWrapper);
if(CollectionUtils.isNotEmpty(tmOdsVdqwWorkuserinfoList)){
return tmOdsVdqwWorkuserinfoList.get(0).getMiddleUserId().toString();
return tmOdsVdqwWorkuserinfoList.get(0);
}
return "";
return null;
}
}

20
ai-analytic-center-biz/src/main/java/com/volvo/ai/analytic/center/service/impl/TmTelephoneCorpusServiceImpl.java
View File

@@ -1,6 +1,5 @@
package com.volvo.ai.analytic.center.service.impl;
import cn.hutool.core.date.DatePattern;
import cn.hutool.core.date.DateUtil;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONArray;
@@ -39,7 +38,10 @@ import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import javax.annotation.Resource;
import java.util.*;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
@@ -96,6 +98,11 @@ public class TmTelephoneCorpusServiceImpl extends ServiceImpl<TmTelephoneCorpusM
JSONObject jsonObject = JSONObject.parseObject( aicorpusTelephone.getDisplay());
JSONArray segments = jsonObject.getJSONArray("segments");
Long audioDuration = jsonObject.getLong("audio_duration"); // 毫秒
if(audioDuration/1000 <=10){
log.info("电话语料时长小于10秒，不进行dify处理");
return;
}
// 遍历 segments
StringBuffer chatList = new StringBuffer();
segments.stream()
@@ -105,9 +112,14 @@ public class TmTelephoneCorpusServiceImpl extends ServiceImpl<TmTelephoneCorpusM
String text = result.getString("text");
JSONObject analysisInfo = result.getJSONObject("analysis_info");
String role = analysisInfo.getString("role");
String title="";
if(role.equals("AGENT")){
title="客服:";
}else{
title="客户:";
}
// 拼接 role 和 text
String chat = role + ": " + text;
String chat = title + ": " + text;
runMaskingRuleInput.setOldStr(chat);
String corpusChat = dataMaskingRuleService.runMaskingRule(runMaskingRuleInput);
chatList.append(corpusChat).append("\n");

12
ai-analytic-center-biz/src/main/resources/mapper/TmOdsVdqwMessagearchivingMapper.xml
View File

@@ -11,8 +11,8 @@
tovm.chat_type as chatType
FROM
`tm_ods_vdqw_messagearchiving` tovm
WHERE
tovm.msg_time between #{statTime} and #{endTime}
WHERE tovm.is_deleted = 0
AND tovm.msg_time between #{statTime} and #{endTime}
AND tovm.chat_type = 0
GROUP BY
tovm.from_user_id,
@@ -29,8 +29,8 @@
tovm.chat_type as chatType
FROM
`tm_ods_vdqw_messagearchiving` tovm
WHERE
tovm.msg_time between #{statTime} and #{endTime}
WHERE tovm.is_deleted = 0
AND tovm.msg_time between #{statTime} and #{endTime}
AND tovm.chat_type = 0
GROUP BY
tovm.from_user_id,
@@ -47,8 +47,8 @@
tovm.accept_user_id
FROM
`tm_ods_vdqw_messagearchiving` tovm
WHERE
tovm.msg_time between #{statTime} and #{endTime}
WHERE tovm.is_deleted = 0
AND tovm.msg_time between #{statTime} and #{endTime}
AND tovm.from_user_id IN
<foreach collection="userIdList" item="userId" open="(" separator="," close=")">
#{userId}