【语音识别处理逻辑流程(初稿)】

This commit is contained in:
weizhihong 2023-04-07 13:06:47 +08:00
parent 78e8f84382
commit 81f89a6aec
22 changed files with 855 additions and 0 deletions

View File

@ -0,0 +1,48 @@
package club.joylink.rtss.services.voice.discriminate;
import club.joylink.rtss.simulation.cbtc.discriminate.OperateResult;
import club.joylink.rtss.simulation.cbtc.discriminate.ReplyResult;
import club.joylink.rtss.simulation.cbtc.discriminate.Stage;
import club.joylink.rtss.simulation.cbtc.discriminate.VoiceDiscriminateResult;
import club.joylink.rtss.simulation.cbtc.ATS.operation.AtsOperationDispatcher;
import club.joylink.rtss.simulation.cbtc.Simulation;
import club.joylink.rtss.simulation.cbtc.conversation.ConversationManagerService;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
/**
* 执行指令
*/
@Service("execCommandServiceImpl")
public class ExecCommandServiceImpl implements VoiceDiscriminateStepService {
@Autowired
private AtsOperationDispatcher atsOperationDispatcher;
@Autowired
private ConversationManagerService conversationManagerService;
@Override
public void doExec(VoiceDiscriminateResult result, Simulation simulation) {
result.setStage(Stage.EXEC_COMMAND);
// 执行操作
try {
if (result.getOperateResult() != null) {
OperateResult operateResult = result.getOperateResult();
atsOperationDispatcher.execute(simulation, operateResult.getMember(), operateResult.getType().name(), operateResult.getParams());
}
} catch (Exception e) {
result.setSuccess(false);
result.setMsg(Stage.EXEC_COMMAND.getDesc() + "执行失败:" + e.getMessage());
return;
}
// 执行回复
if (result.getReplyContentResult() != null) {
ReplyResult replyResult = result.getReplyContentResult();
replyResult.getSimulationMemberList().forEach(member -> {
conversationManagerService.conversationChat(simulation, member, replyResult.getContent(), null);
});
}
}
}

View File

@ -0,0 +1,50 @@
package club.joylink.rtss.services.voice.discriminate;
import club.joylink.rtss.services.voice.VoiceService;
import club.joylink.rtss.simulation.cbtc.discriminate.Stage;
import club.joylink.rtss.simulation.cbtc.discriminate.VoiceDiscriminateResult;
import club.joylink.rtss.simulation.cbtc.Simulation;
import club.joylink.rtss.vo.client.voice.VoiceRecognitionResult;
import club.joylink.rtss.vo.client.voice.VoiceRecognitionVO;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
/**
* 对原始语音文件进行解析
*/
@Slf4j
@Service("loadVoiceFileServiceImpl")
public class LoadVoiceFileServiceImpl implements VoiceDiscriminateStepService {
/**
* 语音解析服务
*/
@Autowired
private VoiceService voiceService;
@Override
public void doExec(VoiceDiscriminateResult result, Simulation simulation) {
result.setStage(Stage.LOAD);
try {
VoiceRecognitionVO vo = VoiceRecognitionVO.load(result.getFile());
VoiceRecognitionResult voiceRecognitionResult = voiceService.voiceRecognition(vo);
String originVoiceContent = voiceRecognitionResult.getResult();
String originVoicePinYin = translateToPinYin(originVoiceContent);
result.setOriginContent(originVoiceContent);
result.setOriginPinYin(originVoicePinYin);
} catch (Exception e) {
result.setSuccess(false);
result.setMsg(e.getCause().getMessage());
log.error(result.getStage().getDesc(), e);
}
}
/**
* 翻译标准
* @param result
* @return
*/
private String translateToPinYin(String result) {
return null;
}
}

View File

@ -0,0 +1,55 @@
package club.joylink.rtss.services.voice.discriminate;
import club.joylink.rtss.simulation.cbtc.discriminate.Stage;
import club.joylink.rtss.simulation.cbtc.discriminate.VoiceDiscriminateResult;
import club.joylink.rtss.simulation.cbtc.discriminate.VoiceDiscriminateRule;
import club.joylink.rtss.simulation.cbtc.Simulation;
import org.springframework.stereotype.Service;
import org.springframework.util.CollectionUtils;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.function.BiFunction;
/**
* 匹配原始指令
*/
@Service("matchCommandServiceImpl")
public class MatchCommandServiceImpl implements VoiceDiscriminateStepService {
/**
* 关键字匹配方法
*/
private final Map<String, BiFunction<String, VoiceDiscriminateRule, List<String>>> keyWordsMatchMethodMap = new HashMap<>();
/**
* 关键字匹配接口
*/
private final BiFunction<String, VoiceDiscriminateRule, List<String>> defaultKeyWordsMatch = (originContent, ruleInfo) -> {
return null;
};
@Override
public void doExec(VoiceDiscriminateResult result, Simulation simulation) {
result.setStage(Stage.MATCH_COMMAND);
// 加载本地图的规则信息列表
List<VoiceDiscriminateRule> ruleList = List.of(new VoiceDiscriminateRule());
// 匹配指令
BiFunction<String, VoiceDiscriminateRule, List<String>> matchMethod = null;
List<String> matchGroupList = null;
for (VoiceDiscriminateRule rule : ruleList) {
matchMethod = keyWordsMatchMethodMap.getOrDefault(rule.getKwMatchMethod(), defaultKeyWordsMatch);
matchGroupList = matchMethod.apply(result.getMatchOriginContent(), rule);
if (!CollectionUtils.isEmpty(matchGroupList)) {
result.setRule(rule);
result.setMatchGroupList(matchGroupList);
return;
}
}
// 出错未匹配到指令
result.setSuccess(false);
result.setMsg(result.getStage().getDesc() + ",未匹配到对应指令");
}
}

View File

@ -0,0 +1,59 @@
package club.joylink.rtss.services.voice.discriminate;
import club.joylink.rtss.simulation.cbtc.Simulation;
import club.joylink.rtss.simulation.cbtc.discriminate.*;
import club.joylink.rtss.simulation.cbtc.member.SimulationMember;
import org.springframework.stereotype.Service;
import org.springframework.util.CollectionUtils;
import java.util.List;
/**
* 操作解析
*/
@Service("operateParseServiceImpl")
public class OperateParseServiceImpl implements VoiceDiscriminateStepService {
@Override
public void doExec(VoiceDiscriminateResult result, Simulation simulation) {
result.setStage(Stage.OPERATE_PARSE);
OperateRule operateRule = result.getRule().getExecOperateRule();
if (operateRule == null) {
return;
}
List<ParamExtractResult> paramExtractResults = result.getParamExtractResultList();
OperateResult operateResult = new OperateResult(operateRule.getType());
result.setOperateResult(operateResult);
for (CommandParamRule paramRule : operateRule.getParamRules()) {
if (paramRule.getIndex() == null) {
operateResult.putParam(paramRule.getName(), paramRule.getParseRule().extractParam(simulation, null));
} else if (paramRule.getIndex() < paramExtractResults.size()) {
operateResult.putParam(paramRule.getName(), paramRule.getParseRule().extractParam(simulation, paramExtractResults.get(paramRule.getIndex())));
} else {
result.setSuccess(false);
result.setMsg(Stage.OPERATE_PARSE.getDesc() + "解析参数出错");
return;
}
}
MemberRule memberRule = operateRule.getMemberRule();
if (memberRule != null) {
List<String> groupList = result.getMatchGroupList();
List<SimulationMember> simulationMemberList = null;
if (memberRule.getIndex() == null) {
simulationMemberList = memberRule.getParseRule().matchMember(simulation, null);
} else if (memberRule.getIndex() < paramExtractResults.size()) {
simulationMemberList = memberRule.getParseRule().matchMember(simulation, groupList.get(memberRule.getIndex()));
} else {
result.setSuccess(false);
result.setMsg(Stage.OPERATE_PARSE.getDesc() + "解析成员出错");
return;
}
if (CollectionUtils.isEmpty(simulationMemberList)) {
result.setSuccess(false);
result.setMsg(Stage.OPERATE_PARSE.getDesc() + "未找到匹配成员");
return;
}
operateResult.setMember(simulationMemberList.get(0));
}
}
}

View File

@ -0,0 +1,37 @@
package club.joylink.rtss.services.voice.discriminate;
import club.joylink.rtss.simulation.cbtc.discriminate.ParamExtractResult;
import club.joylink.rtss.simulation.cbtc.discriminate.ParamExtractRule;
import club.joylink.rtss.simulation.cbtc.discriminate.Stage;
import club.joylink.rtss.simulation.cbtc.discriminate.VoiceDiscriminateResult;
import club.joylink.rtss.simulation.cbtc.Simulation;
import org.springframework.stereotype.Service;
import java.util.ArrayList;
import java.util.List;
/**
* 解析参数
*/
@Service("paramExtractServiceImpl")
public class ParamExtractServiceImpl implements VoiceDiscriminateStepService {
@Override
public void doExec(VoiceDiscriminateResult result, Simulation simulation) {
result.setStage(Stage.PARAM_EXTRACT);
List<String> groupList = result.getMatchGroupList();
List<ParamExtractRule> paramsRules = result.getRule().getParamsRules();
List<ParamExtractResult> paramExtractResults = new ArrayList<>(paramsRules.size());
ParamExtractResult extractResult = null;
for (ParamExtractRule rule : paramsRules) {
if (groupList.size() < rule.getIndex()) {
result.setSuccess(false);
result.setMsg("提取参数出错");
return;
}
extractResult = rule.getParseRule().matchParam(simulation, groupList.get(rule.getIndex()));
paramExtractResults.add(extractResult);
}
result.setParamExtractResultList(paramExtractResults);
}
}

View File

@ -0,0 +1,69 @@
package club.joylink.rtss.services.voice.discriminate;
import club.joylink.rtss.simulation.cbtc.Simulation;
import club.joylink.rtss.simulation.cbtc.discriminate.*;
import club.joylink.rtss.simulation.cbtc.member.SimulationMember;
import org.springframework.stereotype.Service;
import org.springframework.util.CollectionUtils;
import java.util.List;
/**
* 合成回复内容
*/
@Service("replyServiceImpl")
public class ReplyServiceImpl implements VoiceDiscriminateStepService {
@Override
public void doExec(VoiceDiscriminateResult result, Simulation simulation) {
result.setStage(Stage.REPLY_CONTENT);
ReplyRule replyRule = result.getRule().getReplyRule();
if (replyRule == null) {
return;
}
List<ParamExtractResult> paramExtractResults = result.getParamExtractResultList();
ReplyResult replyResult = new ReplyResult(replyRule.getParamRules().size());
result.setReplyContentResult(replyResult);
try {
for (CommandParamRule paramRule : replyRule.getParamRules()) {
if (paramRule.getIndex() == null) {
replyResult.addParams(paramRule.getParseRule().extractParam(simulation, null));
} else if (paramRule.getIndex() < paramExtractResults.size()) {
replyResult.addParams(paramRule.getParseRule().extractParam(simulation, paramExtractResults.get(paramRule.getIndex())));
} else {
result.setSuccess(false);
result.setMsg(Stage.REPLY_CONTENT.getDesc() + "解析参数出错");
return;
}
}
} catch (Exception e) {
result.setSuccess(false);
result.setMsg(Stage.REPLY_CONTENT.getDesc() + "解析参数出错");
}
// 格式化回复语句
Object[] objArr = new Object[replyResult.getParamList().size()];
replyResult.getParamList().toArray(objArr);
replyResult.setContent(String.format(replyRule.getMessageFormat(), objArr));
// 匹配回复人员
MemberRule memberRule = replyRule.getMemberRule();
if (memberRule != null) {
List<String> groupList = result.getMatchGroupList();
List<SimulationMember> simulationMemberList = null;
if (memberRule.getIndex() == null) {
simulationMemberList = memberRule.getParseRule().matchMember(simulation, null);
} else if (memberRule.getIndex() < paramExtractResults.size()) {
simulationMemberList = memberRule.getParseRule().matchMember(simulation, groupList.get(memberRule.getIndex()));
} else {
result.setSuccess(false);
result.setMsg(Stage.REPLY_CONTENT.getDesc() + "解析成员出错");
return;
}
if (CollectionUtils.isEmpty(simulationMemberList)) {
result.setSuccess(false);
result.setMsg(Stage.REPLY_CONTENT.getDesc() + "未找到匹配成员");
return;
}
replyResult.setSimulationMemberList(simulationMemberList);
}
}
}

View File

@ -0,0 +1,82 @@
package club.joylink.rtss.services.voice.discriminate;
import club.joylink.rtss.simulation.cbtc.discriminate.Stage;
import club.joylink.rtss.simulation.cbtc.discriminate.VoiceDiscriminateResult;
import club.joylink.rtss.simulation.cbtc.Simulation;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.stereotype.Service;
import org.springframework.web.multipart.MultipartFile;
import javax.annotation.PostConstruct;
import java.util.*;
/**
* 语音识别服务类(多例)
*/
@Slf4j
@Service
public class VoiceDiscriminateService {
@Autowired
@Qualifier(value = "loadVoiceFileServiceImpl")
private VoiceDiscriminateStepService LoadVoiceFileService;
@Autowired
@Qualifier(value = "matchCommandServiceImpl")
private VoiceDiscriminateStepService MatchCommandService;
@Autowired
@Qualifier(value = "paramExtractServiceImpl")
private VoiceDiscriminateStepService paramExtractService;
@Autowired
@Qualifier(value = "operateParseServiceImpl")
private VoiceDiscriminateStepService operateParseService;
@Autowired
@Qualifier(value = "replyServiceImpl")
private VoiceDiscriminateStepService replyService;
@Autowired
@Qualifier(value = "execCommandServiceImpl")
private VoiceDiscriminateStepService execCommandService;
/**
* 执行步骤
*/
private List<VoiceDiscriminateStepService> stepList = new LinkedList<>();
@PostConstruct
public void initStepList() {
stepList.add(LoadVoiceFileService);
stepList.add(MatchCommandService);
stepList.add(paramExtractService);
stepList.add(operateParseService);
stepList.add(replyService);
stepList.add(execCommandService);
}
/**
* 接受语音文件资源等必要条件解析开始
*/
public VoiceDiscriminateResult doAnalysis(Simulation simulation, MultipartFile file) {
VoiceDiscriminateResult result = new VoiceDiscriminateResult();
result.setSuccess(true);
result.setFile(file);
for (VoiceDiscriminateStepService step : stepList) {
if (result.isSuccess()) {
step.doExec(result, simulation);
} else {
break;
}
}
if (result.isSuccess()) {
result.setStage(Stage.SUCCESS);
}
return result;
}
}

View File

@ -0,0 +1,9 @@
package club.joylink.rtss.services.voice.discriminate;
import club.joylink.rtss.simulation.cbtc.discriminate.VoiceDiscriminateResult;
import club.joylink.rtss.simulation.cbtc.Simulation;
public interface VoiceDiscriminateStepService {
void doExec(VoiceDiscriminateResult result, Simulation simulation);
}

View File

@ -0,0 +1,21 @@
package club.joylink.rtss.simulation.cbtc.discriminate;
import club.joylink.rtss.simulation.cbtc.Simulation;
import lombok.Getter;
/**
* 参数的提取规则
*/
@Getter
public enum CommandParamParseRule {
STATION_NAME {
@Override
public Object extractParam(Simulation simulation, ParamExtractResult extractResult) {
return null;
}
};
public abstract Object extractParam(Simulation simulation, ParamExtractResult extractResult);
}

View File

@ -0,0 +1,30 @@
package club.joylink.rtss.simulation.cbtc.discriminate;
import lombok.Data;
/**
* 指令匹配规则
*/
@Data
public class CommandParamRule {
/**
* 名称
*/
private String name;
/**
* 结果类型
*/
private String type;
/**
* 对应解析参数位置
*/
private Integer index;
/**
* 解析规则
*/
private CommandParamParseRule parseRule;
}

View File

@ -0,0 +1,21 @@
package club.joylink.rtss.simulation.cbtc.discriminate;
import club.joylink.rtss.simulation.cbtc.Simulation;
import lombok.Getter;
/**
* 参数的提取规则
*/
@Getter
public enum ExtractRule {
STATION_NAME {
@Override
public ParamExtractResult matchParam(Simulation simulation, String sourceStr) {
return null;
}
};
public abstract ParamExtractResult matchParam(Simulation simulation, String sourceStr);
}

View File

@ -0,0 +1,23 @@
package club.joylink.rtss.simulation.cbtc.discriminate;
import club.joylink.rtss.simulation.cbtc.Simulation;
import club.joylink.rtss.simulation.cbtc.member.SimulationMember;
import java.util.List;
/**
* 人员匹配规则
*/
public enum MemberParseRule {
STATION_SUPERVISOR {
@Override
public List<SimulationMember> matchMember(Simulation simulation, String sourceStr) {
return null;
}
};
public abstract List<SimulationMember> matchMember(Simulation simulation, String sourceStr);
}

View File

@ -0,0 +1,19 @@
package club.joylink.rtss.simulation.cbtc.discriminate;
import lombok.Data;
/**
* 指令匹配规则
*/
@Data
public class MemberRule {
/**
* 对应group位置
*/
private Integer index;
/**
* 解析规则
*/
private MemberParseRule parseRule;
}

View File

@ -0,0 +1,42 @@
package club.joylink.rtss.simulation.cbtc.discriminate;
import club.joylink.rtss.simulation.cbtc.ATS.operation.Operation;
import club.joylink.rtss.simulation.cbtc.member.SimulationMember;
import lombok.Data;
import java.util.HashMap;
import java.util.Map;
/**
* 指令匹配结果
*/
@Data
public class OperateResult {
/**
* 指令
*/
private Operation.Type type;
/**
* 指令参数
*/
private Map<String, Object> params = new HashMap<>();
/**
* 角色ID
*/
private SimulationMember member;
public OperateResult(Operation.Type type) {
this.type = type;
}
/**
* 参数放入集合
* @param key 集合key
* @param value 参数值
*/
public void putParam(String key, Object value) {
params.put(key, value);
}
}

View File

@ -0,0 +1,27 @@
package club.joylink.rtss.simulation.cbtc.discriminate;
import club.joylink.rtss.simulation.cbtc.ATS.operation.Operation;
import lombok.Data;
import java.util.List;
/**
* 指令匹配规则
*/
@Data
public class OperateRule {
/**
* 操作指令
*/
private Operation.Type type;
/**
* 参数提取规则
*/
private List<CommandParamRule> paramRules;
/**
* 人员匹配规则
*/
private MemberRule memberRule;
}

View File

@ -0,0 +1,19 @@
package club.joylink.rtss.simulation.cbtc.discriminate;
import lombok.Data;
/**
* 参数提取结果
*/
@Data
public class ParamExtractResult {
/**
* 设置名
*/
private String name;
/**
* 数据
*/
private Object value;
}

View File

@ -0,0 +1,29 @@
package club.joylink.rtss.simulation.cbtc.discriminate;
import lombok.Data;
/**
* 参数提取规则
*/
@Data
public class ParamExtractRule {
/**
* 参数名
*/
private String name;
/**
* 是否是设备
*/
private boolean isDevice;
/**
* 解析规则
*/
private ExtractRule parseRule;
/**
* group位置索引
*/
private Integer index;
}

View File

@ -0,0 +1,39 @@
package club.joylink.rtss.simulation.cbtc.discriminate;
import club.joylink.rtss.simulation.cbtc.member.SimulationMember;
import lombok.Data;
import java.util.ArrayList;
import java.util.List;
/**
* 回复内容
*/
@Data
public class ReplyResult {
/**
* 回复人员列表
*/
private List<SimulationMember> simulationMemberList;
/**
* 回复内容
*/
private String content;
/**
* 解析的结果列表
*/
private List<String> paramList;
public ReplyResult(int size) {
this.paramList = new ArrayList<>(size);
}
public void addParams(Object param) {
if (param instanceof String) {
paramList.add(param.toString());
}
throw new IllegalArgumentException();
}
}

View File

@ -0,0 +1,27 @@
package club.joylink.rtss.simulation.cbtc.discriminate;
import lombok.Data;
import java.util.List;
/**
* 回复格式
*/
@Data
public class ReplyRule {
/**
* 人员类型
*/
private MemberRule memberRule;
/**
* 消息格式
*/
private String messageFormat;
/**
* 对应参数列表
*/
private List<CommandParamRule> paramRules;
}

View File

@ -0,0 +1,23 @@
package club.joylink.rtss.simulation.cbtc.discriminate;
import lombok.Getter;
/**
* 阶段
*/
@Getter
public enum Stage {
LOAD("加载文件"),
MATCH_COMMAND("匹配语音指令"),
PARAM_EXTRACT("参数提取"),
OPERATE_PARSE("操作解析"),
REPLY_CONTENT("合成回复内容"),
EXEC_COMMAND("执行指令"),
SUCCESS("执行成功");
private String desc;
Stage(String desc) {
this.desc = desc;
}
}

View File

@ -0,0 +1,88 @@
package club.joylink.rtss.simulation.cbtc.discriminate;
import club.joylink.rtss.simulation.cbtc.member.SimulationMember;
import lombok.Data;
import org.springframework.web.multipart.MultipartFile;
import java.util.List;
/**
* 语音解析结果
*/
@Data
public class VoiceDiscriminateResult {
/**
* 当前所属阶段
*/
private Stage stage;
/**
* 状态是否正常
*/
private boolean success;
/**
* 响应提示语
*/
private String msg;
private MultipartFile file;
/************************* 最初语音识别的结果信息 start *************************/
/**
* 语音原始内容
*/
private String originContent;
/**
* 原始语音拼音
*/
private String originPinYin;
/************************* 最初语音识别的结果信息 end **************************/
/************************* 根据语音识别的指令信息 start ************************/
/**
* 匹配到的指令规则
*/
private VoiceDiscriminateRule rule;
/**
* 匹配指令结果
*/
private List<String> matchGroupList;
/************************* 根据语音识别的指令信息 end *************************/
/************************* 参数提取结果 start *******************************/
/**
* 参数提取集合
*/
private List<ParamExtractResult> paramExtractResultList;
/**
* 匹配人员集合
*/
private List<SimulationMember> simulationMemberList;
/************************* 参数提取结果 end *********************************/
/************************* 合成指令 start **********************************/
/**
* 合成的指令结果
*/
private OperateResult operateResult;
/************************* 合成指令 end *************************************/
/************************* 合成回复信息 start *******************************/
/**
* 回复提取内容
*/
private ReplyResult replyContentResult;
/************************* 合成回复信息 end ********************************/
/**
* 获取匹配指令的语音内容
*/
public String getMatchOriginContent() {
return this.originPinYin;
}
}

View File

@ -0,0 +1,38 @@
package club.joylink.rtss.simulation.cbtc.discriminate;
import lombok.Data;
import java.util.List;
/**
* 语音解析规则
*/
@Data
public class VoiceDiscriminateRule {
private Long id;
/**
* 关键字正则
*/
private String keyWordRules;
/**
* 关键字匹配方法
*/
private String kwMatchMethod;
/**
* 所需参数
*/
private List<ParamExtractRule> paramsRules;
/**
* 执行指令系统操作指令
*/
private OperateRule execOperateRule;
/**
* 回复指令系统指令操作完毕或者无系统指令时交互回复
*/
private ReplyRule replyRule;
}