SH-Arbitrate/Arbitrate-Backend
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

抓取规则修改

Browse Source
This commit is contained in:
18792927508
2023-12-08 11:02:49 +08:00
parent a87fdca103
commit 923d1e0483
6 changed files with 173 additions and 244 deletions
Download Patch File Download Diff File Expand all files Collapse all files
ruoyi-admin/src/main/resources/application.yml
+8 -8
View File
@@ -183,15 +183,15 @@ imConfig:
secretId: AKID3xfHgroY4MQHvLXUXMwIQL1UjmbBX1Tv
# 腾讯云密钥
secretKey: INDrIXcT8YmomZBcsy0oNirnU0LTN4X7
jodconverter:
local:
#jodconverter:
# local:
# host: 121.40.189.20
#暂时关闭预览，启动时会有点慢
enabled: true
# enabled: true
#设置libreoffice主目录 linux地址如：/usr/lib64/libreoffice
office-home: /usr/lib64/libreoffice
# office-home: D:/Program Files/LibreOffice
# office-home: D:\app\libreOffice
# office-home: /usr/lib64/libreoffice/
# office-home: D:\app\libreOffice\
#开启多个libreoffice进程，每个端口对应一个进程
port-numbers: 8100
# port-numbers: 8100
#libreoffice进程重启前的最大进程数
max-tasks-per-process: 100
# max-tasks-per-process: 100
ruoyi-common/pom.xml
+19
View File
@@ -18,7 +18,26 @@
<dependencies>
<!--docx文件下载问题-->
<dependency>
<groupId>org.apache.poi</groupId>
<artifactId>poi-ooxml</artifactId>
<version>4.1.2</version>
</dependency>
<!-- POI依赖，读取.doc型文档-->
<!-- https://mvnrepository.com/artifact/org.apache.poi/poi-scratchpad -->
<dependency>
<groupId>org.apache.poi</groupId>
<artifactId>poi-scratchpad</artifactId>
<version>4.1.1</version>
</dependency>
<dependency>
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
<version>2.6</version>
</dependency>
<dependency>
<groupId>org.jodconverter</groupId>
<artifactId>jodconverter-core</artifactId>
ruoyi-common/src/main/java/com/ruoyi/common/utils/LibreOfficeUtil.java
-119
View File
@@ -1,119 +0,0 @@
package com.ruoyi.common.utils;
import cn.hutool.extra.spring.SpringUtil;
import com.artofsolving.jodconverter.DocumentConverter;
import com.artofsolving.jodconverter.openoffice.connection.OpenOfficeConnection;
import com.artofsolving.jodconverter.openoffice.connection.SocketOpenOfficeConnection;
import com.artofsolving.jodconverter.openoffice.converter.OpenOfficeDocumentConverter;
import com.ruoyi.common.exception.ServiceException;
import lombok.extern.slf4j.Slf4j;
import org.jodconverter.document.DocumentFormat;
import org.jodconverter.office.OfficeException;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.http.MediaType;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
/**
* LibreOffice工具类，用于将word,excel,ppt等格式文件转为pdf预览
*
* @author xuyuxiang
* @date 2020/7/6 14:55
*/
@Slf4j
public class LibreOfficeUtil {
@Value("${jodconverter.local.office-home}")
private long OpenOffice_HOME;
private static DocumentConverter documentConverter;
private static void init() {
try {
documentConverter = SpringUtil.getBean(DocumentConverter.class);
} catch (Exception e) {
throw new ServiceException();
}
}
public static boolean doc2pdf(File docFile, File pdfFile) {
boolean result = false;// 转换结果
if (docFile.exists()) {
if (!pdfFile.exists()) {
OpenOfficeConnection connection = new SocketOpenOfficeConnection(8100);
try {
connection.connect();
DocumentConverter converter = new OpenOfficeDocumentConverter(connection);
converter.convert(docFile, pdfFile);
// 关闭连接
connection.disconnect();
result = true;
log.info("****pdf转换成功，PDF输出：" + pdfFile.getPath() + "****");
} catch (java.net.ConnectException e) {
log.error("openoffice服务未启动", e);
} catch (com.artofsolving.jodconverter.openoffice.connection.OpenOfficeException e) {
log.error("读取转换文件失败", e);
} catch (Exception e) {
log.error("转换失败", e);
}
} else {
result = true;
log.info("****已经转换为pdf，不需要再进行转化****");
}
} else {
log.info("****需要转换的文档不存在，无法转换****");
}
return result;
}
/**
* doc转pdf（程序启动openoffice）
*
* @param inputFile 输入文件
* @param outputFile 输出文件
* @return
*/
public static boolean doc2pdf2(File inputFile, File outputFile) {
boolean result = false;
// OpenOffice的安装目录
String OpenOffice_HOME = "D:\\app\\libreOffice";
if (OpenOffice_HOME.charAt(OpenOffice_HOME.length() - 1) != '/') {
OpenOffice_HOME += "/";
}
Process process = null;
try {
// 启动OpenOffice的服务
String command = OpenOffice_HOME
+ "program/soffice.exe -headless -accept=\"socket,host=127.0.0.1,port=8100;urp;\"";
process = Runtime.getRuntime().exec(command);
// 连接 OpenOffice实例，运行在8100端口
OpenOfficeConnection connection = new SocketOpenOfficeConnection("127.0.0.1", 8100);
connection.connect();
// 转换
DocumentConverter converter = new OpenOfficeDocumentConverter(connection);
converter.convert(inputFile, outputFile);
// 关闭连接
connection.disconnect();
// 销毁OpenOffice服务的进程
process.destroy();
log.info("****pdf转换成功，PDF输出：" + outputFile.getPath() + "****");
return true;
} catch (Exception e) {
log.error("pdf转换失败", e);
} finally {
if (process != null) {
process.destroy();
}
}
return result;
}
}
ruoyi-system/src/main/java/com/ruoyi/wisdomarbitrate/service/impl/AdjudicationServiceImpl.java
+20
View File
@@ -176,6 +176,7 @@ public class AdjudicationServiceImpl implements IAdjudicationService {
Map<String, String> columnValueMap = columnValueList.stream().collect(Collectors.toMap(ColumnValue::getColumn, ColumnValue::getValue));
agentName=columnValueMap.get("agentName");
resName=columnValueMap.get("respondentName");
resName=columnValueMap.get("respondentName");
// 懒得if，暂时这样
//
for (String bookmark : bookmarkList) {
@@ -229,6 +230,23 @@ public class AdjudicationServiceImpl implements IAdjudicationService {
if (objectiJuris!=null&&objectiJuris == 1) {
datas.put("jurisdictionalObjection", jurisdictionalObjection);
}
// 出席庭审人员角色名称
String attendName="秘书、";
boolean isAbsenceFlag = caseApplicationById.getIsAbsence() != null && caseApplicationById.getIsAbsence().equals(0);
boolean appIsAbsenceFlag = caseApplicationById.getAppliIsAbsen() != null && caseApplicationById.getAppliIsAbsen().equals(0);
if(isAbsenceFlag||appIsAbsenceFlag){
if(isAbsenceFlag) {
attendName += "申请代理人" + agentName+"、";
}
if(appIsAbsenceFlag) {
attendName += "被申请人" + resName;
}
if(attendName.endsWith("、")){
agentName=attendName.replace("、","");
}
}
// 仲裁员名称
datas.put("arbitratorName", caseApplicationById.getArbitratorName());
// 审理方式
@@ -237,11 +255,13 @@ public class AdjudicationServiceImpl implements IAdjudicationService {
if (hearDate != null) {
// 审理日期
String hearDateStr = sdf.format(hearDate);
datas.put("hearDate",hearDateStr);
// todo 线上仲裁/线下仲裁方式未选择
//线上开庭时
if (arbitratMethod == 1) {
String replace = onLine.replace(onLineDate, Optional.ofNullable(hearDateStr).orElse(""));
datas.put("onLine", replace);
} else {
//书面仲裁时
String replace = written.replace(writtenDate, Optional.ofNullable(hearDateStr).orElse(""));
ruoyi-system/src/main/java/com/ruoyi/wisdomarbitrate/service/impl/CaseApplicationServiceImpl.java
+123 -114
View File
@@ -49,6 +49,13 @@ import com.tencentyun.TLSSigAPIv2;
import org.apache.pdfbox.pdmodel.PDDocument;
import org.apache.poi.hwpf.extractor.WordExtractor;
import org.apache.poi.ooxml.POIXMLDocument;
import org.apache.poi.ooxml.extractor.POIXMLTextExtractor;
import org.apache.poi.openxml4j.opc.OPCPackage;
import org.apache.poi.xwpf.extractor.XWPFWordExtractor;
import org.apache.poi.xwpf.usermodel.XWPFDocument;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
@@ -1162,11 +1169,6 @@ public class CaseApplicationServiceImpl implements ICaseApplicationService {
maxVersion = 1;
}
caseApplication.setVersion(maxVersion + 1);
// 将修改提交状态改为未提交
// caseApplication.setUpdateSubmitStatus(UpdateSubmitStatus.UNCOMMITTED.getCode());
// 修改案件表的版本号
// caseApplicationMapper.updateVersionById(caseApplication.getId(),caseApplication.getVersion());
// 异步新增案件日志
ThreadPoolUtil.execute(() -> {
try {
@@ -2875,6 +2877,7 @@ public class CaseApplicationServiceImpl implements ICaseApplicationService {
* @param userId
* @return
*/
@Override
public String generateUserSign(String userId) {
TLSSigAPIv2 tlsSigAPIv2 = new TLSSigAPIv2(sdkAppId, sdkSecretKey);
return tlsSigAPIv2.genUserSig(userId, 60 * 60 * 10);
@@ -3016,7 +3019,7 @@ public class CaseApplicationServiceImpl implements ICaseApplicationService {
if (unzipSuccess) {
// 查询抓取规则
// todo 批次需要再上传压缩包时用户填写
List<FatchRule> fatchRuleList = fatchRuleMapper.listByTemplateId(18L);
List<FatchRule> fatchRuleList = fatchRuleMapper.listByTemplateId(templateId);
if (CollectionUtil.isEmpty(fatchRuleList)) {
return error("未设置抓取规则");
}
@@ -3034,14 +3037,13 @@ public class CaseApplicationServiceImpl implements ICaseApplicationService {
if (fatchMap.size() <= 0) {
return error("从压缩包中未抓取到内容，请检查抓取字段配置");
}
if (fatchMap.size() > 0) {
// todo 从压缩包中识别各字段填充到数据库
//调用新增案件的接口
CaseApplication caseApplication = new CaseApplication();
caseApplication.setTemplateId(templateId);
//默认案件标的 todo 案件标的是什么
caseApplication.setCaseSubjectAmount(new BigDecimal(1));
//默认案件标的 todo 案件标的是什么,默认写死
caseApplication.setCaseSubjectAmount(new BigDecimal(10000));
// todo 这些以后要去掉，不在案件基本信息表维护，现在往基本信息表设置字段是因为修改以及查询详情的时候页面中字段是固定的，以后也要动态维护字段
// 仲裁请求
caseApplication.setArbitratClaims(fatchMap.get("arbitrationClaims"));
@@ -3152,14 +3154,8 @@ public class CaseApplicationServiceImpl implements ICaseApplicationService {
caseAffiliate.setResidenAffili(fatchMap.get("applicantHome"));
// 申请人联系地址
caseAffiliate.setContactAddress(fatchMap.get("applicantAddress"));
// if(map.get("职务").size()>0) {
// // 法定代表人职务
// caseAffiliate.setCompLegalperPost(map.get("职务").get(0));
// if(map.get("职务").size()>1) {
// // 代理人职务
// caseAffiliate.setAppliAgentTitle(map.get("职务").get(1));
// }
// }
caseAffiliate.setCompLegalperPost(fatchMap.get("compLegalperPost"));
// 委托代理人
caseAffiliate.setNameAgent(fatchMap.get("agentName"));
// 委托代理人联系电话
@@ -3221,7 +3217,6 @@ public class CaseApplicationServiceImpl implements ICaseApplicationService {
if (null != caseApplication.getId()) {
List<CaseAttach> caseAttachs = new ArrayList<>();
for (Map.Entry<String, String> entry : andConvertPDF.entrySet()) {
if (entry.getValue().contains("证据材料") || entry.getValue().contains("申请书") || entry.getValue().contains("调解协议") || entry.getValue().contains("情况说明")) {
String pdfUrl = entry.getValue();
File file1 = new File(pdfUrl);
CaseAttach caseAttach = new CaseAttach();
@@ -3231,7 +3226,7 @@ public class CaseApplicationServiceImpl implements ICaseApplicationService {
// 申请人提供的证据材料
caseAttach.setAnnexType(2);
caseAttachs.add(caseAttach);
}
}
if (CollectionUtil.isNotEmpty(caseAttachs)) {
// 新增申请人证据材料
@@ -3239,9 +3234,7 @@ public class CaseApplicationServiceImpl implements ICaseApplicationService {
}
return AjaxResult.success("导入成功");
}
} else {
return AjaxResult.error("文件识别内容失败,请检查");
}
} else {
// 没有找到符合条件的文件
@@ -3282,27 +3275,117 @@ public class CaseApplicationServiceImpl implements ICaseApplicationService {
}
}
public static String readerTxtFile(String filePath){
BufferedReader br=null;
StringBuilder result=new StringBuilder();
try {
br = new BufferedReader(new InputStreamReader(new FileInputStream(new File(filePath)),"GBK"));
String line=null;
while ((line=br.readLine())!=null) {
result.append(line).append("\n");
}
} catch (IOException e) {
e.printStackTrace();
} finally {
if (null!=br){
try {
br.close();
} catch (IOException e) {
e.printStackTrace();
}
}
}
return result.toString();
}
public static String readWord(String filePath) throws Exception{
File file = new File(filePath);
if(file.length()==0) return ""; // 需要操作原因是可能会空文件问题，如果不做处理，在下面读取中会报错
StringBuffer sb = new StringBuffer();
String buffer = "";
try {
if (filePath.endsWith(".doc")) {
InputStream is = new FileInputStream(file);
WordExtractor ex = new WordExtractor(is);
buffer = ex.getText();
if(buffer.length() > 0){
//使用回车换行符分割字符串
String [] arry = buffer.split("r\\n");
for (String string : arry) {
sb.append(string.trim());
}
}
} else if (filePath.endsWith(".docx")) {
FileInputStream fis = new FileInputStream(file);
XWPFDocument xdoc = new XWPFDocument(fis);
XWPFWordExtractor extractor = new XWPFWordExtractor(xdoc);
buffer = extractor.getText();
// OPCPackage opcPackage = POIXMLDocument.openPackage(filePath);
// XWPFWordExtractor extractor = new XWPFWordExtractor(opcPackage);
// buffer = extractor.getText();
if(buffer.length() > 0){
//使用换行符分割字符串
String [] arry = buffer.split("\r\n");
for (String string : arry) {
sb.append(string.trim());
}
}
} else {
return null;
}
return sb.toString();
} catch (Exception e) {
System.out.print("error---->"+filePath);
e.printStackTrace();
return null;
}
}
private boolean OCRAndBuildInfo( Map<String, String> andConvertPDF,String mapKey, Map<String, String> map, List<FatchRule> fatchRules) {
String pdfUrl = andConvertPDF.get(mapKey);
if(StrUtil.isNotEmpty(pdfUrl)) {
//获取文件的页数
int fileNumPage = getFileNumPage(pdfUrl);
//文件转成base64
String base64 = OCRUtils.pdfConvertBase64(pdfUrl);
if (base64 == null) {
throw new ServiceException("文件转成base64,转码失败pdfUrl:"+pdfUrl);
// return false;
}
StringBuilder ocrText = new StringBuilder(); // 创建一个StringBuilder对象
for (int i = 1; i <= fileNumPage; i++) {
//对接腾讯云接口.识别里面的数据
String text = OCRUtils.pdfIdentifyText(base64, i , fatchRules);
ocrText.append(text); // 拼接当前的字符串
}
if(StrUtil.isNotEmpty(ocrText)){
OCRUtils.fatchRuleGetContent(ocrText.toString(), fatchRules,map);
if(pdfUrl.endsWith("txt")){
String readerFile = readerTxtFile(pdfUrl);
if(StrUtil.isNotEmpty(readerFile)){
OCRUtils.fatchRuleGetContent(readerFile, fatchRules,map);
}
}else if(pdfUrl.endsWith("doc")||pdfUrl.endsWith("docx")){
// doc,docx,text识别内容
String readerFile = null;
try {
readerFile = readWord(pdfUrl);
} catch (Exception e) {
e.printStackTrace();
}
if(StrUtil.isNotEmpty(readerFile)){
OCRUtils.fatchRuleGetContent(readerFile, fatchRules,map);
}
}else if(pdfUrl.endsWith("pdf")){
//获取文件的页数
int fileNumPage = getFileNumPage(pdfUrl);
//文件转成base64
String base64 = OCRUtils.pdfConvertBase64(pdfUrl);
if (base64 == null) {
throw new ServiceException("文件转成base64,转码失败");
// return false;
}
StringBuilder ocrText = new StringBuilder(); // 创建一个StringBuilder对象
for (int i = 1; i <= fileNumPage; i++) {
//对接腾讯云接口.识别里面的数据
String text = OCRUtils.pdfIdentifyText(base64, i, fatchRules);
ocrText.append(text); // 拼接当前的字符串
if(StrUtil.isNotEmpty(ocrText)){
OCRUtils.fatchRuleGetContent(ocrText.toString(), fatchRules,map);
}
}
}
}
return true;
}
@@ -3367,33 +3450,10 @@ public class CaseApplicationServiceImpl implements ICaseApplicationService {
Map<String,String> pdfPathMap= new HashMap<>();
if (directory.isFile()) {
String path = "";
String fileName = "";
// 如果传入的参数是一个文件
// if (directory.getName().contains("仲裁申请书")) {
if (isPDF(directory)) {
// 如果文件名包含"仲裁申请书"且是PDF格式，直接返回路径
path = directory.getAbsolutePath();
} else {
String extension = getFileExtension(directory);
if( extension.contains("doc")|| extension.contains("docx")) {
// 如果不是PDF格式，进行转换成PDF并返回路径
String pdfPath = convertToPDF(directory);
if (pdfPath != null) {
path = pdfPath;
}
}
}
// 如果文件名包含"仲裁申请书"且是PDF格式，直接返回路径
// 如果是PDF格式，直接添加到列表中
// if(CollectionUtil.isNotEmpty(fatchRuleList)) {
// for (FatchRule fatchRule : fatchRuleList) {
// if(fatchRule.getFileName().contains(directory.getName())){
pdfPathMap.put(directory.getName(), path);
// }
// }
//
// }
// }
pdfPathMap.put(directory.getName(), path);
} else if (directory.isDirectory()) {
searchAndConvertPDF(directory, pdfPathMap);
} else {
@@ -3425,21 +3485,7 @@ public class CaseApplicationServiceImpl implements ICaseApplicationService {
}
if (file.isFile()) {
// 如果是文件且文件名包含"仲裁申请书"
// if (file.getName().contains("仲裁申请书")) {
if (isPDF(file)) {
// 如果是PDF格式，直接添加到列表中
pdfPathMap.put(file.getName(),file.getAbsolutePath());
} else {
// 如果不是PDF格式，进行转换成PDF并添加转换后的路径到列表中
String pdfPath = convertToPDF(file);
if (pdfPath != null) {
// 如果是PDF格式，直接添加到列表中
pdfPathMap.put(file.getName(),pdfPath);
}
}
// }
pdfPathMap.put(file.getName(),file.getAbsolutePath());
} else if (file.isDirectory()) {
// 如果是目录，递归查找
searchAndConvertPDF(file, pdfPathMap);
@@ -3448,43 +3494,6 @@ public class CaseApplicationServiceImpl implements ICaseApplicationService {
}
}
private static String convertToPDF(File file) {
String wordFilePath = file.getAbsolutePath();
// todo
String pdfSaveDirectory = "/home/ruoyi/uploadPath/upload/wordToPDF/";
// String pdfSaveDirectory = "D:/home/ruoyi/uploadPath/upload/wordToPDF/";
File directory = new File(pdfSaveDirectory);
if (!directory.exists()) {
directory.mkdirs();
}
String name = file.getName();
String nameWithoutExtension = name.substring(0, name.lastIndexOf("."));
String pdfFilePath = pdfSaveDirectory + nameWithoutExtension + ".pdf";
File inputWord = new File(wordFilePath);
if(!inputWord.exists()){
throw new ServiceException("文件不存在wordFilePath:"+wordFilePath);
}
File outputFile = new File(pdfFilePath);
try {
LibreOfficeUtil.doc2pdf2(inputWord,outputFile);
//
// InputStream docxInputStream = new FileInputStream(inputWord);
// OutputStream outputStream = new FileOutputStream(outputFile);
// IConverter converter = LocalConverter.builder().build();
// converter.convert(docxInputStream).as(DocumentType.DOCX).to(outputStream).as(DocumentType.PDF).execute();
// docxInputStream.close();
// outputStream.close();
} catch (Exception e) {
throw new ServiceException(e.getMessage()+"wordFilePath:"+wordFilePath+"pdfFilePath:"+pdfFilePath);
// e.printStackTrace();
}
return pdfFilePath;
}
private static int getFileNumPage(String pdfUrl) {
File pdfFile = new File(pdfUrl);
ruoyi-system/src/main/resources/mapper/wisdomarbitrate/FatchRuleMapper.xml
+3 -3
View File
@@ -19,7 +19,7 @@
<select id="selectFatchRuleList" parameterType="FatchRule" resultMap="BaseResultMap">
SELECT f.id ,f.file_name ,f.start_content ,f.end_content ,
f.`column` ,f.is_default ,f.columnName
f.`column` ,f.is_default ,f.`column_name`
FROM template_fatch_rule tf
left join template_manage t on tf.template_id = t.id
LEFT JOIN fatch_rule f on tf.fatch_rule_id = f.id
@@ -32,7 +32,7 @@
<select id="selectFatchRuleListIsDefault" parameterType="FatchRule" resultMap="BaseResultMap">
SELECT f.id ,f.file_name ,f.start_content ,f.end_content ,
f.`column` ,f.is_default ,f.columnName
f.`column` ,f.is_default ,f.`column_name`
FROM fatch_rule f
<where>
<if test="isDefault != null">
@@ -54,7 +54,7 @@
<if test="startContent != null and startContent != ''">start_content,</if>
<if test="endContent != null and endContent != ''">end_content,</if>
<if test="column != null and column != ''">`column`,</if>
<if test="columnName != null and columnName != ''">columnName,</if>
<if test="columnName != null and columnName != ''">`column_name`,</if>
<if test="isDefault != null">is_default</if>
)values(
<if test="fileName != null and fileName != ''">#{fileName},</if>