|
@@ -1,24 +1,35 @@
|
|
|
package cn.cslg.pas.service.upLoadPatent;
|
|
|
|
|
|
+import cn.cslg.pas.common.model.dto.UploadFileDTO;
|
|
|
+import cn.cslg.pas.common.model.vo.ProjectImportPatentVO;
|
|
|
import cn.cslg.pas.common.model.vo.UploadParamsVO;
|
|
|
import cn.cslg.pas.common.model.vo.UploadSettingVO;
|
|
|
+import cn.cslg.pas.common.utils.ExcelUtils;
|
|
|
+import cn.cslg.pas.common.utils.FileUtils;
|
|
|
+import cn.cslg.pas.common.utils.JsonUtils;
|
|
|
import cn.cslg.pas.common.utils.ReadExcelUtils;
|
|
|
import cn.cslg.pas.domain.PatentData;
|
|
|
+import cn.cslg.pas.domain.QueueData;
|
|
|
import cn.cslg.pas.domain.Task;
|
|
|
import cn.cslg.pas.service.TaskService;
|
|
|
+import cn.cslg.pas.service.UploadPatentBatchService;
|
|
|
+import cn.hutool.core.collection.IterUtil;
|
|
|
+import cn.hutool.poi.excel.ExcelUtil;
|
|
|
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
|
|
|
import lombok.RequiredArgsConstructor;
|
|
|
+import org.apache.poi.ss.usermodel.PictureData;
|
|
|
+import org.apache.poi.ss.usermodel.Sheet;
|
|
|
import org.springframework.context.annotation.Lazy;
|
|
|
import org.springframework.stereotype.Component;
|
|
|
import org.springframework.stereotype.Service;
|
|
|
|
|
|
import javax.annotation.PostConstruct;
|
|
|
-import java.util.ArrayList;
|
|
|
-import java.util.LinkedList;
|
|
|
-import java.util.List;
|
|
|
-import java.util.Queue;
|
|
|
+import java.io.IOException;
|
|
|
+import java.util.*;
|
|
|
import java.util.concurrent.CountDownLatch;
|
|
|
|
|
|
+import static org.apache.poi.ss.usermodel.TableStyleType.totalRow;
|
|
|
+
|
|
|
/**
|
|
|
* 将专利信息存入队列或从队列取出
|
|
|
*
|
|
@@ -30,90 +41,217 @@ import java.util.concurrent.CountDownLatch;
|
|
|
public class PantentQueueService {
|
|
|
private final ExcuteDataToVOService excuteDataToVOService;
|
|
|
private final ExcuteUploadSettingService excuteUploadSettingService;
|
|
|
- private final MessageService messageService;
|
|
|
- private final TaskService taskService;
|
|
|
- private Queue<UploadParamsVO> queue = new LinkedList<>();
|
|
|
- private List<Integer> taskQueueList = new ArrayList<>();
|
|
|
+ private final UploadPatentToDBService uploadPatentToDBService;
|
|
|
+ private final UploadPatentBatchService uploadPatentBatchService;
|
|
|
+ private final FileUtils fileUtils;
|
|
|
+ private final MessageService messageService;
|
|
|
+ private final TaskService taskService;
|
|
|
+ private Queue<QueueData> patentImageZhuluQueue = new LinkedList<>();
|
|
|
+ private Queue<QueueData> patentRightInstructionTextQueue = new LinkedList<>();
|
|
|
+ private Queue<QueueData> patentAssoQueue = new LinkedList<>();
|
|
|
+ private List<Integer> taskQueueList = new ArrayList<>();
|
|
|
private Boolean flag = false;
|
|
|
- private CountDownLatch patentLatch = new CountDownLatch(1);
|
|
|
private CountDownLatch taskLatch = new CountDownLatch(1);
|
|
|
+ private CountDownLatch patentImageZhuluLatch = new CountDownLatch(1);
|
|
|
+ private CountDownLatch patentRightInstructionTextLatch = new CountDownLatch(1);
|
|
|
+ private CountDownLatch patentAssoLatch = new CountDownLatch(1);
|
|
|
+
|
|
|
//将专利信息存入队列
|
|
|
public void addPatnetToQueue() {
|
|
|
+ Task task = null;
|
|
|
try {
|
|
|
- while (true){
|
|
|
- //检查任务队列
|
|
|
- if (taskQueueList.size() == 0) {
|
|
|
- taskLatch.await();
|
|
|
- }
|
|
|
- //查找 taskQueueList 中有没有进行中的任务
|
|
|
- long count = taskService.count(new LambdaQueryWrapper<Task>().in(Task::getId, taskQueueList).eq(Task::getStatus, 1));
|
|
|
- //若没有,则取出第一个队列中的任务开始执行
|
|
|
- Task task = null;
|
|
|
- if (count == 0) {
|
|
|
+ while (true) {
|
|
|
+ //检查任务队列
|
|
|
+ if (taskQueueList.size() == 0) {
|
|
|
+ taskLatch.await();
|
|
|
+ }
|
|
|
+
|
|
|
+ //1.从taskQueueList中取出第一个taskId,并将其从taskQueueList中删除
|
|
|
task = taskService.getById(taskQueueList.get(0));
|
|
|
- //任务队列去除该任务
|
|
|
taskQueueList.remove(0);
|
|
|
- //修改该任务状态,改为进行中
|
|
|
- Task currentTask = new Task();
|
|
|
- currentTask.setId(task.getId());
|
|
|
- currentTask.setStatus(1);
|
|
|
- taskService.updateById(currentTask);
|
|
|
- }
|
|
|
- //获得文件路径
|
|
|
- String filePath = task.getUrl();
|
|
|
- //检查文件合法性
|
|
|
- Integer totalRow = ReadExcelUtils.textExcel(filePath);
|
|
|
- //调用解析数据类,根据数据来源id(如1:智慧芽)解析数据源配置文件信息
|
|
|
- List<UploadSettingVO.Column> jsonData = excuteUploadSettingService.ExcuteUploadSetting(1 + "");
|
|
|
- //遍历专利总数量,在循环中保存专利
|
|
|
- for (int i = 1; i <= totalRow; i++) {
|
|
|
- //解析读取一行专利
|
|
|
- PatentData patentData = ReadExcelUtils.readExcelOneRow(filePath, i);
|
|
|
- //调用装载数据类,专利数据转换为VO对象
|
|
|
- UploadParamsVO uploadParamsVO = excuteDataToVOService.fileToPatentVO(patentData, jsonData);
|
|
|
- //一个专利加入队列
|
|
|
- queue.add(uploadParamsVO);
|
|
|
- //通知消费者线程
|
|
|
- patentLatch.countDown();
|
|
|
- //Websocket发送message:通过WebSocket 在每一次循环结束后 向前端发送完成进度
|
|
|
- Long percentage = totalRow == 0 ? 0 : Math.round((totalRow.equals(i) ? (i * 1D) : (i + 1D)) / totalRow * 100D);
|
|
|
- messageService.sendWebsocketMessage(task, totalRow, i, percentage);
|
|
|
+ //获得Excel文件路径
|
|
|
+ String filePath = task.getUrl(); //相对路径
|
|
|
+ filePath = fileUtils.getPath(filePath); //绝对路径
|
|
|
+
|
|
|
+ //List<List<Object>> rowList = new ArrayList<>();
|
|
|
+ //List<Map<Object, Object>> mapList = new ArrayList<>();
|
|
|
+ //解析Excel获取所有行数据 rowList
|
|
|
+ //ExcelUtil.readBySax(filePath, -1, (sheetIndex, rowIndex, row) -> rowList.add(row));
|
|
|
+ //遍历 rowList,将第一行即表头作为key,其余行即专利内容作为value添加进map集合 mapList
|
|
|
+ //for (int i = 1; i < rowList.size(); i++) {
|
|
|
+ // mapList.add(IterUtil.toMap(rowList.get(0), rowList.get(i)));
|
|
|
+ //}
|
|
|
+ //解析Excel获取所有摘要附图
|
|
|
+ //Map<String, PictureData> pictureDataMap = ExcelUtils.getDataFromExcel(filePath);
|
|
|
+
|
|
|
+ //Integer total = task.getTotal();
|
|
|
+ int lastIndex = task.getSuccessNum();
|
|
|
+ String json = task.getPramJson();
|
|
|
+ ProjectImportPatentVO projectImportPatentVO = JsonUtils.jsonToPojo(json, ProjectImportPatentVO.class);
|
|
|
+ //解析数据源类,通过数据来源id(如1:智慧芽)解析数据源配置文件,获得数据源配置文件对象jsonData
|
|
|
+ List<UploadSettingVO.Column> jsonData = excuteUploadSettingService.ExcuteUploadSetting(projectImportPatentVO.getSourceId());
|
|
|
+
|
|
|
+ //解析Excel文件获得工作簿,并取出工作簿总行数作为专利数量
|
|
|
+ Sheet sheet = ReadExcelUtils.readExcel(filePath);
|
|
|
+ int total = sheet.getPhysicalNumberOfRows() - 1;
|
|
|
+
|
|
|
+ //遍历专利总数量,在循环中保存专利
|
|
|
+ for (int i = lastIndex; i < total; i++) {
|
|
|
+ //解析读取一行专利
|
|
|
+ //PatentData patentData = ReadExcelUtils.readExcelOneRow(filePath, i);
|
|
|
+ //PatentData patentData = new PatentData()
|
|
|
+ // .setMap(mapList.get(i))
|
|
|
+ // .setPictureData(pictureDataMap.get(String.valueOf(i + 1)));
|
|
|
+ PatentData patentData = ReadExcelUtils.readExcelOneRow(filePath, sheet, i);
|
|
|
+ //patentData.setPictureData(pictureDataMap.get(String.valueOf(i + 1)));
|
|
|
+ //调用装载数据类,专利数据转换为VO对象
|
|
|
+ UploadParamsVO uploadParamsVO = excuteDataToVOService.fileToPatentVO(patentData, jsonData);
|
|
|
+
|
|
|
+ //保存专利基础数据(专利表"os_patent")
|
|
|
+ uploadPatentBatchService.getOneOrInsertOne(uploadParamsVO);
|
|
|
+
|
|
|
+ //专利分别加入三个消费者队列
|
|
|
+ QueueData queueData = new QueueData()
|
|
|
+ .setUploadParamsVO(uploadParamsVO)
|
|
|
+ .setProjectImportPatentVO(projectImportPatentVO);
|
|
|
+ patentImageZhuluQueue.add(queueData);
|
|
|
+ patentRightInstructionTextQueue.add(queueData);
|
|
|
+ patentAssoQueue.add(queueData);
|
|
|
+
|
|
|
+ //通知消费者线程(三个消费者:保存摘要附图和著录、保存说明书和权要、保存专题库关联专利数据)
|
|
|
+ //保存摘要附图和著录消费者
|
|
|
+ patentImageZhuluLatch.countDown();
|
|
|
+ //保存说明书和权要消费者
|
|
|
+ patentRightInstructionTextLatch.countDown();
|
|
|
+ //保存专题库关联专利数据消费者
|
|
|
+ patentAssoLatch.countDown();
|
|
|
+
|
|
|
+ //任务表更新数据
|
|
|
+ //上一次的位置(保存到第几个专利)
|
|
|
+ task.setLastIndex(i);
|
|
|
+ //成功条数
|
|
|
+ task.setSuccessNum(i);
|
|
|
+ //失败条数
|
|
|
+ task.setDefaultNum(total - i);
|
|
|
+ taskService.updateById(task);
|
|
|
+
|
|
|
+ //Websocket发送message:通过WebSocket 在每一次循环结束后 向前端发送完成进度
|
|
|
+ Long percentage = total == 0 ? 0 : Math.round((total == i ? (i * 1D) : (i + 1D)) / total * 100D);
|
|
|
+ messageService.sendWebsocketMessage(task, total, i, percentage);
|
|
|
+ }
|
|
|
+ //全部循环结束后,发送进度
|
|
|
+ Long percentage = 100L;
|
|
|
+ messageService.sendWebsocketMessage(task, total, total, percentage);
|
|
|
+
|
|
|
+ //任务表更新数据
|
|
|
+ task.setStatus(2);
|
|
|
+ taskService.updateById(task);
|
|
|
}
|
|
|
- //全部循环结束后,发送进度
|
|
|
- Long percentage = 100L;
|
|
|
- messageService.sendWebsocketMessage(task, totalRow, totalRow, percentage);
|
|
|
- }
|
|
|
+
|
|
|
} catch (Exception e) {
|
|
|
e.printStackTrace();
|
|
|
+ //修改任务状态为失败
|
|
|
+ task.setStatus(3);
|
|
|
+ taskService.updateById(task);
|
|
|
} finally {
|
|
|
}
|
|
|
flag = true;
|
|
|
}
|
|
|
- //将专利信息从队列取出
|
|
|
- public void pushPantentToDb() throws InterruptedException {
|
|
|
+
|
|
|
+ /**
|
|
|
+ * 将专利信息从队列取出,摘要附图和著录项目入库
|
|
|
+ */
|
|
|
+ public void pushPatentImageZhuluToDB() throws InterruptedException, IOException {
|
|
|
try {
|
|
|
while (true) {
|
|
|
- if (queue.isEmpty()) {
|
|
|
+ if (patentImageZhuluQueue.isEmpty()) {
|
|
|
if (flag) {
|
|
|
System.out.println("退出循环");
|
|
|
return;
|
|
|
} else {
|
|
|
- patentLatch.await();
|
|
|
+ patentImageZhuluLatch.await();
|
|
|
}
|
|
|
} else {
|
|
|
- UploadParamsVO uploadParamsVO = queue.remove();
|
|
|
- System.out.println("出队列" + uploadParamsVO);
|
|
|
+ System.out.println("开始保存摘要附图和著录项目");
|
|
|
+ QueueData queueData = patentImageZhuluQueue.remove();
|
|
|
+ //摘要附图入库
|
|
|
+ uploadPatentToDBService.uploadPatentImage(queueData.getUploadParamsVO());
|
|
|
+ //著录项目入库
|
|
|
+ uploadPatentToDBService.uploadPatentZhulu(queueData.getUploadParamsVO());
|
|
|
+ System.out.println("出队列");
|
|
|
}
|
|
|
}
|
|
|
} finally {
|
|
|
|
|
|
}
|
|
|
}
|
|
|
+
|
|
|
+ /**
|
|
|
+ * 将专利信息从队列取出,说明书和权要入库
|
|
|
+ */
|
|
|
+ public void pushPatentRightInstructionTextToDB() throws InterruptedException, IOException {
|
|
|
+ try {
|
|
|
+ while (true) {
|
|
|
+ if (patentRightInstructionTextQueue.isEmpty()) {
|
|
|
+ if (flag) {
|
|
|
+ System.out.println("退出循环");
|
|
|
+ return;
|
|
|
+ } else {
|
|
|
+ patentRightInstructionTextLatch.await();
|
|
|
+ }
|
|
|
+ } else {
|
|
|
+ System.out.println("开始保存说明书和权要");
|
|
|
+ QueueData queueData = patentRightInstructionTextQueue.remove();
|
|
|
+ //说明书入库
|
|
|
+ uploadPatentToDBService.uploadPatentInstructionText(queueData.getUploadParamsVO());
|
|
|
+ //权要入库
|
|
|
+ uploadPatentToDBService.uploadPatentRight(queueData.getUploadParamsVO());
|
|
|
+
|
|
|
+ System.out.println("出队列");
|
|
|
+ }
|
|
|
+ }
|
|
|
+ } finally {
|
|
|
+
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ /**
|
|
|
+ * 将专利信息从队列取出,与专利关联的数据入库
|
|
|
+ */
|
|
|
+ public void pushPatentAssoToDB() throws InterruptedException, IOException {
|
|
|
+ try {
|
|
|
+ while (true) {
|
|
|
+ if (patentAssoQueue.isEmpty()) {
|
|
|
+ if (flag) {
|
|
|
+ System.out.println("退出循环");
|
|
|
+ return;
|
|
|
+ } else {
|
|
|
+ patentAssoLatch.await();
|
|
|
+ }
|
|
|
+ } else {
|
|
|
+ System.out.println("开始保存与专利关联的数据");
|
|
|
+ QueueData queueData = patentAssoQueue.remove();
|
|
|
+ //专题库与专利关联入库
|
|
|
+ uploadPatentToDBService.uploadAssoThemaPat(queueData.getUploadParamsVO(), queueData.getProjectImportPatentVO());
|
|
|
+ //自定义字段标引与专利关联入库
|
|
|
+ uploadPatentToDBService.uploadAssoFieldPat(queueData.getUploadParamsVO(), queueData.getProjectImportPatentVO());
|
|
|
+ //文件夹与专利关联入库
|
|
|
+ uploadPatentToDBService.uploadAssoPorPat(queueData.getUploadParamsVO(), queueData.getProjectImportPatentVO());
|
|
|
+
|
|
|
+ System.out.println("出队列");
|
|
|
+ }
|
|
|
+ }
|
|
|
+ } finally {
|
|
|
+
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
public void queueAddTask(List<Integer> taskQueueList) {
|
|
|
this.taskQueueList.addAll(taskQueueList);
|
|
|
}
|
|
|
- public void awakeTasktch(){
|
|
|
- this.taskLatch.countDown();
|
|
|
+
|
|
|
+ public void awakeTasktch() {
|
|
|
+ this.taskLatch.countDown();
|
|
|
|
|
|
}
|
|
|
}
|