12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094 |
- package cn.cslg.pas.service.business.es;
- import cn.cslg.pas.common.dto.GetUnselectedDTO;
- import cn.cslg.pas.common.dto.business.EsCountDTO;
- import cn.cslg.pas.common.dto.business.EsCountDetailDTO;
- import cn.cslg.pas.common.dto.es.EsCustomFieldValueDTO;
- import cn.cslg.pas.common.dto.es.EsDateRangeDTO;
- import cn.cslg.pas.common.utils.parseQueryToTree.expressManager;
- import cn.cslg.pas.common.utils.parseQueryToTree.operateNode;
- import cn.cslg.pas.common.utils.parseQueryToTree.treeNode;
- import cn.cslg.pas.common.vo.EsConfigVO;
- import cn.cslg.pas.common.vo.business.EsAllCountVO;
- import cn.cslg.pas.common.vo.business.EsCountVO;
- import cn.cslg.pas.common.vo.es.EsDateRangeVO;
- import cn.cslg.pas.domain.es.Patent;
- import cn.cslg.pas.factorys.EsCountAnalyseBuilderFactory.EsCountAnalysisBuilderFactory;
- import cn.cslg.pas.factorys.EsCountAnalyseBuilderFactory.IEsCountAnalysisBuilder;
- import cn.cslg.pas.service.business.CommonService;
- import cn.cslg.pas.service.query.FormatQueryService;
- import co.elastic.clients.elasticsearch.ElasticsearchClient;
- import co.elastic.clients.elasticsearch._types.aggregations.*;
- import co.elastic.clients.elasticsearch._types.query_dsl.IdsQuery;
- import co.elastic.clients.elasticsearch._types.query_dsl.Query;
- import co.elastic.clients.elasticsearch._types.query_dsl.QueryBuilders;
- import co.elastic.clients.elasticsearch.core.SearchRequest;
- import co.elastic.clients.elasticsearch.core.SearchResponse;
- import co.elastic.clients.elasticsearch.core.search.Hit;
- import com.alibaba.fastjson.JSON;
- import lombok.RequiredArgsConstructor;
- import org.apache.commons.lang3.StringUtils;
- import org.springframework.beans.factory.annotation.Autowired;
- import org.springframework.context.annotation.Lazy;
- import org.springframework.stereotype.Service;
- import org.springframework.util.CollectionUtils;
- import java.io.IOException;
- import java.text.ParseException;
- import java.text.SimpleDateFormat;
- import java.util.*;
- import java.util.stream.Collectors;
- @Service
- @RequiredArgsConstructor(onConstructor_ = {@Lazy})
- public class EsCountService {
- private final List<String> childList = Arrays.asList("field");
- private final List<String> nestedList = Arrays.asList("PA", "IN", "PE", "SAT", "SRH","PRCO");
- private final List<String> dateList = Arrays.asList("PD", "AD", "GD");
- private final List<String> numberList = Arrays.asList("QPN", "QDPN", "SFN", "IFN", "PFN");
- private final List<String> nestDateList = Arrays.asList("PRD");
- private final List<String> nestChildList = Arrays.asList("MAT","MRH","MIN");
- private final ElasticsearchClient client;
- @Autowired
- private EsCountAnalysisBuilderFactory esCountAnalysisBuilderFactory;
- @Autowired
- private FormatQueryService formatQueryService;
- @Autowired
- private EsService esService;
- /**
- * 聚合统计
- *
- * @param vo
- * @return
- * @throws Exception
- */
- public EsCountDTO esCountAnalysis(EsAllCountVO vo) throws Exception {
- List<EsCountVO> countVOS = vo.getCountVOS();
- String searchCondition = vo.getCondition();
- List<EsCustomFieldValueDTO> customFields = vo.getCustomFields();
- Integer taskId = vo.getTaskId();
- Integer projectId = vo.getProjectId();
- EsCountDTO esCountDTO = new EsCountDTO();
- HashMap<String, List<EsCountDetailDTO>> map = new HashMap<>();
- List<EsCountDetailDTO> detailDTOS = new ArrayList<>();
- if (countVOS.size() > 1) {
- EsCountVO countVO = countVOS.get(0);
- List<String> values = countVO.getValues();
- values.removeIf(i -> i.equals("其他"));
- List<EsCountVO> esCountVOS = new ArrayList<>();
- esCountVOS.add(countVOS.get(1));
- if (StringUtils.isNotEmpty(countVO.getFieldId())) {
- if (!CollectionUtils.isEmpty(values)) {
- for (String value : values) {
- EsCustomFieldValueDTO valueDTO = new EsCustomFieldValueDTO();
- valueDTO.setFieldId(countVO.getFieldId());
- valueDTO.setFieldValue(Arrays.asList(value));
- customFields.add(valueDTO);
- this.getReturnData(searchCondition, "",customFields, taskId, projectId,
- esCountVOS, detailDTOS, esCountDTO, value, map);
- }
- }
- } else {
- for (String value : values) {
- String condition = "";
- if (dateList.contains(countVO.getField())) {
- condition = this.getDateFormat(countVO.getField(), value);
- } else {
- condition = countVO.getField() + " = " + value;
- }
- this.getReturnData(searchCondition, condition, customFields, taskId, projectId,
- esCountVOS, detailDTOS, esCountDTO, value, map);
- }
- }
- } else {
- this.getReturnData(searchCondition, "", customFields, taskId, projectId, countVOS,
- detailDTOS, esCountDTO, "", map);
- }
- esCountDTO.setDetailDTOS(detailDTOS);
- return esCountDTO;
- }
- public String getDateFormat(String field,String value) {
- String condition = "";
- if (value.contains("Q") || value.contains("H") || value.length() == 9) {
- String year = value.substring(0, value.indexOf("-"));
- int nextYear = Integer.parseInt(year) + 1;
- String start = "";
- String end = "";
- if (value.contains("Q1")) {
- start = year + "-01";
- end = year + "-04";
- } else if (value.contains("Q2")) {
- start = year + "-04";
- end = year + "-07";
- } else if (value.contains("Q3")) {
- start = year + "-07";
- end = year + "-10";
- } else if (value.contains("Q4")) {
- start = year + "-10";
- end = nextYear + "-01";
- } else if (value.contains("H1")) {
- start = year + "-01";
- end = year + "-07";
- } else if (value.contains("H2")) {
- start = year + "-07";
- end = nextYear + "-01";
- } else if (value.length() == 9) {
- start = year;
- end = value.substring(value.indexOf("-") + 1);
- }
- condition = field + ">=" + start + " " + "AND" + " " + field + "<" + end;
- } else {
- condition = field + " = " + value;
- }
- return condition;
- }
- public void getReturnData(String searchCondition,String condition, List<EsCustomFieldValueDTO> customFields, Integer taskId,
- Integer projectId, List<EsCountVO> countVOS, List<EsCountDetailDTO> detailDTOS,
- EsCountDTO esCountDTO, String firstName, Map<String, List<EsCountDetailDTO>> map) throws Exception {
- if (StringUtils.isNotEmpty(condition)) {
- if (searchCondition != null && !"".equals(searchCondition.trim())) {
- searchCondition = condition + " AND " + searchCondition;
- } else {
- searchCondition = condition;
- }
- }
- if (!CollectionUtils.isEmpty(customFields)) {
- searchCondition = esService.parseCustomField(customFields,projectId,taskId);
- }
- searchCondition = this.getCondition(searchCondition, taskId, projectId);
- SearchRequest.Builder builder = new SearchRequest.Builder();
- //设置查询索引
- builder.index("patent");
- //设置查询索引
- Query query = null;
- if (StringUtils.isNotEmpty(searchCondition)) {
- //1. 解析检索条件
- treeNode tree = expressManager.getInstance().Parse(searchCondition, false);
- //格式化检索式
- //3. 从es中检索数据
- query = formatQueryService.EsQueryToQuery((operateNode) tree, "patent");
- }
- for (EsCountVO countVO : countVOS) {
- String field = countVO.getField();
- Integer topN = countVO.getTopN();
- Integer fieldType = null;
- String type = countVO.getFieldType();
- if (StringUtils.isNotEmpty(type) && type.equals("tree")) {
- fieldType = 6;
- }
- String format = countVO.getFormat();
- List<String> values = countVO.getValues();
- Aggregation aggregation = this.getAggregation(countVO, projectId, taskId);
- if (query != null) {
- Query finalQuery = query;
- Aggregation finalAggregation = aggregation;
- Aggregation filtersAgg = new Aggregation.Builder().filters(new FiltersAggregation.Builder()
- .filters(i -> i.array(Arrays.asList(finalQuery))).build())
- .aggregations(new HashMap() {{
- put("filters_agg", finalAggregation);
- }}).build();
- builder.aggregations("Agg", filtersAgg);
- } else {
- builder.aggregations("Agg", aggregation);
- }
- SearchResponse<Patent> response = client.search(builder.build(), Patent.class);
- Aggregate agg = response.aggregations().get("Agg");
- if (query != null) {
- String finalSearchCondition = searchCondition;
- if (StringUtils.isNotEmpty(field)) {
- List<FiltersBucket> filtersBuckets = agg.filters().buckets().array();
- if (dateList.contains(field)) {
- if (CollectionUtils.isEmpty(values)) {
- filtersBuckets.forEach(filtersBucket -> {
- Aggregate filtersAgg = filtersBucket.aggregations().get("filters_agg");
- try {
- this.getDateCountDTOS(filtersAgg, field, topN, format, detailDTOS, esCountDTO);
- } catch (ParseException e) {
- e.printStackTrace();
- }
- });
- } else {
- filtersBuckets.forEach(filtersBucket -> {
- Aggregate filtersAgg = filtersBucket.aggregations().get("filters_agg");
- this.getDateAnalysisDTOS(filtersAgg, field, topN, detailDTOS, esCountDTO, firstName, map, values);
- });
- }
- } else if (nestedList.contains(field)) {
- filtersBuckets.forEach(filtersBucket -> {
- Aggregate filtersAgg = filtersBucket.aggregations().get("filters_agg");
- this.getNestedCountDTOS(filtersAgg, field, topN, detailDTOS, esCountDTO, firstName, map, values);
- if (!CollectionUtils.isEmpty(values)) {
- esCountDTO.setAllNumber(filtersAgg.nested().docCount());
- }
- });
- } else if (nestDateList.contains(field)) {
- if (CollectionUtils.isEmpty(values)) {
- filtersBuckets.forEach(filtersBucket -> {
- Aggregate filtersAgg = filtersBucket.aggregations().get("filters_agg");
- Aggregate termsAgg = filtersAgg.nested().aggregations().get("terms_agg");
- try {
- this.getDateCountDTOS(termsAgg, field, topN, format, detailDTOS, esCountDTO);
- } catch (ParseException e) {
- e.printStackTrace();
- }
- });
- } else {
- filtersBuckets.forEach(filtersBucket -> {
- Aggregate filtersAgg = filtersBucket.aggregations().get("filters_agg");
- Aggregate termsAgg = filtersAgg.nested().aggregations().get("terms_agg");
- this.getDateAnalysisDTOS(termsAgg, field, topN, detailDTOS, esCountDTO, firstName, map, values);
- });
- }
- } else if (nestChildList.contains(field)) {
- filtersBuckets.forEach(filtersBucket -> {
- Aggregate filtersAgg = filtersBucket.aggregations().get("filters_agg");
- Aggregate childAgg = filtersAgg.children().aggregations().get("childAgg");
- this.getNestedCountDTOS(childAgg, field, topN, detailDTOS, esCountDTO, firstName, map, values);
- if (!CollectionUtils.isEmpty(values)) {
- esCountDTO.setAllNumber(childAgg.nested().docCount());
- }
- });
- } else if (childList.contains(field)) {
- if (CollectionUtils.isEmpty(values)) {
- Integer finalFieldType = fieldType;
- filtersBuckets.forEach(filtersBucket -> {
- Aggregate filtersAgg = filtersBucket.aggregations().get("filters_agg");
- try {
- this.getChildCountDTOS(filtersAgg, field, finalFieldType, topN, detailDTOS, esCountDTO,projectId,taskId);
- } catch (Exception e) {
- e.printStackTrace();
- }
- });
- } else {
- Integer finalFieldType1 = fieldType;
- filtersBuckets.forEach(filtersBucket -> {
- Aggregate filtersAgg = filtersBucket.aggregations().get("filters_agg");
- this.getChildAnalysisDTOS(filtersAgg, field, finalFieldType1, topN,
- detailDTOS, esCountDTO, firstName, map, values);
- });
- }
- } else if (numberList.contains(field)) {
- if (CollectionUtils.isEmpty(values)) {
- } else {
- filtersBuckets.forEach(filtersBucket -> {
- Aggregate filtersAgg = filtersBucket.aggregations().get("filters_agg");
- this.getNumberAnalysisDTOS(filtersAgg, field, topN, detailDTOS, esCountDTO, firstName, map, values);
- });
- }
- } else {
- filtersBuckets.forEach(filtersBucket -> {
- Aggregate filtersAgg = filtersBucket.aggregations().get("filters_agg");
- this.getTermCountDTOS(filtersAgg, field, topN, detailDTOS, esCountDTO, firstName, map, values);
- });
- }
- } else {
- this.getFilterCountDTO(agg, finalSearchCondition, detailDTOS);
- }
- } else {
- if (dateList.contains(field)) {
- if (CollectionUtils.isEmpty(values)) {
- this.getDateCountDTOS(agg, field, topN, format, detailDTOS, esCountDTO);
- } else {
- this.getDateAnalysisDTOS(agg, field, topN, detailDTOS, esCountDTO, firstName, map, values);
- }
- } else if (nestedList.contains(field)) {
- this.getNestedCountDTOS(agg, field, topN, detailDTOS, esCountDTO, firstName, map, values);
- if (!CollectionUtils.isEmpty(values)) {
- esCountDTO.setAllNumber(agg.nested().docCount());
- }
- } else if (nestDateList.contains(field)) {
- Aggregate termsAgg = agg.nested().aggregations().get("terms_agg");
- this.getDateCountDTOS(termsAgg, field, topN, format, detailDTOS, esCountDTO);
- } else if (nestChildList.contains(field)) {
- Aggregate childAgg = agg.children().aggregations().get("childAgg");
- this.getNestedCountDTOS(childAgg, field, topN, detailDTOS, esCountDTO, firstName, map, values);
- if (!CollectionUtils.isEmpty(values)) {
- esCountDTO.setAllNumber(childAgg.nested().docCount());
- }
- } else if (childList.contains(field)) {
- if (CollectionUtils.isEmpty(values)) {
- this.getChildCountDTOS(agg, field, fieldType, topN, detailDTOS, esCountDTO, projectId, taskId);
- } else {
- this.getChildAnalysisDTOS(agg, field, fieldType, topN,
- detailDTOS, esCountDTO, firstName, map, values);
- }
- } else if (numberList.contains(field)) {
- this.getNumberAnalysisDTOS(agg, field, topN, detailDTOS, esCountDTO, firstName, map, values);
- } else {
- this.getTermCountDTOS(agg, field, topN, detailDTOS, esCountDTO, firstName, map, values);
- }
- }
- esCountDTO.setCondition(searchCondition);
- esCountDTO.setAnalyseMap(map);
- }
- }
- public EsDateRangeDTO getFieldRange(EsDateRangeVO rangeVO) throws Exception {
- String searchCondition = rangeVO.getCondition();
- String field = rangeVO.getField();
- List<EsCustomFieldValueDTO> customFields = rangeVO.getCustomFields();
- if (!CollectionUtils.isEmpty(customFields)) {
- searchCondition = esService.parseCustomField(customFields, rangeVO.getProjectId(), rangeVO.getTaskId());
- }
- Integer taskId = rangeVO.getTaskId();
- Integer projectId = rangeVO.getProjectId();
- searchCondition = this.getCondition(searchCondition, taskId, projectId);
- SearchRequest.Builder builder = new SearchRequest.Builder();
- //设置查询索引
- builder.index("patent");
- //设置查询索引
- Query query = null;
- if (StringUtils.isNotEmpty(searchCondition)) {
- //1. 解析检索条件
- treeNode tree = expressManager.getInstance().Parse(searchCondition, false);
- //格式化检索式
- //3. 从es中检索数据
- query = formatQueryService.EsQueryToQuery((operateNode) tree, "patent");
- }
- String esField = "";
- if (StringUtils.isNotEmpty(field)) {
- switch (field) {
- case "AD":
- esField = "app_date";
- break;
- case "PD":
- esField = "public_date";
- break;
- case "GD":
- esField = "grant_date";
- break;
- }
- }
- EsDateRangeDTO rangeDTO = new EsDateRangeDTO();
- if (StringUtils.isNotEmpty(esField)) {
- String finalEsField = esField;
- Query finalQuery = query;
- Aggregation min = AggregationBuilders.min(i -> i.field(finalEsField).format("yyyy"));
- Aggregation filtersAgg = new Aggregation.Builder().filters(new FiltersAggregation.Builder()
- .filters(i -> i.array(Arrays.asList(finalQuery))).build())
- .aggregations(new HashMap() {{
- put("filters_agg", min);
- }}).build();
- Aggregation max = AggregationBuilders.max(i -> i.field(finalEsField).format("yyyy"));
- Aggregation maxFilters = new Aggregation.Builder().filters(new FiltersAggregation.Builder()
- .filters(i -> i.array(Arrays.asList(finalQuery))).build())
- .aggregations(new HashMap() {{
- put("max_agg", max);
- }}).build();
- builder.aggregations("minAgg", filtersAgg);
- builder.aggregations("maxAgg", maxFilters);
- SearchResponse<Patent> response = client.search(builder.build(), Patent.class);
- Aggregate minAgg = response.aggregations().get("minAgg");
- if (minAgg != null) {
- List<FiltersBucket> list = minAgg.filters().buckets().array();
- for (FiltersBucket bucket : list) {
- Aggregate aggregate = bucket.aggregations().get("filters_agg");
- String minValue = aggregate.min().valueAsString();
- rangeDTO.setMinDate(minValue);
- }
- }
- Aggregate maxAgg = response.aggregations().get("maxAgg");
- if (maxAgg != null) {
- List<FiltersBucket> list = maxAgg.filters().buckets().array();
- for (FiltersBucket bucket : list) {
- Aggregate aggregate = bucket.aggregations().get("max_agg");
- String maxValue = aggregate.max().valueAsString();
- rangeDTO.setMaxDate(maxValue);
- }
- }
- }
- return rangeDTO;
- }
- /**
- * 查询共用Aggregation
- *
- * @param countVO
- * @return
- * @throws Exception
- */
- public Aggregation getAggregation(EsCountVO countVO, Integer projectId, Integer taskId) throws Exception {
- String field = countVO.getField();
- Integer topN = countVO.getTopN();
- Boolean ifHaveChild = countVO.getIfHaveChild();
- String fieldId = countVO.getFieldId();
- Integer fieldType = null;
- String type = countVO.getFieldType();
- if (StringUtils.isNotEmpty(type) && type.equals("tree")) {
- fieldType = 6;
- }
- String format = countVO.getFormat();
- String valueOne = countVO.getValueOne();
- String valueTwo = countVO.getValueTwo();
- List<String> values = countVO.getValues();
- IEsCountAnalysisBuilder iEsCountAnalysisBuilder = null;
- String json = CommonService.readJsonFile("esCountAnalysis.json");
- List<EsConfigVO> esConfigVOS = JSON.parseArray(json, EsConfigVO.class);
- EsConfigVO esConfigVO = esConfigVOS.stream().filter(item -> item.getField().equals(field))
- .findFirst().orElse(null);
- Aggregation aggregation = null;
- if (esConfigVO != null) {
- iEsCountAnalysisBuilder = esCountAnalysisBuilderFactory.getClass(esConfigVO.getEsClass());
- iEsCountAnalysisBuilder.setField(esConfigVO.getEsField());
- iEsCountAnalysisBuilder.setValueOne(valueOne);
- iEsCountAnalysisBuilder.setValueTwo(valueTwo);
- iEsCountAnalysisBuilder.setValues(values);
- iEsCountAnalysisBuilder.setFieldId(fieldId);
- iEsCountAnalysisBuilder.setFieldType(String.valueOf(fieldType));
- iEsCountAnalysisBuilder.setTopN(topN);
- iEsCountAnalysisBuilder.setFormat(format);
- iEsCountAnalysisBuilder.setProjectId(projectId);
- iEsCountAnalysisBuilder.setTaskId(taskId);
- iEsCountAnalysisBuilder.setIfHaveChild(ifHaveChild);
- if (iEsCountAnalysisBuilder.getField().contains(".")) {
- String path = iEsCountAnalysisBuilder.getField()
- .substring(0, iEsCountAnalysisBuilder.getField().indexOf("."));
- iEsCountAnalysisBuilder.setPath(path);
- }
- aggregation = iEsCountAnalysisBuilder.createCountAnalyseAgg();
- }
- return aggregation;
- }
- /**
- * 获取筛选条件
- *
- * @param taskId
- * @param projectId
- * @return
- */
- public String getCondition(String searchCondition, Integer taskId, Integer projectId) {
- if (taskId != null) {
- if (searchCondition != null && !"".equals(searchCondition.trim())) {
- searchCondition = "taskId = " + taskId + " AND " + searchCondition;
- } else {
- searchCondition = "taskId = " + taskId;
- }
- } else {
- if (projectId != null) {
- if (searchCondition != null && !"".equals(searchCondition.trim())) {
- searchCondition = "projectId = " + projectId + " AND " + searchCondition;
- } else {
- searchCondition = "projectId = " + projectId;
- }
- }
- }
- return searchCondition;
- }
- /**
- * 获取Filter聚合返回数据
- *
- * @param agg
- * @param condition
- * @return
- */
- public void getFilterCountDTO(Aggregate agg, String condition, List<EsCountDetailDTO> detailDTOS) {
- EsCountDetailDTO filterDTO = new EsCountDetailDTO();
- filterDTO.setField("condition");
- filterDTO.setName(condition);
- filterDTO.setNumber(agg.filter().docCount());
- if (filterDTO.getNumber() > 0) {
- detailDTOS.add(filterDTO);
- }
- }
- /**
- * 获取Terms聚合后数据
- *
- * @param agg
- * @param field
- * @param detailDTOS
- */
- public void getTermCountDTOS(Aggregate agg, String field, Integer topN, List<EsCountDetailDTO> detailDTOS,
- EsCountDTO esCountDTO, String firstName,
- Map<String, List<EsCountDetailDTO>> map,List<String> values) {
- List<EsCountDetailDTO> countDetailDTOS = new ArrayList<>();
- EsCountDetailDTO countDTO = new EsCountDetailDTO();
- countDTO.setField(field);
- countDTO.setName("其他");
- countDTO.setTopN(topN);
- Long count = agg.sterms().sumOtherDocCount();
- if (count > 0) {
- countDTO.setNumber(count);
- detailDTOS.add(countDTO);
- }
- List<StringTermsBucket> list = agg.sterms().buckets().array();
- list.forEach(bucket -> {
- EsCountDetailDTO dto = new EsCountDetailDTO();
- dto.setField(field);
- Aggregate aggregate = bucket.aggregations().get("filter_agg");
- dto.setName(bucket.key().stringValue());
- dto.setNumber(bucket.docCount());
- dto.setTopN(topN);
- dto.setFirstName(firstName);
- if (aggregate != null) {
- dto.setNumber(aggregate.filter().docCount());
- }
- if (dto.getNumber() > 0) {
- countDetailDTOS.add(dto);
- }
- });
- List<String> strs = new ArrayList<>();
- if (!CollectionUtils.isEmpty(values)) {
- strs.addAll(values);
- }
- if (StringUtils.isNotEmpty(firstName)) {
- countDetailDTOS.removeIf(dto -> dto.getName().equals("其他"));
- if (countDetailDTOS.size() != values.size()) {
- for (EsCountDetailDTO detailDTO : countDetailDTOS) {
- strs.removeIf(i -> i.equals(detailDTO.getName()));
- }
- for (String value : strs) {
- EsCountDetailDTO detailDTO = new EsCountDetailDTO();
- detailDTO.setName(value);
- detailDTO.setFirstName(firstName);
- detailDTO.setField(field);
- detailDTO.setNumber(0L);
- detailDTO.setTopN(topN);
- countDetailDTOS.add(detailDTO);
- }
- } else if (countDetailDTOS.isEmpty()) {
- for (String value : values) {
- EsCountDetailDTO detailDTO = new EsCountDetailDTO();
- detailDTO.setName(value);
- detailDTO.setFirstName(firstName);
- detailDTO.setField(field);
- detailDTO.setNumber(0L);
- detailDTO.setTopN(topN);
- countDetailDTOS.add(detailDTO);
- }
- }
- map.put(firstName, countDetailDTOS);
- } else {
- detailDTOS.addAll(countDetailDTOS);
- }
- }
- /**
- * 获取children聚合后数据
- *
- * @param agg
- * @param field
- * @param detailDTOS
- */
- public void getChildCountDTOS(Aggregate agg, String field, Integer fieldType, Integer topN,
- List<EsCountDetailDTO> detailDTOS, EsCountDTO esCountDTO,Integer projectId, Integer taskId) throws Exception {
- esCountDTO.setAllNumber(agg.children().docCount());
- Aggregate childAgg = agg.children().aggregations().get("childAgg");
- List<StringTermsBucket> list = childAgg.sterms().buckets().array();
- list.forEach(bucket -> {
- Aggregate aggregate = bucket.aggregations().get("termAgg");
- List<StringTermsBucket> termsBuckets = aggregate.sterms().buckets().array();
- termsBuckets.forEach(termsBucket -> {
- Aggregate termAgg = termsBucket.aggregations().get("filterAgg");
- long count = termAgg.filter().docCount();
- if (count > 0) {
- EsCountDetailDTO dto = new EsCountDetailDTO();
- dto.setField(field);
- dto.setName(termsBucket.key().stringValue());
- dto.setNumber(termsBucket.docCount());
- dto.setNumber(count);
- dto.setTopN(topN);
- if (dto.getNumber() > 0) {
- detailDTOS.add(dto);
- }
- }
- });
- });
- if (fieldType == null || fieldType != 6) {
- GetUnselectedDTO unselectedDTO = this.getUnselectedCustomNum(projectId, taskId);
- EsCountDetailDTO detail = new EsCountDetailDTO();
- detail.setField(field);
- detail.setName("未选择");
- detail.setNumber(unselectedDTO.getNumber().longValue());
- detail.setTopN(topN);
- if (!detailDTOS.contains(detail)) {
- detailDTOS.add(detail);
- }
- }
- }
- public GetUnselectedDTO getUnselectedCustomNum(Integer projectId, Integer taskId) throws Exception {
- GetUnselectedDTO dto = new GetUnselectedDTO();
- SearchRequest.Builder builder = new SearchRequest.Builder();
- //设置查询索引
- builder.index("patent");
- Query q = null;
- if (taskId != null) {
- Query query = QueryBuilders.term(i -> i.field("project_task.task_id").value(taskId));
- q = QueryBuilders.hasChild(i -> i.type("task").query(query));
- } else {
- Query query = QueryBuilders.term(i -> i.field("project_id").value(projectId));
- q = QueryBuilders.hasChild(i -> i.type("project").query(query));
- }
- builder.size(9999);
- builder.query(q);
- SearchResponse<Patent> response = client.search(builder.build(), Patent.class);
- List<Hit<Patent>> hits = response.hits().hits();
- List<String> list = new ArrayList<>();
- for (Hit<Patent> hit : hits) {
- String id = hit.id();
- list.add(id);
- }
- if (!CollectionUtils.isEmpty(list)) {
- dto = this.ifExistChild(list);
- }
- return dto;
- }
- public GetUnselectedDTO ifExistChild(List<String> list) throws IOException {
- GetUnselectedDTO dto = new GetUnselectedDTO();
- List<String> existChildIds = new ArrayList<>();
- List<String> beinglessChildIds = new ArrayList<>();
- int count = 0;
- for (String id : list) {
- SearchRequest.Builder builder = new SearchRequest.Builder();
- //设置查询索引
- builder.index("patent");
- Query q1 = QueryBuilders.parentId(i -> i.type("project_customfield").id(id));
- Query q2 = QueryBuilders.exists(i -> i.field("custom_field"));
- Query bool = QueryBuilders.bool(i -> i.must(q1, q2));
- builder.query(bool);
- SearchResponse<Patent> response = client.search(builder.build(), Patent.class);
- Long total = response.hits().total().value();
- if (total > 0) {
- existChildIds.add(id);
- } else {
- beinglessChildIds.add(id);
- count++;
- }
- }
- dto.setNumber(count);
- dto.setExistChildIds(existChildIds);
- dto.setBeinglessChildIds(beinglessChildIds);
- return dto;
- }
- /**
- * 获取children分析后数据
- *
- * @param agg
- * @param field
- * @param detailDTOS
- */
- public void getChildAnalysisDTOS(Aggregate agg, String field, Integer fieldType, Integer topN,
- List<EsCountDetailDTO> detailDTOS, EsCountDTO esCountDTO,
- String firstName, Map<String, List<EsCountDetailDTO>> map,List<String> values) {
- List<EsCountDetailDTO> countDetailDTOS = new ArrayList<>();
- Aggregate childAgg = agg.children().aggregations().get("childAgg");
- List<StringTermsBucket> list = childAgg.sterms().buckets().array();
- list.forEach(bucket -> {
- esCountDTO.setAllNumber(bucket.docCount());
- Aggregate aggregate = bucket.aggregations().get("termAgg");
- List<StringTermsBucket> termsBuckets = aggregate.sterms().buckets().array();
- termsBuckets.forEach(termsBucket -> {
- EsCountDetailDTO dto = new EsCountDetailDTO();
- dto.setField(field);
- dto.setName(termsBucket.key().stringValue());
- dto.setNumber(termsBucket.docCount());
- dto.setFirstName(firstName);
- dto.setTopN(topN);
- countDetailDTOS.add(dto);
- if (fieldType == null || fieldType != 6) {
- Long docCount = aggregate.sterms().sumOtherDocCount();
- EsCountDetailDTO detail = new EsCountDetailDTO();
- detail.setField(field);
- detail.setName("未选择");
- detail.setNumber(docCount);
- detail.setTopN(topN);
- if (!countDetailDTOS.contains(detail)) {
- countDetailDTOS.add(detail);
- }
- }
- });
- });
- List<String> strs = new ArrayList<>();
- if (!CollectionUtils.isEmpty(values)) {
- strs.addAll(values);
- }
- if (StringUtils.isNotEmpty(firstName)) {
- countDetailDTOS.removeIf(dto -> dto.getName().equals("其他"));
- if (countDetailDTOS.size() != values.size()) {
- for (EsCountDetailDTO detailDTO : countDetailDTOS) {
- strs.removeIf(i -> i.equals(detailDTO.getName()));
- }
- for (String value : strs) {
- EsCountDetailDTO detailDTO = new EsCountDetailDTO();
- detailDTO.setName(value);
- detailDTO.setFirstName(firstName);
- detailDTO.setField(field);
- detailDTO.setNumber(0L);
- detailDTO.setTopN(topN);
- countDetailDTOS.add(detailDTO);
- }
- } else if (countDetailDTOS.isEmpty()) {
- for (String value : values) {
- EsCountDetailDTO detailDTO = new EsCountDetailDTO();
- detailDTO.setName(value);
- detailDTO.setFirstName(firstName);
- detailDTO.setField(field);
- detailDTO.setNumber(0L);
- detailDTO.setTopN(topN);
- countDetailDTOS.add(detailDTO);
- }
- }
- map.put(firstName, countDetailDTOS);
- } else {
- detailDTOS.addAll(countDetailDTOS);
- }
- }
- /**
- * 获取range分析后数据
- *
- * @param agg
- * @param field
- * @param detailDTOS
- */
- public void getNumberAnalysisDTOS(Aggregate agg, String field, Integer topN, List<EsCountDetailDTO> detailDTOS,
- EsCountDTO esCountDTO, String firstName,
- Map<String, List<EsCountDetailDTO>> map,List<String> values) {
- List<EsCountDetailDTO> countDetailDTOS = new ArrayList<>();
- List<RangeBucket> list = agg.range().buckets().array();
- for (RangeBucket bucket : list) {
- EsCountDetailDTO dto = new EsCountDetailDTO();
- dto.setField(field);
- dto.setName(bucket.key());
- dto.setNumber(bucket.docCount());
- dto.setTopN(topN);
- dto.setFirstName(firstName);
- countDetailDTOS.add(dto);
- // if (dto.getNumber() > 0) {
- // detailDTOS.add(dto);
- // }
- }
- List<String> strs = new ArrayList<>();
- if (!CollectionUtils.isEmpty(values)) {
- strs.addAll(values);
- }
- if (StringUtils.isNotEmpty(firstName)) {
- countDetailDTOS.removeIf(dto -> dto.getName().equals("其他"));
- if (countDetailDTOS.size() != values.size()) {
- for (EsCountDetailDTO detailDTO : countDetailDTOS) {
- strs.removeIf(i -> i.equals(detailDTO.getName()));
- }
- for (String value : strs) {
- EsCountDetailDTO detailDTO = new EsCountDetailDTO();
- detailDTO.setName(value);
- detailDTO.setFirstName(firstName);
- detailDTO.setField(field);
- detailDTO.setNumber(0L);
- detailDTO.setTopN(topN);
- countDetailDTOS.add(detailDTO);
- }
- } else if (countDetailDTOS.isEmpty()) {
- for (String value : values) {
- EsCountDetailDTO detailDTO = new EsCountDetailDTO();
- detailDTO.setName(value);
- detailDTO.setFirstName(firstName);
- detailDTO.setField(field);
- detailDTO.setNumber(0L);
- detailDTO.setTopN(topN);
- countDetailDTOS.add(detailDTO);
- }
- }
- map.put(firstName, countDetailDTOS);
- } else {
- detailDTOS.addAll(countDetailDTOS);
- }
- }
- /**
- * 获取dateHistogram聚合后数据
- *
- * @param agg
- * @param field
- * @param detailDTOS
- */
- public void getDateCountDTOS(Aggregate agg, String field, Integer topN, String format, List<EsCountDetailDTO> detailDTOS, EsCountDTO esCountDTO) throws ParseException {
- List<DateHistogramBucket> list = agg.dateHistogram().buckets().array();
- esCountDTO.setAllNumber(Long.valueOf(String.valueOf(list.size())));
- List<EsCountDetailDTO> esCountDetailDTOS = new ArrayList<>();
- if (StringUtils.isNotEmpty(format) && format.equals("半年")) {
- List<EsCountDetailDTO> detailDTOList = new ArrayList<>();
- for (DateHistogramBucket bucket : list) {
- String year = bucket.keyAsString();
- SimpleDateFormat monthFormat = new SimpleDateFormat("yyyy-MM");
- String startMonth = year + "-01";
- Calendar calendar = Calendar.getInstance();
- Date date = monthFormat.parse(startMonth);
- calendar.setTime(date);
- calendar.add(Calendar.MONTH, 6);
- Date halfYearDate = calendar.getTime();
- Aggregate aggregate = bucket.aggregations().get("halfYearAgg");
- List<DateHistogramBucket> buckets = aggregate.dateHistogram().buckets().array();
- long h1Count = 0;
- long h2Count = 0;
- for (DateHistogramBucket histogramBucket : buckets) {
- String key = histogramBucket.keyAsString();
- Date month = monthFormat.parse(key);
- EsCountDetailDTO dto = new EsCountDetailDTO();
- dto.setField(field);
- dto.setName(year + "-H1");
- dto.setNumber(h1Count);
- dto.setTopN(topN);
- EsCountDetailDTO dto2 = new EsCountDetailDTO();
- dto2.setField(field);
- dto2.setName(year + "-H2");
- dto2.setNumber(h2Count);
- dto2.setTopN(topN);
- if (month.before(halfYearDate)) {
- h1Count += histogramBucket.docCount();
- dto.setNumber(h1Count);
- }
- if (month.after(halfYearDate)) {
- h2Count += histogramBucket.docCount();
- dto2.setNumber(h2Count);
- }
- detailDTOList.add(dto);
- detailDTOList.add(dto2);
- }
- }
- Map<String, List<EsCountDetailDTO>> map = detailDTOList.stream().collect(Collectors.groupingBy(EsCountDetailDTO::getName));
- for (String key : map.keySet()) {
- List<EsCountDetailDTO> dtoList = map.get(key);
- long sum = dtoList.stream().mapToLong(EsCountDetailDTO::getNumber).sum();
- EsCountDetailDTO detailDTO = dtoList.get(0);
- detailDTO.setNumber(sum);
- esCountDetailDTOS.add(detailDTO);
- }
- } else if (StringUtils.isNotEmpty(format) && format.equals("季")) {
- for (DateHistogramBucket bucket : list) {
- String yearMonth = bucket.keyAsString();
- String year = yearMonth.substring(0, yearMonth.indexOf("-"));
- String firstQuarter = year + "-01";
- String secondQuarter = year + "-04";
- String thirdQuarter = year + "-07";
- String forthQuarter = year + "-10";
- String firstQ = year + "-Q1";
- String secondQ = year + "-Q2";
- String thirdQ = year + "-Q3";
- String forthQ = year + "-Q4";
- EsCountDetailDTO dto = new EsCountDetailDTO();
- dto.setField(field);
- Aggregate aggregate = bucket.aggregations().get("filter_agg");
- if (yearMonth.equals(firstQuarter)) {
- dto.setName(firstQ);
- } else if (yearMonth.equals(secondQuarter)) {
- dto.setName(secondQ);
- } else if (yearMonth.equals(thirdQuarter)) {
- dto.setName(thirdQ);
- } else if (yearMonth.equals(forthQuarter)) {
- dto.setName(forthQ);
- }
- dto.setNumber(bucket.docCount());
- dto.setTopN(topN);
- if (aggregate != null) {
- dto.setNumber(aggregate.filter().docCount());
- }
- esCountDetailDTOS.add(dto);
- }
- } else {
- list.forEach(bucket -> {
- EsCountDetailDTO dto = new EsCountDetailDTO();
- dto.setField(field);
- Aggregate aggregate = bucket.aggregations().get("filter_agg");
- dto.setName(bucket.keyAsString());
- dto.setNumber(bucket.docCount());
- dto.setTopN(topN);
- if (aggregate != null) {
- dto.setNumber(aggregate.filter().docCount());
- }
- esCountDetailDTOS.add(dto);
- });
- }
- if (!CollectionUtils.isEmpty(esCountDetailDTOS)) {
- List<EsCountDetailDTO> collect = esCountDetailDTOS.stream()
- .sorted(Comparator.comparing(EsCountDetailDTO::getName).reversed()).limit(topN).collect(Collectors.toList());
- detailDTOS.addAll(collect);
- }
- // EsCountDetailDTO countDTO = new EsCountDetailDTO();
- // countDTO.setField(field);
- // countDTO.setName("其他");
- // if (list.size() > topN) {
- // int sum = list.size() - topN;
- // countDTO.setNumber(Long.valueOf(String.valueOf(sum)));
- //
- // } else {
- // countDTO.setNumber(0L);
- // }
- // countDTO.setTopN(topN);
- // detailDTOS.add(countDTO);
- }
- /**
- * 获取dateHistogram分析后数据
- *
- * @param agg
- * @param field
- * @param topN
- * @param detailDTOS
- */
- public void getDateAnalysisDTOS(Aggregate agg, String field, Integer topN, List<EsCountDetailDTO> detailDTOS,
- EsCountDTO esCountDTO, String firstName,
- Map<String, List<EsCountDetailDTO>> map,List<String> values) {
- List<RangeBucket> list1 = agg.dateRange().buckets().array();
- List<EsCountDetailDTO> esCountDetailDTOS = new ArrayList<>();
- for (RangeBucket bucket : list1) {
- EsCountDetailDTO dto = new EsCountDetailDTO();
- dto.setFirstName(firstName);
- dto.setField(field);
- dto.setName(bucket.key());
- dto.setNumber(bucket.docCount());
- dto.setTopN(topN);
- esCountDetailDTOS.add(dto);
- }
- if (!CollectionUtils.isEmpty(esCountDetailDTOS)) {
- List<EsCountDetailDTO> collect = esCountDetailDTOS.stream()
- .sorted(Comparator.comparing(EsCountDetailDTO::getName).reversed()).limit(topN).collect(Collectors.toList());
- List<String> strs = new ArrayList<>();
- if (!CollectionUtils.isEmpty(values)) {
- strs.addAll(values);
- }
- if (StringUtils.isNotEmpty(firstName)) {
- collect.removeIf(dto -> dto.getName().equals("其他"));
- if (collect.size() != values.size()) {
- for (EsCountDetailDTO detailDTO : collect) {
- strs.removeIf(i -> i.equals(detailDTO.getName()));
- }
- for (String value : strs) {
- EsCountDetailDTO detailDTO = new EsCountDetailDTO();
- detailDTO.setName(value);
- detailDTO.setFirstName(firstName);
- detailDTO.setField(field);
- detailDTO.setNumber(0L);
- detailDTO.setTopN(topN);
- collect.add(detailDTO);
- }
- } else if (collect.isEmpty()) {
- for (String value : values) {
- EsCountDetailDTO detailDTO = new EsCountDetailDTO();
- detailDTO.setName(value);
- detailDTO.setFirstName(firstName);
- detailDTO.setField(field);
- detailDTO.setNumber(0L);
- detailDTO.setTopN(topN);
- collect.add(detailDTO);
- }
- }
- map.put(firstName, collect);
- } else {
- detailDTOS.addAll(collect);
- }
- }
- }
- /**
- * 获取nested聚合后数据
- *
- * @param agg
- * @param field
- * @param detailDTOS
- */
- public void getNestedCountDTOS(Aggregate agg, String field, Integer topN, List<EsCountDetailDTO> detailDTOS,
- EsCountDTO esCountDTO, String firstName,
- Map<String, List<EsCountDetailDTO>> map, List<String> values) {
- Aggregate termsAgg = agg.nested().aggregations().get("terms_agg");
- List<EsCountDetailDTO> countDetailDTOS = new ArrayList<>();
- EsCountDetailDTO countDTO = new EsCountDetailDTO();
- countDTO.setField(field);
- countDTO.setName("其他");
- countDTO.setTopN(topN);
- Long count = termsAgg.sterms().sumOtherDocCount();
- if (count > 0) {
- countDTO.setNumber(count);
- countDetailDTOS.add(countDTO);
- }
- List<StringTermsBucket> list = termsAgg.sterms().buckets().array();
- list.forEach(bucket -> {
- EsCountDetailDTO dto = new EsCountDetailDTO();
- dto.setField(field);
- Aggregate aggregate = bucket.aggregations().get("filter_agg");
- String value = bucket.key().stringValue();
- if (StringUtils.isNotEmpty(value)) {
- dto.setName(value);
- dto.setNumber(bucket.docCount());
- dto.setTopN(topN);
- dto.setFirstName(firstName);
- if (aggregate != null) {
- dto.setNumber(aggregate.filter().docCount());
- }
- countDetailDTOS.add(dto);
- }
- // if (dto.getNumber() > 0) {
- // detailDTOS.add(dto);
- // }
- });
- List<String> strs = new ArrayList<>();
- if (!CollectionUtils.isEmpty(values)) {
- strs.addAll(values);
- }
- if (StringUtils.isNotEmpty(firstName)) {
- countDetailDTOS.removeIf(dto -> dto.getName().equals("其他"));
- if (countDetailDTOS.size() != values.size()) {
- for (EsCountDetailDTO detailDTO : countDetailDTOS) {
- strs.removeIf(i -> i.equals(detailDTO.getName()));
- }
- for (String value : strs) {
- EsCountDetailDTO detailDTO = new EsCountDetailDTO();
- detailDTO.setName(value);
- detailDTO.setFirstName(firstName);
- detailDTO.setField(field);
- detailDTO.setNumber(0L);
- detailDTO.setTopN(topN);
- countDetailDTOS.add(detailDTO);
- }
- } else if (countDetailDTOS.isEmpty()) {
- for (String value : values) {
- EsCountDetailDTO detailDTO = new EsCountDetailDTO();
- detailDTO.setName(value);
- detailDTO.setFirstName(firstName);
- detailDTO.setField(field);
- detailDTO.setNumber(0L);
- detailDTO.setTopN(topN);
- countDetailDTOS.add(detailDTO);
- }
- }
- map.put(firstName, countDetailDTOS);
- } else {
- detailDTOS.addAll(countDetailDTOS);
- }
- }
- }
|