springboot3 多个es

转载
mob64ca13fa6a3c 2024-11-27 16:54:51
文章标签 springboot3 多个es elasticsearch java 搜索引擎 spring 文章分类 架构后端开发


pom.xml 文件：

<dependency>
    <groupId>org.elasticsearch.client</groupId>
    <artifactId>elasticsearch-rest-high-level-client</artifactId>
    <version>7.13.3</version>
</dependency>

yml文件：

# 开发环境数据源配置
spring:
    elasticsearch:
        rest:
            uris: http://192.168.1.198:9200
    datasource:
        type: com.alibaba.druid.pool.DruidDataSource
        driverClassName: com.mysql.cj.jdbc.Driver
        druid:
            # 主库数据源
            master:
                url: jdbc:mysql://rm-uf658ly8804f04r25ro.mysql.rds.aliyuncs.com:3306/realize-vue?useUnicode=true&characterEncoding=utf8&zeroDateTimeBehavior=convertToNull&useSSL=true&serverTimezone=GMT%2B8
                username: root
                password: 123456
            # 从库数据源
            slave:
                # 从数据源开关/默认关闭
                enabled: false
                url:
                username:
                password:
            # 初始连接数
            initialSize: 5
            # 最小连接池数量
            minIdle: 10
            # 最大连接池数量
            maxActive: 20
            # 配置获取连接等待超时的时间
            maxWait: 60000
            # 配置间隔多久才进行一次检测，检测需要关闭的空闲连接，单位是毫秒
            timeBetweenEvictionRunsMillis: 60000
            # 配置一个连接在池中最小生存的时间，单位是毫秒
            minEvictableIdleTimeMillis: 300000
            # 配置一个连接在池中最大生存的时间，单位是毫秒
            maxEvictableIdleTimeMillis: 900000
            # 配置检测连接是否有效
            validationQuery: SELECT 1 FROM DUAL
            testWhileIdle: true
            testOnBorrow: false
            testOnReturn: false
            webStatFilter:
                enabled: true
            statViewServlet:
                enabled: true
                # 设置白名单，不填则允许所有访问
                allow:
                url-pattern: /druid/*
                # 控制台管理用户名和密码
                login-username: ruoyi
                login-password: 123456
            filter:
                stat:
                    enabled: true
                    # 慢SQL记录
                    log-slow-sql: true
                    slow-sql-millis: 1000
                    merge-sql: true
                wall:
                    config:
                        multi-statement-allow: true
    # redis 配置
    redis:
        # 地址
        host: 192.168.1.11
        # 端口，默认为6379
        port: 6379
        # 数据库索引
        database: 0
        # 密码
        password:
        # 连接超时时间
        timeout: 10s
        lettuce:
            pool:
                # 连接池中的最小空闲连接
                min-idle: 0
                # 连接池中的最大空闲连接
                max-idle: 8
                # 连接池的最大数据库连接数
                max-active: 8
                # #连接池最大阻塞等待时间（使用负值表示没有限制）
                max-wait: -1ms
##es地址
data:
    elasticsearch:
        cluster-nodes: 127.0.0.1:9300
        cluster-name: my-application
        repositories:
            enabled: true
http.cors.enabled: true
http.cors.allow-origin: "*"




package com.realize.project.system.controller;

import com.realize.framework.web.controller.EsBaseController;
import com.realize.framework.web.page.TableDataInfo;
import com.realize.project.system.service.ILawsAndRegulationsService;
import com.realize.project.system.service.ISysESLawsAndRegulationsService;
import org.apache.lucene.search.TotalHits;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.transaction.annotation.Transactional;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;

import java.io.IOException;
import java.util.List;
import java.util.Map;

/**
 * @Author hzj
 * @Date 2022/1/6 18:44
 * @Desc Es
 **/
@RestController
@Transactional(rollbackFor = Exception.class)
@RequestMapping("/system/SysESLawsAndRegulationsController")
public class SysESLawsAndRegulationsController extends EsBaseController {
    @Autowired
    ISysESLawsAndRegulationsService ISysESLawsAndRegulationsService;

    @Autowired
    ILawsAndRegulationsService iLawsAndRegulationsService;

    @GetMapping("/a")
    public String a() {
        return "a";
    }
    //删除索引
    @GetMapping("/deleteIndex")
    public Boolean deleteIndex() {
        return iLawsAndRegulationsService.deleteIndex();
    }

    //创建索引，存入数据
    @GetMapping("/create")
    public Boolean create() throws IOException {
        return iLawsAndRegulationsService.esTest();
    }




    /**
     * @Author HZJ
     * @Description 法规条件检索
     * @date  2022/1/11 10:51
     * @param marketNum 市场板块参数
     * @param incentiveNum 激励工具参数
     * @param enterpriseNum 企业性质参数
     * @param title 标题关键字
     * @param fullText 全文关键字
     * @param theValidity 有效性
     * @return  com.realize.framework.web.page.TableDataInfo
     **/
    @GetMapping("/getESLawsAndRegulations")
    public TableDataInfo getText(String marketNum,
                                 String incentiveNum,
                                 String enterpriseNum,
                                 String title,
                                 String fullText,
                                 Integer theValidity,
                                 Integer pageNum,
                                 Integer pageSize
    ) throws IOException {
//        System.out.println("参数"+marketNum +","+ incentiveNum +","+ enterpriseNum +","+title +","+fullText +","+theValidity);
//        System.out.println("分页："+pageNum+","+pageSize);
        List<Map<String, Object>> esList = ISysESLawsAndRegulationsService.getEsLawsAndRegulations(marketNum, incentiveNum, enterpriseNum, title, fullText, theValidity,pageNum,pageSize);
        Long esTotal = ISysESLawsAndRegulationsService.getEsTotal(marketNum, incentiveNum, enterpriseNum, title, fullText, theValidity, pageNum, pageSize);
        return getDataTable(esList,esTotal);

    }


}


-----------------es 分页查询 service----------------

package com.realize.project.system.service.impl;

import com.realize.project.system.service.ISysESLawsAndRegulationsService;
import org.elasticsearch.action.search.SearchRequest;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.client.RequestOptions;
import org.elasticsearch.client.RestHighLevelClient;
import org.elasticsearch.common.text.Text;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.index.query.*;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.SearchHits;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder;
import org.elasticsearch.search.fetch.subphase.highlight.HighlightField;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;

import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.concurrent.TimeUnit;

/**
 * @Author hzj
 * @Date 2022/1/9 18:18
 * @Desc
 **/
@Service
public class SysESLawsAndRegulationsServiceImpl implements ISysESLawsAndRegulationsService {

    private static final Logger logger = LoggerFactory.getLogger(SysESLawsAndRegulationsServiceImpl.class);

    //集群名称
    //public static final String CLUSTER_NAME = "my-application";
    @Value("${data.elasticsearch.cluster-name}")
    String cluster_name ;

    @Autowired
    RestHighLevelClient restHighLevelClient;

    /**
     * @Author HZJ
     * @Description ES法规（条件）检索
     * @date  2022/1/10 10:43
     * @param marketNum 市场板块
     * @param incentiveNum 激励工具
     * @param enterpriseNum 企业性质
     * @param title 标题关键字
     * @param fullText 全文关键字
     * @param theValidity 有效性
     * @return  List<Map<String,Object>> 数据结果集
     **/
    @Override
    public List<Map<String, Object>> getEsLawsAndRegulations(
            String marketNum,
            String incentiveNum,
            String enterpriseNum,
            String title,
            String fullText,
            Integer theValidity,
            Integer pageNum,
            Integer pageSize
    ) throws IOException {
        System.out.println(cluster_name);
        //创建索引请求
        SearchRequest searchRequest = new SearchRequest(cluster_name);
        //实例化
        SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
        //实例化
        BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();
        //初始化每个字段的属性
        MatchPhraseQueryBuilder marketNumTQ = null;
        MatchPhraseQueryBuilder incentiveNumTQ = null;
        MatchPhraseQueryBuilder enterpriseNumTQ = null;
        MatchPhraseQueryBuilder titleTQ = null;
        MatchPhraseQueryBuilder fullTextTQ = null;
        MatchPhraseQueryBuilder theValidityTQ = null;

        if (marketNum != null && incentiveNum != null && enterpriseNum != null && theValidity != null &&
                marketNum != "" && incentiveNum != "" && enterpriseNum != ""
        ) {
            //如果是全部，则市场板块不成为条件，如果不是全部，则成为条件
            if (!marketNum.equals("AllMarket")) {
                marketNumTQ = QueryBuilders.matchPhraseQuery(marketNum,1);
                boolQueryBuilder.must(marketNumTQ);
            }
            //如果是全部，则市场板块不成为条件，如果不是全部，则成为条件
            if (!incentiveNum.equals("Allincentive")) {
                incentiveNumTQ = QueryBuilders.matchPhraseQuery(incentiveNum,1);
                boolQueryBuilder.must(incentiveNumTQ);
            }
            //如果是全部，则市场板块不成为条件，如果不是全部，则成为条件
            if (!enterpriseNum.equals("Allenterprise")) {
                enterpriseNumTQ = QueryBuilders.matchPhraseQuery(enterpriseNum,1);
                boolQueryBuilder.must(enterpriseNumTQ);
            }
            //有效性为全部，则不用添加有效性条件
            if (theValidity != 2) {
                //matchPhraseQuery不分词
                theValidityTQ = QueryBuilders.matchPhraseQuery("theValidity",theValidity);
                boolQueryBuilder.must(theValidityTQ);
            }
            if (title != null && title != "") {
                titleTQ = QueryBuilders.matchPhraseQuery("lawsRegulationsName",title);
                boolQueryBuilder.must(titleTQ);
            }
            if (fullText != null && fullText != "") {
                fullTextTQ = QueryBuilders.matchPhraseQuery("contentInfo",fullText);
                boolQueryBuilder.must(fullTextTQ);
            }
        }

        //全文高亮 标题高亮
        String preTags="<span style='font-weight: bolder;color:red'>";
        String postTags="</span>";
        if ((fullText !=null & fullText !="") || (title !=null && title != "")) {
            HighlightBuilder highlightBuilder = new HighlightBuilder();
            highlightBuilder.field("contentInfo").field("lawsRegulationsName");
            highlightBuilder.preTags("<span style='font-weight: bolder;color:red'>");
            highlightBuilder.postTags("</span>");
            searchSourceBuilder.highlighter(highlightBuilder);
        }
        //分页
        if(pageNum<=0){
            pageNum = 1;
        }
        if(pageSize<=0){
            pageSize=10;
        }
        //起始记录下标
        int from = (pageNum-1) * pageSize;
        searchSourceBuilder.from(from);
        searchSourceBuilder.size(pageSize);

        //执行查询
        searchSourceBuilder.query(boolQueryBuilder);

        //打印
        System.out.println("检索：   " + searchSourceBuilder.toString());
        //不能超过60秒
        searchSourceBuilder.timeout(new TimeValue(60, TimeUnit.SECONDS));
        searchSourceBuilder.trackTotalHits(true);
        //检索
        searchRequest.source(searchSourceBuilder);
        //数据结果集初始化
        List<Map<String, Object>> list = new ArrayList<>();
        //客户端调用检索语法
        SearchResponse searchResponse = restHighLevelClient.search(searchRequest, RequestOptions.DEFAULT);
        //获取响应结果
        SearchHits hits = searchResponse.getHits();
        //记录总数
        Long totalHits = hits.getTotalHits().value;
//        System.out.println("命中数：" + totalHits);
        SearchHit[] hitss = searchResponse.getHits().getHits();
//        System.out.println("命中：" + hitss.length);

        //循环存入结果集
        for (SearchHit hit : searchResponse.getHits().getHits()) {
            if ((title != null && title != "") || (fullText != null && fullText != "")) {
                //设置标题高亮
                Map<String, HighlightField> titlehighlightFields = hit.getHighlightFields();
                HighlightField titleHight = titlehighlightFields.get("lawsRegulationsName");
                //设置全文高亮
                Map<String, HighlightField> fullTexthighlightFields = hit.getHighlightFields();
                HighlightField fullTextHight = fullTexthighlightFields.get("contentInfo");

                Map<String, Object> sourceAsMap = hit.getSourceAsMap();
                String contentInfo = (String) sourceAsMap.get("contentInfo");
                //标题-高亮词替换
                if (titleHight != null ){
                    Text[] fragments = titleHight.fragments();
                    StringBuilder newtitle = new StringBuilder();
                    for (Text text:fragments) {
                        newtitle.append(text);
                    }
                    sourceAsMap.replace("lawsRegulationsName", newtitle);
                }
                //全文-高亮词替换
                if (fullTextHight != null ){
                    Text[] fragments = fullTextHight.getFragments();
                    StringBuilder newfullText = new StringBuilder();
                    for (Text text:fragments) {
                        newfullText.append(text);
                    }
                    String strc = contentInfo.replace(fullText,preTags+ fullText +postTags);
                    sourceAsMap.put("contentInfo", strc);
                }
                list.add(sourceAsMap);
            } else {
                list.add(hit.getSourceAsMap());
            }
        }
        System.out.println("结果集长度：" + list.size());

        //返回结果集
        return list;
    }

    /**
     * @Author HZJ
     * @Description ES法规检索命中条数
     * @date  2022/1/15 15:03
     * @param marketNum
     * @param incentiveNum
     * @param enterpriseNum
     * @param title
     * @param fullText
     * @param theValidity
     * @param pageNum
     * @param pageSize
     * @return  int  命中条数
     **/
    @Override
    public Long getEsTotal(String marketNum, String incentiveNum, String enterpriseNum, String title, String fullText, Integer theValidity, Integer pageNum, Integer pageSize) throws IOException {
        //创建索引请求
        SearchRequest searchRequest = new SearchRequest(cluster_name);
        //实例化
        SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
        //实例化
        BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();
        //初始化每个字段的属性
        MatchPhraseQueryBuilder marketNumTQ = null;
        MatchPhraseQueryBuilder incentiveNumTQ = null;
        MatchPhraseQueryBuilder enterpriseNumTQ = null;
        MatchPhraseQueryBuilder titleTQ = null;
        MatchPhraseQueryBuilder fullTextTQ = null;
        MatchPhraseQueryBuilder theValidityTQ = null;

        if (marketNum != null && incentiveNum != null && enterpriseNum != null && theValidity != null &&
                marketNum != "" && incentiveNum != "" && enterpriseNum != ""
        ) {
            //如果是全部，则市场板块不成为条件，如果不是全部，则成为条件
            if (!marketNum.equals("AllMarket")) {
                marketNumTQ = QueryBuilders.matchPhraseQuery(marketNum,1);
                boolQueryBuilder.must(marketNumTQ);
            }
            //如果是全部，则市场板块不成为条件，如果不是全部，则成为条件
            if (!incentiveNum.equals("Allincentive")) {
                incentiveNumTQ = QueryBuilders.matchPhraseQuery(incentiveNum,1);
                boolQueryBuilder.must(incentiveNumTQ);
            }
            //如果是全部，则市场板块不成为条件，如果不是全部，则成为条件
            if (!enterpriseNum.equals("Allenterprise")) {
                enterpriseNumTQ = QueryBuilders.matchPhraseQuery(enterpriseNum,1);
                boolQueryBuilder.must(enterpriseNumTQ);
            }
            //有效性为全部，则不用添加有效性条件
            if (theValidity != 2) {
                //matchPhraseQuery不分词
                theValidityTQ = QueryBuilders.matchPhraseQuery("theValidity",theValidity);
                boolQueryBuilder.must(theValidityTQ);
            }
            if (title != null && title != "") {
                titleTQ = QueryBuilders.matchPhraseQuery("lawsRegulationsName",title);
                boolQueryBuilder.must(titleTQ);
            }
            if (fullText != null && fullText != "") {
                fullTextTQ = QueryBuilders.matchPhraseQuery("contentInfo",fullText);
                boolQueryBuilder.must(fullTextTQ);
            }
        }
        //分页
        if(pageNum<=0){
            pageNum = 1;
        }
        if(pageSize<=0){
            pageSize=10;
        }
        //起始记录下标
        int from = (pageNum-1) * pageSize;
        searchSourceBuilder.from(from);
        searchSourceBuilder.size(pageSize);

        //执行查询
        searchSourceBuilder.query(boolQueryBuilder);

        //打印
        System.out.println("检索：   " + searchSourceBuilder.toString());
        //不能超过60秒
        searchSourceBuilder.timeout(new TimeValue(60, TimeUnit.SECONDS));
        //检索
        searchRequest.source(searchSourceBuilder);
        //客户端调用检索语法
        SearchResponse searchResponse = restHighLevelClient.search(searchRequest, RequestOptions.DEFAULT);
        //获取响应结果
        SearchHits hits = searchResponse.getHits();
        Long totalHits = hits.getTotalHits().value;
        //记录总数
//        System.out.println("命中数："+totalHits);
        //返回总记录数
        return totalHits;
    }

}


--------------mysql表数据同步到es  service -----------------------------
package com.realize.project.system.service.impl;

import com.alibaba.fastjson.JSON;
import com.realize.project.system.domain.SyslawsAndRegulations;
import com.realize.project.system.domain.dto.LawsAndRegulationsDto;
import com.realize.project.system.mapper.LawsAndRegulationsMapper;
import com.realize.project.system.service.ILawsAndRegulationsService;
import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest;
import org.elasticsearch.action.bulk.BulkRequest;
import org.elasticsearch.action.bulk.BulkResponse;
import org.elasticsearch.action.index.IndexRequest;
import org.elasticsearch.action.search.SearchRequest;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.action.support.IndicesOptions;
import org.elasticsearch.action.support.master.AcknowledgedResponse;
import org.elasticsearch.client.RequestOptions;
import org.elasticsearch.client.RestHighLevelClient;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.index.query.*;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;

import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.TimeUnit;

/**
 * @Author hzj
 * @Date 2021/12/27 17:06
 * @Desc
 **/
@Service
public class SysLawsAndRegulationsServiceImpl implements ILawsAndRegulationsService {
    @Value("${data.elasticsearch.cluster-name}")
    String cluster_name ;

    @Autowired
    LawsAndRegulationsMapper lawsAndRegulationsMapper;
    @Autowired
    RestHighLevelClient restHighLevelClient;

    /**
     * @Author HZJ
     * @Description 法规列表
     * @date  2021/12/27 17:06
     * @param lawsAndRegulationsDto 法规列表参数对象 为空查询则所有
     * @return  List<LawsAndRegulationsInfo> 法规数据列表
     **/
    @Override
    public List<SyslawsAndRegulations> select(LawsAndRegulationsDto lawsAndRegulationsDto) {
        List<SyslawsAndRegulations> lawsAndRegulationsInfoList = lawsAndRegulationsMapper.select(lawsAndRegulationsDto);
        return lawsAndRegulationsInfoList;
    }

    /**
     * @Author HZJ
     * @Description 查询表中数据存入到ES
     * @date  2022/1/10 10:28
     * @return  java.lang.Boolean
     **/
    @Override
    public Boolean esTest() throws IOException {
        //查询出所有数据
        List<SyslawsAndRegulations> searchList = lawsAndRegulationsMapper.searchText();


        try{
            BulkRequest bulkRequest = new BulkRequest();
            bulkRequest.timeout("2m");
            for (int i = 0; i < searchList.size(); i++) {
                bulkRequest.add(
                        new IndexRequest(cluster_name).source(JSON.toJSONString(searchList.get(i)),XContentType.JSON)
                );
            }
            BulkResponse bulk = restHighLevelClient.bulk(bulkRequest, RequestOptions.DEFAULT);
            return !bulk.hasFailures();
        }catch (Exception e){
            e.printStackTrace();
        }
        return null;
    }

    /**
     * @Author HZJ
     * @Description 删除索引
     * @date  2022/1/11 10:58
     * @return  java.lang.Boolean
     **/
    @Override
    public Boolean deleteIndex(){
        boolean acknowledged = false;
        try {
            DeleteIndexRequest deleteIndexRequest = new DeleteIndexRequest(cluster_name);
            deleteIndexRequest.indicesOptions(IndicesOptions.LENIENT_EXPAND_OPEN);
            AcknowledgedResponse delete = restHighLevelClient.indices().delete(deleteIndexRequest, RequestOptions.DEFAULT);
            acknowledged = delete.isAcknowledged();
            if (acknowledged=true) {
                System.out.println("删除完成");
            }
        } catch (IOException e) {
            e.printStackTrace();
        }
        return acknowledged;
    }

    /**
     * @Author HZJ
     * @Description 获取索引根据条件查询
     * @date  2022/1/10 10:28
     * @param lawsAndRegulationsDto
     * @return  java.util.List<java.util.Map<java.lang.String,java.lang.Object>>
     **/
    @Override
    public List<Map<String, Object>> getEs(LawsAndRegulationsDto lawsAndRegulationsDto) throws IOException {
        //获取索引请求
        SearchRequest searchRequest = new SearchRequest(cluster_name);
        //搜索条件
        SearchSourceBuilder sourceBuilder = new SearchSourceBuilder();

        //TermQueryBuilder termQueryBuilder1 = QueryBuilders.termQuery("lawsRegulationsName", lawsAndRegulationsDto.getTitle());
        QueryBuilder queryBuilder = QueryBuilders.boolQuery()
                .must(QueryBuilders.matchQuery("lawsRegulationsName",lawsAndRegulationsDto.getTitle()).operator(Operator.AND))
                .must(QueryBuilders.matchQuery("contentInfo",lawsAndRegulationsDto.getFullText()).operator(Operator.AND));

        //执行查询
        sourceBuilder.query(queryBuilder);
        sourceBuilder.timeout(new TimeValue(60,TimeUnit.SECONDS));
        searchRequest.source(sourceBuilder);
        List<Map<String, Object>> list = new ArrayList<>();
        SearchResponse searchResponse = restHighLevelClient.search(searchRequest, RequestOptions.DEFAULT);
        for (SearchHit documentFields : searchResponse.getHits().getHits()) {
            list.add(documentFields.getSourceAsMap());
        }
        return list;
    }

}
本文章为转载内容，我们尊重原作者对文章享有的著作权。如有内容错误或侵权问题，欢迎原作者联系我们进行内容更正或删除文章。