• 首页
  • 邻居
  • 关于
  • 归档
  • 搜索
  • 夜间模式
    ©2020-2026  我的学习笔记 Theme by OneBlog

    我的学习笔记博客

    搜索
    标签
    # 随笔 # Java # 教程 # openwrt # Mysql # SQL # 爬虫 # post # Js调优 # MAVEN
  • 首页>
  • 随笔>
  • 正文
  • 怎么把达梦数据库文件sql文件导入es中

    2024年04月09日 1.4 k 阅读 0 评论 30145 字

    我是把达梦数据库文件导出到sql文件中
    然后再解析sql文件中分批导入到es中

    package com.jinw.es;
    
    import java.io.BufferedReader;
    import java.io.FileReader;
    import java.io.IOException;
    import java.util.ArrayList;
    import java.util.HashMap;
    import java.util.List;
    import java.util.Map;
    import java.util.regex.Matcher;
    import java.util.regex.Pattern;
    
    import cn.hutool.core.io.FileUtil;
    import cn.hutool.core.lang.Assert;
    import cn.hutool.core.util.ObjectUtil;
    import cn.hutool.core.util.StrUtil;
    import com.github.xiaoymin.knife4j.annotations.ApiOperationSupport;
    import com.github.xiaoymin.knife4j.annotations.ApiSort;
    import com.google.common.collect.Lists;
    import com.jinw.activity.service.IProcessFormService;
    import com.jinw.base.model.PageInfo;
    import com.jinw.base.model.ProcessForm;
    import com.jinw.base.model.QueryVo;
    import com.jinw.base.model.RestResult;
    import com.jinw.modules.applyCar.domain.ApplyCarParameterVo;
    import com.jinw.modules.business.service.IFlowCommonService;
    import com.jinw.modules.car.domain.UserCarVo;
    import com.jinw.modules.card.domain.UserCardVo;
    import com.jinw.modules.excelLogs.domain.IssueVo;
    import com.jinw.utils.StringUtils;
    import com.jinw.utils.poi.ExcelUtil;
    import io.swagger.annotations.Api;
    import io.swagger.annotations.ApiImplicitParam;
    import io.swagger.annotations.ApiImplicitParams;
    import io.swagger.annotations.ApiOperation;
    import lombok.extern.slf4j.Slf4j;
    import org.elasticsearch.action.support.WriteRequest;
    import org.elasticsearch.client.core.CountRequest;
    import org.elasticsearch.client.core.CountResponse;
    import org.elasticsearch.core.TimeValue;
    import org.elasticsearch.index.query.BoolQueryBuilder;
    import org.elasticsearch.index.query.MatchPhrasePrefixQueryBuilder;
    import org.elasticsearch.index.query.QueryBuilders;
    import org.elasticsearch.search.builder.SearchSourceBuilder;
    import org.elasticsearch.search.sort.SortBuilders;
    import org.elasticsearch.search.sort.SortOrder;
    import org.springframework.beans.factory.annotation.Autowired;
    import org.springframework.security.access.prepost.PreAuthorize;
    import org.springframework.web.bind.annotation.*;
    import org.springframework.context.annotation.Lazy;
    import com.jinw.base.controller.BaseCommonController;
    import com.jinw.modules.excelLogs.domain.BTExcelLogs;
    import com.jinw.modules.excelLogs.service.IBTExcelLogsService;
    import org.springframework.web.multipart.MultipartFile;
    
    import javax.servlet.http.HttpServletRequest;
    
    /**
     * @author liuxin
     * @Description:
     * @date 2024-04-08
     */
    
    @RestController
    @RequestMapping("/es")
    @Api(tags = "es")
    @Slf4j
    public class EsController {
    
        @Autowired
        private EsDataOperation esDataOperation;
        @Autowired
        private EsIndexOperation esIndexOperation;
        @Autowired
        private EsQueryOperation esQueryOperation;
    
        @PostMapping("createIndex")
        public void createIndex(String indexName, String filePath) {
      
            boolean b = esIndexOperation.createIndex(indexName, filePath);
            Assert.isTrue(b);
        }
    
        @PostMapping("insert")
        public void insert(String indexName, String filePath) throws IOException {
          
            parseSqlFile(filePath, indexName);
        }
    
        public void parseSqlFile(String filePath, String indexName) {
            int batchSize = 100000; // 设置批次大小
            List<Map<String, Object>> dataList = null;
            try (BufferedReader reader = new BufferedReader(new FileReader(filePath))) {
                int totalLines = FileUtil.getTotalLines(FileUtil.file(filePath)); // 获取总行数
                System.out.println("Total lines in SQL file: " + totalLines);
                int batchCount = (int) Math.ceil((double) totalLines / batchSize); // 计算批次数量
                System.out.println("Total batches to execute: " + batchCount);
                // 循环读取数据
                for (int batchIndex = 0; batchIndex < batchCount; batchIndex++) {
                    int linesRead = 0;
                    dataList = Lists.newArrayList();
                    System.out.println("Batch " + (batchIndex + 1) + "/" + batchCount + ":");
    
                    // 读取每个批次的数据
                    String line;
                    while ((line = reader.readLine()) != null && linesRead < batchSize) {
                        if (!line.trim().isEmpty()) { // 判断是否为空行
                            linesRead++;
                            // 在这里处理每一行的数据,例如执行 SQL 操作等
                            // System.out.println("line:" + line);
                            Map<String, Object> map = parseInsertStatement(line);
                            dataList.add(map);
                        }
                    }
                    if (ObjectUtil.isNotEmpty(dataList)) {
                        boolean flag = esDataOperation.batchInsert(indexName, dataList);
                        System.out.println("flag:" + flag);
                    }
                    System.out.println("Batch " + (batchIndex + 1) + " processed. Lines read: " + linesRead);
                }
                System.out.println("Total lines read: " + totalLines);
            } catch (IOException e) {
                e.printStackTrace();
            }
        }
    
        private static Map<String, Object> parseInsertStatement(String line) {
            line = line.replaceAll("INSERT INTO \"xxx\".\"xx_INOUT_LOGS\"\\(", "")
                    .replaceAll("\\)", "").replaceAll(";", "");
            line = line.replaceAll("INSERT INTO \"xxx\".\"xx_GETOUT_LOGS\"\\(", "")
                    .replaceAll("\\)", "").replaceAll(";", "");
            String[] values = line.split("VALUES\\(");
            Map<String, Object> dataMap = new HashMap<>();
            String[] split = values[0].split(",");
            String[] split1 = values[1].split(",");
            for (int i = 0; i < split.length; i++) {
                String stringWithoutQuotes = StrUtil.toCamelCase(split[i].toLowerCase().replaceAll("\"", ""));
                String stringWithoutQuotes2 = split1[i].replaceAll("\"", "").replaceAll("'", "");
                dataMap.put(stringWithoutQuotes, stringWithoutQuotes2);
            }
            return dataMap;
        }
    
        @PostMapping("update")
        public void update() {
            String indexName = "b_t_people_inout_logs";
            HashMap<String, Object> hashMap = new HashMap<>();
            hashMap.put("id", 5);
            hashMap.put("name", "jack7");
            boolean update = esDataOperation.update(indexName, hashMap, WriteRequest.RefreshPolicy.WAIT_UNTIL);
            Assert.isTrue(update);
        }
    
        /**
         * 查询总数
         */
        @GetMapping("count")
        public Long count(String indexName) {
            return esQueryOperation.count(indexName);
        }
    
        @GetMapping("list")
        public List<Map<String, Object>> list(String indexName) {
            // 查询条件
            SearchSourceBuilder sourceBuilder = new SearchSourceBuilder();
    
            //精确查询
            //  BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery();
            //  queryBuilder.must(QueryBuilders.termQuery("personName", "xxx"));
    
            //模糊查询
            //  MatchPhrasePrefixQueryBuilder matchQueryBuilder = QueryBuilders.matchPhrasePrefixQuery("personName", "杨");
    
    
            // 设置模糊查询
            //  sourceBuilder.query(QueryBuilders.fuzzyQuery("personName", "马中明"));
    
            // 设置排序规则
            //sourceBuilder.sort(SortBuilders.fieldSort("eventTime").order(SortOrder.DESC));
            // 设置超时时间
            sourceBuilder.timeout(TimeValue.timeValueSeconds(10));
    
            // 分页查询
            sourceBuilder.from(0);
            sourceBuilder.size(10);
            List<Map<String, Object>> list = esQueryOperation.list(indexName, sourceBuilder);
            return list;
        }
    }
    

    另外如果要排序的话要把字段类型 设置为keyword
    字段映射配置:

    {
        "aliases": {
            "btcgl": {}
        },
        "mappings": {
            "properties": {
                "userId": {
                    "type": "text",
                    "fields": {
                        "keyword": {
                            "type": "keyword",
                            "ignore_above": 256
                        }
                    }
                },
                "quartersId": {
                    "type": "text",
                    "fields": {
                        "keyword": {
                            "type": "keyword",
                            "ignore_above": 256
                        }
                    }
                },
                "carName": {
                    "type": "text",
                    "fields": {
                        "keyword": {
                            "type": "keyword",
                            "ignore_above": 256
                        }
                    }
                },
                "quartersName": {
                    "type": "text",
                    "fields": {
                        "keyword": {
                            "type": "keyword",
                            "ignore_above": 256
                        }
                    }
                },
                "crossTime": {
                    "type": "keyword",
                    "fields": {
                        "keyword": {
                            "type": "keyword",
                            "ignore_above": 256
                        }
                    }
                },
                "plateNo": {
                    "type": "text",
                    "fields": {
                        "keyword": {
                            "type": "keyword",
                            "ignore_above": 256
                        }
                    }
                },
                "crossRecordSyscode": {
                    "type": "text",
                    "fields": {
                        "keyword": {
                            "type": "keyword",
                            "ignore_above": 256
                        }
                    }
                },
                "parkSyscode": {
                    "type": "text",
                    "fields": {
                        "keyword": {
                            "type": "keyword",
                            "ignore_above": 256
                        }
                    }
                },
                "parkName": {
                    "type": "text",
                    "fields": {
                        "keyword": {
                            "type": "keyword",
                            "ignore_above": 256
                        }
                    }
                },
                "entranceSyscode": {
                    "type": "text",
                    "fields": {
                        "keyword": {
                            "type": "keyword",
                            "ignore_above": 256
                        }
                    }
                },
                "entranceName": {
                    "type": "text",
                    "fields": {
                        "keyword": {
                            "type": "keyword",
                            "ignore_above": 256
                        }
                    }
                },
                "roadwaySyscode": {
                    "type": "text",
                    "fields": {
                        "keyword": {
                            "type": "keyword",
                            "ignore_above": 256
                        }
                    }
                },
                "roadwayName": {
                    "type": "text",
                    "fields": {
                        "keyword": {
                            "type": "keyword",
                            "ignore_above": 256
                        }
                    }
                },
                "vehicleOut": {
                    "type": "text",
                    "fields": {
                        "keyword": {
                            "type": "keyword",
                            "ignore_above": 256
                        }
                    }
                },
                "cardNo": {
                    "type": "text",
                    "fields": {
                        "keyword": {
                            "type": "keyword",
                            "ignore_above": 256
                        }
                    }
                },
                "releaseMode": {
                    "type": "text",
                    "fields": {
                        "keyword": {
                            "type": "keyword",
                            "ignore_above": 256
                        }
                    }
                },
                "releaseResult": {
                    "type": "text",
                    "fields": {
                        "keyword": {
                            "type": "keyword",
                            "ignore_above": 256
                        }
                    }
                },
                "releaseWay": {
                    "type": "text",
                    "fields": {
                        "keyword": {
                            "type": "keyword",
                            "ignore_above": 256
                        }
                    }
                },
                "releaseReason": {
                    "type": "text",
                    "fields": {
                        "keyword": {
                            "type": "keyword",
                            "ignore_above": 256
                        }
                    }
                },
                "vehicleColor": {
                    "type": "text",
                    "fields": {
                        "keyword": {
                            "type": "keyword",
                            "ignore_above": 256
                        }
                    }
                },
                "vehicleType": {
                    "type": "text",
                    "fields": {
                        "keyword": {
                            "type": "keyword",
                            "ignore_above": 256
                        }
                    }
                },
                "plateColor": {
                    "type": "text",
                    "fields": {
                        "keyword": {
                            "type": "keyword",
                            "ignore_above": 256
                        }
                    }
                },
                "plateType": {
                    "type": "text",
                    "fields": {
                        "keyword": {
                            "type": "keyword",
                            "ignore_above": 256
                        }
                    }
                },
                "carCategory": {
                    "type": "text",
                    "fields": {
                        "keyword": {
                            "type": "keyword",
                            "ignore_above": 256
                        }
                    }
                },
                "carCategoryName": {
                    "type": "text",
                    "fields": {
                        "keyword": {
                            "type": "keyword",
                            "ignore_above": 256
                        }
                    }
                },
                "orgId": {
                    "type": "text",
                    "fields": {
                        "keyword": {
                            "type": "keyword",
                            "ignore_above": 256
                        }
                    }
                }
            }
        },
        "settings": {
            "index": {
                "number_of_shards": "1",
                "number_of_replicas": "1"
            }
        }
    }
    package com.jinw.modules.utils;
    
    import cn.hutool.core.util.ObjectUtil;
    import com.google.common.collect.Lists;
    import com.jinw.es.EsDataOperation;
    import com.jinw.es.EsQueryOperation;
    import com.jinw.utils.spring.SpringUtils;
    import org.elasticsearch.core.TimeValue;
    import org.elasticsearch.index.query.QueryBuilders;
    import org.elasticsearch.index.query.TermQueryBuilder;
    import org.elasticsearch.search.builder.SearchSourceBuilder;
    import org.elasticsearch.search.sort.SortBuilders;
    import org.elasticsearch.search.sort.SortOrder;
    import org.springframework.beans.factory.annotation.Autowired;
    import org.springframework.stereotype.Component;
    
    import java.util.List;
    import java.util.Map;
    
    /**
     * @author @liuxin
     * @classname ESUtils
     * @date 2024/4/9 8:40
     * @description TODO
     */
    @Component
    public class ESUtils {
        private static EsQueryOperation esQueryOperation = SpringUtils.getBean(EsQueryOperation.class);
        private static EsDataOperation esDataOperation = SpringUtils.getBean(EsDataOperation.class);
    
        public static List<Map<String, Object>> selectByPage(String indexName, SearchSourceBuilder sourceBuilder, int pageNum, int pageSize) {
            // 设置超时时间
            sourceBuilder.timeout(TimeValue.timeValueSeconds(10));
            // 分页查询
            sourceBuilder.from((pageNum - 1) * pageSize);
            sourceBuilder.size(pageSize);
            sourceBuilder.trackTotalHits(true);
            List<Map<String, Object>> result = esQueryOperation.list(indexName, sourceBuilder);
            return ObjectUtil.isNotEmpty(result) ? result : Lists.newArrayList();
        }
    
        public static Long selectByCount(String indexName, SearchSourceBuilder sourceBuilder) {
            return esQueryOperation.countByCondition(indexName, sourceBuilder);
        }
    
    
        public static Map<String, Object> selectOneById(
                String indexName, String recordId) {
            SearchSourceBuilder sourceBuilder = new SearchSourceBuilder();
            TermQueryBuilder termQueryBuilder = QueryBuilders.termQuery("id", recordId);
            // 添加到布尔查询的 should 子句中
            sourceBuilder.query(termQueryBuilder);
            List<Map<String, Object>> list = esQueryOperation.list(indexName, sourceBuilder);
            Map<String, Object> entity = null;
            if (ObjectUtil.isNotEmpty(list)) {
                entity = list.get(0);
            }
            return entity;
        }
    
        public static boolean deleteById(
                String indexName, String recordId) {
            return esDataOperation.delete(indexName, recordId);
        }
    
    
        public static boolean insert(String indexName, Map<String, Object> dataMap) {
            return esDataOperation.insert(indexName, dataMap);
        }
    }
    
    package com.jinw.es.config;
    
    import lombok.Data;
    import lombok.extern.slf4j.Slf4j;
    import org.apache.http.HttpHost;
    import org.apache.http.auth.AuthScope;
    import org.apache.http.auth.UsernamePasswordCredentials;
    import org.apache.http.client.CredentialsProvider;
    import org.apache.http.impl.client.BasicCredentialsProvider;
    import org.elasticsearch.client.RestClient;
    import org.elasticsearch.client.RestClientBuilder;
    import org.elasticsearch.client.RestHighLevelClient;
    import org.springframework.boot.context.properties.ConfigurationProperties;
    import org.springframework.context.annotation.Bean;
    import org.springframework.context.annotation.Configuration;
    
    import java.util.ArrayList;
    import java.util.List;
    
    /**
     * restHighLevelClient 客户端配置类
     */
    @Slf4j
    @Data
    @Configuration
    @ConfigurationProperties(prefix = "elasticsearch")
    public class ElasticsearchConfig {
    
        /**
         * es host ip 地址(集群)
         */
        private String hosts;
        /**
         * es用户名
         */
        private String userName;
        /**
         * es密码
         */
        private String password;
        /**
         * es 请求方式
         */
        private String scheme;
        /**
         * es 连接超时时间
         */
        private int connectTimeOut;
        /**
         * es socket 连接超时时间
         */
        private int socketTimeOut;
        /**
         * es 请求超时时间
         */
        private int connectionRequestTimeOut;
        /**
         * es 最大连接数
         */
        private int maxConnectNum;
        /**
         * es 每个路由的最大连接数
         */
        private int maxConnectNumPerRoute;
    
    
        /**
         * 如果@Bean没有指定bean的名称,那么方法名就是bean的名称
         */
        @Bean(name = "restHighLevelClient")
        public RestHighLevelClient restHighLevelClient() {
            // 构建连接对象
            RestClientBuilder builder = RestClient.builder(getEsHost());
    
            // 连接延时配置
            builder.setRequestConfigCallback(requestConfigBuilder -> {
                requestConfigBuilder.setConnectTimeout(connectTimeOut);
                requestConfigBuilder.setSocketTimeout(socketTimeOut);
                requestConfigBuilder.setConnectionRequestTimeout(connectionRequestTimeOut);
                return requestConfigBuilder;
            });
    
            // 连接数配置
            builder.setHttpClientConfigCallback(httpClientBuilder -> {
                httpClientBuilder.setMaxConnTotal(maxConnectNum);
                httpClientBuilder.setMaxConnPerRoute(maxConnectNumPerRoute);
                httpClientBuilder.setDefaultCredentialsProvider(getCredentialsProvider());
                return httpClientBuilder;
            });
    
            return new RestHighLevelClient(builder);
        }
    
        private HttpHost[] getEsHost() {
            // 拆分地址(es为多节点时,不同host以逗号间隔)
            List<HttpHost> hostLists = new ArrayList<>();
            String[] hostList = hosts.split(",");
            for (String addr : hostList) {
                String host = addr.split(":")[0];
                String port = addr.split(":")[1];
                hostLists.add(new HttpHost(host, Integer.parseInt(port), scheme));
            }
            // 转换成 HttpHost 数组
            return hostLists.toArray(new HttpHost[]{});
        }
    
        private CredentialsProvider getCredentialsProvider() {
            // 设置用户名、密码
            CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
            credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(userName, password));
            return credentialsProvider;
        }
    }
    package com.jinw.es;
    
    import lombok.extern.slf4j.Slf4j;
    import org.elasticsearch.action.bulk.BulkRequest;
    import org.elasticsearch.action.bulk.BulkResponse;
    import org.elasticsearch.action.delete.DeleteRequest;
    import org.elasticsearch.action.index.IndexRequest;
    import org.elasticsearch.action.search.ClearScrollRequest;
    import org.elasticsearch.action.search.SearchRequest;
    import org.elasticsearch.action.search.SearchResponse;
    import org.elasticsearch.action.search.SearchScrollRequest;
    import org.elasticsearch.action.support.WriteRequest;
    import org.elasticsearch.action.update.UpdateRequest;
    import org.elasticsearch.client.RequestOptions;
    import org.elasticsearch.client.RestHighLevelClient;
    import org.elasticsearch.core.TimeValue;
    import org.elasticsearch.index.query.QueryBuilders;
    import org.elasticsearch.search.Scroll;
    import org.elasticsearch.search.SearchHit;
    import org.elasticsearch.search.builder.SearchSourceBuilder;
    import org.elasticsearch.xcontent.XContentType;
    import org.springframework.stereotype.Service;
    
    import javax.annotation.Resource;
    import java.io.IOException;
    import java.util.List;
    import java.util.Map;
    
    /**
     * 增删改数据
     */
    @Slf4j
    @Service
    public class EsDataOperation {
    
        @Resource
        private RestHighLevelClient client;
        private final RequestOptions options = RequestOptions.DEFAULT;
    
        /**
         * 写入数据
         */
        public boolean insert(String indexName, Map<String, Object> dataMap) {
            try {
                BulkRequest request = new BulkRequest();
                request.add(new IndexRequest(indexName).opType("create")
                        .id(dataMap.get("id").toString())
                        .source(dataMap, XContentType.JSON));
                this.client.bulk(request, options);
                return Boolean.TRUE;
            } catch (Exception e) {
                log.error("EsDataOperation insert error.", e);
            }
            return Boolean.FALSE;
        }
    
        /**
         * 批量写入数据
         */
        public boolean batchInsert(String indexName, List<Map<String, Object>> userIndexList) {
            try {
                BulkRequest request = new BulkRequest();
                for (Map<String, Object> dataMap : userIndexList) {
                    request.add(new IndexRequest(indexName).opType("create")
                            .id(dataMap.get("id").toString())
                            .source(dataMap, XContentType.JSON));
                }
                this.client.bulk(request, options);
                return Boolean.TRUE;
            } catch (Exception e) {
                log.error("EsDataOperation batchInsert error.", e);
            }
            return Boolean.FALSE;
        }
    
        /**
         * 根据id更新数据,可以直接修改索引结构
         *
         * @param refreshPolicy 数据刷新策略
         */
        public boolean update(String indexName, Map<String, Object> dataMap, WriteRequest.RefreshPolicy refreshPolicy) {
            try {
                UpdateRequest updateRequest = new UpdateRequest(indexName, dataMap.get("id").toString());
                updateRequest.setRefreshPolicy(refreshPolicy);
                updateRequest.doc(dataMap);
                this.client.update(updateRequest, options);
                return Boolean.TRUE;
            } catch (Exception e) {
                log.error("EsDataOperation update error.", e);
            }
            return Boolean.FALSE;
        }
    
        /**
         * 删除数据
         */
        public boolean delete(String indexName, String id) {
            try {
                DeleteRequest deleteRequest = new DeleteRequest(indexName, id);
                this.client.delete(deleteRequest, options);
                return Boolean.TRUE;
            } catch (Exception e) {
                log.error("EsDataOperation delete error.", e);
            }
            return Boolean.FALSE;
        }
    
    
        /**
         * 批量删除数据
         */
        public boolean batchDelete(String indexName, String fieldName, String fieldValue) {
            try {
                // 设置滚动参数
                Scroll scroll = new Scroll(TimeValue.timeValueMinutes(1L));
    
                // 构建初始查询请求
                SearchRequest searchRequest = new SearchRequest(indexName);
                searchRequest.scroll(scroll);
    
                // 构建查询条件
                SearchSourceBuilder sourceBuilder = new SearchSourceBuilder();
                sourceBuilder.query(QueryBuilders.rangeQuery(fieldName).lt(fieldValue));
                searchRequest.source(sourceBuilder);
    
                // 执行初始查询请求
                SearchResponse searchResponse = client.search(searchRequest, options);
    
                // 获取滚动ID
                String scrollId = searchResponse.getScrollId();
    
                // 构建批量删除请求
                BulkRequest bulkRequest = new BulkRequest();
    
                // 循环获取滚动结果并添加到批量删除请求中
                while (true) {
                    for (SearchHit hit : searchResponse.getHits().getHits()) {
                        bulkRequest.add(new DeleteRequest(indexName, hit.getId()));
                    }
    
                    // 执行批量删除请求
                    BulkResponse bulkResponse = client.bulk(bulkRequest, options);
    
                    // 检查是否还有滚动结果
                    if (bulkResponse.getItems().length == 0) {
                        break;
                    }
    
                    // 继续滚动搜索
                    SearchScrollRequest scrollRequest = new SearchScrollRequest(scrollId);
                    scrollRequest.scroll(scroll);
                    searchResponse = client.scroll(scrollRequest, options);
                }
    
                // 清除滚动ID
                ClearScrollRequest clearScrollRequest = new ClearScrollRequest();
                clearScrollRequest.addScrollId(scrollId);
                client.clearScroll(clearScrollRequest, options);
                return Boolean.TRUE;
            } catch (Exception e) {
                log.error("EsDataOperation delete error.", e);
            }
            return Boolean.FALSE;
        }
    }
    package com.jinw.es;
    
    import cn.hutool.core.io.FileUtil;
    import com.alibaba.fastjson.JSON;
    import com.alibaba.fastjson.JSONObject;
    import lombok.extern.slf4j.Slf4j;
    import org.apache.commons.io.FileUtils;
    import org.apache.commons.lang3.StringUtils;
    import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest;
    import org.elasticsearch.action.support.master.AcknowledgedResponse;
    import org.elasticsearch.client.RequestOptions;
    import org.elasticsearch.client.RestHighLevelClient;
    import org.elasticsearch.client.indices.CreateIndexRequest;
    import org.elasticsearch.client.indices.GetIndexRequest;
    import org.elasticsearch.xcontent.XContentType;
    import org.springframework.stereotype.Service;
    
    import javax.annotation.Resource;
    import java.io.IOException;
    
    /**
     * 操作ES索引
     */
    @Slf4j
    @Service
    public class EsIndexOperation {
    
        @Resource
        private RestHighLevelClient restHighLevelClient;
        private final RequestOptions options = RequestOptions.DEFAULT;
    
        /**
         * 判断索引是否存在
         */
        public boolean checkIndex(String index) {
            try {
                return restHighLevelClient.indices().exists(new GetIndexRequest(index), options);
            } catch (Exception e) {
                log.error("EsIndexOperation checkIndex error.", e);
            }
            return Boolean.FALSE;
        }
    
        /**
         * 创建索引
         *
         * @param indexName         es索引名
         * @param esSettingFilePath es索引的alias、settings和mapping的配置文件
         */
        public boolean createIndex(String indexName, String esSettingFilePath) {
            String aliases = null;
            String mappings = null;
            String settings = null;
            if (StringUtils.isNotBlank(esSettingFilePath)) {
                try {
                    String fileContent = FileUtil.readUtf8String(esSettingFilePath);
                    if (StringUtils.isNotBlank(fileContent)) {
                        JSONObject jsonObject = JSON.parseObject(fileContent);
                        aliases = jsonObject.getString("aliases");
                        mappings = jsonObject.getString("mappings");
                        settings = jsonObject.getString("settings");
                    }
                } catch (Exception e) {
                    log.error("createIndex error.", e);
                    return false;
                }
            }
    
            if (checkIndex(indexName)) {
                log.error("createIndex indexName:[{}]已存在", indexName);
                return false;
            }
    
            CreateIndexRequest request = new CreateIndexRequest(indexName);
            if ((StringUtils.isNotBlank(aliases))) {
                request.aliases(aliases, XContentType.JSON);
            }
    
            if (StringUtils.isNotBlank(mappings)) {
                request.mapping(mappings, XContentType.JSON);
            }
    
            if (StringUtils.isNotBlank(settings)) {
                request.settings(settings, XContentType.JSON);
            }
    
            try {
                this.restHighLevelClient.indices().create(request, options);
                return true;
            } catch (IOException e) {
                log.error("EsIndexOperation createIndex error.", e);
                return false;
            }
        }
    
        /**
         * 删除索引
         */
        public boolean deleteIndex(String indexName) {
            try {
                if (checkIndex(indexName)) {
                    DeleteIndexRequest request = new DeleteIndexRequest(indexName);
                    AcknowledgedResponse response = restHighLevelClient.indices().delete(request, options);
                    return response.isAcknowledged();
                }
            } catch (Exception e) {
                log.error("EsIndexOperation deleteIndex error.", e);
            }
            return Boolean.FALSE;
        }
    }
    package com.jinw.es;
    
    import lombok.extern.slf4j.Slf4j;
    import org.elasticsearch.action.search.SearchRequest;
    import org.elasticsearch.action.search.SearchResponse;
    import org.elasticsearch.client.RequestOptions;
    import org.elasticsearch.client.RestHighLevelClient;
    import org.elasticsearch.client.core.CountRequest;
    import org.elasticsearch.client.core.CountResponse;
    import org.elasticsearch.search.SearchHit;
    import org.elasticsearch.search.builder.SearchSourceBuilder;
    import org.springframework.stereotype.Service;
    
    import javax.annotation.Resource;
    import java.util.ArrayList;
    import java.util.List;
    import java.util.Map;
    
    /**
     * 查询操作
     */
    @Slf4j
    @Service
    public class EsQueryOperation {
    
        @Resource
        private RestHighLevelClient client;
        private final RequestOptions options = RequestOptions.DEFAULT;
    
        /**
         * 查询总数
         */
        public Long count(String indexName) {
            CountRequest countRequest = new CountRequest(indexName);
            try {
                CountResponse countResponse = client.count(countRequest, options);
                return countResponse.getCount();
            } catch (Exception e) {
                log.error("EsQueryOperation count error.", e);
            }
            return 0L;
        }
    
    
        /**
         * 查询总数
         */
        public Long countByCondition(String indexName, SearchSourceBuilder sourceBuilder) {
            CountRequest countRequest = new CountRequest(indexName);
            countRequest.source(sourceBuilder);
            try {
                CountResponse countResponse = client.count(countRequest, options);
                return countResponse.getCount();
            } catch (Exception e) {
                log.error("EsQueryOperation count error.", e);
            }
            return 0L;
        }
    
        /**
         * 查询数据集
         */
        public List<Map<String, Object>> list(String indexName, SearchSourceBuilder sourceBuilder) {
            SearchRequest searchRequest = new SearchRequest(indexName);
            searchRequest.source(sourceBuilder);
            try {
                SearchResponse searchResp = client.search(searchRequest, options);
                List<Map<String, Object>> data = new ArrayList<>();
                SearchHit[] searchHitArr = searchResp.getHits().getHits();
                for (SearchHit searchHit : searchHitArr) {
                    Map<String, Object> temp = searchHit.getSourceAsMap();
                    temp.put("id", searchHit.getId());
                    data.add(temp);
                }
                return data;
            } catch (Exception e) {
                log.error("EsQueryOperation list error.", e);
            }
            return null;
        }
    
    
    }
    本文著作权归作者 [ admin ] 享有,未经作者书面授权,禁止转载,封面图片来源于 [ 互联网 ] ,本文仅供个人学习、研究和欣赏使用。如有异议,请联系博主及时处理。
    JavaSQLes
    取消回复

    发表留言
    回复

    首页邻居关于归档
    Copyright©2020-2026  All Rights Reserved.  Load:0.017 s
    京ICP备18019712号
    Theme by OneBlog V3.6.5
    夜间模式

    开源不易,请尊重作者版权,保留基本的版权信息。